[nltk] 01/02: Imported Upstream version 3.0.0b1
Daniel Stender
danstender-guest at moszumanska.debian.org
Mon Jul 28 19:02:01 UTC 2014
This is an automated email from the git hooks/post-receive script.
danstender-guest pushed a commit to branch master
in repository nltk.
commit 6a117b09de9d69443460931796bd4cb374579d01
Author: Daniel Stender <debian at danielstender.com>
Date: Sun Jul 27 18:43:23 2014 +0200
Imported Upstream version 3.0.0b1
---
.gitattributes | 1 +
.gitignore | 24 +
ChangeLog | 981 +++
INSTALL.txt | 6 +
LICENSE.txt | 13 +
MANIFEST.in | 6 +
Makefile | 62 +
README.md | 164 +
README.txt | 55 +
RELEASE-HOWTO | 69 +
emacs/doctest-mode.el | 1078 +++
emacs/psvn.el | 6225 +++++++++++++++
emacs/pycomplete.el | 36 +
emacs/pycomplete.py | 97 +
emacs/python-mode.el | 3922 ++++++++++
emacs/rst-mode.el | 698 ++
emacs/rst.el | 3431 +++++++++
examples/grammars/Makefile | 38 +
examples/grammars/basque_grammars/basque1.cfg | 11 +
examples/grammars/basque_grammars/basque1.fcfg | 22 +
examples/grammars/basque_grammars/basque1.pcfg | 21 +
examples/grammars/basque_grammars/basque1.regexp | 1 +
examples/grammars/basque_grammars/basque2.cfg | 10 +
examples/grammars/basque_grammars/basque2.fcfg | 26 +
examples/grammars/basque_grammars/basque2.pcfg | 4 +
examples/grammars/basque_grammars/basque2.regexp | 2 +
examples/grammars/basque_grammars/basque3.cfg | 4 +
examples/grammars/basque_grammars/basque3.fcfg | 36 +
examples/grammars/basque_grammars/basque3.regexp | 3 +
examples/grammars/basque_grammars/basque4.regexp | 3 +
examples/grammars/basque_grammars/basque5.regexp | 5 +
examples/grammars/book_grammars/background.fol | 22 +
examples/grammars/book_grammars/discourse.fcfg | 125 +
examples/grammars/book_grammars/drt.fcfg | 82 +
examples/grammars/book_grammars/feat0.fcfg | 49 +
examples/grammars/book_grammars/feat1.fcfg | 55 +
examples/grammars/book_grammars/german.fcfg | 86 +
examples/grammars/book_grammars/simple-sem.fcfg | 65 +
examples/grammars/book_grammars/sql0.fcfg | 32 +
examples/grammars/book_grammars/sql1.fcfg | 44 +
examples/grammars/book_grammars/storage.fcfg | 54 +
examples/grammars/sample_grammars/background0.fol | 19 +
examples/grammars/sample_grammars/bindop.fcfg | 42 +
examples/grammars/sample_grammars/chat80.fcfg | 95 +
examples/grammars/sample_grammars/chat_pnames.fcfg | 545 ++
examples/grammars/sample_grammars/dep_test2.dep | 4 +
examples/grammars/sample_grammars/drt_glue.semtype | 61 +
.../sample_grammars/drt_glue_event.semtype | 62 +
examples/grammars/sample_grammars/event.fcfg | 72 +
examples/grammars/sample_grammars/glue.semtype | 59 +
.../grammars/sample_grammars/glue_event.semtype | 66 +
examples/grammars/sample_grammars/glue_train.conll | 27 +
.../grammars/sample_grammars/gluesemantics.fcfg | 131 +
examples/grammars/sample_grammars/hole.fcfg | 23 +
examples/grammars/sample_grammars/np.fcfg | 12 +
examples/grammars/sample_grammars/sem0.fcfg | 14 +
examples/grammars/sample_grammars/sem1.fcfg | 19 +
examples/grammars/sample_grammars/sem2.fcfg | 68 +
examples/grammars/sample_grammars/sql.fcfg | 27 +
examples/grammars/sample_grammars/toy.cfg | 9 +
examples/grammars/sample_grammars/valuation1.val | 15 +
examples/grammars/spanish_grammars/spanish1.cfg | 10 +
examples/grammars/spanish_grammars/spanish1.fcfg | 32 +
examples/grammars/spanish_grammars/spanish1.pcfg | 9 +
examples/grammars/spanish_grammars/spanish1.regexp | 2 +
examples/grammars/spanish_grammars/spanish2.cfg | 8 +
examples/grammars/spanish_grammars/spanish2.fcfg | 17 +
examples/grammars/spanish_grammars/spanish2.pcfg | 4 +
examples/grammars/spanish_grammars/spanish2.regexp | 4 +
examples/grammars/spanish_grammars/spanish3.cfg | 4 +
examples/grammars/spanish_grammars/spanish3.regexp | 4 +
examples/grammars/spanish_grammars/spanish4.regexp | 5 +
examples/grammars/spanish_grammars/spanish5.regexp | 6 +
examples/school/README | 3 +
examples/school/categories.py | 222 +
examples/school/count.py | 16 +
examples/school/generate.py | 14 +
examples/school/parse1.py | 14 +
examples/school/parse2.py | 17 +
examples/school/parse3.py | 22 +
examples/school/parser.py | 18 +
examples/school/search.py | 6 +
examples/school/words.py | 105 +
examples/semantics/chat.db | Bin 0 -> 274432 bytes
examples/semantics/chat80.cfg | 96 +
examples/semantics/chat_pnames.cfg | 545 ++
examples/semantics/chat_sentences | 17 +
examples/semantics/demo_sentences | 14 +
examples/semantics/model0.py | 44 +
examples/semantics/model1.py | 27 +
examples/semantics/sem0.cfg | 14 +
examples/semantics/sem1.cfg | 18 +
examples/semantics/sem2.cfg | 68 +
examples/semantics/sem3.cfg | 17 +
examples/semantics/syn2sem.py | 118 +
jenkins-job-config.xml | 342 +
jenkins.sh | 48 +
nltk/VERSION | 1 +
nltk/__init__.py | 173 +
nltk/align/__init__.py | 19 +
nltk/align/api.py | 298 +
nltk/align/bleu.py | 148 +
nltk/align/gale_church.py | 238 +
nltk/align/ibm1.py | 135 +
nltk/align/ibm2.py | 200 +
nltk/align/ibm3.py | 403 +
nltk/align/util.py | 7 +
nltk/app/__init__.py | 53 +
nltk/app/chartparser_app.py | 2276 ++++++
nltk/app/chunkparser_app.py | 1263 ++++
nltk/app/collocations_app.py | 348 +
nltk/app/concordance_app.py | 570 ++
nltk/app/nemo_app.py | 155 +
nltk/app/rdparser_app.py | 894 +++
nltk/app/srparser_app.py | 809 ++
nltk/app/wordfreq_app.py | 32 +
nltk/app/wordnet_app.py | 968 +++
nltk/book.py | 89 +
nltk/ccg/__init__.py | 22 +
nltk/ccg/api.py | 335 +
nltk/ccg/chart.py | 361 +
nltk/ccg/combinator.py | 308 +
nltk/ccg/lexicon.py | 245 +
nltk/chat/__init__.py | 49 +
nltk/chat/eliza.py | 244 +
nltk/chat/iesha.py | 140 +
nltk/chat/rude.py | 92 +
nltk/chat/suntsu.py | 117 +
nltk/chat/util.py | 120 +
nltk/chat/zen.py | 282 +
nltk/chunk/__init__.py | 190 +
nltk/chunk/api.py | 51 +
nltk/chunk/named_entity.py | 331 +
nltk/chunk/regexp.py | 1384 ++++
nltk/chunk/util.py | 595 ++
nltk/classify/__init__.py | 96 +
nltk/classify/api.py | 193 +
nltk/classify/decisiontree.py | 295 +
nltk/classify/maxent.py | 1484 ++++
nltk/classify/megam.py | 179 +
nltk/classify/naivebayes.py | 240 +
nltk/classify/positivenaivebayes.py | 177 +
nltk/classify/rte_classify.py | 183 +
nltk/classify/scikitlearn.py | 152 +
nltk/classify/svm.py | 15 +
nltk/classify/tadm.py | 112 +
nltk/classify/util.py | 311 +
nltk/classify/weka.py | 343 +
nltk/cluster/__init__.py | 86 +
nltk/cluster/api.py | 70 +
nltk/cluster/em.py | 247 +
nltk/cluster/gaac.py | 168 +
nltk/cluster/kmeans.py | 221 +
nltk/cluster/util.py | 290 +
nltk/collocations.py | 366 +
nltk/compat.py | 499 ++
nltk/corpus/__init__.py | 282 +
nltk/corpus/europarl_raw.py | 44 +
nltk/corpus/reader/__init__.py | 131 +
nltk/corpus/reader/aligned.py | 114 +
nltk/corpus/reader/api.py | 434 ++
nltk/corpus/reader/bnc.py | 252 +
nltk/corpus/reader/bracket_parse.py | 184 +
nltk/corpus/reader/chasen.py | 139 +
nltk/corpus/reader/childes.py | 482 ++
nltk/corpus/reader/chunked.py | 210 +
nltk/corpus/reader/cmudict.py | 95 +
nltk/corpus/reader/conll.py | 521 ++
nltk/corpus/reader/dependency.py | 101 +
nltk/corpus/reader/framenet.py | 2062 +++++
nltk/corpus/reader/ieer.py | 111 +
nltk/corpus/reader/indian.py | 88 +
nltk/corpus/reader/ipipan.py | 331 +
nltk/corpus/reader/knbc.py | 164 +
nltk/corpus/reader/lin.py | 156 +
nltk/corpus/reader/nombank.py | 421 ++
nltk/corpus/reader/nps_chat.py | 73 +
nltk/corpus/reader/pl196x.py | 278 +
nltk/corpus/reader/plaintext.py | 228 +
nltk/corpus/reader/ppattach.py | 95 +
nltk/corpus/reader/propbank.py | 481 ++
nltk/corpus/reader/rte.py | 146 +
nltk/corpus/reader/semcor.py | 256 +
nltk/corpus/reader/senseval.py | 201 +
nltk/corpus/reader/sentiwordnet.py | 134 +
nltk/corpus/reader/sinica_treebank.py | 75 +
nltk/corpus/reader/string_category.py | 61 +
nltk/corpus/reader/switchboard.py | 119 +
nltk/corpus/reader/tagged.py | 294 +
nltk/corpus/reader/timit.py | 450 ++
nltk/corpus/reader/toolbox.py | 68 +
nltk/corpus/reader/udhr.py | 80 +
nltk/corpus/reader/util.py | 797 ++
nltk/corpus/reader/verbnet.py | 396 +
nltk/corpus/reader/wordlist.py | 37 +
nltk/corpus/reader/wordnet.py | 1960 +++++
nltk/corpus/reader/xmldocs.py | 387 +
nltk/corpus/reader/ycoe.py | 242 +
nltk/corpus/util.py | 127 +
nltk/data.py | 1404 ++++
nltk/decorators.py | 217 +
nltk/downloader.py | 2264 ++++++
nltk/draw/__init__.py | 27 +
nltk/draw/cfg.py | 777 ++
nltk/draw/dispersion.py | 58 +
nltk/draw/table.py | 1090 +++
nltk/draw/tree.py | 966 +++
nltk/draw/util.py | 2356 ++++++
nltk/examples/__init__.py | 0
nltk/examples/pt.py | 49 +
nltk/featstruct.py | 2500 ++++++
nltk/grammar.py | 1515 ++++
nltk/help.py | 56 +
nltk/inference/__init__.py | 20 +
nltk/inference/api.py | 589 ++
nltk/inference/discourse.py | 608 ++
nltk/inference/mace.py | 311 +
nltk/inference/nonmonotonic.py | 509 ++
nltk/inference/prover9.py | 431 ++
nltk/inference/resolution.py | 687 ++
nltk/inference/tableau.py | 607 ++
nltk/internals.py | 889 +++
nltk/jsontags.py | 63 +
nltk/lazyimport.py | 142 +
nltk/metrics/__init__.py | 28 +
nltk/metrics/agreement.py | 401 +
nltk/metrics/artstein_poesio_example.txt | 200 +
nltk/metrics/association.py | 409 +
nltk/metrics/confusionmatrix.py | 206 +
nltk/metrics/distance.py | 192 +
nltk/metrics/scores.py | 228 +
nltk/metrics/segmentation.py | 235 +
nltk/metrics/spearman.py | 68 +
nltk/misc/__init__.py | 11 +
nltk/misc/babelfish.py | 10 +
nltk/misc/chomsky.py | 133 +
nltk/misc/minimalset.py | 83 +
nltk/misc/sort.py | 157 +
nltk/misc/wordfinder.py | 129 +
nltk/parse/__init__.py | 77 +
nltk/parse/api.py | 66 +
nltk/parse/broker_test.cfg | 10 +
nltk/parse/chart.py | 1681 +++++
nltk/parse/dependencygraph.py | 516 ++
nltk/parse/earleychart.py | 451 ++
nltk/parse/featurechart.py | 576 ++
nltk/parse/generate.py | 76 +
nltk/parse/malt.py | 283 +
nltk/parse/nonprojectivedependencyparser.py | 640 ++
nltk/parse/pchart.py | 485 ++
nltk/parse/projectivedependencyparser.py | 540 ++
nltk/parse/recursivedescent.py | 656 ++
nltk/parse/shiftreduce.py | 459 ++
nltk/parse/stanford.py | 255 +
nltk/parse/test.cfg | 10 +
nltk/parse/util.py | 168 +
nltk/parse/viterbi.py | 401 +
nltk/probability.py | 2225 ++++++
nltk/sem/__init__.py | 61 +
nltk/sem/boxer.py | 1226 +++
nltk/sem/chat80.py | 781 ++
nltk/sem/cooper_storage.py | 118 +
nltk/sem/drt.py | 1254 ++++
nltk/sem/drt_glue_demo.py | 483 ++
nltk/sem/evaluate.py | 712 ++
nltk/sem/glue.py | 661 ++
nltk/sem/hole.py | 374 +
nltk/sem/lfg.py | 199 +
nltk/sem/linearlogic.py | 449 ++
nltk/sem/logic.py | 1918 +++++
nltk/sem/relextract.py | 483 ++
nltk/sem/skolemize.py | 101 +
nltk/sem/util.py | 317 +
nltk/stem/__init__.py | 36 +
nltk/stem/api.py | 28 +
nltk/stem/isri.py | 348 +
nltk/stem/lancaster.py | 313 +
nltk/stem/porter.py | 697 ++
nltk/stem/regexp.py | 64 +
nltk/stem/rslp.py | 145 +
nltk/stem/snowball.py | 3727 +++++++++
nltk/stem/wordnet.py | 54 +
nltk/tag/__init__.py | 114 +
nltk/tag/api.py | 84 +
nltk/tag/brill.py | 421 ++
nltk/tag/brill_trainer.py | 630 ++
nltk/tag/brill_trainer_orig.py | 414 +
nltk/tag/hmm.py | 1280 ++++
nltk/tag/hunpos.py | 134 +
nltk/tag/mapping.py | 100 +
nltk/tag/senna.py | 286 +
nltk/tag/sequential.py | 746 ++
nltk/tag/stanford.py | 173 +
nltk/tag/tnt.py | 607 ++
nltk/tag/util.py | 75 +
nltk/tbl/__init__.py | 28 +
nltk/tbl/api.py | 1 +
nltk/tbl/demo.py | 367 +
nltk/tbl/erroranalysis.py | 39 +
nltk/tbl/feature.py | 265 +
nltk/tbl/rule.py | 314 +
nltk/tbl/template.py | 311 +
nltk/test/FX8.xml | 17 +
nltk/test/Makefile | 23 +
nltk/test/__init__.py | 18 +
nltk/test/align.doctest | 234 +
nltk/test/align_fixt.py | 4 +
nltk/test/all.py | 23 +
nltk/test/bnc.doctest | 55 +
nltk/test/ccg.doctest | 277 +
nltk/test/chat80.doctest | 234 +
nltk/test/childes.doctest | 184 +
nltk/test/childes_fixt.py | 13 +
nltk/test/chunk.doctest | 373 +
nltk/test/classify.doctest | 185 +
nltk/test/classify_fixt.py | 10 +
nltk/test/collocations.doctest | 276 +
nltk/test/compat.doctest | 138 +
nltk/test/compat_fixt.py | 8 +
nltk/test/corpus.doctest | 2048 +++++
nltk/test/corpus_fixt.py | 4 +
nltk/test/data.doctest | 374 +
nltk/test/dependency.doctest | 119 +
nltk/test/discourse.doctest | 546 ++
nltk/test/discourse_fixt.py | 13 +
nltk/test/doctest_nose_plugin.py | 156 +
nltk/test/drt.doctest | 517 ++
nltk/test/featgram.doctest | 607 ++
nltk/test/featstruct.doctest | 1230 +++
nltk/test/floresta.txt | 7924 ++++++++++++++++++++
nltk/test/framenet.doctest | 239 +
nltk/test/generate.doctest | 67 +
nltk/test/gluesemantics.doctest | 381 +
nltk/test/gluesemantics_malt.doctest | 68 +
nltk/test/gluesemantics_malt_fixt.py | 11 +
nltk/test/grammar.doctest | 48 +
nltk/test/grammartestsuites.doctest | 109 +
nltk/test/index.doctest | 100 +
nltk/test/inference.doctest | 533 ++
nltk/test/inference_fixt.py | 11 +
nltk/test/internals.doctest | 140 +
nltk/test/japanese.doctest | 48 +
nltk/test/logic.doctest | 1098 +++
nltk/test/metrics.doctest | 270 +
nltk/test/misc.doctest | 118 +
nltk/test/nonmonotonic.doctest | 286 +
nltk/test/nonmonotonic_fixt.py | 11 +
nltk/test/onto1.fol | 6 +
nltk/test/parse.doctest | 884 +++
nltk/test/portuguese.doctest_latin1 | 300 +
nltk/test/portuguese_en.doctest | 565 ++
nltk/test/portuguese_en_fixt.py | 11 +
nltk/test/probability.doctest | 266 +
nltk/test/probability_fixt.py | 12 +
nltk/test/propbank.doctest | 176 +
nltk/test/relextract.doctest | 263 +
nltk/test/resolution.doctest | 221 +
nltk/test/runtests.py | 64 +
nltk/test/segmentation_fixt.py | 11 +
nltk/test/sem3.cfg | 14 +
nltk/test/semantics.doctest | 665 ++
nltk/test/semantics_fixt.py | 7 +
nltk/test/sentiwordnet.doctest | 39 +
nltk/test/simple.doctest | 85 +
nltk/test/stem.doctest | 81 +
nltk/test/tag.doctest | 22 +
nltk/test/tokenize.doctest | 102 +
nltk/test/toolbox.doctest | 307 +
nltk/test/toy.cfg | 9 +
nltk/test/tree.doctest | 1077 +++
nltk/test/treetransforms.doctest | 156 +
nltk/test/unit/__init__.py | 0
nltk/test/unit/test_2x_compat.py | 26 +
nltk/test/unit/test_classify.py | 53 +
nltk/test/unit/test_collocations.py | 79 +
nltk/test/unit/test_corpora.py | 181 +
nltk/test/unit/test_corpus_views.py | 45 +
nltk/test/unit/test_hmm.py | 88 +
nltk/test/unit/test_naivebayes.py | 26 +
.../unit/test_seekable_unicode_stream_reader.py | 124 +
nltk/test/unit/test_stem.py | 27 +
nltk/test/unit/test_tag.py | 20 +
nltk/test/unit/utils.py | 42 +
nltk/test/util.doctest | 48 +
nltk/test/wordnet.doctest | 580 ++
nltk/test/wordnet_fixt.py | 6 +
nltk/test/wordnet_lch.doctest | 53 +
nltk/test/wsd.doctest | 52 +
nltk/text.py | 610 ++
nltk/tokenize/__init__.py | 98 +
nltk/tokenize/api.py | 78 +
nltk/tokenize/punkt.py | 1648 ++++
nltk/tokenize/regexp.py | 208 +
nltk/tokenize/sexpr.py | 145 +
nltk/tokenize/simple.py | 139 +
nltk/tokenize/stanford.py | 106 +
nltk/tokenize/texttiling.py | 454 ++
nltk/tokenize/treebank.py | 104 +
nltk/tokenize/util.py | 93 +
nltk/toolbox.py | 497 ++
nltk/tree.py | 1585 ++++
nltk/treetransforms.py | 309 +
nltk/util.py | 1100 +++
nltk/wsd.py | 66 +
papers/acl-02/.cvsignore | 7 +
papers/acl-02/Makefile | 62 +
papers/acl-02/acl-02.tex | 705 ++
papers/acl-02/acl.bst | 1322 ++++
papers/acl-02/acl2002.sty | 340 +
papers/acl-02/chartparse.eps.gz | Bin 0 -> 26079 bytes
papers/acl-02/contest.ps.gz | Bin 0 -> 7096 bytes
papers/acl-02/nltk.bib | 154 +
papers/acl-04/.cvsignore | 12 +
papers/acl-04/Makefile | 45 +
papers/acl-04/acl-04.tex | 508 ++
papers/acl-04/acl.bst | 1323 ++++
papers/acl-04/acl04.sty | 361 +
papers/acl-04/chart-matrix.gif | Bin 0 -> 12520 bytes
papers/acl-04/chart.eps.gz | Bin 0 -> 46621 bytes
papers/acl-04/nltk.bib | 50 +
papers/acl-06/acl-06.tex | 405 +
papers/acl-06/acl.bst | 1322 ++++
papers/acl-06/colacl06.sty | 368 +
papers/acl-06/rdparser.eps.gz | Bin 0 -> 17440 bytes
papers/acl-06/srparser.eps.gz | Bin 0 -> 23349 bytes
papers/acl-08/acl-08.bib | 204 +
papers/acl-08/acl-08.tex | 749 ++
papers/acl-08/acl08.sty | 344 +
papers/acl-08/grammar1.py | 22 +
papers/acl-08/grammar2.py | 22 +
papers/acl-08/police.py | 19 +
papers/altw-06/acl.bst | 1322 ++++
papers/altw-06/altw-06.bib | 53 +
papers/altw-06/altw-06.tex | 864 +++
papers/altw-06/colacl06.sty | 368 +
papers/icon-05/acl.bst | 1322 ++++
papers/icon-05/acl2005.sty | 338 +
papers/icon-05/icon-05.tex | 657 ++
papers/iwcs-08/drs.png | Bin 0 -> 9656 bytes
papers/iwcs-08/garrette-klein.tar.gz | Bin 0 -> 54079 bytes
papers/iwcs-08/iwcs.doctest | 169 +
papers/iwcs-08/lingmacros.sty | 262 +
papers/iwcs-08/modules.graffle | Bin 0 -> 2680 bytes
papers/iwcs-08/modules.pdf | Bin 0 -> 39925 bytes
papers/iwcs-08/nltk_iwcs_09.bib | 72 +
papers/iwcs-08/nltk_iwcs_09.tex | 708 ++
pip-req.txt | 8 +
setup.cfg | 0
setup.py | 82 +
tools/find_deprecated.py | 226 +
tools/global_replace.py | 53 +
tools/nltk_term_index.py | 102 +
tools/nltk_term_index.stoplist | 106 +
tools/run_doctests.py | 16 +
tools/svnmime.py | 50 +
tox.ini | 101 +
web/Makefile | 161 +
web/api/nltk.rst | 142 +
web/conf.py | 246 +
web/contribute.rst | 37 +
web/data.rst | 49 +
web/dev/jenkins.rst | 113 +
web/dev/local_testing.rst | 167 +
web/dev/python3porting.rst | 269 +
web/images/book.gif | Bin 0 -> 7170 bytes
web/images/tree.gif | Bin 0 -> 14369 bytes
web/index.rst | 88 +
web/install.rst | 27 +
web/news.rst | 236 +
469 files changed, 155004 insertions(+)
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..176a458
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1 @@
+* text=auto
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1572169
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,24 @@
+*.pyc
+*.pyo
+.tox
+
+*.class
+*.jar
+
+*.egg
+build/
+dist/
+nltk.egg-info/
+web/_build
+
+# generated by tests
+*.errs
+.noseids
+.coverage
+
+# editor temporary files
+*.*.sw[op]
+.idea
+
+# git mergetools backups
+*.orig
diff --git a/ChangeLog b/ChangeLog
new file mode 100644
index 0000000..84c98ae
--- /dev/null
+++ b/ChangeLog
@@ -0,0 +1,981 @@
+Version 3.0b1 2013-07-11
+* Added SentiWordNet corpus and corpus reader
+* Fixed support for 10-column dependency file format
+* Changed Tree initialization to use fromstring
+
+Thanks to the following contributors to 3.0b1: Mark Amery, Ivan
+Barria, Ingolf Becker, Francis Bond, Lars Buitinck, Arthur Darcet,
+Michelle Fullwood, Dan Garrette, Dougal Graham, Dan Garrette, Dougal
+Graham, Tyler Hartley, Ofer Helman, Bruce Hill, Marcus Huderle, Nancy
+Ide, Nick Johnson, Angelos Katharopoulos, Ewan Klein, Mikhail Korobov,
+Chris Liechti, Peter Ljunglof, Joseph Lynch, Haejoong Lee, Peter
+Ljunglöf, Dean Malmgren, Rob Malouf, Thorsten Marek, Dmitrijs
+Milajevs, Shari A’aidil Nasruddin, Lance Nathan, Joel Nothman, Alireza
+Nourian, Alexander Oleynikov, Ted Pedersen, Jacob Perkins, Will
+Roberts, Alex Rudnick, Nathan Schneider, Geraldine Sim Wei Ying, Lynn
+Soe, Liling Tan, Louis Tiao, Marcus Uneson, Yu Usami, Steven Xu, Zhe
+Wang, Chuck Wooters, isnowfy, onesandzeros, pquentin, wvanlint
+
+Version 3.0a4 2013-05-25
+* IBM Models 1-3, BLEU, Gale-Church aligner
+* Lesk algorithm for WSD
+* Open Multilingual WordNet
+* New implementation of Brill Tagger
+* Extend BNCCorpusReader to parse the whole BNC
+* MASC Tagged Corpus and corpus reader
+* Interface to Stanford Parser
+* Code speed-ups and clean-ups
+* API standardisation, including fromstring method for many objects
+* Improved regression testing setup
+* Removed PyYAML dependency
+
+Thanks to the following contributors to 3.0a4:
+Ivan Barria, Ingolf Becker, Francis Bond, Arthur Darcet, Dan Garrette,
+Ofer Helman, Dougal Graham, Nancy Ide, Ewan Klein, Mikhail Korobov,
+Chris Liechti, Peter Ljunglof, Joseph Lynch, Rob Malouf, Thorsten Marek,
+Dmitrijs Milajevs, Shari A’aidil Nasruddin, Lance Nathan, Joel Nothman,
+Jacob Perkins, Lynn Soe, Liling Tan, Louis Tiao, Marcus Uneson, Steven Xu,
+Geraldine Sim Wei Ying
+
+Version 3.0a3 2013-11-02
+* support for FrameNet contributed by Chuck Wooters
+* support for Universal Declaration of Human Rights Corpus (udhr2)
+* major API changes:
+ - Tree.node -> Tree.label() / Tree.set_label()
+ - Chunk parser: top_node -> root_label; chunk_node -> chunk_label
+ - WordNet properties are now access methods, e.g. Synset.definition -> Synset.definition()
+ - relextract: show_raw_rtuple() -> rtuple(), show_clause() -> clause()
+* bugfix in texttiling
+* replaced simplify_tags with support for universal tagset (simplify_tags=True -> tagset='universal')
+* Punkt default behavior changed to realign sentence boundaries after trailing parenthesis and quotes
+* deprecated classify.svm (use scikit-learn instead)
+* various efficiency improvements
+
+Thanks to the following contributors to 3.0a3:
+Lars Buitinck, Marcus Huderle, Nick Johnson, Dougal Graham, Ewan Klein,
+Mikhail Korobov, Haejoong Lee, Peter Ljunglöf, Dean Malmgren, Lance Nathan,
+Alexander Oleynikov, Nathan Schneider, Chuck Wooters, Yu Usami, Steven Xu,
+pquentin, wvanlint
+
+Version 3.0a2 2013-07-12
+* speed improvements in word_tokenize, GAAClusterer, TnT tagger, Baum Welch, HMM tagger
+* small improvements in collocation finders, probability, modelling, Porter Stemmer
+* bugfix in lowest common hypernyn calculation (used in path similarity measures)
+* code cleanups, docstring cleanups, demo fixes
+
+Thanks to the following contributors to 3.0a2:
+Mark Amery, Lars Buitinck, Michelle Fullwood, Dan Garrette, Dougal Graham,
+Tyler Hartley, Bruce Hill, Angelos Katharopoulos, Mikhail Korobov,
+Rob Malouf, Joel Nothman, Ted Pedersen, Will Roberts, Alex Rudnick,
+Steven Xu, isnowfy, onesandzeros
+
+Version 3.0a1 2013-02-14
+* reinstated tkinter support (Haejoong Lee)
+
+Version 3.0a0 2013-01-14
+* alpha release of first version to support Python 2.6, 2.7, and 3.
+
+Version 2.0.4 2012-11-07
+* minor bugfix (removed numpy dependency)
+
+Version 2.0.3 2012-09-24
+
+* fixed corpus/reader/util.py to support Python 2.5
+* make MaltParser safe to use in parallel
+* fixed bug in inter-annotator agreement
+* updates to various doctests (nltk/test)
+* minor bugfixes
+
+Thanks to the following contributors to 2.0.3:
+Robin Cooper, Pablo Duboue, Christian Federmann, Dan Garrette, Ewan Klein,
+Pierre-François Laquerre, Max Leonov, Peter Ljunglöf, Nitin Madnani, Ceri Stagg
+
+Version 2.0.2 2012-07-05
+
+* improvements to PropBank, NomBank, and SemCor corpus readers
+* interface to full Penn Treebank Corpus V3 (corpus.ptb)
+* made wordnet.lemmas case-insensitive
+* more flexible padding in model.ngram
+* minor bugfixes and documentation enhancements
+* better support for automated testing
+
+Thanks to the following contributors to 2.0.2:
+Daniel Blanchard, Mikhail Korobov, Nitin Madnani, Duncan McGreggor,
+Morten Neergaard, Nathan Schneider, Rico Sennrich.
+
+Version 2.0.1 2012-05-15
+
+* moved NLTK to GitHub: http://github.com/nltk
+* set up integration testing: https://jenkins.shiningpanda.com/nltk/ (Morten Neergaard)
+* converted documentation to Sphinx format: http://nltk.github.com/api/nltk.html
+* dozens of minor enhancements and bugfixes: https://github.com/nltk/nltk/commits/
+* dozens of fixes for conformance with PEP-8
+* dozens of fixes to ensure operation with Python 2.5
+* added interface to Lin's Dependency Thesaurus (Dan Blanchard)
+* added interface to scikit-learn classifiers (Lars Buitinck)
+* added segmentation evaluation measures (David Doukhan)
+
+Thanks to the following contributors to 2.0.1 (since 2.0b9, July 2010):
+Rami Al-Rfou', Yonatan Becker, Steven Bethard, Daniel Blanchard, Lars
+Buitinck, David Coles, Lucas Cooper, David Doukhan, Dan Garrette,
+Masato Hagiwara, Michael Hansen, Michael Heilman, Rebecca Ingram,
+Sudharshan Kaushik, Mikhail Korobov, Peter Ljunglof, Nitin Madnani,
+Rob Malouf, Tomonori Nagano, Morten Neergaard, David Nemeskey,
+Joel Nothman, Jacob Perkins, Alessandro Presta, Alex Rudnick,
+Nathan Schneider, Stefano Lattarini, Peter Stahl, Jason Yoder
+
+Version 2.0.1 (rc1) 2011-04-11
+
+NLTK:
+* added interface to the Stanford POS Tagger
+* updates to sem.Boxer, sem.drt.DRS
+* allow unicode strings in grammars
+* allow non-string features in classifiers
+* modifications to HunposTagger
+* issues with DRS printing
+* fixed bigram collocation finder for window_size > 2
+* doctest paths no longer presume unix-style pathname separators
+* fixed issue with NLTK's tokenize module colliding with the Python tokenize module
+* fixed issue with stemming Unicode strings
+* changed ViterbiParser.nbest_parse to parse
+* ChaSen and KNBC Japanese corpus readers
+* preserve case in concordance display
+* fixed bug in simplification of Brown tags
+* a version of IBM Model 1 as described in Koehn 2010
+* new class AlignedSent for aligned sentence data and evaluation metrics
+* new nltk.util.set_proxy to allow easy configuration of HTTP proxy
+* improvements to downloader user interface to catch URL and HTTP errors
+* added CHILDES corpus reader
+* created special exception hierarchy for Prover9 errors
+* significant changes to the underlying code of the boxer interface
+* path-based wordnet similarity metrics use a fake root node for verbs, following the Perl version
+* added ability to handle multi-sentence discourses in Boxer
+* added the 'english' Snowball stemmer
+* simplifications and corrections of Earley Chart Parser rules
+* several changes to the feature chart parsers for correct unification
+* bugfixes: FreqDist.plot, FreqDist.max, NgramModel.entropy, CategorizedCorpusReader, DecisionTreeClassifier
+* removal of Python >2.4 language features for 2.4 compatibility
+* removal of deprecated functions and associated warnings
+* added semantic domains to wordnet corpus reader
+* changed wordnet similarity functions to include instance hyponyms
+* updated to use latest version of Boxer
+
+Data:
+* JEITA Public Morphologically Tagged Corpus (in ChaSen format)
+* KNB Annotated corpus of Japanese blog posts
+* Fixed some minor bugs in alvey.fcfg, and added number of parse trees in alvey_sentences.txt
+* added more comtrans data
+
+Documentation:
+* minor fixes to documentation
+* NLTK Japanese book (chapter 12) by Masato Hagiwara
+
+NLTK-Contrib:
+* Viethen and Dale referring expression algorithms
+
+
+Version 2.0b9 2010-07-25
+
+NLTK:
+* many code and documentation cleanups
+* Added port of Snowball stemmers
+* Fixed loading of pickled tokenizers (issue 556)
+* DecisionTreeClassifier now handles unknown features (issue 570)
+* Added error messages to LogicParser
+* Replaced max_models with end_size to prevent Mace from hanging
+* Added interface to Boxer
+* Added nltk.corpus.semcor to give access to SemCor 3.0 corpus (issue 530)
+* Added support for integer- and float-valued features in maxent classifiers
+* Permit NgramModels to be pickled
+* Added Sourced Strings (see test/sourcedstring.doctest for details)
+* Fixed bugs in with Good-Turing and Simple Good-Turing Estimation (issue 26)
+* Add support for span tokenization, aka standoff annotation of segmentation (incl Punkt)
+* allow unicode nodes in Tree.productions()
+* Fixed WordNet's morphy to be consistent with the original implementation,
+ taking the shortest returned form instead of an arbitrary one (issues 427, 487)
+* Fixed bug in MaxentClassifier
+* Accepted bugfixes for YCOE corpus reader (issue 435)
+* Added test to _cumulative_frequencies() to correctly handle the case when no arguments are supplied
+* Added a TaggerI interface to the HunPos open-source tagger
+* Return 0, not None, when no count is present for a lemma in WordNet
+* fixed pretty-printing of unicode leaves
+* More efficient calculation of the leftcorner relation for left corner parsers
+* Added two functions for graph calculations: transitive closure and inversion.
+* FreqDist.pop() and FreqDist.popitems() now invalidate the caches (issue 511)
+
+Data:
+* Added SemCor 3.0 corpus (Brown Corpus tagged with WordNet synsets)
+* Added LanguageID corpus (trigram counts for 451 languages)
+* Added grammar for a^n b^n c^n
+
+NLTK-Contrib:
+* minor updates
+
+Thanks to the following contributors to 2.0b9:
+
+Steven Bethard, Francis Bond, Dmitry Chichkov, Liang Dong, Dan Garrette,
+Simon Greenhill, Bjorn Maeland, Rob Malouf, Joel Nothman, Jacob Perkins,
+Alberto Planas, Alex Rudnick, Geoffrey Sampson, Kevin Scannell, Richard Sproat
+
+
+Version 2.0b8 2010-02-05
+
+NLTK:
+* fixed copyright and license statements
+* removed PyYAML, and added dependency to installers and download instructions
+* updated to LogicParser, DRT (Dan Garrette)
+* WordNet similarity metrics return None instead of -1 when
+ they fail to find a path (Steve Bethard)
+* shortest_path_distance uses instance hypernyms (Jordan Boyd-Graber)
+* clean_html improved (Bjorn Maeland)
+* batch_parse, batch_interpret and batch_evaluate functions allow
+ grammar or grammar filename as argument
+* more Portuguese examples (portuguese_en.doctest, examples/pt.py)
+
+NLTK-Contrib:
+* Aligner implementations (Christopher Crowner, Torsten Marek)
+* ScriptTranscriber package (Richard Sproat and Kristy Hollingshead)
+
+Book:
+* updates for second printing, correcting errata
+ http://nltk.googlecode.com/svn/trunk/nltk/doc/book/errata.txt
+
+Data:
+* added Europarl sample, with 10 docs for each of 11 langs (Nitin Madnani)
+* added SMULTRON sample corpus (Torsten Marek, Martin Volk)
+
+
+Version 2.0b7 2009-11-09
+
+NLTK:
+* minor bugfixes and enhancements: data loader, inference package, FreqDist, Punkt
+* added Portuguese example module, similar to nltk.book for English (examples/pt.py)
+* added all_lemma_names() method to WordNet corpus reader
+* added update() and __add__() extensions to FreqDist (enhances alignment with Python 3.0 counters)
+* reimplemented clean_html
+* added test-suite runner for automatic/manual regression testing
+
+NLTK-Data:
+* updated Punkt models for sentence segmentation
+* added corpus of the works of Machado de Assis (Brazilian Portuguese)
+
+Book:
+* Added translation of preface into Portuguese, contributed by Tiago Tresoldi.
+
+Version 2.0b6 2009-09-20
+
+NLTK:
+* minor fixes for Python 2.4 compatibility
+* added words() method to XML corpus reader
+* minor bugfixes and code clean-ups
+* fixed downloader to put data in %APPDATA% on Windows
+
+Data:
+* Updated Punkt models
+* Fixed utf8 encoding issues with UDHR and Stopwords Corpora
+* Renamed CoNLL "cat" files to "esp" (different language)
+* Added Alvey NLT feature-based grammar
+* Added Polish PL196x corpus
+
+Version 2.0b5 2009-07-19
+
+NLTK:
+* minor bugfixes (incl FreqDist, Python eggs)
+* added reader for Europarl Corpora (contributed by Nitin Madnani)
+* added reader for IPI PAN Polish Corpus (contributed by Konrad Goluchowski)
+* fixed data.py so that it doesn't generate a warning for Windows Python 2.6
+
+NLTK-Contrib:
+* updated Praat reader (contributed by Margaret Mitchell)
+
+Version 2.0b4 2009-07-10
+
+NLTK:
+* switched to Apache License, Version 2.0
+* minor bugfixes in semantics and inference packages
+* support for Python eggs
+* fixed stale regression tests
+
+Data:
+* added NomBank 1.0
+* uppercased feature names in some grammars
+
+Version 2.0b3 2009-06-25
+
+NLTK:
+* several bugfixes
+* added nombank corpus reader (Paul Bedaride)
+
+Version 2.0b2 2009-06-15
+
+NLTK:
+* minor bugfixes and optimizations for parsers, updated some doctests
+* added bottom-up filtered left corner parsers,
+ LeftCornerChartParser and IncrementalLeftCornerChartParser.
+* fixed dispersion plot bug which prevented empty plots
+
+Version 2.0b1 2009-06-09
+
+NLTK:
+* major refactor of chart parser code and improved API (Peter Ljungl喃)
+* added new bottom-up left-corner chart parser strategy
+* misc bugfixes (ChunkScore, chart rules, chatbots, jcn-similarity)
+* improved efficiency of "import nltk" using lazy module imports
+* moved CCG package and ISRI Arabic stemmer from NLTK-Contrib into core NLTK
+* misc code cleanups
+
+Contrib:
+* moved out of the main NLTK distribution into a separate distribution
+
+Book:
+* Ongoing polishing ahead of print publication
+
+Version 0.9.9 2009-05-06
+
+NLTK:
+* Finalized API for NLTK 2.0 and the book, incl dozens of small fixes
+* Names of the form nltk.foo.Bar now available as nltk.Bar
+ for significant functionality; in some cases the name was modified
+ (using old names will produce a deprecation warning)
+* Bugfixes in downloader, WordNet
+* Expanded functionality in DecisionTree
+* Bigram collocations extended for discontiguous bigrams
+* Translation toy nltk.misc.babelfish
+* New module nltk.help giving access to tagset documentation
+* Fix imports so that NLTK builds without Tkinter (Bjorn Maeland)
+
+Data:
+* new maxent NE chunker model
+* updated grammar packages for the book
+* data for new tagsets collection, documenting several tagsets
+* added lolcat translation to the Genesis collection
+
+Contrib (work in progress):
+* Updates to coreference package (Joseph Frazee)
+* New ISRI Arabic stemmer (Hosam Algasaier)
+* Updates to Toolbox package (Greg Aumann)
+
+Book:
+* Substantial editorial corrections ahead of final submission
+
+Version 0.9.8 2009-02-18
+
+NLTK:
+* New off-the-shelf tokenizer, POS tagger, and named-entity tagger
+* New metrics package with inter-annotator agreement scores,
+ distance metrics, rank correlation
+* New collocations package (Joel Nothman)
+* Many clean-ups to WordNet package (Steven Bethard, Jordan Boyd-Graber)
+* Moved old pywordnet-based WordNet package to nltk_contrib
+* WordNet browser (Paul Bone)
+* New interface to dependency treebank corpora
+* Moved MinimalSet class into nltk.misc package
+* Put NLTK applications in new nltk.app package
+* Many other improvements incl semantics package, toolbox, MaltParser
+* Misc changes to many API names in preparation for 1.0, old names deprecated
+* Most classes now available in the top-level namespace
+* Work on Python egg distribution (Brandon Rhodes)
+* Removed deprecated code remaining from 0.8.* versions
+* Fixes for Python 2.4 compatibility
+
+Data:
+* Corrected identifiers in Dependency Treebank corpus
+* Basque and Catalan Dependency Treebanks (CoNLL 2007)
+* PE08 Parser Evalution data
+* New models for POS tagger and named-entity tagger
+
+Book:
+* Substantial editorial corrections
+
+Version 0.9.7 2008-12-19
+
+NLTK:
+* fixed problems with accessing zipped corpora
+* improved design and efficiency of grammars and chart parsers
+ including new bottom-up combine strategy and a redesigned
+ Earley strategy (Peter Ljunglof)
+* fixed bugs in smoothed probability distributions and added
+ regression tests (Peter Ljunglof)
+* improvements to Punkt (Joel Nothman)
+* improvements to text classifiers
+* simple word-overlap RTE classifier
+
+Data:
+* A new package of large grammars (Peter Ljunglof)
+* A small gazetteer corpus and corpus reader
+* Organized example grammars into separate packages
+* Childrens' stories added to gutenberg package
+
+Contrib (work in progress):
+* fixes and demonstration for named-entity feature extractors in nltk_contrib.coref
+
+Book:
+* extensive changes throughout, including new chapter 5 on classification
+ and substantially revised chapter 11 on managing linguistic data
+
+Version 0.9.6 2008-12-07
+
+NLTK:
+* new WordNet corpus reader (contributed by Steven Bethard)
+* incorporated dependency parsers into NLTK (was NLTK-Contrib) (contributed by Jason Narad)
+* moved nltk/cfg.py to nltk/grammar.py and incorporated dependency grammars
+* improved efficiency of unification algorithm
+* various enhancements to the semantics package
+* added plot() and tabulate() methods to FreqDist and ConditionalFreqDist
+* FreqDist.keys() and list(FreqDist) provide keys reverse-sorted by value,
+ to avoid the confusion caused by FreqDist.sorted()
+* new downloader module to support interactive data download: nltk.download()
+ run using "python -m nltk.downloader all"
+* fixed WordNet bug that caused min_depth() to sometimes give incorrect result
+* added nltk.util.Index as a wrapper around defaultdict(list) plus
+ a functional-style initializer
+* fixed bug in Earley chart parser that caused it to break
+* added basic TnT tagger nltk.tag.tnt
+* new corpus reader for CoNLL dependency format (contributed by Kepa Sarasola and Iker Manterola)
+* misc other bugfixes
+
+Contrib (work in progress):
+* TIGERSearch implementation by Torsten Marek
+* extensions to hole and glue semantics modules by Dan Garrette
+* new coreference package by Joseph Frazee
+* MapReduce interface by Xinfan Meng
+
+Data:
+* Corpora are stored in compressed format if this will not compromise speed of access
+* Swadesh Corpus of comparative wordlists in 23 languages
+* Split grammar collection into separate packages
+* New Basque and Spanish grammar samples (contributed by Kepa Sarasola and Iker Manterola)
+* Brown Corpus sections now have meaningful names (e.g. 'a' is now 'news')
+* Fixed bug that forced users to manually unzip the WordNet corpus
+* New dependency-parsed version of Treebank corpus sample
+* Added movie script "Monty Python and the Holy Grail" to webtext corpus
+* Replaced words corpus data with a much larger list of English words
+* New URL for list of available NLTK corpora
+ http://nltk.googlecode.com/svn/trunk/nltk_data/index.xml
+
+Book:
+* complete rewrite of first three chapters to make the book accessible
+ to a wider audience
+* new chapter on data-intensive language processing
+* extensive reworking of most chapters
+* Dropped subsection numbering; moved exercises to end of chapters
+
+Distributions:
+* created Portfile to support Mac installation
+
+
+Version 0.9.5 2008-08-27
+
+NLTK:
+* text module with support for concordancing, text generation, plotting
+* book module
+* Major reworking of the automated theorem proving modules (Dan Garrette)
+* draw.dispersion now uses pylab
+* draw.concordance GUI tool
+* nltk.data supports for reading corpora and other data files from within zipfiles
+* trees can be constructed from strings with Tree(s) (cf Tree.parse(s))
+
+Contrib (work in progress):
+* many updates to student projects
+ - nltk_contrib.agreement (Thomas Lippincott)
+ - nltk_contrib.coref (Joseph Frazee)
+ - nltk_contrib.depparser (Jason Narad)
+ - nltk_contrib.fuf (Petro Verkhogliad)
+ - nltk_contrib.hadoop (Xinfan Meng)
+* clean-ups: deleted stale files; moved some packages to misc
+
+Data
+* Cleaned up Gutenberg text corpora
+* added Moby Dick; removed redundant copy of Blake songs.
+* more tagger models
+* renamed to nltk_data to facilitate installation
+* stored each corpus as a zip file for quicker installation
+ and access, and to solve a problem with the Propbank
+ corpus including a file with an illegal name for MSWindows
+ (con.xml).
+
+Book:
+* changed filenames to chNN format
+* reworked opening chapters (work in progress)
+
+Distributions:
+* fixed problem with mac installer that arose when Python binary
+ couldn't be found
+* removed dependency of NLTK on nltk_data so that NLTK code can be
+ installed before the data
+
+Version 0.9.4 2008-08-01
+
+NLTK:
+- Expanded semantics package for first order logic, linear logic,
+ glue semantics, DRT, LFG (Dan Garrette)
+- new WordSense class in wordnet.synset supporting access to synsets
+ from sense keys and accessing sense counts (Joel Nothman)
+- interface to Mallet's linear chain CRF implementation (nltk.tag.crf)
+- misc bugfixes incl Punkt, synsets, maxent
+- improved support for chunkers incl flexible chunk corpus reader,
+ new rule type: ChunkRuleWithContext
+- new GUI for pos-tagged concordancing nltk.draw.pos_concordance
+- new GUI for developing regexp chunkers nltk.draw.rechunkparser
+- added bio_sents() and bio_words() methods to ConllChunkCorpusReader in conll.py
+ to allow reading (word, tag, chunk_typ) tuples off of CoNLL-2000 corpus. Also
+ modified ConllChunkCorpusView to support these changes.
+- feature structures support values with custom unification methods
+- new flag on tagged corpus readers to use simplified tagsets
+- new package for ngram language modeling with Katz backoff nltk.model
+- added classes for single-parented and multi-parented trees that
+ automatically maintain parent pointers (nltk.tree.ParentedTree and
+ nltk.tree.MultiParentedTree)
+- new WordNet browser GUI (Jussi Salmela, Paul Bone)
+- improved support for lazy sequences
+- added generate() method to probability distributions
+- more flexible parser for converting bracketed strings to trees
+- made fixes to docstrings to improve API documentation
+
+Contrib (work in progress)
+- new NLG package, FUF/SURGE (Petro Verkhogliad)
+- new dependency parser package (Jason Narad)
+- new Coreference package, incl support for
+ ACE-2, MUC-6 and MUC-7 corpora (Joseph Frazee)
+- CCG Parser (Graeme Gange)
+- first order resolution theorem prover (Dan Garrette)
+
+Data:
+- Nnw NPS Chat Corpus and corpus reader (nltk.corpus.nps_chat)
+- ConllCorpusReader can now be used to read CoNLL 2004 and 2005 corpora.
+- Implemented HMM-based Treebank POS tagger and phrase chunker for
+ nltk_contrib.coref in api.py. Pickled versions of these objects are checked
+ in in data/taggers and data/chunkers.
+
+Book:
+- misc corrections in response to feedback from readers
+
+Version 0.9.3 2008-06-03
+
+NLTK:
+- modified WordNet similarity code to use pre-built information content files
+- new classifier-based tagger, BNC corpus reader
+- improved unicode support for corpus readers
+- improved interfaces to Weka, Prover9/Mace4
+- new support for using MEGAM and SciPy to train maxent classifiers
+- rewrite of Punkt sentence segmenter (Joel Nothman)
+- bugfixes for WordNet information content module (Jordan Boyd-Graber)
+- code clean-ups throughout
+
+Book:
+- miscellaneous fixes in response to feedback from readers
+
+Contrib:
+- implementation of incremental algorithm for generating
+ referring expressions (contributed by Margaret Mitchell)
+- refactoring WordNet browser (Paul Bone)
+
+Corpora:
+- included WordNet information content files
+
+Version 0.9.2 2008-03-04
+
+NLTK:
+- new theorem-prover and model-checker module nltk.inference,
+ including interface to Prover9/Mace4 (Dan Garrette, Ewan Klein)
+- bugfix in Reuters corpus reader that causes Python
+ to complain about too many open files
+- VerbNet and PropBank corpus readers
+
+Data:
+- VerbNet Corpus version 2.1: hierarchical, verb lexicon linked to WordNet
+- PropBank Corpus: predicate-argument structures, as stand-off annotation of Penn Treebank
+
+Contrib:
+- New work on WordNet browser, incorporating a client-server model (Jussi Salmela)
+
+Distributions:
+- Mac OS 10.5 distribution
+
+Version 0.9.1 2008-01-24
+
+NLTK:
+- new interface for text categorization corpora
+- new corpus readers: RTE, Movie Reviews, Question Classification, Brown Corpus
+- bugfix in ConcatenatedCorpusView that caused iteration to fail if it didn't start from the beginning of the corpus
+
+Data:
+- Question classification data, included with permission of Li & Roth
+- Reuters 21578 Corpus, ApteMod version, from CPAN
+- Movie Reviews corpus (sentiment polarity), included with permission of Lillian Lee
+- Corpus for Recognising Textual Entailment (RTE) Challenges 1, 2 and 3
+- Brown Corpus (reverted to original file structure: ca01-cr09)
+- Penn Treebank corpus sample (simplified implementation, new readers treebank_raw and treebank_chunk)
+- Minor redesign of corpus readers, to use filenames instead of "items" to identify parts of a corpus
+
+Contrib:
+- theorem_prover: Prover9, tableau, MaltParser, Mace4, glue semantics, docs (Dan Garrette, Ewan Klein)
+- drt: improved drawing, conversion to FOL (Dan Garrette)
+- gluesemantics: GUI demonstration, abstracted LFG code, documentation (Dan Garrette)
+- readability: various text readability scores (Thomas Jakobsen, Thomas Skardal)
+- toolbox: code to normalize toolbox databases (Greg Aumann)
+
+Book:
+- many improvements in early chapters in response to reader feedback
+- updates for revised corpus readers
+- moved unicode section to chapter 3
+- work on engineering.txt (not included in 0.9.1)
+
+Distributions:
+- Fixed installation for Mac OS 10.5 (Joshua Ritterman)
+- Generalize doctest_driver to work with doc_contrib
+
+Version 0.9 2007-10-12
+
+NLTK:
+- New naming of packages and modules, and more functions imported into
+ top-level nltk namespace, e.g. nltk.chunk.Regexp -> nltk.RegexpParser,
+ nltk.tokenize.Line -> nltk.LineTokenizer, nltk.stem.Porter -> nltk.PorterStemmer,
+ nltk.parse.ShiftReduce -> nltk.ShiftReduceParser
+- processing class names changed from verbs to nouns, e.g.
+ StemI -> StemmerI, ParseI -> ParserI, ChunkParseI -> ChunkParserI, ClassifyI -> ClassifierI
+- all tokenizers are now available as subclasses of TokenizeI,
+ selected tokenizers are also available as functions, e.g. wordpunct_tokenize()
+- rewritten ngram tagger code, collapsed lookup tagger with unigram tagger
+- improved tagger API, permitting training in the initializer
+- new system for deprecating code so that users are notified of name changes.
+- support for reading feature cfgs to parallel reading cfgs (parse_featcfg())
+- text classifier package, maxent (GIS, IIS), naive Bayes, decision trees, weka support
+- more consistent tree printing
+- wordnet's morphy stemmer now accessible via stemmer package
+- RSLP Portuguese stemmer (originally developed by Viviane Moreira Orengo, reimplemented by Tiago Tresoldi)
+- promoted ieer_rels.py to the sem package
+- improvements to WordNet package (Jussi Salmela)
+- more regression tests, and support for checking coverage of tests
+- miscellaneous bugfixes
+- remove numpy dependency
+
+Data:
+- new corpus reader implementation, refactored syntax corpus readers
+- new data package: corpora, grammars, tokenizers, stemmers, samples
+- CESS-ESP Spanish Treebank and corpus reader
+- CESS-CAT Catalan Treebank and corpus reader
+- Alpino Dutch Treebank and corpus reader
+- MacMorpho POS-tagged Brazilian Portuguese news text and corpus reader
+- trained model for Portuguese sentence segmenter
+- Floresta Portuguese Treebank version 7.4 and corpus reader
+- TIMIT player audio support
+
+Contrib:
+- BioReader (contributed by Carlos Rodriguez)
+- TnT tagger (contributed by Sam Huston)
+- wordnet browser (contributed by Jussi Salmela, requires wxpython)
+- lpath interpreter (contributed by Haejoong Lee)
+- timex -- regular expression-based temporal expression tagger
+
+Book:
+- polishing of early chapters
+- introductions to parts 1, 2, 3
+- improvements in book processing software (xrefs, avm & gloss formatting, javascript clipboard)
+- updates to book organization, chapter contents
+- corrections throughout suggested by readers (acknowledged in preface)
+- more consistent use of US spelling throughout
+- all examples redone to work with single import statement: "import nltk"
+- reordered chapters: 5->7->8->9->11->12->5
+ * language engineering in part 1 to broaden the appeal
+ of the earlier part of the book and to talk more about
+ evaluation and baselines at an earlier stage
+ * concentrate the partial and full parsing material in part 2,
+ and remove the specialized feature-grammar material into part 3
+
+Distributions:
+- streamlined mac installation (Joshua Ritterman)
+- included mac distribution with ISO image
+
+Version 0.8 2007-07-01
+
+Code:
+- changed nltk.__init__ imports to explicitly import names from top-level modules
+- changed corpus.util to use the 'rb' flag for opening files, to fix problems
+ reading corpora under MSWindows
+- updated stale examples in engineering.txt
+- extended feature stucture interface to permit chained features, e.g. fs['F','G']
+- further misc improvements to test code plus some bugfixes
+Tutorials:
+- rewritten opening section of tagging chapter
+- reorganized some exercises
+
+Version 0.8b2 2007-06-26
+
+Code (major):
+- new corpus package, obsoleting old corpora package
+ - supports caching, slicing, corpus search path
+ - more flexible API
+ - global updates so all NLTK modules use new corpus package
+- moved nltk/contrib to separate top-level package nltk_contrib
+- changed wordpunct tokenizer to use \w instead of a-zA-Z0-9
+ as this will be more robust for languages other than English,
+ with implications for many corpus readers that use it
+- known bug: certain re-entrant structures in featstruct
+- known bug: when the LHS of an edge contains an ApplicationExpression,
+ variable values in the RHS bindings aren't copied over when the
+ fundamental rule applies
+- known bug: HMM tagger is broken
+Tutorials:
+- global updates to NLTK and docs
+- ongoing polishing
+Corpora:
+- treebank sample reverted to published multi-file structure
+Contrib:
+- DRT and Glue Semantics code (nltk_contrib.drt, nltk_contrib.gluesemantics, by Dan Garrette)
+
+Version 0.8b1 2007-06-18
+
+Code (major):
+- changed package name to nltk
+- import all top-level modules into nltk, reducing need for import statements
+- reorganization of sub-package structures to simplify imports
+- new featstruct module, unifying old featurelite and featurestructure modules
+- FreqDist now inherits from dict, fd.count(sample) becomes fd[sample]
+- FreqDist initializer permits: fd = FreqDist(len(token) for token in text)
+- made numpy optional
+Code (minor):
+- changed GrammarFile initializer to accept filename
+- consistent tree display format
+- fixed loading process for WordNet and TIMIT that prevented code installation if data not installed
+- taken more care with unicode types
+- incorporated pcfg code into cfg module
+- moved cfg, tree, featstruct to top level
+- new filebroker module to make handling of example grammar files more transparent
+- more corpus readers (webtext, abc)
+- added cfg.covers() to check that a grammar covers a sentence
+- simple text-based wordnet browser
+- known bug: parse/featurechart.py uses incorrect apply() function
+Corpora:
+- csv data file to document NLTK corpora
+Contrib:
+- added Glue semantics code (contrib.glue, by Dan Garrette)
+- Punkt sentence segmenter port (contrib.punkt, by Willy)
+- added LPath interpreter (contrib.lpath, by Haejoong Lee)
+- extensive work on classifiers (contrib.classifier*, Sumukh Ghodke)
+Tutorials:
+- polishing on parts I, II
+- more illustrations, data plots, summaries, exercises
+- continuing to make prose more accessible to non-linguistic audience
+- new default import that all chapters presume: from nltk.book import *
+Distributions:
+- updated to latest version of numpy
+- removed WordNet installation instructions as WordNet is now included in corpus distribution
+- added pylab (matplotlib)
+
+Version 0.7.5 2007-05-16
+
+Code:
+- improved WordNet and WordNet-Similarity interface
+- the Lancaster Stemmer (contributed by Steven Tomcavage)
+Corpora:
+- Web text samples
+- BioCreAtIvE-PPI - a corpus for protein-protein interactions
+- Switchboard Telephone Speech Corpus Sample (via Talkbank)
+- CMU Problem Reports Corpus sample
+- CONLL2002 POS+NER data
+- Patient Information Leaflet corpus
+- WordNet 3.0 data files
+- English wordlists: basic English, frequent words
+Tutorials:
+- more improvements to text and images
+
+Version 0.7.4 2007-05-01
+
+Code:
+- Indian POS tagged corpus reader: corpora.indian
+- Sinica Treebank corpus reader: corpora.sinica_treebank
+- new web corpus reader corpora.web
+- tag package now supports pickling
+- added function to utilities.py to guess character encoding
+Corpora:
+- Rotokas texts from Stuart Robinson
+- POS-tagged corpora for several Indian languages (Bangla, Hindi, Marathi, Telugu) from A Kumaran
+Tutorials:
+- Substantial work on Part II of book on structured programming, parsing and grammar
+- More bibliographic citations
+- Improvements in typesetting, cross references
+- Redimensioned images and tables for better use of page space
+- Moved project list to wiki
+Contrib:
+- validation of toolbox entries using chunking
+- improved classifiers
+Distribution:
+- updated for Python 2.5.1, Numpy 1.0.2
+
+Version 0.7.3 2007-04-02
+
+* Code:
+ - made chunk.Regexp.parse() more flexible about its input
+ - developed new syntax for PCFG grammars, e.g. A -> B C [0.3] | D [0.7]
+ - fixed CFG parser to support grammars with slash categories
+ - moved beta classify package from main NLTK to contrib
+ - Brill taggers loaded correctly
+ - misc bugfixes
+* Corpora:
+ - Shakespeare XML corpus sample and corpus reader
+* Tutorials:
+ - improvements to prose, exercises, plots, images
+ - expanded and reorganized tutorial on structured programming
+ - formatting improvements for Python listings
+ - improved plots (using pylab)
+ - categorization of problems by difficulty
+Contrib:
+ - more work on kimmo lexicon and grammar
+ - more work on classifiers
+
+Version 0.7.2 2007-03-01
+
+* Code:
+ - simple feature detectors (detect module)
+ - fixed problem when token generators are passed to a parser (parse package)
+ - fixed bug in Grammar.productions() (identified by Lucas Champollion and Mitch Marcus)
+ - fixed import bug in category.GrammarFile.earley_parser
+ - added utilities.OrderedDict
+ - initial port of old NLTK classifier package (by Sam Huston)
+ - UDHR corpus reader
+* Corpora:
+ - added UDHR corpus (Universal Declaration of Human Rights)
+ with 10k text samples in 300+ languages
+* Tutorials:
+ - improved images
+ - improved book formatting, including new support for:
+ - javascript to copy program examples to clipboard in HTML version,
+ - bibliography, chapter cross-references, colorization, index, table-of-contents
+
+* Contrib:
+ - new Kimmo system: contrib.mit.six863.kimmo (Rob Speer)
+ - fixes for: contrib.fsa (Rob Speer)
+ - demonstration of text classifiers trained on UDHR corpus for
+ language identification: contrib.langid (Sam Huston)
+ - new Lambek calculus system: contrib.lambek
+ - new tree implementation based on elementtree: contrib.tree
+
+Version 0.7.1 2007-01-14
+
+* Code:
+ - bugfixes (HMM, WordNet)
+
+Version 0.7 2006-12-22
+
+* Code:
+ - bugfixes, including fixed bug in Brown corpus reader
+ - cleaned up wordnet 2.1 interface code and similarity measures
+ - support for full Penn treebank format contributed by Yoav Goldberg
+* Tutorials:
+ - expanded tutorials on advanced parsing and structured programming
+ - checked all doctest code
+ - improved images for chart parsing
+
+Version 0.7b1 2006-12-06
+
+* Code:
+ - expanded semantic interpretation package
+ - new high-level chunking interface, with cascaded chunking
+ - split chunking code into new chunk package
+ - updated wordnet package to support version 2.1 of Wordnet.
+ - prototyped basic wordnet similarity measures
+ (path distance, Wu + Palmer and Leacock + Chodorow, Resnik similarity measures.)
+ - bugfixes (tag.Window, tag.ngram)
+ - more doctests
+* Contrib:
+ - toolbox language settings module
+* Tutorials:
+ - rewrite of chunking chapter, switched from Treebank to CoNLL format as main focus,
+ simplified evaluation framework, added ngram chunking section
+ - substantial updates throughout (esp programming and semantics chapters)
+* Corpora:
+ - Chat-80 Prolog data files provided as corpora, plus corpus reader
+
+Version 0.7a2 2006-11-13
+
+* Code:
+ - more doctest
+ - code to read Chat-80 data
+ - HMM bugfix
+* Tutorials:
+ - continued updates and polishing
+* Corpora:
+ - toolbox MDF sample data
+
+Version 0.7a1 2006-10-29
+
+* Code:
+ - new toolbox module (Greg Aumann)
+ - new semantics package (Ewan Klein)
+ - bugfixes
+* Tutorials
+ - substantial revision, especially in preface, introduction, words,
+ and semantics chapters.
+
+Version 0.6.6 2006-10-06
+
+* Code:
+ - bugfixes (probability, shoebox, draw)
+* Contrib:
+ - new work on shoebox package (Stuart Robinson)
+* Tutorials:
+ - continual expansion and revision, especially on introduction to
+ programming, advanced programming and the feature-based grammar chapters.
+
+Version 0.6.5 2006-07-09
+
+* Code:
+ - improvements to shoebox module (Stuart Robinson, Greg Aumann)
+ - incorporated feature-based parsing into core NLTK-Lite
+ - corpus reader for Sinica treebank sample
+ - new stemmer package
+* Contrib:
+ - hole semantics implementation (Peter Wang)
+ - Incorporating yaml
+ - new work on feature structures, unification, lambda calculus
+ - new work on shoebox package (Stuart Robinson, Greg Aumann)
+* Corpora:
+ - Sinica treebank sample
+* Tutorials:
+ - expanded discussion throughout, incl: left-recursion, trees, grammars,
+ feature-based grammar, agreement, unification, PCFGs,
+ baseline performance, exercises, improved display of trees
+
+Version 0.6.4 2006-04-20
+
+* Code:
+ - corpus readers for Senseval 2 and TIMIT
+ - clusterer (ported from old NLTK)
+ - support for cascaded chunkers
+ - bugfix suggested by Brent Payne
+ - new SortedDict class for regression testing
+* Contrib:
+ - CombinedTagger tagger and marshalling taggers, contributed by Tiago Tresoldi
+* Corpora:
+ - new: Senseval 2, TIMIT sample
+* Tutorials:
+ - major revisions to programming, words, tagging, chunking, and parsing tutorials
+ - many new exercises
+ - formatting improvements, including colorized program examples
+ - fixed problem with testing on training data, reported by Jason Baldridge
+
+Version 0.6.3 2006-03-09
+
+* switch to new style classes
+* repair FSA model sufficiently for Kimmo module to work
+* port of MIT Kimmo morphological analyzer; still needs lots of code clean-up and inline docs
+* expanded support for shoebox format, developed with Stuart Robinson
+* fixed bug in indexing CFG productions, for empty right-hand-sides
+* efficiency improvements, suggested by Martin Ranang
+* replaced classeq with isinstance, for efficiency improvement, as suggested by Martin Ranang
+* bugfixes in chunk eval
+* simplified call to draw_trees
+* names, stopwords corpora
+
+Version 0.6.2 2006-01-29
+
+* Peter Spiller's concordancer
+* Will Hardy's implementation of Penton's paradigm visualization system
+* corpus readers for presidential speeches
+* removed NLTK dependency
+* generalized CFG terminals to permit full range of characters
+* used fully qualified names in demo code, for portability
+* bugfixes from Yoav Goldberg, Eduardo Pereira Habkost
+* fixed obscure quoting bug in tree displays and conversions
+* simplified demo code, fixed import bug
diff --git a/INSTALL.txt b/INSTALL.txt
new file mode 100644
index 0000000..048ac9e
--- /dev/null
+++ b/INSTALL.txt
@@ -0,0 +1,6 @@
+To install NLTK, run setup.py from an administrator account, e.g.:
+
+ sudo python setup.py install
+
+For full installation instructions, please see http://nltk.github.com/install.html
+
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..3172938
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,13 @@
+Copyright (C) 2001-2014 NLTK Project
+
+Licensed under the Apache License, Version 2.0 (the 'License');
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an 'AS IS' BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
diff --git a/MANIFEST.in b/MANIFEST.in
new file mode 100644
index 0000000..7925410
--- /dev/null
+++ b/MANIFEST.in
@@ -0,0 +1,6 @@
+include LICENSE.txt INSTALL.txt README.txt MANIFEST.in
+include setup.py
+include nltk/test/*.doctest
+include nltk/VERSION
+recursive-include *.txt Makefile
+global-exclude *~
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..51a712a
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,62 @@
+# Natural Language Toolkit: source Makefile
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+PYTHON = python
+VERSION = $(shell $(PYTHON) -c 'import nltk; print(nltk.__version__)' | sed '/^Warning: */d')
+NLTK_URL = $(shell $(PYTHON) -c 'import nltk; print(nltk.__url__)' | sed '/^Warning: */d')
+
+.PHONY: all clean clean_code
+
+all: dist
+
+########################################################################
+# TESTING
+########################################################################
+
+DOCTEST_DRIVER = nltk/test/doctest_driver.py
+DOCTEST_FLAGS = --ellipsis --normalize_whitespace
+DOCTEST_FILES = nltk/test/*.doctest
+DOCTEST_CODE_FILES = nltk/*.py nltk/*/*.py
+
+doctest:
+ $(PYTHON) $(DOCTEST_DRIVER) $(DOCTEST_FLAGS) $(DOCTEST_FILES)
+
+doctest_code:
+ $(PYTHON) $(DOCTEST_DRIVER) $(DOCTEST_FLAGS) $(DOCTEST_CODE_FILES)
+
+demotest:
+ find nltk -name "*.py"\
+ -and -not -path *misc* \
+ -and -not -name brown_ic.py \
+ -exec echo ==== '{}' ==== \; -exec python '{}' \;
+
+########################################################################
+# DISTRIBUTIONS
+########################################################################
+
+dist: zipdist gztardist windist
+
+gztardist: clean_code
+ $(PYTHON) setup.py -q sdist --format=gztar
+zipdist: clean_code
+ $(PYTHON) setup.py -q sdist --format=zip
+windist: clean_code
+ $(PYTHON) setup.py -q bdist --format=wininst --plat-name=win32
+
+########################################################################
+# CLEAN
+########################################################################
+
+clean: clean_code
+ rm -rf build iso dist api MANIFEST nltk-$(VERSION) nltk.egg-info
+
+clean_code:
+ rm -f `find nltk -name '*.pyc'`
+ rm -f `find nltk -name '*.pyo'`
+ rm -f `find . -name '*~'`
+ rm -f MANIFEST # regenerate manifest from MANIFEST.in
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..6ab9cbb
--- /dev/null
+++ b/README.md
@@ -0,0 +1,164 @@
+Natural Language Toolkit (NLTK) nltk.org
+====================================
+
+NLTK -- the Natural Language Toolkit -- is a suite of open source
+Python modules, data sets and tutorials supporting research and
+development in Natural Language Processing.
+
+Copyright (C) 2001-2014 NLTK Project
+
+For license information, see LICENSE.txt
+
+For documentation, please visit http://nltk.org/
+
+Redistributing
+----------------------
+NLTK source code is distributed under the Apache 2.0 License.
+NLTK documentation is distributed under the Creative Commons Attribution-Noncommercial-No Derivative Works 3.0 United States license.
+NLTK corpora are provided under the terms given in the README file for each corpus; all are redistributable, and available for non-commercial use.
+NLTK may be freely redistributed, subject to the provisions of these licenses.
+
+Testing
+-------
+
+The CI test suite previously running at [Shining Panda](http://shiningpanda.com)
+is down due to them having taken their Clap de Fin. There have been some
+investigations into moving to other CI, but no official build is running right
+now. This may change in the near future.
+
+nltk/test/runtests.py is a good starting point for running tests locally, but
+note that the suite is currently failing.
+
+Contributing
+------------
+
+[How to contribute to NLTK](http://www.nltk.org/contribute.html).
+
+The following people have contributed to NLTK:
+
+Rami Al-Rfou',
+Mark Amery,
+Greg Aumann,
+Yonatan Becker,
+Paul Bedaride,
+Steven Bethard,
+Robert Berwick,
+Dan Blanchard,
+Nathan Bodenstab,
+Francis Bond,
+Paul Bone,
+Jordan Boyd-Graber,
+Daniel Blanchard,
+Phil Blunsom,
+Lars Buitinck,
+Steve Cassidy,
+Chen-Fu Chiang,
+Dmitry Chichkov,
+Jinyoung Choi,
+Andrew Clausen,
+Lucas Champollion,
+Trevor Cohn,
+David Coles,
+Lucas Cooper,
+Robin Cooper,
+Chris Crowner,
+James Curran,
+Dariel Dato-on,
+Selina Dennis,
+Leon Derczynski,
+Alexis Dimitriadis,
+Nikhil Dinesh,
+Liang Dong,
+David Doukhan,
+Rebecca Dridan,
+Pablo Duboue,
+Christian Federmann,
+Michelle Fullwood,
+Dan Garrette,
+Jean Mark Gawron,
+Sumukh Ghodke,
+Yoav Goldberg,
+Dougal Graham,
+Brent Gray,
+Simon Greenhill,
+Eduardo Pereira Habkost,
+Masato Hagiwara,
+Michael Hansen,
+Yurie Hara,
+Will Hardy,
+Tyler Hartley,
+Peter Hawkins,
+Michael Heilman,
+Bruce Hill,
+Amy Holland,
+Kristy Hollingshead,
+Baden Hughes,
+Rebecca Ingram,
+Edward Ivanovic,
+Thomas Jakobsen,
+Piotr Kasprzyk,
+Angelos Katharopoulos,
+Sudharshan Kaushik,
+Chris Koenig,
+Mikhail Korobov,
+Stefano Lattarini,
+Pierre-François Laquerre,
+Stefano Lattarini,
+Haejoong Lee,
+Max Leonov,
+Tom Lippincott,
+Peter Ljunglöf,
+Nitin Madnani,
+Bjørn Mæland,
+Christopher Maloof,
+Rob Malouf,
+Iker Manterola,
+Carl de Marcken,
+Mitch Marcus,
+Torsten Marek,
+Robert Marshall,
+Duncan McGreggor,
+Xinfan Meng,
+Margaret Mitchell,
+Tomonori Nagano,
+Jason Narad,
+Morten Neergaard,
+David Nemeskey,
+Eric Nichols,
+Joel Nothman,
+Ted Pedersen,
+Jacob Perkins,
+Alberto Planas,
+Alessandro Presta,
+Martin Thorsen Ranang,
+Brandon Rhodes,
+Joshua Ritterman,
+Will Roberts,
+Stuart Robinson,
+Carlos Rodriguez,
+Alex Rudnick,
+Jussi Salmela,
+Geoffrey Sampson,
+Kepa Sarasola,
+Kevin Scannell,
+Nathan Schneider,
+Rico Sennrich,
+Thomas Skardal,
+Eric Smith,
+Rob Speer,
+Peter Spiller,
+Richard Sproat,
+Ceri Stagg,
+Peter Stahl,
+Oliver Steele,
+Jan Strunk,
+Claire Taylor,
+Steven Tomcavage,
+Tiago Tresoldi,
+Petro Verkhogliad,
+Peter Wang,
+Charlotte Wilson,
+Steven Xu,
+Beracah Yankama,
+Patrick Ye,
+Jason Yoder.
diff --git a/README.txt b/README.txt
new file mode 100644
index 0000000..cbc5379
--- /dev/null
+++ b/README.txt
@@ -0,0 +1,55 @@
+Natural Language Toolkit (NLTK) nltk.org
+
+Authors: Steven Bird <stevenbird1 at gmail.com>
+ Edward Loper <edloper at gmail.com>
+ Ewan Klein <ewan at inf.ed.ac.uk>
+
+Copyright (C) 2001-2014 NLTK Project
+
+For license information, see LICENSE.txt
+
+NLTK -- the Natural Language Toolkit -- is a suite of open source
+Python modules, data sets and tutorials supporting research and
+development in Natural Language Processing.
+
+Documentation: A substantial amount of documentation about how
+to use NLTK, including a textbook and API documention, is
+available from the NLTK website: http://nltk.org/
+
+ - The book covers a wide range of introductory topics in NLP, and
+ shows how to do all the processing tasks using the toolkit.
+
+ - The toolkit's reference documentation describes every module,
+ interface, class, method, function, and variable in the toolkit.
+ This documentation should be useful to both users and developers.
+
+Mailing Lists: There are several mailing lists associated with NLTK:
+
+ - nltk: Public information and announcements about NLTK (very low volume)
+ http://groups.google.com/group/nltk
+ - nltk-users: Discussions amongst NLTK users
+ http://groups.google.com/group/nltk-users
+ - nltk-dev: Discussions amongst NLTK developers
+ http://groups.google.com/group/nltk-dev
+ - nltk-translation: Discussions about translating the NLTK book
+ http://groups.google.com/group/nltk-translation
+ - nltk-commits: Subversion commit logs for NLTK
+ http://groups.google.com/group/nltk-commits
+
+Contributing: If you would like to contribute to NLTK,
+ please see http://nltk.org/contribute
+
+Donating: Have you found the toolkit helpful? Please support NLTK development
+ by donating to the project via PayPal, using the link on the NLTK homepage.
+
+Redistributing: NLTK source code is distributed under the Apache 2.0 License.
+ NLTK documentation is distributed under the Creative Commons
+ Attribution-Noncommercial-No Derivative Works 3.0 United States license.
+ NLTK corpora are provided under the terms given in the README file
+ for each corpus; all are redistributable, and available for non-commercial use.
+ NLTK may be freely redistributed, subject to the provisions of these licenses.
+
+Citing: If you publish work that uses NLTK, please cite the NLTK book, as follows:
+
+ Bird, Steven, Edward Loper and Ewan Klein (2009).
+ Natural Language Processing with Python. O'Reilly Media Inc.
diff --git a/RELEASE-HOWTO b/RELEASE-HOWTO
new file mode 100644
index 0000000..4b55b67
--- /dev/null
+++ b/RELEASE-HOWTO
@@ -0,0 +1,69 @@
+Building an NLTK distribution
+----------------------------------
+
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+@@@ BUILD
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+
+A. PREPARATION
+
+ 1. Check that installation instructions are up-to-date
+ 2. Update the data index (make data_index) and commit
+ 3. Update the ChangeLog (for nltk, nltk_data)
+ git log --since=20XX-YY-ZZ
+ 4. install the new version, since its the installed code that is checked
+ 5. cd nltk/test; make (run the tests in nltk.test)
+ 6. make demotest (run the demonstration code included in many modules)
+
+B. BUILD
+
+ 1. Modify nltk/VERSION with the version number and commit
+ 2. Make dist
+ ?. (cd ../nltk_contrib; make dist???)
+
+D. RELEASE
+
+ 1. Update the news page in nltk/web/news.rst
+ 2. git tag -a 3.X.Y -m "version 3.X.Y"
+ git push --tags
+ 3. Put up the release on github
+ https://github.com/nltk/nltk/releases
+ 4. sudo python setup.py register
+ 5. Log in to http://pypi.python.org/pypi and upload distributions
+ 6. post announcement to NLTK the mailing lists:
+ nltk-dev (for beta releases)
+ nltk (for final releases)
+ 7. post announcement to external mailing lists, for major N.N releases only
+ CORPORA at uib.no, linguist at linguistlist.org,
+ PythonSIL at lists.sil.org, edu-sig at python.org
+ mailing lists for any local courses using NLTK
+
+
+
+
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+@@@ BOOK BUILD
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+
+The build requires docutils, pdflatex, python imaging library, epydoc,
+ cdrtools, ImageMagick
+
+ 1. Check out a clean copy of the subversion repository (or make clean)
+ and install locally with sudo python setup.py install; make clean
+ 2. make doc (slow; see doc/ for the results) and commit
+
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+@@@ INSTALL
+@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+
+D. INSTALLATION
+
+ 1. download and install new version on all machines
+ 2. contact relevant sysads to install new version
+ 3. copy dist directory to memory stick
+
+E. NEW VERSION NUMBER (optional)
+
+ 1. update the version numbers in the repository so that builds
+ off the repository don't have the same version as the release,
+ e.g. after release 0.9.6, update repository version to 0.9.7a (alpha)
diff --git a/emacs/doctest-mode.el b/emacs/doctest-mode.el
new file mode 100644
index 0000000..65ee728
--- /dev/null
+++ b/emacs/doctest-mode.el
@@ -0,0 +1,1078 @@
+;;; doctest-mode.el --- Major mode for editing Python doctest files
+
+;; Copyright (C) 2004 Edward Loper
+
+;; Author: Edward Loper
+;; Maintainer: edloper at alum.mit.edu
+;; Created: Aug 2004
+;; Keywords: python doctest unittest test docstring
+
+(defconst doctest-version "0.3"
+ "`doctest-mode' version number.")
+
+;; This software is provided as-is, without express or implied
+;; warranty. Permission to use, copy, modify, distribute or sell this
+;; software, without fee, for any purpose and by any individual or
+;; organization, is hereby granted, provided that the above copyright
+;; notice and this paragraph appear in all copies.
+
+;; This is a major mode for editing text files that contain Python
+;; doctest examples. Doctest is a testing framework for Python that
+;; emulates an interactive session, and checks the result of each
+;; command. For more information, see the Python library reference:
+;; <http://docs.python.org/lib/module-doctest.html>
+
+;; Known bugs:
+;; - Some places assume prompts are 4 chars (but they can be 3
+;; if they're bare).
+;; - String literals are not colored correctly. (We need to color
+;; string literals on source lines, but *not* output lines or
+;; text lines; this is hard to do.)
+;; - Output lines starting with "..." are mistakenly interpreted
+;; as (continuation) source lines.
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Customizable Constants
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defgroup doctest nil
+ "Support for the Python doctest framework"
+ :group 'languages
+ :prefix "doctest-")
+
+(defcustom doctest-default-margin 4
+ "The default pre-prompt margin for doctest examples."
+ :type 'integer
+ :group 'doctest)
+
+(defcustom doctest-avoid-trailing-whitespace t
+ "If true, then delete trailing whitespace when inserting a newline."
+ :type 'boolean
+ :group 'doctest)
+
+(defcustom doctest-temp-directory
+ (let ((ok '(lambda (x)
+ (and x
+ (setq x (expand-file-name x)) ; always true
+ (file-directory-p x)
+ (file-writable-p x)
+ x))))
+ (or (funcall ok (getenv "TMPDIR"))
+ (funcall ok "/usr/tmp")
+ (funcall ok "/tmp")
+ (funcall ok "/var/tmp")
+ (funcall ok ".")
+ (error (concat "Couldn't find a usable temp directory -- "
+ "set `doctest-temp-directory'"))))
+
+ "*Directory used for temporary files created when running doctest.
+By default, the first directory from this list that exists and that you
+can write into: the value (if any) of the environment variable TMPDIR,
+/usr/tmp, /tmp, /var/tmp, or the current directory."
+ :type 'string
+ :group 'doctest)
+
+(defcustom hide-example-source t
+ "If true, then don't display the example source code for each
+failure in the results buffer."
+ :type 'boolean
+ :group 'doctest)
+
+(defcustom doctest-python-command "python"
+ "Shell command used to start the python interpreter"
+ :type 'string
+ :group 'doctest)
+
+(defcustom doctest-results-buffer-name "*doctest-output*"
+ "The name of the buffer used to store the output of the doctest
+command."
+ :type 'string
+ :group 'doctest)
+
+(defcustom doctest-optionflags '()
+ "Option flags for doctest"
+ :group 'doctest
+ :type '(repeat (choice (const :tag "Select an option..." "")
+ (const :tag "Normalize whitespace"
+ "NORMALIZE_WHITESPACE")
+ (const :tag "Ellipsis"
+ "ELLIPSIS")
+ (const :tag "Don't accept True for 1"
+ DONT_ACCEPT_TRUE_FOR_1)
+ (const :tag "Don't accept <BLANKLINE>"
+ DONT_ACCEPT_BLANKLINE)
+ (const :tag "Ignore Exception detail"
+ IGNORE_EXCEPTION_DETAIL)
+ (const :tag "Report only first failure"
+ REPORT_ONLY_FIRST_FAILURE)
+ )))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Fonts
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defface doctest-prompt-face
+ '((((class color) (background dark))
+ (:foreground "#68f"))
+ (t (:foreground "#226")))
+ "Face for Python prompts in doctest examples."
+ :group 'doctest)
+
+(defface doctest-output-face
+ '((((class color) (background dark))
+ (:foreground "#afd"))
+ (t (:foreground "#262")))
+ "Face for the output of doctest examples."
+ :group 'doctest)
+
+(defface doctest-output-marker-face
+ '((((class color) (background dark))
+ (:foreground "#0f0"))
+ (t (:foreground "#080")))
+ "Face for markers in the output of doctest examples."
+ :group 'doctest)
+
+(defface doctest-output-traceback-face
+ '((((class color) (background dark))
+ (:foreground "#f88"))
+ (t (:foreground "#622")))
+ "Face for traceback headers in the output of doctest examples."
+ :group 'doctest)
+
+(defface doctest-results-divider-face
+ '((((class color) (background dark))
+ (:foreground "#08f"))
+ (t (:foreground "#00f")))
+ "Face for dividers in the doctest results window."
+ :group 'doctest)
+
+(defface doctest-results-loc-face
+ '((((class color) (background dark))
+ (:foreground "#0f8"))
+ (t (:foreground "#084")))
+ "Face for location headers in the doctest results window."
+ :group 'doctest)
+
+(defface doctest-results-header-face
+ '((((class color) (background dark))
+ (:foreground "#8ff"))
+ (t (:foreground "#088")))
+ "Face for sub-headers in the doctest results window."
+ :group 'doctest)
+
+(defface doctest-results-selection-face
+ '((((class color) (background dark))
+ (:foreground "#ff0" :background "#008"))
+ (t (:background "#088" :foreground "#fff")))
+ "Face for selected failure's location header in the results window."
+ :group 'doctest)
+
+(defface doctest-selection-face
+ '((((class color) (background dark))
+ (:foreground "#ff0" :background "#00f" :bold t))
+ (t (:foreground "#f00")))
+ "Face for selected example's prompt"
+ :group 'doctest)
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Constants
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defconst doctest-prompt-re
+ "^\\([ \t]*\\)\\(>>> ?\\|[.][.][.] ?\\)\\([ \t]*\\)"
+ "Regular expression for doctest prompts. It defines three groups:
+the pre-prompt margin; the prompt; and the post-prompt indentation.")
+
+(defconst doctest-open-block-re
+ "[^\n]+:[ \t]*\\(#.*\\)?$"
+ "Regular expression for a line that opens a block")
+
+(defconst doctest-close-block-re
+ "\\(return\\|raise\\|break\\|continue\\|pass\\)\\b"
+ "Regular expression for a line that closes a block")
+
+(defconst doctest-outdent-re
+ (concat "\\(" (mapconcat 'identity
+ '("else:"
+ "except\\(\\s +.*\\)?:"
+ "finally:"
+ "elif\\s +.*:")
+ "\\|")
+ "\\)")
+ "Regular expression for a line that should be outdented. Any line
+that matches `doctest-outdent-re', but does not follow a line matching
+`doctest-no-outdent-re', will be outdented.")
+
+(defconst doctest-no-outdent-re
+ (concat
+ "\\("
+ (mapconcat 'identity
+ (list "try:"
+ "except\\(\\s +.*\\)?:"
+ "while\\s +.*:"
+ "for\\s +.*:"
+ "if\\s +.*:"
+ "elif\\s +.*:"
+ "\\(return\\|raise\\|break\\|continue\\|pass\\)[ \t\n]"
+ )
+ "\\|")
+ "\\)")
+ "Regular expression matching lines not to outdent after. Any line
+that matches `doctest-outdent-re', but does not follow a line matching
+`doctest-no-outdent-re', will be outdented.")
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Colorization support (font-lock mode)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Define the font-lock keyword table.
+(defconst doctest-font-lock-keywords
+ (let ((prompt "^[ \t]*\\(>>>\\|\\.\\.\\.\\)")
+ (kw1 (mapconcat 'identity
+ '("and" "assert" "break" "class"
+ "continue" "def" "del" "elif"
+ "else" "except" "exec" "for"
+ "from" "global" "if" "import"
+ "in" "is" "lambda" "not"
+ "or" "pass" "print" "raise"
+ "return" "while" "yield"
+ )
+ "\\|"))
+ (kw2 (mapconcat 'identity
+ '("else:" "except:" "finally:" "try:")
+ "\\|"))
+ (kw3 (mapconcat 'identity
+ '("ArithmeticError" "AssertionError"
+ "AttributeError" "DeprecationWarning" "EOFError"
+ "Ellipsis" "EnvironmentError" "Exception" "False"
+ "FloatingPointError" "FutureWarning" "IOError"
+ "ImportError" "IndentationError" "IndexError"
+ "KeyError" "KeyboardInterrupt" "LookupError"
+ "MemoryError" "NameError" "None" "NotImplemented"
+ "NotImplementedError" "OSError" "OverflowError"
+ "OverflowWarning" "PendingDeprecationWarning"
+ "ReferenceError" "RuntimeError" "RuntimeWarning"
+ "StandardError" "StopIteration" "SyntaxError"
+ "SyntaxWarning" "SystemError" "SystemExit"
+ "TabError" "True" "TypeError" "UnboundLocalError"
+ "UnicodeDecodeError" "UnicodeEncodeError"
+ "UnicodeError" "UnicodeTranslateError"
+ "UserWarning" "ValueError" "Warning"
+ "ZeroDivisionError" "__debug__"
+ "__import__" "__name__" "abs" "apply" "basestring"
+ "bool" "buffer" "callable" "chr" "classmethod"
+ "cmp" "coerce" "compile" "complex" "copyright"
+ "delattr" "dict" "dir" "divmod"
+ "enumerate" "eval" "execfile" "exit" "file"
+ "filter" "float" "getattr" "globals" "hasattr"
+ "hash" "hex" "id" "input" "int" "intern"
+ "isinstance" "issubclass" "iter" "len" "license"
+ "list" "locals" "long" "map" "max" "min" "object"
+ "oct" "open" "ord" "pow" "property" "range"
+ "raw_input" "reduce" "reload" "repr" "round"
+ "setattr" "slice" "staticmethod" "str" "sum"
+ "super" "tuple" "type" "unichr" "unicode" "vars"
+ "xrange" "zip")
+ "\\|"))
+ (pseudokw (mapconcat 'identity
+ '("self" "None" "True" "False" "Ellipsis")
+ "\\|"))
+ (brk "\\([ \t(]\\|$\\)")
+ )
+ `(
+ ;; The following pattern colorizes source lines. In particular,
+ ;; it first matches prompts, and then looks for any of the
+ ;; following matches *on the same line* as the prompt. It uses
+ ;; the form:
+ ;;
+ ;; (MATCHER MATCH-HIGHLIGHT
+ ;; (ANCHOR-MATCHER nil nil MATCH-HIGHLIGHT)
+ ;; ...
+ ;; (ANCHOR-MATCHER nil nil MATCH-HIGHLIGHT))
+ ;;
+ ;; See the variable documentation for font-lock-keywords for a
+ ;; description of what each of those means.
+ (,prompt (1 'doctest-prompt-face)
+ ;; classes
+ ("\\b\\(class\\)[ \t]+\\([a-zA-Z_]+[a-zA-Z0-9_]*\\)"
+ nil nil (1 'font-lock-keyword-face)
+ (2 'font-lock-type-face))
+ ;; functions
+ ("\\b\\(def\\)[ \t]+\\([a-zA-Z_]+[a-zA-Z0-9_]*\\)"
+ nil nil (1 'font-lock-keyword-face) (2 'font-lock-type-face))
+ ;; keywords
+ (,(concat "\\b\\(" kw1 "\\)" brk)
+ nil nil (1 'font-lock-keyword-face))
+ ;; builtins when they don't appear as object attributes
+ (,(concat "\\(\\b\\|[.]\\)\\(" kw3 "\\)" brk)
+ nil nil (2 'font-lock-keyword-face))
+ ;; block introducing keywords with immediately
+ ;; following colons. Yes "except" is in both lists.
+ (,(concat "\\b\\(" kw2 "\\)" brk)
+ nil nil (1 'font-lock-keyword-face))
+ ;; `as' but only in "import foo as bar"
+ ("[ \t]*\\(\\bfrom\\b.*\\)?\\bimport\\b.*\\b\\(as\\)\\b"
+ nil nil (2 'font-lock-keyword-face))
+ ;; pseudo-keywords
+ (,(concat "\\b\\(" pseudokw "\\)" brk)
+ nil nil (1 'font-lock-keyword-face))
+ ;; comments
+ ("\\(#.*\\)"
+ nil nil (1 'font-lock-comment-face)))
+
+ ;; The following pattern colorizes output lines. In particular,
+ ;; it uses doctest-output-line-matcher to check if this is an
+ ;; output line, and if so, it colorizes it, and any special
+ ;; markers it contains.
+ (doctest-output-line-matcher
+ (0 'doctest-output-face t)
+ ("\\.\\.\\." (beginning-of-line) (end-of-line)
+ (0 'doctest-output-marker-face t))
+ ("<BLANKLINE>" (beginning-of-line) (end-of-line)
+ (0 'doctest-output-marker-face t))
+ ("^Traceback (most recent call last):" (beginning-of-line) (end-of-line)
+ (0 'doctest-output-traceback-face t))
+ ("^Traceback (innermost last):" (beginning-of-line) (end-of-line)
+ (0 'doctest-output-traceback-face t))
+ )
+
+ ;; A PS1 prompt followed by a non-space is an error.
+ ("^[ \t]*\\(>>>[^ \t\n][^\n]*\\)" (1 'font-lock-warning-face t))
+
+ ;; Selected example (to highlight selected failure)
+ (doctest-selection-matcher (0 'doctest-selection-face t))
+ ))
+ "Expressions to highlight in Doctest mode.")
+
+(defun doctest-output-line-matcher (limit)
+ "A `font-lock-keyword' MATCHER that returns t if the current
+line is the expected output for a doctest example, and if so,
+sets `match-data' so that group 0 spans the current line."
+ ;; The real work is done by find-doctest-output-line.
+ (when (find-doctest-output-line limit)
+ ;; If we found one, then mark the entire line.
+ (beginning-of-line)
+ (search-forward-regexp "[^\n]*" limit)))
+
+;; [XX] Under construction.
+(defun doctest-selection-matcher (limit)
+ (let (found-it)
+ (while (and (not found-it)
+ (search-forward-regexp "^[ \t]*\\(>>>\\|[.][.][.]\\)"
+ limit t))
+ (if (get-text-property (point) 'doctest-selected)
+ (setq found-it t)))
+ found-it))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Source line indentation
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defun doctest-indent-source-line (&optional dedent-only)
+ "Re-indent the current line, as doctest source code. I.e., add a
+prompt to the current line if it doesn't have one, and re-indent the
+source code (to the right of the prompt). If `dedent-only' is true,
+then don't increase the indentation level any."
+ (interactive "*")
+ (let ((indent-end nil))
+ (save-excursion
+ (beginning-of-line)
+ (let ((new-indent (doctest-current-source-line-indentation dedent-only))
+ (new-margin (doctest-current-source-line-margin))
+ (line-had-prompt (looking-at doctest-prompt-re)))
+ ;; Delete the old prompt (if any).
+ (when line-had-prompt
+ (goto-char (match-end 1))
+ (delete-char 4))
+ ;; Delete the old indentation.
+ (delete-backward-char (skip-chars-forward " \t"))
+ ;; If it's a continuation line, or a new PS1 prompt,
+ ;; then copy the margin.
+ (when (or new-indent (not line-had-prompt))
+ (beginning-of-line)
+ (delete-backward-char (skip-chars-forward " \t"))
+ (insert-char ?\ new-margin))
+ ;; Add the new prompt.
+ (insert-string (if new-indent "... " ">>> "))
+ ;; Add the new indentation
+ (if new-indent (insert-char ?\ new-indent))
+ (setq indent-end (point))))
+ ;; If we're left of the indentation end, then move up to the
+ ;; indentation end.
+ (if (< (point) indent-end) (goto-char indent-end))))
+
+(defun doctest-current-source-line-indentation (&optional dedent-only)
+ "Return the post-prompt indent to use for this line. This is an
+integer for a continuation lines, and nil for non-continuation lines."
+ (save-excursion
+ (let ((prev-line-indent 0)
+ (curr-line-indent 0)
+ (prev-line-opens-block nil)
+ (prev-line-closes-block nil)
+ (curr-line-outdented nil))
+ ;; Examine this doctest line.
+ (beginning-of-line)
+ (when (looking-at doctest-prompt-re)
+ (setq curr-line-indent (- (match-end 3) (match-beginning 3)))
+ (goto-char (match-end 3)))
+ (setq curr-line-outdented (looking-at doctest-outdent-re))
+ ;; Examine the previous line.
+ (when (= (forward-line -1) 0) ; move up a line
+ (when (looking-at doctest-prompt-re) ; is it a source line?
+ (let ((indent-beg (column-at-char (match-beginning 3)))
+ (indent-end (column-at-char (match-end 3))))
+ (setq prev-line-indent (- indent-end indent-beg))
+ (goto-char (match-end 3))
+ (if (looking-at doctest-open-block-re)
+ (setq prev-line-opens-block t))
+ (if (looking-at doctest-close-block-re)
+ (setq prev-line-closes-block t))
+ (if (looking-at doctest-no-outdent-re)
+ (setq curr-line-outdented nil))
+ )))
+ (let ((indent (+ prev-line-indent
+ (if curr-line-outdented -4 0)
+ (if prev-line-opens-block 4 0)
+ (if prev-line-closes-block -4 0))))
+ ;; If dedent-only is true, then make sure we don't indent.
+ (when dedent-only
+ (setq indent (min indent curr-line-indent)))
+ ;; If indent=0 and we're not outdented, then set indent to
+ ;; nil (to signify the start of a new source example).
+ (when (and (= indent 0) (not curr-line-outdented))
+ (setq indent nil))
+ ;; Return the indentation.
+ indent))))
+
+(defun doctest-current-source-line-margin ()
+ "Return the pre-prompt margin to use for this source line. This is
+copied from the most recent source line, or set to
+`doctest-default-margin' if there are no preceeding source lines."
+ (save-excursion
+ (beginning-of-line)
+ (if (search-backward-regexp doctest-prompt-re nil t)
+ (let ((margin-beg (column-at-char (match-beginning 1)))
+ (margin-end (column-at-char (match-end 1))))
+ (- margin-end margin-beg))
+ doctest-default-margin)))
+
+(defun doctest-electric-backspace ()
+ "Delete the preceeding character, level of indentation, or
+prompt.
+
+If point is at the leftmost column, delete the preceding newline.
+
+Otherwise, if point is at the first non-whitespace character
+following an indented source line's prompt, then reduce the
+indentation to the next multiple of 4; and update the source line's
+prompt, when necessary.
+
+Otherwise, if point is at the first non-whitespace character
+following an unindented source line's prompt, then remove the
+prompt (converting the line to an output line or text line).
+
+Otherwise, if point is at the first non-whitespace character of a
+line, the delete the line's indentation.
+
+Otherwise, delete the preceeding character.
+"
+ (interactive "*")
+ (cond
+ ;; Beginning of line: delete preceeding newline.
+ ((bolp) (backward-delete-char 1))
+
+ ;; First non-ws char following prompt: dedent or remove prompt.
+ ((and (looking-at "[^ \t\n]\\|$") (doctest-looking-back doctest-prompt-re))
+ (let* ((prompt-beg (match-beginning 2))
+ (indent-beg (match-beginning 3)) (indent-end (match-end 3))
+ (old-indent (- indent-end indent-beg))
+ (new-indent (* (/ (- old-indent 1) 4) 4)))
+ (cond
+ ;; Indented source line: dedent it.
+ ((> old-indent 0)
+ (goto-char indent-beg)
+ (delete-region indent-beg indent-end)
+ (insert-char ?\ new-indent)
+ ;; Change prompt to PS1, when appropriate.
+ (when (and (= new-indent 0) (not (looking-at doctest-outdent-re)))
+ (delete-backward-char 4)
+ (insert-string ">>> ")))
+ ;; Non-indented source line: remove prompt.
+ (t
+ (goto-char indent-end)
+ (delete-region prompt-beg indent-end)))))
+
+ ;; First non-ws char of a line: delete all indentation.
+ ((and (looking-at "[^ \n\t]\\|$") (doctest-looking-back "^[ \t]+"))
+ (delete-region (match-beginning 0) (match-end 0)))
+
+ ;; Otherwise: delete a character.
+ (t
+ (backward-delete-char 1))))
+
+(defun doctest-newline-and-indent ()
+ "Insert a newline, and indent the new line appropriately.
+
+If the current line is a source line containing a bare prompt,
+then clear the current line, and insert a newline.
+
+Otherwise, if the current line is a source line, then insert a
+newline, and add an appropriately indented prompt to the new
+line.
+
+Otherwise, if the current line is an output line, then insert a
+newline and indent the new line to match the example's margin.
+
+Otherwise, insert a newline.
+
+If `doctest-avoid-trailing-whitespace' is true, then clear any
+whitespace to the left of the point before inserting a newline.
+"
+ (interactive "*")
+ ;; If we're avoiding trailing spaces, then delete WS before point.
+ (if doctest-avoid-trailing-whitespace
+ (delete-char (- (skip-chars-backward " \t"))))
+ (cond
+ ;; If we're on an empty prompt, delete it.
+ ((on-empty-doctest-source-line)
+ (delete-region (match-beginning 0) (match-end 0))
+ (insert-char ?\n 1))
+ ;; If we're on a doctest line, add a new prompt.
+ ((on-doctest-source-line)
+ (insert-char ?\n 1)
+ (doctest-indent-source-line))
+ ;; If we're in doctest output, indent to the margin.
+ ((on-doctest-output-line)
+ (insert-char ?\n 1)
+ (insert-char ?\ (doctest-current-source-line-margin)))
+ ;; Otherwise, just add a newline.
+ (t (insert-char ?\n 1))))
+
+(defun doctest-electric-colon ()
+ "Insert a colon, and dedent the line when appropriate."
+ (interactive "*")
+ (insert-char ?: 1)
+ (when (on-doctest-source-line)
+ (doctest-indent-source-line t)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Code Execution
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defun doctest-execute-buffer (&optional diff)
+ "Run doctest on the current buffer, and display the results in the
+*doctest-output* buffer."
+ (interactive "*")
+ (setq doctest-results-buffer (get-buffer-create doctest-results-buffer-name))
+ (let* ((temp (concat (doctest-temp-name) ".py"))
+ (tempfile (expand-file-name temp doctest-temp-directory))
+ (cur-buf (current-buffer))
+ (in-buf (get-buffer-create "*doctest-input*"))
+ (beg (point-min)) (end (point-max))
+ (flags (reduce (lambda (a b) (if (equal b "") a (concat a "|" b)))
+ doctest-optionflags))
+ (script (concat "from doctest import *\n"
+ "doc = open('" tempfile "').read()\n"
+ "test = DocTestParser().get_doctest("
+ "doc, {}, '" (buffer-name) "', '"
+ (buffer-file-name) "', 0)\n"
+ "r = DocTestRunner(optionflags=" flags
+ (if diff "+REPORT_UDIFF" "")
+ ")\n"
+ "r.run(test)\n"
+ "print\n" ;; <- so the buffer won't be empty
+ ))
+ (cmd (concat doctest-python-command " -c \"" script "\"")))
+ ;; Write buffer to a file.
+ (save-excursion
+ (set-buffer in-buf)
+ (insert-buffer-substring cur-buf beg end)
+ (write-file tempfile))
+ ;; Run doctest
+ (shell-command cmd doctest-results-buffer)
+ ;; Delete the temp file
+ (delete-file tempfile)
+ ;; Delete the input buffer.
+ (if (buffer-live-p in-buf)
+ (kill-buffer in-buf))
+ ;; Set mode on output buffer.
+ (save-excursion
+ (set-buffer doctest-results-buffer)
+ (doctest-results-mode))
+ ;; If any tests failed, display them.
+ (cond ((> (buffer-size doctest-results-buffer) 1)
+ (display-buffer doctest-results-buffer)
+ (doctest-postprocess-results)
+ (message "Test failed!"))
+ (t
+ (display-buffer doctest-results-buffer)
+ (if (get-buffer-window doctest-results-buffer)
+ (delete-window (get-buffer-window doctest-results-buffer)))
+ (message "Test passed!")
+ ))))
+
+(defun doctest-execute-buffer-with-diff ()
+ "Run doctest on the current buffer, and display the results in the
+*doctest-output* buffer, using the diff format."
+ (interactive "*")
+ (doctest-execute-buffer t))
+
+(defun doctest-postprocess-results ()
+ (doctest-next-failure 1)
+ (if hide-example-source
+ (hide-example-source)))
+
+(defun doctest-next-failure (count)
+ "Move to the top of the next failing example, and highlight the
+example's failure description in *doctest-output*."
+ (interactive "p")
+ (let (lineno)
+ (cond
+ ((not (buffer-live-p doctest-results-buffer))
+ (message "Run doctest first! (C-c C-c)"))
+ (t
+ (let ((orig-window (selected-window))
+ (results-window (display-buffer doctest-results-buffer)))
+ (save-excursion
+ (set-buffer doctest-results-buffer)
+ ;; Switch to the results window (so its point gets updated)
+ (if results-window (select-window results-window))
+ ;; Pick up where we left off.
+ ;; (nb: doctest-selected-failure is buffer-local)
+ (goto-char (or doctest-selected-failure (point-min)))
+ ;; Skip past anything on *this* line.
+ (if (>= count 0) (end-of-line) (beginning-of-line))
+ ;; Look for the next failure
+ (if (>= count 0)
+ (re-search-forward doctest-results-loc-re nil t count)
+ (re-search-backward doctest-results-loc-re nil t (- count)))
+ (cond
+ ;; We found a failure:
+ ((match-string 2)
+ (let ((old-selected-failure doctest-selected-failure))
+ ;; Extract the line number for the doctest file.
+ (setq lineno (string-to-int (match-string 2)))
+ ;; Store our position for next time.
+ (beginning-of-line)
+ (setq doctest-selected-failure (point))
+ ;; Update selection.
+ (doctest-fontify-line old-selected-failure)
+ (doctest-fontify-line doctest-selected-failure)))
+ ;; We didn't find a failure:
+ (t
+ (message "No failures found!"))))
+ ;; Return to the original window
+ (select-window orig-window))))
+
+ (when lineno
+ ;; Move point to the selected failure.
+ (goto-line lineno)
+; ;; Highlight it. [XX] Under construction.
+; (let ((beg (save-excursion (beginning-of-line) (point)))
+; (end (save-excursion (end-of-line) (point))))
+; (add-text-properties (point-min) (point-max) '(doctest-selected nil))
+; (add-text-properties beg end '(doctest-selected t))
+; (doctest-fontify-line (point)))
+ )))
+
+(defun doctest-prev-failure (count)
+ "Move to the top of the previous failing example, and highlight
+the example's failure description in *doctest-output*."
+ (interactive "p")
+ (doctest-next-failure (- count)))
+
+(defun doctest-first-failure ()
+ (interactive "")
+ (if (buffer-live-p doctest-results-buffer)
+ (save-excursion
+ (set-buffer doctest-results-buffer)
+ (let ((old-selected-failure doctest-selected-failure))
+ (setq doctest-selected-failure (point-min))
+ (doctest-fontify-line old-selected-failure))))
+ (doctest-next-failure 1))
+
+(defun doctest-last-failure ()
+ (interactive "")
+ (if (buffer-live-p doctest-results-buffer)
+ (save-excursion
+ (set-buffer doctest-results-buffer)
+ (let ((old-selected-failure doctest-selected-failure))
+ (setq doctest-selected-failure (point-max))
+ (doctest-fontify-line old-selected-failure))))
+ (doctest-next-failure -1))
+
+(defconst doctest-example-source-re
+ "^Failed example:\n\\(\n\\| [^\n]*\n\\)+")
+(defun hide-example-source ()
+ "Delete the source code listings from the results buffer (since it's
+easy enough to see them in the original buffer)"
+ (save-excursion
+ (set-buffer doctest-results-buffer)
+ (toggle-read-only nil)
+ (beginning-of-buffer)
+ (while (re-search-forward doctest-example-source-re nil t)
+ (replace-match "" nil nil))
+ (toggle-read-only t)))
+
+
+;; Unfortunately, the `replace-regexp-in-string' is not provided by all
+;; versions of Emacs. But this will do the job:
+(defun doctest-replace-regexp-in-string (regexp replacement text)
+ "Return the result of replacing all mtaches of REGEXP with
+REPLACEMENT in TEXT. (Since replace-regexp-in-string is not available
+under all versions of emacs, and is called different names in
+different versions, this compatibility function will emulate it if
+it's not available."
+ (let ((start 0) (repl-len (length replacement)))
+ (while (string-match regexp text start)
+ (setq start (+ (match-beginning 0) repl-len 1))
+ (setq text (replace-match replacement t nil text)))
+ text))
+
+(defun doctest-results-next-header ()
+ (if (re-search-forward (concat doctest-results-header-re "\\|"
+ doctest-results-divider-re) nil t)
+ (let ((result (match-string 0)))
+ (if (string-match doctest-results-header-re result)
+ result
+ nil))
+ nil))
+
+(defun doctest-replace-output ()
+ "Move to the top of the closest example, and replace its output
+with the 'got' output from the *doctest-output* buffer. An error is
+displayed if the chosen example is not listed in *doctest-output*, or
+if the 'expected' output for the example does not exactly match the
+output listed in the source buffer. The user is asked to confirm the
+replacement."
+ (interactive)
+ ;; Move to the beginning of the example.
+ (cond
+ ((not (buffer-live-p doctest-results-buffer))
+ (message "Run doctest first! (C-c C-c)"))
+ (t
+ (save-excursion
+ (let ((orig-buffer (current-buffer)))
+ ;; Find the doctest case closest to the cursor.
+ (end-of-line)
+ (re-search-backward "^\\( *\\)>>> " nil t)
+ ;; Find the corresponding doctest in the output buffer.
+ (let ((prompt-indent (match-string 1))
+ (output-re (format "^File .*, line %d," (line-number)))
+ (doctest-got nil) (doctest-expected nil) (header nil))
+ (set-buffer doctest-results-buffer)
+
+ ;; Find the corresponding example in the output.
+ (goto-char (point-min))
+ (if (not (re-search-forward output-re nil t))
+ (error "Could not find corresponding output"))
+
+ ;; Get the output's 'expected' & 'got' texts.
+ (while (setq header (doctest-results-next-header))
+ (cond
+ ((equal header "Failed example:")
+ t)
+ ((equal header "Expected nothing")
+ (setq doctest-expected ""))
+ ((equal header "Expected:")
+ (re-search-forward "^\\(\\( \\).*\n\\)*")
+ (setq doctest-expected (doctest-replace-regexp-in-string
+ "^ " prompt-indent (match-string 0))))
+ ((equal header "Got nothing")
+ (setq doctest-got ""))
+ ((or (equal header "Got:") (equal header "Exception raised:"))
+ (re-search-forward "^\\(\\( \\).*\n\\)*")
+ (setq doctest-got (doctest-replace-regexp-in-string
+ "^ " prompt-indent (match-string 0))))
+ (t (error "Unexpected header %s" header))))
+
+ ;; Go back to the source buffer.
+ (set-buffer orig-buffer)
+
+ ;; Skip ahead to the output.
+ (re-search-forward "^ *>>>.*\n\\( *\\.\\.\\..*\n\\)*")
+
+ ;; Check that the output matches.
+ (let ((start (point)) end)
+ (re-search-forward "^ *\\(>>>.*\\|$\\)")
+ (setq end (match-beginning 0))
+ (if doctest-expected
+ (if (not (equal (buffer-substring start end) doctest-expected))
+ (error "Output does mot match 'expected'")))
+ (setq doctest-expected (buffer-substring start end))
+ (goto-char end))
+
+ (let ((confirm-buffer (get-buffer-create "*doctest-confirm*")))
+ (set-buffer confirm-buffer)
+ ;; Erase anything left over in the buffer.
+ (delete-region (point-min) (point-max))
+ ;; Write a confirmation message
+ (if (equal doctest-expected "")
+ (insert-string "Replace nothing\n")
+ (insert-string (concat "Replace:\n" doctest-expected)))
+ (if (equal doctest-got "")
+ (insert-string "With nothing\n")
+ (insert-string (concat "With:\n" doctest-got)))
+ (let ((confirm-window (display-buffer confirm-buffer nil nil t)))
+ ;; Get confirmation
+ (set-buffer orig-buffer)
+ ;; [XX]
+ (if doctest-expected
+ (search-backward doctest-expected)
+ t)
+ (when (y-or-n-p "Ok to replace? ")
+ (replace-match doctest-got t t))
+ (kill-buffer confirm-buffer)
+ (delete-window confirm-window)))))))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Doctest Results Mode (output of doctest-execute-buffer)
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; [XX] Todo:
+;; - Make it read-only?
+;; - Hitting enter goes to the corresponding error
+;; - Clicking goes to corresponding error (not as useful)
+
+
+(defconst doctest-results-divider-re
+ "^\\([*]\\{60,\\}\\)$")
+
+(defconst doctest-results-loc-re
+ "^File \"\\([^\"]+\\)\", line \\([0-9]+\\), in \\([^\n]+\\)")
+
+(defconst doctest-results-header-re
+ "^\\([a-zA-Z0-9 ]+:\\|Expected nothing\\|Got nothing\\)$")
+
+(defconst doctest-results-font-lock-keywords
+ `((,doctest-results-divider-re
+ (0 'doctest-results-divider-face))
+ (,doctest-results-loc-re
+ (0 'doctest-results-loc-face))
+ (,doctest-results-header-re
+ (0 'doctest-results-header-face))
+ (doctest-results-selection-matcher
+ (0 'doctest-results-selection-face t))))
+
+(defun doctest-results-selection-matcher (limit)
+ "Matches from `doctest-selected-failure' to the end of the
+line. This is used to highlight the currently selected failure."
+ (when (and doctest-selected-failure
+ (<= (point) doctest-selected-failure)
+ (< doctest-selected-failure limit))
+ (goto-char doctest-selected-failure)
+ (search-forward-regexp "[^\n]+" limit)))
+
+;; Register the font-lock keywords (xemacs)
+(put 'doctest-results-mode 'font-lock-defaults
+ '(doctest-results-font-lock-keywords))
+
+;; Register the font-lock keywords (gnu emacs)
+(defvar font-lock-defaults-alist nil) ; in case we're in xemacs
+(setq font-lock-defaults-alist
+ (append font-lock-defaults-alist
+ `((doctest-results-mode
+ doctest-results-font-lock-keywords
+ nil nil nil nil))))
+
+;; Define the mode
+(define-derived-mode doctest-results-mode text-mode "Doctest Results"
+ "docstring"
+ ;; Enable font-lock mode.
+ (if (featurep 'font-lock) (font-lock-mode 1))
+ ;; Keep track of which failure is selected
+ (set (make-local-variable 'doctest-selected-failure) nil)
+ ;; Make the buffer read-only.
+ (toggle-read-only t))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Helper functions
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defun on-doctest-source-line ()
+ "Return true if the current line is a source line."
+ (save-excursion
+ (beginning-of-line)
+ (looking-at doctest-prompt-re)))
+
+(defun on-empty-doctest-source-line ()
+ "Return true if the current line contains a bare prompt."
+ (save-excursion
+ (beginning-of-line)
+ (looking-at (concat doctest-prompt-re "$"))))
+
+(defun on-doctest-output-line ()
+ "Return true if the current line is an output line."
+ (save-excursion
+ (beginning-of-line)
+ (let ((prompt-or-blankline (concat doctest-prompt-re "\\|" "^[ \t]*\n")))
+ ;; The line must not be blank or start with a prompt.
+ (when (not (looking-at prompt-or-blankline))
+ ;; The line must follow a line starting with a prompt, with
+ ;; no intervening blank lines.
+ (search-backward-regexp prompt-or-blankline nil t)
+ (looking-at doctest-prompt-re)))))
+
+(defun find-doctest-output-line (&optional limit)
+ "Move forward to the next doctest output line (staying within
+the given bounds). Return the character position of the doctest
+output line if one was found, and false otherwise."
+ (let ((found-it nil) ; point where we found an output line
+ (limit (or limit (point-max)))) ; default value for limit
+ (save-excursion
+ ;; Keep moving forward, one line at a time, until we find a
+ ;; doctest output line.
+ (while (and (not found-it) (< (point) limit) (not (eobp)))
+ (if (and (not (eolp)) (on-doctest-output-line))
+ (setq found-it (point))
+ (forward-line))))
+ ;; If we found a doctest output line, then go to it.
+ (if found-it (goto-char found-it))))
+
+(defun doctest-version ()
+ "Echo the current version of `doctest-mode' in the minibuffer."
+ (interactive)
+ (message "Using `doctest-mode' version %s" doctest-version))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Utility functions
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defvar doctest-serial-number 0) ;used if broken-temp-names.
+(defun doctest-temp-name ()
+ (if (memq 'broken-temp-names features)
+ (let
+ ((sn doctest-serial-number)
+ (pid (and (fboundp 'emacs-pid) (emacs-pid))))
+ (setq doctest-serial-number (1+ doctest-serial-number))
+ (if pid
+ (format "doctest-%d-%d" sn pid)
+ (format "doctest-%d" sn)))
+ (make-temp-name "doctest-")))
+
+(defun column-at-char (pos)
+ "Return the column of the given character position"
+ (save-excursion (goto-char pos) (current-column)))
+
+(defun doctest-looking-back (regexp)
+ "Return True if the text before point matches the given regular
+expression. Like looking-at except backwards and slower. (This
+is available as `looking-back' in GNU emacs and
+`looking-at-backwards' in XEmacs, but it's easy enough to define
+from scratch such that it works under both.)"
+ (save-excursion
+ (let ((orig-pos (point)))
+ ;; Search backwards for the regexp.
+ (if (re-search-backward regexp nil t)
+ ;; Check if it ends at the original point.
+ (= orig-pos (match-end 0))))))
+
+(defun doctest-fontify-line (charpos)
+ "Run font-lock-fontify-region on the line containing the given
+position."
+ (if charpos
+ (save-excursion
+ (goto-char charpos)
+ (let ((beg (progn (beginning-of-line) (point)))
+ (end (progn (end-of-line) (point))))
+ (font-lock-fontify-region beg end)))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Syntax Table
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; We do *NOT* currently use this, because it applies too
+;; indiscrimanantly. In particular, we don't want "'" and '"' treated
+;; as quote marks on text lines. But there's no good way to prevent
+;; it.
+(defvar doctest-syntax-alist nil
+ "Syntax alist used in `doctest-mode' buffers.")
+(setq doctest-syntax-alist '((?\( . "()") (?\[ . "(]") (?\{ . "(}")
+ (?\) . ")(") (?\] . ")[") (?\} . "){")
+ (?\$ . "." ) (?\% . "." ) (?\& . "." )
+ (?\* . "." ) (?\+ . "." ) (?\- . "." )
+ (?\/ . "." ) (?\< . "." ) (?\= . "." )
+ (?\> . "." ) (?\| . "." ) (?\_ . "w" )
+ (?\' . "\"") (?\" . "\"") (?\` . "$" )
+ (?\# . "<" ) (?\n . ">" )))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Key Bindings
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defconst doctest-mode-map
+ (let ((map (make-keymap)))
+ (define-key map [backspace] 'doctest-electric-backspace)
+ (define-key map [return] 'doctest-newline-and-indent)
+ (define-key map [tab] 'doctest-indent-source-line)
+ (define-key map ":" 'doctest-electric-colon)
+ (define-key map "\C-c\C-v" 'doctest-version)
+ (define-key map "\C-c\C-c" 'doctest-execute-buffer)
+ (define-key map "\C-c\C-d" 'doctest-execute-buffer-with-diff)
+ (define-key map "\C-c\C-n" 'doctest-next-failure)
+ (define-key map "\C-c\C-p" 'doctest-prev-failure)
+ (define-key map "\C-c\C-a" 'doctest-first-failure)
+ (define-key map "\C-c\C-z" 'doctest-last-failure)
+ (define-key map "\C-c\C-r" 'doctest-replace-output)
+ map)
+ "Keymap for doctest-mode.")
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Define the mode
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Register the font-lock keywords (xemacs)
+(put 'doctest-mode 'font-lock-defaults '(doctest-font-lock-keywords))
+
+;; Register the font-lock keywords (gnu emacs)
+(defvar font-lock-defaults-alist nil) ; in case we're in xemacs
+(setq font-lock-defaults-alist
+ (append font-lock-defaults-alist
+ `((doctest-mode doctest-font-lock-keywords nil nil nil nil))))
+
+(defvar doctest-results-buffer nil
+ "The output buffer for doctest-mode")
+
+;; Use doctest mode for files ending in .doctest
+;;;###autoload
+(add-to-list 'auto-mode-alist '("\\.doctest$" . doctest-mode))
+
+;;;###autoload
+(define-derived-mode doctest-mode text-mode "Doctest"
+ "A major mode for editing text files that contain Python
+doctest examples. Doctest is a testing framework for Python that
+emulates an interactive session, and checks the result of each
+command. For more information, see the Python library reference:
+<http://docs.python.org/lib/module-doctest.html>
+
+`doctest-mode' defines three kinds of line, each of which is
+treated differently:
+
+ - 'Source lines' are lines consisting of a Python prompt
+ ('>>>' or '...'), followed by source code. Source lines are
+ colored (similarly to `python-mode') and auto-indented.
+
+ - 'Output lines' are non-blank lines immediately following
+ source lines. They are colored using several doctest-
+ specific output faces.
+
+ - 'Text lines' are any other lines. They are not processed in
+ any special way.
+
+\\{doctest-mode-map}
+"
+ ;; Enable auto-fill mode.
+ (auto-fill-mode 1)
+
+ ;; Enable font-lock mode.
+ (if (featurep 'font-lock) (font-lock-mode 1))
+
+ ;; Register our indentation function.
+ (set (make-local-variable 'indent-line-function)
+ 'doctest-indent-source-line)
+
+ ;; Keep track of our results buffer.
+ (set (make-local-variable 'doctest-results-buffer) nil)
+ )
+
+(provide 'doctest-mode)
+;;; doctest-mode.el ends here
diff --git a/emacs/psvn.el b/emacs/psvn.el
new file mode 100644
index 0000000..9f25d5a
--- /dev/null
+++ b/emacs/psvn.el
@@ -0,0 +1,6225 @@
+;;; psvn.el --- Subversion interface for emacs
+;; Copyright (C) 2002-2008 by Stefan Reichoer
+
+;; Author: Stefan Reichoer <stefan at xsteve.at>
+
+;; psvn.el is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 2, or (at your option)
+;; any later version.
+
+;; psvn.el is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+
+;; You should have received a copy of the GNU General Public License
+;; along with GNU Emacs; see the file COPYING. If not, write to
+;; the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+;; Boston, MA 02111-1307, USA.
+
+;;; Commentary
+
+;; psvn.el is tested with GNU Emacs 21.3 on windows, debian linux,
+;; freebsd5, red hat el4, ubuntu edgy with svn 1.4.0
+
+;; psvn.el needs at least svn 1.1.0
+;; if you upgrade to a higher version, you need to do a fresh checkout
+
+;; psvn.el is an interface for the revision control tool subversion
+;; (see http://subversion.tigris.org)
+;; psvn.el provides a similar interface for subversion as pcl-cvs for cvs.
+;; At the moment the following commands are implemented:
+;;
+;; M-x svn-status: run 'svn -status -v'
+;; M-x svn-examine (like pcl-cvs cvs-examine) is alias for svn-status
+;;
+;; and show the result in the svn-status-buffer-name buffer (normally: *svn-status*).
+;; If svn-status-verbose is set to nil, only "svn status" without "-v"
+;; is run. Currently you have to toggle this variable manually.
+;; This buffer uses svn-status mode in which the following keys are defined:
+;; g - svn-status-update: run 'svn status -v'
+;; M-s - svn-status-update: run 'svn status -v'
+;; C-u g - svn-status-update: run 'svn status -vu'
+;; = - svn-status-show-svn-diff run 'svn diff'
+;; l - svn-status-show-svn-log run 'svn log'
+;; i - svn-status-info run 'svn info'
+;; r - svn-status-revert run 'svn revert'
+;; X v - svn-status-resolved run 'svn resolved'
+;; U - svn-status-update-cmd run 'svn update'
+;; M-u - svn-status-update-cmd run 'svn update'
+;; c - svn-status-commit run 'svn commit'
+;; a - svn-status-add-file run 'svn add --non-recursive'
+;; A - svn-status-add-file-recursively run 'svn add'
+;; + - svn-status-make-directory run 'svn mkdir'
+;; R - svn-status-mv run 'svn mv'
+;; C - svn-status-cp run 'svn cp'
+;; D - svn-status-rm run 'svn rm'
+;; M-c - svn-status-cleanup run 'svn cleanup'
+;; k - svn-status-lock run 'svn lock'
+;; K - svn-status-unlock run 'svn unlock'
+;; b - svn-status-blame run 'svn blame'
+;; X e - svn-status-export run 'svn export'
+;; RET - svn-status-find-file-or-examine-directory
+;; ^ - svn-status-examine-parent
+;; ~ - svn-status-get-specific-revision
+;; E - svn-status-ediff-with-revision
+;; X X - svn-status-resolve-conflicts
+;; S g - svn-status-grep-files
+;; S s - svn-status-search-files
+;; s - svn-status-show-process-buffer
+;; h - svn-status-pop-to-partner-buffer
+;; e - svn-status-toggle-edit-cmd-flag
+;; ? - svn-status-toggle-hide-unknown
+;; _ - svn-status-toggle-hide-unmodified
+;; m - svn-status-set-user-mark
+;; u - svn-status-unset-user-mark
+;; $ - svn-status-toggle-elide
+;; w - svn-status-copy-current-line-info
+;; DEL - svn-status-unset-user-mark-backwards
+;; * ! - svn-status-unset-all-usermarks
+;; * ? - svn-status-mark-unknown
+;; * A - svn-status-mark-added
+;; * M - svn-status-mark-modified
+;; * P - svn-status-mark-modified-properties
+;; * D - svn-status-mark-deleted
+;; * * - svn-status-mark-changed
+;; * . - svn-status-mark-by-file-ext
+;; * % - svn-status-mark-filename-regexp
+;; . - svn-status-goto-root-or-return
+;; f - svn-status-find-file
+;; o - svn-status-find-file-other-window
+;; C-o - svn-status-find-file-other-window-noselect
+;; v - svn-status-view-file-other-window
+;; I - svn-status-parse-info
+;; V - svn-status-svnversion
+;; P l - svn-status-property-list
+;; P s - svn-status-property-set
+;; P d - svn-status-property-delete
+;; P e - svn-status-property-edit-one-entry
+;; P i - svn-status-property-ignore-file
+;; P I - svn-status-property-ignore-file-extension
+;; P C-i - svn-status-property-edit-svn-ignore
+;; P k - svn-status-property-set-keyword-list
+;; P K i - svn-status-property-set-keyword-id
+;; P K d - svn-status-property-set-keyword-date
+;; P y - svn-status-property-set-eol-style
+;; P x - svn-status-property-set-executable
+;; P m - svn-status-property-set-mime-type
+;; H - svn-status-use-history
+;; x - svn-status-update-buffer
+;; q - svn-status-bury-buffer
+
+;; C-x C-j - svn-status-dired-jump
+
+;; The output in the buffer contains this header to ease reading
+;; of svn output:
+;; FPH BASE CMTD Author em File
+;; F = Filemark
+;; P = Property mark
+;; H = History mark
+;; BASE = local base revision
+;; CMTD = last committed revision
+;; Author = author of change
+;; em = "**" or "(Update Available)" [see `svn-status-short-mod-flag-p']
+;; if file can be updated
+;; File = path/filename
+;;
+
+;; To use psvn.el put the following line in your .emacs:
+;; (require 'psvn)
+;; Start the svn interface with M-x svn-status
+
+;; The latest version of psvn.el can be found at:
+;; http://www.xsteve.at/prg/emacs/psvn.el
+;; Or you can check it out from the subversion repository:
+;; svn co http://svn.collab.net/repos/svn/trunk/contrib/client-side/emacs emacs-svn
+
+;; TODO:
+;; * shortcut for svn propset svn:keywords "Date" psvn.el
+;; * docstrings for the functions
+;; * perhaps shortcuts for ranges, dates
+;; * when editing the command line - offer help from the svn client
+;; * finish svn-status-property-set
+;; * Add repository browser
+;; * Get rid of all byte-compiler warnings
+;; * SVK working copy support
+;; * multiple independent buffers in svn-status-mode
+;; There are "TODO" comments in other parts of this file as well.
+
+;; Overview over the implemented/not (yet) implemented svn sub-commands:
+;; * add implemented
+;; * blame implemented
+;; * cat implemented
+;; * checkout (co) implemented
+;; * cleanup implemented
+;; * commit (ci) implemented
+;; * copy (cp) implemented
+;; * delete (del, remove, rm) implemented
+;; * diff (di) implemented
+;; * export implemented
+;; * help (?, h)
+;; * import used (in svn-admin-create-trunk-directory)
+;; * info implemented
+;; * list (ls) implemented
+;; * lock implemented
+;; * log implemented
+;; * merge
+;; * mkdir implemented
+;; * move (mv, rename, ren) implemented
+;; * propdel (pdel) implemented
+;; * propedit (pedit, pe) not needed
+;; * propget (pget, pg) used (in svn-status-property-edit)
+;; * proplist (plist, pl) implemented
+;; * propset (pset, ps) used (in svn-prop-edit-do-it)
+;; * resolved implemented
+;; * revert implemented
+;; * status (stat, st) implemented
+;; * switch (sw)
+;; * unlock implemented
+;; * update (up) implemented
+
+;; For the not yet implemented commands you should use the command line
+;; svn client. If there are user requests for any missing commands I will
+;; probably implement them.
+
+;; There is also limited support for the web-based software project management and bug/issue tracking system trac
+;; Trac ticket links can be enabled in the *svn-log* buffers when using the following:
+;; (setq svn-log-link-handlers '(trac-ticket-short))
+
+;; ---------------------------
+;; Frequently asked questions:
+;; ---------------------------
+
+;; Q1: I need support for user names with blanks/spaces
+;; A1: Add the user names to svn-user-names-including-blanks and set the
+;; svn-pre-parse-status-hook.
+;; The problem is, that the user names and the file names from the svn status
+;; output can both contain blanks. Blanks in file names are supported.
+;; the svn-user-names-including-blanks list is used to replace the spaces
+;; in the user names with - to overcome this problem
+
+;; Q2: My svn-update command it taking a really long time. How can I
+;; see what's going on?
+;; A2: In the *svn-status* buffer press "s".
+
+;; Q3: How do I enter a username and password?
+;; A3: In the *svn-status* buffer press "s", switch to the
+;; *svn-process* buffer and press enter. You will be prompted for
+;; username and password.
+
+;; Q4: What does "?", "M", and "C" in the first column of the
+;; *svn-status* buffer mean?
+;; A4: "?" means the file(s) is not under Subversion control
+;; "M" means you have a locally modified file
+;; "C" means there is a conflict
+;; "@$&#!" means someone is saying nasty things to you
+
+
+;; Comments / suggestions and bug reports are welcome!
+
+;; Development notes
+;; -----------------
+
+;; "svn-" is the package prefix used in psvn.el. There are also longer
+;; prefixes which clarify the code and help symbol completion, but they
+;; are not intended to prevent name clashes with other packages. All
+;; interactive commands meant to be used only in a specific mode should
+;; have names beginning with the name of that mode: for example,
+;; "svn-status-add-file" in "svn-status-mode". "psvn" should be used
+;; only in names of files, customization groups, and features. If SVK
+;; support is ever added, it should use "svn-svk-" when no existing
+;; prefix is applicable.
+
+;; Many of the variables marked as `risky-local-variable' are probably
+;; impossible to abuse, as the commands that read them are used only in
+;; buffers that are not visiting any files. Better safe than sorry.
+
+;;; Code:
+
+(require 'easymenu)
+
+(eval-when-compile (require 'dired))
+(eval-when-compile (require 'ediff-util))
+(eval-when-compile (require 'ediff-wind))
+(eval-when-compile (require 'elp))
+(eval-when-compile (require 'pp))
+
+(condition-case nil
+ (progn
+ (require 'diff-mode))
+ (error nil))
+
+(defconst svn-psvn-revision "$Id: psvn.el 32032 2008-07-08 17:21:58Z xsteve $"
+ "The revision number of psvn.")
+
+;;; user setable variables
+(defcustom svn-status-verbose t
+ "*Add '-v' to svn status call.
+This can be toggled with \\[svn-status-toggle-svn-verbose-flag]."
+ :type 'boolean
+ :group 'psvn)
+(defcustom svn-log-edit-file-name "++svn-log++"
+ "*Name of a saved log file.
+This can be either absolute, or relative to the default directory
+of the `svn-log-edit-buffer-name' buffer."
+ :type 'file
+ :group 'psvn)
+(put 'svn-log-edit-file-name 'risky-local-variable t)
+(defcustom svn-log-edit-insert-files-to-commit t
+ "*Insert the filelist to commit in the *svn-log* buffer"
+ :type 'boolean
+ :group 'psvn)
+(defcustom svn-log-edit-show-diff-for-commit nil
+ "*Show the diff being committed when you run `svn-status-commit.'."
+ :type 'boolean
+ :group 'psvn)
+(defcustom svn-log-edit-use-log-edit-mode
+ (and (condition-case nil (require 'log-edit) (error nil)) t)
+ "*Use log-edit-mode as base for svn-log-edit-mode
+This variable takes effect only when psvn.el is being loaded."
+ :type 'boolean
+ :group 'psvn)
+(defcustom svn-log-edit-paragraph-start
+ "$\\|[ \t]*$\\|##.*$\\|\\*.*:.*$\\|[ \t]+(.+):.*$"
+ "*Value used for `paragraph-start' in `svn-log-edit-buffer-name' buffer."
+ :type 'regexp
+ :group 'psvn)
+(defcustom svn-log-edit-paragraph-separate "$\\|##.*$"
+ "*Value used for `paragraph-separate' in `svn-log-edit-buffer-name' buffer."
+ :type 'regexp
+ :group 'psvn)
+(defcustom svn-status-hide-unknown nil
+ "*Hide unknown files in `svn-status-buffer-name' buffer.
+This can be toggled with \\[svn-status-toggle-hide-unknown]."
+ :type 'boolean
+ :group 'psvn)
+(defcustom svn-status-hide-unmodified nil
+ "*Hide unmodified files in `svn-status-buffer-name' buffer.
+This can be toggled with \\[svn-status-toggle-hide-unmodified]."
+ :type 'boolean
+ :group 'psvn)
+(defcustom svn-status-sort-status-buffer t
+ "*Whether to sort the `svn-status-buffer-name' buffer.
+
+Setting this variable to nil speeds up \\[M-x svn-status], however the
+listing may then become incorrect.
+
+This can be toggled with \\[svn-status-toggle-sort-status-buffer]."
+ :type 'boolean
+ :group 'psvn)
+
+(defcustom svn-status-ediff-delete-temporary-files nil
+ "*Whether to delete temporary ediff files. If set to ask, ask the user"
+ :type '(choice (const t)
+ (const nil)
+ (const ask))
+ :group 'psvn)
+
+(defcustom svn-status-changelog-style 'changelog
+ "*The changelog style that is used for `svn-file-add-to-changelog'.
+Possible values are:
+ 'changelog: use `add-change-log-entry-other-window'
+ 'svn-dev: use commit messages that are used by the svn developers
+ a function: This function is called to add a new entry to the changelog file.
+"
+ :type '(set (const changelog)
+ (const svn-dev))
+ :group 'psvn)
+
+(defcustom svn-status-unmark-files-after-list '(commit revert)
+ "*List of operations after which all user marks will be removed.
+Possible values are: commit, revert."
+ :type '(set (const commit)
+ (const revert))
+ :group 'psvn)
+
+(defcustom svn-status-preserve-window-configuration t
+ "*Try to preserve the window configuration."
+ :type 'boolean
+ :group 'psvn)
+
+(defcustom svn-status-auto-revert-buffers t
+ "*Auto revert buffers that have changed on disk."
+ :type 'boolean
+ :group 'psvn)
+
+(defcustom svn-status-fancy-file-state-in-modeline t
+ "*Show a color dot in the modeline that describes the state of the current file."
+ :type 'boolean
+ :group 'psvn)
+
+(defcustom svn-status-negate-meaning-of-arg-commands '()
+ "*List of operations that should use a negated meaning of the prefix argument.
+The supported functions are `svn-status' and `svn-status-set-user-mark'."
+ :type '(set (function-item svn-status)
+ (function-item svn-status-set-user-mark))
+ :group 'psvn)
+
+(defcustom svn-status-svn-executable "svn"
+ "*The name of the svn executable.
+This can be either absolute or looked up on `exec-path'."
+ ;; Don't use (file :must-match t). It doesn't know about `exec-path'.
+ :type 'file
+ :group 'psvn)
+(put 'svn-status-svn-executable 'risky-local-variable t)
+
+(defcustom svn-status-default-export-directory "~/" "*The default directory that is suggested svn export."
+ :type 'file
+ :group 'psvn)
+
+(defcustom svn-status-svn-environment-var-list '("LC_MESSAGES=C" "LC_ALL=")
+ "*A list of environment variables that should be set for that svn process.
+Each element is either a string \"VARIABLE=VALUE\" which will be added to
+the environment when svn is run, or just \"VARIABLE\" which causes that
+variable to be entirely removed from the environment.
+
+The default setting is '(\"LC_MESSAGES=C\" \"LC_ALL=\"). This ensures that the svn command
+line client does not output localized strings. psvn.el relies on the english
+messages."
+ :type '(repeat string)
+ :group 'psvn)
+(put 'svn-status-svn-environment-var-list 'risky-local-variable t)
+
+(defcustom svn-browse-url-function nil
+ ;; If the user hasn't changed `svn-browse-url-function', then changing
+ ;; `browse-url-browser-function' should affect psvn even after it has
+ ;; been loaded.
+ "Function to display a Subversion related WWW page in a browser.
+So far, this is used only for \"trac\" issue tracker integration.
+By default, this is nil, which means use `browse-url-browser-function'.
+Any non-nil value overrides that variable, with the same syntax."
+ ;; It would be nice to show the full list of browsers supported by
+ ;; browse-url, but (custom-variable-type 'browse-url-browser-function)
+ ;; returns just `function' if browse-url has not yet been loaded,
+ ;; and there seems to be no easy way to autoload browse-url when
+ ;; the custom-type of svn-browse-url-function is actually needed.
+ ;; So I'll only offer enough choices to cover all supported types.
+ :type `(choice (const :tag "Specified by `browse-url-browser-function'" nil)
+ (function :value browse-url-default-browser
+ ;; In XEmacs 21.4.17, the `function' widget matches
+ ;; all objects. Constrain it here so that alists
+ ;; fall through to the next choice. Accept either
+ ;; a symbol (fbound or not) or a lambda expression.
+ :match ,(lambda (widget value)
+ (or (symbolp value) (functionp value))))
+ (svn-alist :tag "Regexp/function association list"
+ :key-type regexp :value-type function
+ :value (("." . browse-url-default-browser))))
+ :link '(emacs-commentary-link "browse-url")
+ :group 'psvn)
+;; (put 'svn-browse-url-function 'risky-local-variable t)
+;; already implied by "-function" suffix
+
+(defcustom svn-status-window-alist
+ '((diff "*svn-diff*") (log "*svn-log*") (info t) (blame t) (proplist t) (update t))
+ "An alist to specify which windows should be used for svn command outputs.
+The following keys are supported: diff, log, info, blame, proplist, update.
+The following values can be given:
+nil ... show in `svn-process-buffer-name' buffer
+t ... show in dedicated *svn-info* buffer
+invisible ... don't show the buffer (eventually useful for update)
+a string ... show in a buffer named string"
+ :type '(svn-alist
+ :key-type symbol
+ :value-type (group
+ (choice
+ (const :tag "Show in *svn-process* buffer" nil)
+ (const :tag "Show in dedicated *svn-info* buffer" t)
+ (const :tag "Don't show the output" invisible)
+ (string :tag "Show in a buffer named"))))
+ :options '(diff log info blame proplist update)
+ :group 'psvn)
+
+(defcustom svn-status-short-mod-flag-p t
+ "*Whether the mark for out of date files is short or long.
+
+If this variable is is t, and a file is out of date (i.e., there is a newer
+version in the repository than the working copy), then the file will
+be marked by \"**\"
+
+If this variable is nil, and the file is out of date then the longer phrase
+\"(Update Available)\" is used.
+
+In either case the mark gets the face
+`svn-status-update-available-face', and will only be visible if
+`\\[svn-status-update]' is run with a prefix argument"
+ :type '(choice (const :tag "Short \"**\"" t)
+ (const :tag "Long \"(Update Available)\"" nil))
+ :group 'psvn)
+
+(defvar svn-status-debug-level 0 "The psvn.el debugging verbosity level.
+The higher the number, the more debug messages are shown.
+
+See `svn-status-message' for the meaning of values for that variable.")
+
+(defvar svn-bookmark-list nil "A list of locations for a quick access via `svn-status-via-bookmark'")
+;;(setq svn-bookmark-list '(("proj1" . "~/work/proj1")
+;; ("doc1" . "~/docs/doc1")))
+
+(defvar svn-status-buffer-name "*svn-status*" "Name for the svn status buffer")
+(defvar svn-process-buffer-name " *svn-process*" "Name for the svn process buffer")
+(defvar svn-log-edit-buffer-name "*svn-log-edit*" "Name for the svn log-edit buffer")
+
+(defcustom svn-status-use-header-line
+ (if (boundp 'header-line-format) t 'inline)
+ "*Whether a header line should be used.
+When t: Use the emacs header line
+When 'inline: Insert the header line in the `svn-status-buffer-name' buffer
+Otherwise: Don't display a header line"
+ :type '(choice (const :tag "Show column titles as a header line" t)
+ (const :tag "Insert column titles as text in the buffer" inline)
+ (other :tag "No column titles" nil))
+ :group 'psvn)
+
+;;; default arguments to pass to svn commands
+;; TODO: When customizing, an option menu or completion might be nice....
+(defcustom svn-status-default-log-arguments '("-v")
+ "*List of arguments to pass to svn log.
+\(used in `svn-status-show-svn-log'; override these by giving prefixes\)."
+ :type '(repeat string)
+ :group 'psvn)
+(put 'svn-status-default-log-arguments 'risky-local-variable t)
+
+(defcustom svn-status-default-commit-arguments '()
+ "*List of arguments to pass to svn commit.
+If you don't like recursive commits, set this value to (\"-N\")
+or mark the directory before committing it.
+Do not put an empty string here, except as an argument of an option:
+Subversion and the operating system may treat that as a file name
+equivalent to \".\", so you would commit more than you intended."
+ :type '(repeat string)
+ :group 'psvn)
+(put 'svn-status-default-commit-arguments 'risky-local-variable t)
+
+(defcustom svn-status-default-diff-arguments '("-x" "--ignore-eol-style")
+ "*A list of arguments that is passed to the svn diff command.
+When the built in diff command is used,
+the following options are available: --ignore-eol-style, --ignore-space-change,
+--ignore-all-space, --ignore-eol-style.
+The following setting ignores eol style changes and all white space changes:
+'(\"-x\" \"--ignore-eol-style --ignore-all-space\")
+
+If you'd like to suppress whitespace changes using the external diff command
+use the following value:
+'(\"--diff-cmd\" \"diff\" \"-x\" \"-wbBu\")
+
+"
+ :type '(repeat string)
+ :group 'psvn)
+(put 'svn-status-default-diff-arguments 'risky-local-variable t)
+
+(defcustom svn-status-default-status-arguments '()
+ "*A list of arguments that is passed to the svn status command.
+The following options are available: --ignore-externals
+
+"
+ :type '(repeat string)
+ :group 'psvn)
+(put 'svn-status-default-status-arguments 'risky-local-variable t)
+
+(defcustom svn-status-default-blame-arguments '("-x" "--ignore-eol-style")
+ "*A list of arguments that is passed to the svn blame command.
+See `svn-status-default-diff-arguments' for some examples."
+ :type '(repeat string)
+ :group 'psvn)
+
+(put 'svn-status-default-blame-arguments 'risky-local-variable t)
+
+(defvar svn-trac-project-root nil
+ "Path for an eventual existing trac issue tracker.
+This can be set with \\[svn-status-set-trac-project-root].")
+
+(defvar svn-status-module-name nil
+ "*A short name for the actual project.
+This can be set with \\[svn-status-set-module-name].")
+
+(defvar svn-status-branch-list nil
+ "*A list of known branches for the actual project
+This can be set with \\[svn-status-set-branch-list].
+
+The list contains full repository paths or shortcuts starting with \#
+\# at the beginning is replaced by the repository url.
+\#1\# has the special meaning that all paths below the given directory
+will be considered for interactive selections.
+
+A useful setting might be: '\(\"\#trunk\" \"\#1\#tags\" \"\#1\#branches\")")
+
+(defvar svn-status-load-state-before-svn-status t
+ "*Whether to automatically restore state from ++psvn.state file before running svn-status.")
+
+(defvar svn-log-link-handlers nil "A list of link handlers in *svn-log* buffers.
+These link handlers must be registered via `svn-log-register-link-handler'")
+
+;;; hooks
+(defvar svn-status-mode-hook nil "Hook run when entering `svn-status-mode'.")
+(defvar svn-log-edit-mode-hook nil "Hook run when entering `svn-log-edit-mode'.")
+(defvar svn-log-edit-done-hook nil "Hook run after commiting files via svn.")
+;; (put 'svn-log-edit-mode-hook 'risky-local-variable t)
+;; (put 'svn-log-edit-done-hook 'risky-local-variable t)
+;; already implied by "-hook" suffix
+
+(defvar svn-post-process-svn-output-hook nil "Hook that can be used to preprocess the output from svn.
+The function `svn-status-remove-control-M' can be useful for that hook")
+
+(when (eq system-type 'windows-nt)
+ (add-hook 'svn-post-process-svn-output-hook 'svn-status-remove-control-M))
+
+(defvar svn-status-svn-process-coding-system (when (boundp 'locale-coding-system) locale-coding-system)
+ "The coding system that is used for the svn command line client.
+It is used in svn-run, if it is not nil.")
+
+(defvar svn-status-svn-file-coding-system 'undecided-unix
+ "The coding system that is used to save files that are loaded as
+parameter or data files via the svn command line client.
+It is used in the following functions: `svn-prop-edit-do-it', `svn-log-edit-done'.
+You could set it to 'utf-8")
+
+(defcustom svn-status-use-ido-completion
+ (fboundp 'ido-completing-read)
+ "*Use ido completion functionality."
+ :type 'boolean
+ :group 'psvn)
+
+(defvar svn-status-completing-read-function
+ (if svn-status-use-ido-completion 'ido-completing-read 'completing-read))
+
+;;; experimental features
+(defvar svn-status-track-user-input nil "Track user/password queries.
+This feature is implemented via a process filter.
+It is an experimental feature.")
+
+(defvar svn-status-refresh-info nil "Whether `svn-status-update-buffer' should call `svn-status-parse-info'.")
+
+;;; Customize group
+(defgroup psvn nil
+ "Subversion interface for Emacs."
+ :group 'tools)
+
+(defgroup psvn-faces nil
+ "psvn faces."
+ :group 'psvn)
+
+
+(eval-and-compile
+ (require 'cl)
+ (defconst svn-xemacsp (featurep 'xemacs))
+ (if svn-xemacsp
+ (require 'overlay)
+ (require 'overlay nil t)))
+
+(defcustom svn-status-display-full-path nil
+ "Specifies how the filenames look like in the listing.
+If t, their full path name will be displayed, else only the filename."
+ :type 'boolean
+ :group 'psvn)
+
+(defcustom svn-status-prefix-key [(control x) (meta s)]
+ "Prefix key for the psvn commands in the global keymap."
+ :type '(choice (const [(control x) ?v ?S])
+ (const [(super s)])
+ (const [(hyper s)])
+ (const [(control x) ?v])
+ (const [(control x) ?V])
+ (sexp))
+ :group 'psvn
+ :set (lambda (var value)
+ (if (boundp var)
+ (global-unset-key (symbol-value var)))
+ (set var value)
+ (global-set-key (symbol-value var) 'svn-global-keymap)))
+
+(defcustom svn-admin-default-create-directory "~/"
+ "*The default directory that is suggested for `svn-admin-create'."
+ :type 'string
+ :group 'psvn)
+
+(defvar svn-status-custom-hide-function nil
+ "A function that receives a line-info and decides whether to hide that line.
+See psvn.el for an example function.")
+;; (put 'svn-status-custom-hide-function 'risky-local-variable t)
+;; already implied by "-function" suffix
+
+
+;; Use the normally used mode for files ending in .~HEAD~, .~BASE~, ...
+(add-to-list 'auto-mode-alist '("\\.~?\\(HEAD\\|BASE\\|PREV\\)~?\\'" ignore t))
+
+;;; internal variables
+(defvar svn-status-directory-history nil "List of visited svn working directories.")
+(defvar svn-process-cmd nil)
+(defvar svn-status-info nil)
+(defvar svn-status-filename-to-buffer-position-cache (make-hash-table :test 'equal :weakness t))
+(defvar svn-status-base-info nil "The parsed result from the svn info command.")
+(defvar svn-status-initial-window-configuration nil)
+(defvar svn-status-default-column 23)
+(defvar svn-status-default-revision-width 4)
+(defvar svn-status-default-author-width 9)
+(defvar svn-status-line-format " %c%c%c %4s %4s %-9s")
+(defvar svn-start-of-file-list-line-number 0)
+(defvar svn-status-files-to-commit nil
+ "List of files to commit at `svn-log-edit-done'.
+This is always set together with `svn-status-recursive-commit'.")
+(defvar svn-status-recursive-commit nil
+ "Non-nil if the next commit should be recursive.
+This is always set together with `svn-status-files-to-commit'.")
+(defvar svn-log-edit-update-log-entry nil
+ "Revision number whose log entry is being edited.
+This is nil if the log entry is for a new commit.")
+(defvar svn-status-pre-commit-window-configuration nil)
+(defvar svn-status-pre-propedit-window-configuration nil)
+(defvar svn-status-head-revision nil)
+(defvar svn-status-root-return-info nil)
+(defvar svn-status-property-edit-must-match-flag nil)
+(defvar svn-status-propedit-property-name nil "The property name for the actual svn propset command")
+(defvar svn-status-propedit-file-list nil)
+(defvar svn-status-mode-line-process "")
+(defvar svn-status-mode-line-process-status "")
+(defvar svn-status-mode-line-process-edit-flag "")
+(defvar svn-status-edit-svn-command nil)
+(defvar svn-status-update-previous-process-output nil)
+(defvar svn-pre-run-asynch-recent-keys nil)
+(defvar svn-pre-run-mode-line-process nil)
+(defvar svn-status-temp-dir
+ (expand-file-name
+ (or
+ (when (boundp 'temporary-file-directory) temporary-file-directory) ;emacs
+ ;; XEmacs 21.4.17 can return "/tmp/kalle" from (temp-directory).
+ ;; `file-name-as-directory' adds a slash so we can append a file name.
+ (when (fboundp 'temp-directory) (file-name-as-directory (temp-directory)))
+ "/tmp/")) "The directory that is used to store temporary files for psvn.")
+;; Because `temporary-file-directory' is not a risky local variable in
+;; GNU Emacs 22.0.51, we don't mark `svn-status-temp-dir' as such either.
+(defvar svn-temp-suffix (make-temp-name "."))
+(put 'svn-temp-suffix 'risky-local-variable t)
+(defvar svn-status-temp-file-to-remove nil)
+(put 'svn-status-temp-file-to-remove 'risky-local-variable t)
+(defvar svn-status-temp-arg-file (concat svn-status-temp-dir "svn.arg" svn-temp-suffix))
+(put 'svn-status-temp-arg-file 'risky-local-variable t)
+(defvar svn-status-options nil)
+(defvar svn-status-remote)
+(defvar svn-status-commit-rev-number nil)
+(defvar svn-status-update-rev-number nil)
+(defvar svn-status-operated-on-dot nil)
+(defvar svn-status-last-commit-author nil)
+(defvar svn-status-elided-list nil)
+(defvar svn-status-last-output-buffer-name nil "The buffer name for the buffer that holds the output from the last executed svn command")
+(defvar svn-status-pre-run-svn-buffer nil)
+(defvar svn-status-update-list nil)
+(defvar svn-transient-buffers)
+(defvar svn-ediff-windows)
+(defvar svn-ediff-result)
+(defvar svn-status-last-diff-options nil)
+(defvar svn-status-blame-file-name nil)
+(defvar svn-admin-last-repository-dir nil "The last repository url for various operations.")
+(defvar svn-last-cmd-ring (make-ring 30) "Ring that holds the last executed svn commands (for debugging purposes)")
+(defvar svn-status-cached-version-string nil)
+(defvar svn-client-version nil "The version number of the used svn client")
+(defvar svn-status-get-line-information-for-file nil)
+(defvar svn-status-base-dir-cache (make-hash-table :test 'equal :weakness nil))
+(defvar svn-log-registered-link-handlers (make-hash-table :test 'eql :weakness nil))
+
+(defvar svn-status-partner-buffer nil "The partner buffer for this svn related buffer")
+(make-variable-buffer-local 'svn-status-partner-buffer)
+
+;; Emacs 21 defines these in ediff-init.el but it seems more robust
+;; to just declare the variables here than try to load that file.
+;; It is Ediff's job to declare these as risky-local-variable if needed.
+(defvar ediff-buffer-A)
+(defvar ediff-buffer-B)
+(defvar ediff-buffer-C)
+(defvar ediff-quit-hook)
+
+;; Ditto for log-edit.el.
+(defvar log-edit-initial-files)
+(defvar log-edit-callback)
+(defvar log-edit-listfun)
+
+;; Ediff does not use this variable in GNU Emacs 20.7, GNU Emacs 21.4,
+;; nor XEmacs 21.4.17. However, pcl-cvs (a.k.a. pcvs) does.
+;; TODO: Check if this should be moved into the "svn-" namespace.
+(defvar ediff-after-quit-destination-buffer)
+
+;; That is an example for the svn-status-custom-hide-function:
+;; Note: For many cases it is a better solution to ignore files or
+;; file extensions via the svn-ignore properties (on P i, P I)
+;; (setq svn-status-custom-hide-function 'svn-status-hide-pyc-files)
+;; (defun svn-status-hide-pyc-files (info)
+;; "Hide all pyc files in the `svn-status-buffer-name' buffer."
+;; (let* ((fname (svn-status-line-info->filename-nondirectory info))
+;; (fname-len (length fname)))
+;; (and (> fname-len 4) (string= (substring fname (- fname-len 4)) ".pyc"))))
+
+;;; faces
+(defface svn-status-marked-face
+ '((((type tty) (class color)) (:foreground "green" :weight light))
+ (((class color) (background light)) (:foreground "green3"))
+ (((class color) (background dark)) (:foreground "palegreen2"))
+ (t (:weight bold)))
+ "Face to highlight the mark for user marked files in svn status buffers."
+ :group 'psvn-faces)
+
+(defface svn-status-marked-popup-face
+ '((((type tty) (class color)) (:foreground "green" :weight light))
+ (((class color) (background light)) (:foreground "green3"))
+ (((class color) (background dark)) (:foreground "palegreen2"))
+ (t (:weight bold)))
+ "Face to highlight the actual file, if a popup menu is activated."
+ :group 'psvn-faces)
+
+(defface svn-status-update-available-face
+ '((((type tty) (class color)) (:foreground "magenta" :weight light))
+ (((class color) (background light)) (:foreground "magenta"))
+ (((class color) (background dark)) (:foreground "yellow"))
+ (t (:weight bold)))
+ "Face used to highlight the 'out of date' mark.
+\(i.e., the mark used when there is a newer version in the repository
+than the working copy.\)
+
+See also `svn-status-short-mod-flag-p'."
+ :group 'psvn-faces)
+
+;based on cvs-filename-face
+(defface svn-status-directory-face
+ '((((type tty) (class color)) (:foreground "lightblue" :weight light))
+ (((class color) (background light)) (:foreground "blue4"))
+ (((class color) (background dark)) (:foreground "lightskyblue1"))
+ (t (:weight bold)))
+ "Face for directories in *svn-status* buffers.
+See `svn-status--line-info->directory-p' for what counts as a directory."
+ :group 'psvn-faces)
+
+;based on font-lock-comment-face
+(defface svn-status-filename-face
+ '((((class color) (background light)) (:foreground "chocolate"))
+ (((class color) (background dark)) (:foreground "beige")))
+ "Face for non-directories in *svn-status* buffers.
+See `svn-status--line-info->directory-p' for what counts as a directory."
+ :group 'psvn-faces)
+
+;not based on anything, may be horribly ugly!
+(defface svn-status-symlink-face
+ '((((class color) (background light)) (:foreground "cornflower blue"))
+ (((class color) (background dark)) (:foreground "cyan")))
+ "Face for symlinks in *svn-status* buffers.
+
+This is the face given to the actual link (i.e., the versioned item),
+the target of the link gets either `svn-status-filename-face' or
+`svn-status-directory-face'."
+ :group 'psvn-faces)
+
+;based on font-lock-warning-face
+(defface svn-status-locked-face
+ '((t
+ (:weight bold :foreground "Red")))
+ "Face for the phrase \"[ LOCKED ]\" `svn-status-buffer-name' buffers."
+ :group 'psvn-faces)
+
+;based on vhdl-font-lock-directive-face
+(defface svn-status-switched-face
+ '((((class color)
+ (background light))
+ (:foreground "CadetBlue"))
+ (((class color)
+ (background dark))
+ (:foreground "Aquamarine"))
+ (t
+ (:bold t :italic t)))
+ "Face for the phrase \"(switched)\" non-directories in svn status buffers."
+ :group 'psvn-faces)
+
+(if svn-xemacsp
+ (defface svn-status-blame-highlight-face
+ '((((type tty) (class color)) (:foreground "green" :weight light))
+ (((class color) (background light)) (:foreground "green3"))
+ (((class color) (background dark)) (:foreground "palegreen2"))
+ (t (:weight bold)))
+ "Default face for highlighting a line in svn status blame mode."
+ :group 'psvn-faces)
+ (defface svn-status-blame-highlight-face
+ '((t :inherit highlight))
+ "Default face for highlighting a line in svn status blame mode."
+ :group 'psvn-faces))
+
+(defface svn-status-blame-rev-number-face
+ '((((class color) (background light)) (:foreground "DarkGoldenrod"))
+ (((class color) (background dark)) (:foreground "LightGoldenrod"))
+ (t (:weight bold :slant italic)))
+ "Face to highlight revision numbers in the svn-blame mode."
+ :group 'psvn-faces)
+
+(defvar svn-highlight t)
+;; stolen from PCL-CVS
+(defun svn-add-face (str face &optional keymap)
+ "Return string STR decorated with the specified FACE.
+If `svn-highlight' is nil then just return STR."
+ (when svn-highlight
+ ;; Do not use `list*'; cl.el might not have been loaded. We could
+ ;; put (require 'cl) at the top but let's try to manage without.
+ (add-text-properties 0 (length str)
+ `(face ,face
+ mouse-face highlight)
+;; 18.10.2004: the keymap parameter is not used (yet) in psvn.el
+;; ,@(when keymap
+;; `(mouse-face highlight
+;; local-map ,keymap)))
+ str))
+ str)
+
+(defun svn-status-maybe-add-face (condition text face)
+ "If CONDITION then add FACE to TEXT.
+Else return TEXT unchanged."
+ (if condition
+ (svn-add-face text face)
+ text))
+
+(defun svn-status-choose-face-to-add (condition text face1 face2)
+ "If CONDITION then add FACE1 to TEXT, else add FACE2 to TEXT."
+ (if condition
+ (svn-add-face text face1)
+ (svn-add-face text face2)))
+
+(defun svn-status-maybe-add-string (condition string face)
+ "If CONDITION then return STRING decorated with FACE.
+Otherwise, return \"\"."
+ (if condition
+ (svn-add-face string face)
+ ""))
+
+;; compatibility
+;; emacs 20
+(defalias 'svn-point-at-eol
+ (if (fboundp 'point-at-eol) 'point-at-eol 'line-end-position))
+(defalias 'svn-point-at-bol
+ (if (fboundp 'point-at-bol) 'point-at-bol 'line-beginning-position))
+(defalias 'svn-read-directory-name
+ (if (fboundp 'read-directory-name) 'read-directory-name 'read-file-name))
+
+(eval-when-compile
+ (if (not (fboundp 'gethash))
+ (require 'cl-macs)))
+(defalias 'svn-puthash (if (fboundp 'puthash) 'puthash 'cl-puthash))
+
+;; emacs 21
+(if (fboundp 'line-number-at-pos)
+ (defalias 'svn-line-number-at-pos 'line-number-at-pos)
+ (defun svn-line-number-at-pos (&optional pos)
+ "Return (narrowed) buffer line number at position POS.
+If POS is nil, use current buffer location."
+ (let ((opoint (or pos (point))) start)
+ (save-excursion
+ (goto-char (point-min))
+ (setq start (point))
+ (goto-char opoint)
+ (forward-line 0)
+ (1+ (count-lines start (point)))))))
+
+(defun svn-substring-no-properties (string &optional from to)
+ (if (fboundp 'substring-no-properties)
+ (substring-no-properties string from to)
+ (substring string from to)))
+
+; xemacs
+;; Evaluate the defsubst at compile time, so that the byte compiler
+;; knows the definition and can inline calls. It cannot detect the
+;; defsubst automatically from within the if form.
+(eval-and-compile
+ (if (fboundp 'match-string-no-properties)
+ (defalias 'svn-match-string-no-properties 'match-string-no-properties)
+ (defsubst svn-match-string-no-properties (match)
+ (buffer-substring-no-properties (match-beginning match) (match-end match)))))
+
+;; XEmacs 21.4.17 does not have an `alist' widget. Define a replacement.
+;; To find out whether the `alist' widget exists, we cannot check just
+;; (get 'alist 'widget-type), because GNU Emacs 21.4 defines it in
+;; "wid-edit.el", which is not preloaded; it will be autoloaded when
+;; `widget-create' is called. Instead, we call `widgetp', which is
+;; also autoloaded from "wid-edit.el". XEmacs 21.4.17 does not have
+;; `widgetp' either, so we check that first.
+(if (and (fboundp 'widgetp) (widgetp 'alist))
+ (define-widget 'svn-alist 'alist
+ "An association list.
+Use this instead of `alist', for XEmacs 21.4 compatibility.")
+ (define-widget 'svn-alist 'list
+ "An association list.
+Use this instead of `alist', for XEmacs 21.4 compatibility."
+ :convert-widget 'svn-alist-convert-widget
+ :tag "Association List"
+ :key-type 'sexp
+ :value-type 'sexp)
+ (defun svn-alist-convert-widget (widget)
+ (let* ((value-type (widget-get widget :value-type))
+ (option-widgets (loop for option in (widget-get widget :options)
+ collect `(cons :format "%v"
+ (const :format "%t: %v\n"
+ :tag "Key"
+ ,option)
+ ,value-type))))
+ (widget-put widget :args
+ `(,@(when option-widgets
+ `((set :inline t :format "%v"
+ , at option-widgets)))
+ (editable-list :inline t
+ (cons :format "%v"
+ ,(widget-get widget :key-type)
+ ,value-type)))))
+ widget))
+
+;; process launch functions
+(defvar svn-call-process-function (if (fboundp 'process-file) 'process-file 'call-process))
+(defvar svn-start-process-function (if (fboundp 'start-file-process) 'start-file-process 'start-process))
+
+
+;;; keymaps
+
+(defvar svn-global-keymap nil "Global keymap for psvn.el.
+To bind this to a different key, customize `svn-status-prefix-key'.")
+(put 'svn-global-keymap 'risky-local-variable t)
+(when (not svn-global-keymap)
+ (setq svn-global-keymap (make-sparse-keymap))
+ (define-key svn-global-keymap (kbd "v") 'svn-status-version)
+ (define-key svn-global-keymap (kbd "s") 'svn-status-this-directory)
+ (define-key svn-global-keymap (kbd "b") 'svn-status-via-bookmark)
+ (define-key svn-global-keymap (kbd "h") 'svn-status-use-history)
+ (define-key svn-global-keymap (kbd "u") 'svn-status-update-cmd)
+ (define-key svn-global-keymap (kbd "=") 'svn-status-show-svn-diff)
+ (define-key svn-global-keymap (kbd "f =") 'svn-file-show-svn-diff)
+ (define-key svn-global-keymap (kbd "f e") 'svn-file-show-svn-ediff)
+ (define-key svn-global-keymap (kbd "f l") 'svn-status-show-svn-log)
+ (define-key svn-global-keymap (kbd "f b") 'svn-status-blame)
+ (define-key svn-global-keymap (kbd "f a") 'svn-file-add-to-changelog)
+ (define-key svn-global-keymap (kbd "c") 'svn-status-commit)
+ (define-key svn-global-keymap (kbd "S") 'svn-status-switch-to-status-buffer)
+ (define-key svn-global-keymap (kbd "o") 'svn-status-pop-to-status-buffer))
+
+(defvar svn-status-diff-mode-map ()
+ "Keymap used in `svn-status-diff-mode' for additional commands that are not defined in diff-mode.")
+(put 'svn-status-diff-mode-map 'risky-local-variable t) ;for Emacs 20.7
+
+(when (not svn-status-diff-mode-map)
+ (setq svn-status-diff-mode-map (copy-keymap diff-mode-shared-map))
+ (define-key svn-status-diff-mode-map [?g] 'revert-buffer)
+ (define-key svn-status-diff-mode-map [?s] 'svn-status-pop-to-status-buffer)
+ (define-key svn-status-diff-mode-map [?c] 'svn-status-diff-pop-to-commit-buffer)
+ (define-key svn-status-diff-mode-map [?w] 'svn-status-diff-save-current-defun-as-kill))
+
+(defvar svn-global-trac-map ()
+ "Subkeymap used in `svn-global-keymap' for trac issue tracker commands.")
+(put 'svn-global-trac-map 'risky-local-variable t) ;for Emacs 20.7
+(when (not svn-global-trac-map)
+ (setq svn-global-trac-map (make-sparse-keymap))
+ (define-key svn-global-trac-map (kbd "w") 'svn-trac-browse-wiki)
+ (define-key svn-global-trac-map (kbd "t") 'svn-trac-browse-timeline)
+ (define-key svn-global-trac-map (kbd "m") 'svn-trac-browse-roadmap)
+ (define-key svn-global-trac-map (kbd "s") 'svn-trac-browse-source)
+ (define-key svn-global-trac-map (kbd "r") 'svn-trac-browse-report)
+ (define-key svn-global-trac-map (kbd "i") 'svn-trac-browse-ticket)
+ (define-key svn-global-trac-map (kbd "c") 'svn-trac-browse-changeset)
+ (define-key svn-global-keymap (kbd "t") svn-global-trac-map))
+
+;; The setter of `svn-status-prefix-key' makes a binding in the global
+;; map refer to the `svn-global-keymap' symbol, rather than directly
+;; to the keymap. Emacs then implicitly uses the symbol-function.
+;; This has the advantage that `describe-bindings' (C-h b) can show
+;; the name of the keymap and link to its documentation.
+(defalias 'svn-global-keymap svn-global-keymap)
+;; `defalias' of GNU Emacs 21.4 doesn't allow a docstring argument.
+(put 'svn-global-keymap 'function-documentation
+ '(documentation-property 'svn-global-keymap 'variable-documentation t))
+
+
+;; named after SVN_WC_ADM_DIR_NAME in svn_wc.h
+(defun svn-wc-adm-dir-name ()
+ "Return the name of the \".svn\" subdirectory or equivalent."
+ (if (and (eq system-type 'windows-nt)
+ (getenv "SVN_ASP_DOT_NET_HACK"))
+ "_svn"
+ ".svn"))
+
+(defun svn-log-edit-file-name (&optional curdir)
+ "Get the name of the saved log edit file
+If curdir, return `svn-log-edit-file-name'
+Otherwise position svn-log-edit-file-name in the root directory of this working copy"
+ (if curdir
+ svn-log-edit-file-name
+ (concat (svn-status-base-dir) svn-log-edit-file-name)))
+
+(defun svn-status-message (level &rest args)
+ "If LEVEL is lower than `svn-status-debug-level' print ARGS using `message'.
+
+Guideline for numbers:
+1 - error messages, 3 - non-serious error messages, 5 - messages for things
+that take a long time, 7 - not very important messages on stuff, 9 - messages
+inside loops."
+ (if (<= level svn-status-debug-level)
+ (apply 'message args)))
+
+(defun svn-status-flatten-list (list)
+ "Flatten any lists within ARGS, so that there are no sublists."
+ (loop for item in list
+ if (listp item) nconc (svn-status-flatten-list item)
+ else collect item))
+
+(defun svn-status-window-line-position (w)
+ "Return the window line at point for window W, or nil if W is nil."
+ (svn-status-message 3 "About to count lines; selected window is %s" (selected-window))
+ (and w (count-lines (window-start w) (point))))
+
+;;;###autoload
+(defun svn-checkout (repos-url path)
+ "Run svn checkout REPOS-URL PATH."
+ (interactive (list (read-string "Checkout from repository Url: ")
+ (svn-read-directory-name "Checkout to directory: ")))
+ (svn-run t t 'checkout "checkout" repos-url (expand-file-name path)))
+
+;;;###autoload (defalias 'svn-examine 'svn-status)
+(defalias 'svn-examine 'svn-status)
+
+;;;###autoload
+(defun svn-status (dir &optional arg)
+ "Examine the status of Subversion working copy in directory DIR.
+If ARG is -, allow editing of the parameters. One could add -N to
+run svn status non recursively to make it faster.
+For every other non nil ARG pass the -u argument to `svn status', which
+asks svn to connect to the repository and check to see if there are updates
+there.
+
+If there is no .svn directory, examine if there is CVS and run
+`cvs-examine'. Otherwise ask if to run `dired'."
+ (interactive (list (svn-read-directory-name "SVN status directory: "
+ nil default-directory nil)
+ current-prefix-arg))
+ (let ((svn-dir (format "%s%s"
+ (file-name-as-directory dir)
+ (svn-wc-adm-dir-name)))
+ (cvs-dir (format "%sCVS" (file-name-as-directory dir))))
+ (cond
+ ((file-directory-p svn-dir)
+ (setq arg (svn-status-possibly-negate-meaning-of-arg arg 'svn-status))
+ (svn-status-1 dir arg))
+ ((and (file-directory-p cvs-dir)
+ (fboundp 'cvs-examine))
+ (cvs-examine dir nil))
+ (t
+ (when (y-or-n-p
+ (format
+ (concat
+ "%s "
+ "is not Subversion controlled (missing %s "
+ "directory). "
+ "Run dired instead? ")
+ dir
+ (svn-wc-adm-dir-name)))
+ (dired dir))))))
+
+(defvar svn-status-display-new-status-buffer nil)
+(defun svn-status-1 (dir &optional arg)
+ "Examine DIR. See `svn-status' for more information."
+ (unless (file-directory-p dir)
+ (error "%s is not a directory" dir))
+ (setq dir (file-name-as-directory dir))
+ (when svn-status-load-state-before-svn-status
+ (unless (string= dir (car svn-status-directory-history))
+ (let ((default-directory dir)) ;otherwise svn-status-base-dir looks in the wrong place
+ (svn-status-load-state t))))
+ (setq svn-status-directory-history (delete dir svn-status-directory-history))
+ (add-to-list 'svn-status-directory-history dir)
+ (if (string= (buffer-name) svn-status-buffer-name)
+ (setq svn-status-display-new-status-buffer nil)
+ (setq svn-status-display-new-status-buffer t)
+ ;;(message "psvn: Saving initial window configuration")
+ (setq svn-status-initial-window-configuration
+ (current-window-configuration)))
+ (let* ((cur-buf (current-buffer))
+ (status-buf (get-buffer-create svn-status-buffer-name))
+ (proc-buf (get-buffer-create svn-process-buffer-name))
+ (want-edit (eq arg '-))
+ (status-option (if want-edit
+ (if svn-status-verbose "-v" "")
+ (if svn-status-verbose
+ (if arg "-uv" "-v")
+ (if arg "-u" "")))))
+ (save-excursion
+ (set-buffer status-buf)
+ (setq default-directory dir)
+ (set-buffer proc-buf)
+ (setq default-directory dir
+ svn-status-remote (when arg t))
+ (set-buffer cur-buf)
+ (if want-edit
+ (let (svn-status-edit-svn-command t)
+ (svn-run t t 'status "status" svn-status-default-status-arguments status-option))
+ (svn-run t t 'status "status" svn-status-default-status-arguments status-option)))))
+
+(defun svn-status-this-directory (arg)
+ "Run `svn-status' for the `default-directory'"
+ (interactive "P")
+ (svn-status default-directory arg))
+
+(defun svn-status-use-history ()
+ "Interactively select a different directory from `svn-status-directory-history'."
+ (interactive)
+ (let* ((in-status-buffer (eq major-mode 'svn-status-mode))
+ (hist (if in-status-buffer (cdr svn-status-directory-history) svn-status-directory-history))
+ (dir (funcall svn-status-completing-read-function "svn-status on directory: " hist))
+ (svn-status-buffer (get-buffer svn-status-buffer-name))
+ (svn-buffer-available (and svn-status-buffer
+ (with-current-buffer svn-status-buffer-name (string= default-directory dir)))))
+ (if (file-directory-p dir)
+ (if svn-buffer-available
+ (svn-status-switch-to-status-buffer)
+ (unless svn-status-refresh-info
+ (setq svn-status-refresh-info 'once))
+ (svn-status dir))
+ (error "%s is not a directory" dir))))
+
+(defun svn-had-user-input-since-asynch-run ()
+ (not (equal (recent-keys) svn-pre-run-asynch-recent-keys)))
+
+(defun svn-process-environment ()
+ "Construct the environment for the svn process.
+It is a combination of `svn-status-svn-environment-var-list' and
+the usual `process-environment'."
+ ;; If there are duplicate elements in `process-environment', then GNU
+ ;; Emacs 21.4 guarantees that the first one wins; but GNU Emacs 20.7
+ ;; and XEmacs 21.4.17 don't document what happens. We'll just remove
+ ;; any duplicates ourselves, then. This also gives us an opportunity
+ ;; to handle the "VARIABLE" syntax that none of them supports.
+ (loop with found = '()
+ for elt in (append svn-status-svn-environment-var-list
+ process-environment)
+ for has-value = (string-match "=" elt)
+ for name = (substring elt 0 has-value)
+ unless (member name found)
+ do (push name found)
+ and when has-value
+ collect elt))
+
+(defun svn-run (run-asynchron clear-process-buffer cmdtype &rest arglist)
+ "Run svn with arguments ARGLIST.
+
+If RUN-ASYNCHRON is t then run svn asynchronously.
+
+If CLEAR-PROCESS-BUFFER is t then erase the contents of the
+`svn-process-buffer-name' buffer before commencing.
+
+CMDTYPE is a symbol such as 'mv, 'revert, or 'add, representing the
+command to run.
+
+ARGLIST is a list of arguments \(which must include the command name,
+for example: '(\"revert\" \"file1\"\)
+ARGLIST is flattened and any every nil value is discarded.
+
+If the variable `svn-status-edit-svn-command' is non-nil then the user
+can edit ARGLIST before running svn.
+
+The hook svn-pre-run-hook allows to monitor/modify the ARGLIST."
+ (setq arglist (svn-status-flatten-list arglist))
+ (if (eq (process-status "svn") nil)
+ (progn
+ (when svn-status-edit-svn-command
+ (setq arglist (append
+ (list (car arglist))
+ (split-string
+ (read-from-minibuffer
+ (format "svn %s flags: " (car arglist))
+ (mapconcat 'identity (cdr arglist) " ")))))
+ (when (eq svn-status-edit-svn-command t)
+ (svn-status-toggle-edit-cmd-flag t))
+ (message "svn-run %s: %S" cmdtype arglist))
+ (run-hooks 'svn-pre-run-hook)
+ (unless (eq mode-line-process 'svn-status-mode-line-process)
+ (setq svn-pre-run-mode-line-process mode-line-process)
+ (setq mode-line-process 'svn-status-mode-line-process))
+ (setq svn-status-pre-run-svn-buffer (current-buffer))
+ (let* ((proc-buf (get-buffer-create svn-process-buffer-name))
+ (svn-exe svn-status-svn-executable)
+ (svn-proc))
+ (when (listp (car arglist))
+ (setq arglist (car arglist)))
+ (save-excursion
+ (set-buffer proc-buf)
+ (unless (file-executable-p default-directory)
+ (message "psvn: workaround in %s needed: %s no longer exists" (current-buffer) default-directory)
+ (cd (expand-file-name "~")))
+ (setq buffer-read-only nil)
+ (buffer-disable-undo)
+ (fundamental-mode)
+ (if clear-process-buffer
+ (delete-region (point-min) (point-max))
+ (goto-char (point-max)))
+ (setq svn-process-cmd cmdtype)
+ (setq svn-status-last-commit-author nil)
+ (setq svn-status-mode-line-process-status (format " running %s" cmdtype))
+ (svn-status-update-mode-line)
+ (sit-for 0.1)
+ (ring-insert svn-last-cmd-ring (list (current-time-string) arglist default-directory))
+ (if run-asynchron
+ (progn
+ ;;(message "running asynchron: %s %S" svn-exe arglist)
+ (setq svn-pre-run-asynch-recent-keys (recent-keys))
+ (let ((process-environment (svn-process-environment))
+ (process-connection-type nil))
+ ;; Communicate with the subprocess via pipes rather
+ ;; than via a pseudoterminal, so that if the svn+ssh
+ ;; scheme is being used, SSH will not ask for a
+ ;; passphrase via stdio; psvn.el is currently unable
+ ;; to answer such prompts. Instead, SSH will run
+ ;; x11-ssh-askpass if possible. If Emacs is being
+ ;; run on a TTY without $DISPLAY, this will fail; in
+ ;; such cases, the user should start ssh-agent and
+ ;; then run ssh-add explicitly.
+ (setq svn-proc (apply svn-start-process-function "svn" proc-buf svn-exe arglist)))
+ (when svn-status-svn-process-coding-system
+ (set-process-coding-system svn-proc svn-status-svn-process-coding-system
+ svn-status-svn-process-coding-system))
+ (set-process-sentinel svn-proc 'svn-process-sentinel)
+ (when svn-status-track-user-input
+ (set-process-filter svn-proc 'svn-process-filter)))
+ ;;(message "running synchron: %s %S" svn-exe arglist)
+ (let ((process-environment (svn-process-environment)))
+ ;; `call-process' ignores `process-connection-type' and
+ ;; never opens a pseudoterminal.
+ (apply svn-call-process-function svn-exe nil proc-buf nil arglist))
+ (setq svn-status-last-output-buffer-name svn-process-buffer-name)
+ (run-hooks 'svn-post-process-svn-output-hook)
+ (setq svn-status-mode-line-process-status "")
+ (svn-status-update-mode-line)
+ (when svn-pre-run-mode-line-process
+ (setq mode-line-process svn-pre-run-mode-line-process)
+ (setq svn-pre-run-mode-line-process nil))))))
+ (error "You can only run one svn process at once!")))
+
+(defun svn-process-sentinel-fixup-path-seperators ()
+ "Convert all path separators to UNIX style.
+\(This is a no-op unless `system-type' is windows-nt\)"
+ (when (eq system-type 'windows-nt)
+ (save-excursion
+ (goto-char (point-min))
+ (while (search-forward "\\" nil t)
+ (replace-match "/")))))
+
+(defun svn-process-sentinel (process event)
+ ;;(princ (format "Process: %s had the event `%s'" process event)))
+ ;;(save-excursion
+ (let ((act-buf (current-buffer)))
+ (when svn-pre-run-mode-line-process
+ (with-current-buffer svn-status-pre-run-svn-buffer
+ (setq mode-line-process svn-pre-run-mode-line-process))
+ (setq svn-pre-run-mode-line-process nil))
+ (set-buffer (process-buffer process))
+ (setq svn-status-mode-line-process-status "")
+ (svn-status-update-mode-line)
+ (cond ((string= event "finished\n")
+ (run-hooks 'svn-post-process-svn-output-hook)
+ (cond ((eq svn-process-cmd 'status)
+ ;;(message "svn status finished")
+ (svn-process-sentinel-fixup-path-seperators)
+ (svn-parse-status-result)
+ (svn-status-apply-elide-list)
+ (when svn-status-update-previous-process-output
+ (set-buffer (process-buffer process))
+ (delete-region (point-min) (point-max))
+ (insert "Output from svn command:\n")
+ (insert svn-status-update-previous-process-output)
+ (goto-char (point-min))
+ (setq svn-status-update-previous-process-output nil))
+ (when svn-status-update-list
+ ;; (message "Using svn-status-update-list: %S" svn-status-update-list)
+ (save-excursion
+ (svn-status-update-with-command-list svn-status-update-list))
+ (setq svn-status-update-list nil))
+ (when svn-status-display-new-status-buffer
+ (set-window-configuration svn-status-initial-window-configuration)
+ (if (svn-had-user-input-since-asynch-run)
+ (message "svn status finished")
+ (switch-to-buffer svn-status-buffer-name))))
+ ((eq svn-process-cmd 'log)
+ (svn-status-show-process-output 'log t)
+ (pop-to-buffer svn-status-last-output-buffer-name)
+ (svn-log-view-mode)
+ (forward-line 2)
+ (unless (looking-at "Changed paths:")
+ (forward-line 1))
+ (font-lock-fontify-buffer)
+ (message "svn log finished"))
+ ((eq svn-process-cmd 'info)
+ (svn-status-show-process-output 'info t)
+ (message "svn info finished"))
+ ((eq svn-process-cmd 'ls)
+ (svn-status-show-process-output 'info t)
+ (message "svn ls finished"))
+ ((eq svn-process-cmd 'diff)
+ (svn-status-activate-diff-mode)
+ (message "svn diff finished"))
+ ((eq svn-process-cmd 'parse-info)
+ (svn-status-parse-info-result))
+ ((eq svn-process-cmd 'blame)
+ (svn-status-show-process-output 'blame t)
+ (when svn-status-pre-run-svn-buffer
+ (with-current-buffer svn-status-pre-run-svn-buffer
+ (unless (eq major-mode 'svn-status-mode)
+ (let ((src-line-number (svn-line-number-at-pos)))
+ (pop-to-buffer (get-buffer svn-status-last-output-buffer-name))
+ (goto-line src-line-number)))))
+ (with-current-buffer (get-buffer svn-status-last-output-buffer-name)
+ (svn-status-activate-blame-mode))
+ (message "svn blame finished"))
+ ((eq svn-process-cmd 'commit)
+ (svn-process-sentinel-fixup-path-seperators)
+ (svn-status-remove-temp-file-maybe)
+ (when (member 'commit svn-status-unmark-files-after-list)
+ (svn-status-unset-all-usermarks))
+ (svn-status-update-with-command-list (svn-status-parse-commit-output))
+ (svn-revert-some-buffers)
+ (run-hooks 'svn-log-edit-done-hook)
+ (setq svn-status-files-to-commit nil
+ svn-status-recursive-commit nil)
+ (if (null svn-status-commit-rev-number)
+ (message "No revision to commit.")
+ (message "svn: Committed revision %s." svn-status-commit-rev-number)))
+ ((eq svn-process-cmd 'update)
+ (svn-status-show-process-output 'update t)
+ (setq svn-status-update-list (svn-status-parse-update-output))
+ (svn-revert-some-buffers)
+ (svn-status-update)
+ (if (car svn-status-update-rev-number)
+ (message "svn: Updated to revision %s." (cadr svn-status-update-rev-number))
+ (message "svn: At revision %s." (cadr svn-status-update-rev-number))))
+ ((eq svn-process-cmd 'add)
+ (svn-status-update-with-command-list (svn-status-parse-ar-output))
+ (message "svn add finished"))
+ ((eq svn-process-cmd 'lock)
+ (svn-status-update)
+ (message "svn lock finished"))
+ ((eq svn-process-cmd 'unlock)
+ (svn-status-update)
+ (message "svn unlock finished"))
+ ((eq svn-process-cmd 'mkdir)
+ (svn-status-update)
+ (message "svn mkdir finished"))
+ ((eq svn-process-cmd 'revert)
+ (when (member 'revert svn-status-unmark-files-after-list)
+ (svn-status-unset-all-usermarks))
+ (svn-revert-some-buffers)
+ (svn-status-update)
+ (message "svn revert finished"))
+ ((eq svn-process-cmd 'resolved)
+ (svn-status-update)
+ (message "svn resolved finished"))
+ ((eq svn-process-cmd 'rm)
+ (svn-status-update-with-command-list (svn-status-parse-ar-output))
+ (message "svn rm finished"))
+ ((eq svn-process-cmd 'cleanup)
+ (message "svn cleanup finished"))
+ ((eq svn-process-cmd 'proplist)
+ (svn-status-show-process-output 'proplist t)
+ (message "svn proplist finished"))
+ ((eq svn-process-cmd 'checkout)
+ (svn-status default-directory))
+ ((eq svn-process-cmd 'proplist-parse)
+ (svn-status-property-parse-property-names))
+ ((eq svn-process-cmd 'propset)
+ (svn-status-remove-temp-file-maybe)
+ (if (member svn-status-propedit-property-name '("svn:keywords"))
+ (svn-status-update-with-command-list (svn-status-parse-property-output))
+ (svn-status-update)))
+ ((eq svn-process-cmd 'propdel)
+ (svn-status-update))))
+ ((string= event "killed\n")
+ (message "svn process killed"))
+ ((string-match "exited abnormally" event)
+ (while (accept-process-output process 0 100))
+ ;; find last error message and show it.
+ (goto-char (point-max))
+ (if (re-search-backward "^svn: \\(.*\\)" nil t)
+ (svn-process-handle-error (match-string 1))
+ (message "svn failed: %s" event)))
+ (t
+ (message "svn process had unknown event: %s" event))
+ (svn-status-show-process-output nil t))))
+
+(defvar svn-process-handle-error-msg nil)
+(defun svn-process-handle-error (error-msg)
+ (let ((svn-process-handle-error-msg error-msg))
+ (electric-helpify 'svn-process-help-with-error-msg)))
+
+(defun svn-process-help-with-error-msg ()
+ (interactive)
+ (let ((help-msg (cadr (assoc svn-process-handle-error-msg
+ '(("Cannot non-recursively commit a directory deletion"
+ "Please unmark all files and position point at the directory you would like to remove.\nThen run commit again."))))))
+ (if help-msg
+ (save-excursion
+ (with-output-to-temp-buffer (help-buffer)
+ (princ (format "svn failed: %s\n\n%s" svn-process-handle-error-msg help-msg))))
+ (message "svn failed: %s" svn-process-handle-error-msg))))
+
+
+(defun svn-process-filter (process str)
+ "Track the svn process output and ask user questions in the minibuffer when appropriate."
+ (save-window-excursion
+ (set-buffer svn-process-buffer-name)
+ ;;(message "svn-process-filter: %s" str)
+ (goto-char (point-max))
+ (insert str)
+ (save-excursion
+ (goto-char (svn-point-at-bol))
+ (when (looking-at "Password for '\\(.+\\)': ")
+ ;(svn-status-show-process-buffer)
+ (let ((passwd (read-passwd
+ (format "Enter svn password for %s: " (match-string 1)))))
+ (svn-process-send-string-and-newline passwd t)))
+ (when (looking-at "Username: ")
+ (let ((user-name (read-string "Username for svn operation: ")))
+ (svn-process-send-string-and-newline user-name)))
+ (when (looking-at "(R)eject, accept (t)emporarily or accept (p)ermanently")
+ (svn-status-show-process-buffer)
+ (let ((answer (read-string "(R)eject, accept (t)emporarily or accept (p)ermanently? ")))
+ (svn-process-send-string (substring answer 0 1)))))))
+
+(defun svn-revert-some-buffers (&optional tree)
+ "Reverts all buffers visiting a file in TREE that aren't modified.
+To be run after a commit, an update or a merge."
+ (interactive)
+ (let ((tree (or (svn-status-base-dir) tree)))
+ (dolist (buffer (buffer-list))
+ (with-current-buffer buffer
+ (when (not (buffer-modified-p))
+ (let ((file (buffer-file-name)))
+ (when file
+ (let ((root (svn-status-base-dir (file-name-directory file)))
+ (point-pos (point)))
+ (when (and root
+ (string= root tree)
+ ;; buffer is modified and in the tree TREE.
+ svn-status-auto-revert-buffers)
+ (when svn-status-fancy-file-state-in-modeline
+ (svn-status-update-modeline))
+ ;; (message "svn-revert-some-buffers: %s %s" (buffer-file-name) (verify-visited-file-modtime (current-buffer)))
+ ;; Keep the buffer if the file doesn't exist
+ (when (and (file-exists-p file) (not (verify-visited-file-modtime (current-buffer))))
+ (revert-buffer t t)
+ (goto-char point-pos)))))))))))
+
+(defun svn-parse-rev-num (str)
+ (if (and str (stringp str)
+ (save-match-data (string-match "^[0-9]+" str)))
+ (string-to-number str)
+ -1))
+
+(defsubst svn-status-make-ui-status ()
+ "Make a ui-status structure for a file in a svn working copy.
+The initial values in the structure returned by this function
+are good for a file or directory that the user hasn't seen before.
+
+The ui-status structure keeps track of how the file or directory
+should be displayed in svn-status mode. Updating the svn-status
+buffer from the working copy preserves the ui-status if possible.
+User commands modify this structure; each file or directory must
+thus have its own copy.
+
+Currently, the ui-status is a list (USER-MARK USER-ELIDE).
+USER-MARK is non-nil iff the user has marked the file or directory,
+ typically with `svn-status-set-user-mark'. To read USER-MARK,
+ call `svn-status-line-info->has-usermark'.
+USER-ELIDE is non-nil iff the user has elided the file or directory
+ from the svn-status buffer, typically with `svn-status-toggle-elide'.
+ To read USER-ELIDE, call `svn-status-line-info->user-elide'.
+
+Call `svn-status-line-info->ui-status' to access the whole ui-status
+structure."
+ (list nil nil))
+
+(defun svn-status-make-dummy-dirs (dir-list old-ui-information)
+ "Calculate additionally necessary directories that were not shown in the output
+of 'svn status'"
+ ;; (message "svn-status-make-dummy-dirs %S" dir-list)
+ (let ((candidate)
+ (base-dir))
+ (dolist (dir dir-list)
+ (setq base-dir (file-name-directory dir))
+ (while base-dir
+ ;;(message "dir: %S dir-list: %S, base-dir: %S" dir dir-list base-dir)
+ (setq candidate (replace-regexp-in-string "/+$" "" base-dir))
+ (setq base-dir (file-name-directory candidate))
+ ;; (message "dir: %S, candidate: %S" dir candidate)
+ (add-to-list 'dir-list candidate))))
+ ;; (message "svn-status-make-dummy-dirs %S" dir-list)
+ (append (mapcar (lambda (dir)
+ (svn-status-make-line-info
+ dir
+ (gethash dir old-ui-information)))
+ dir-list)
+ svn-status-info))
+
+(defun svn-status-make-line-info (&optional
+ path
+ ui
+ file-mark prop-mark
+ local-rev last-change-rev
+ author
+ update-mark
+ locked-mark
+ with-history-mark
+ switched-mark
+ locked-repo-mark
+ psvn-extra-info)
+ "Create a new line-info from the given arguments
+Anything left nil gets a sensible default.
+nb: LOCKED-MARK refers to the kind of locks you get after an error,
+ LOCKED-REPO-MARK is the kind managed with `svn lock'"
+ (list (or ui (svn-status-make-ui-status))
+ (or file-mark ? )
+ (or prop-mark ? )
+ (or path "")
+ (or local-rev ? )
+ (or last-change-rev ? )
+ (or author "")
+ update-mark
+ locked-mark
+ with-history-mark
+ switched-mark
+ locked-repo-mark
+ psvn-extra-info))
+
+(defvar svn-user-names-including-blanks nil "A list of svn user names that include blanks.
+To add support for the names \"feng shui\" and \"mister blank\", place the following in your .emacs:
+ (setq svn-user-names-including-blanks '(\"feng shui\" \"mister blank\"))
+ (add-hook 'svn-pre-parse-status-hook 'svn-status-parse-fixup-user-names-including-blanks)
+")
+;;(setq svn-user-names-including-blanks '("feng shui" "mister blank"))
+;;(add-hook 'svn-pre-parse-status-hook 'svn-status-parse-fixup-user-names-including-blanks)
+
+(defun svn-status-parse-fixup-user-names-including-blanks ()
+ "Helper function to allow user names that include blanks.
+Add this function to the `svn-pre-parse-status-hook'. The variable
+`svn-user-names-including-blanks' must be configured to hold all user names that contain
+blanks. This function replaces the blanks with '-' to allow further processing with
+the usual parsing functionality in `svn-parse-status-result'."
+ (when svn-user-names-including-blanks
+ (goto-char (point-min))
+ (let ((search-string (concat " \\(" (mapconcat 'concat svn-user-names-including-blanks "\\|") "\\) ")))
+ (save-match-data
+ (save-excursion
+ (while (re-search-forward search-string (point-max) t)
+ (replace-match (replace-regexp-in-string " " "-" (match-string 1)) nil nil nil 1)))))))
+
+(defun svn-parse-status-result ()
+ "Parse the `svn-process-buffer-name' buffer.
+The results are used to build the `svn-status-info' variable."
+ (setq svn-status-head-revision nil)
+ (save-excursion
+ (let ((old-ui-information (svn-status-ui-information-hash-table))
+ (svn-marks)
+ (svn-file-mark)
+ (svn-property-mark)
+ (svn-wc-locked-mark)
+ (svn-repo-locked-mark)
+ (svn-with-history-mark)
+ (svn-switched-mark)
+ (svn-update-mark)
+ (local-rev)
+ (last-change-rev)
+ (author)
+ (path)
+ (dir)
+ (revision-width svn-status-default-revision-width)
+ (author-width svn-status-default-author-width)
+ (svn-marks-length (if svn-status-verbose
+ (if svn-status-remote
+ 8 6)
+ (if svn-status-remote
+ ;; not verbose
+ 8 7)))
+ (dir-set '("."))
+ (externals-map (make-hash-table :test 'equal))
+ (skip-double-external-dir-entry-name nil))
+ (set-buffer svn-process-buffer-name)
+ (setq svn-status-info nil)
+ (run-hooks 'svn-pre-parse-status-hook)
+ (goto-char (point-min))
+ (while (< (point) (point-max))
+ (cond
+ ((= (svn-point-at-eol) (svn-point-at-bol)) ;skip blank lines
+ nil)
+ ((looking-at "Status against revision:[ ]+\\([0-9]+\\)")
+ ;; the above message appears for the main listing plus once for each svn:externals entry
+ (unless svn-status-head-revision
+ (setq svn-status-head-revision (match-string 1))))
+ ((looking-at "Performing status on external item at '\\(.*\\)'")
+ ;; The *next* line has info about the directory named in svn:externals
+ ;; [ie the directory in (match-string 1)]
+ ;; we should parse it, and merge the info with what we have already know
+ ;; but for now just ignore the line completely
+ ; (forward-line)
+ ;; Actually, this seems to not always be the case
+ ;; I have an example where we are in an svn:external which
+ ;; is itself inside a svn:external, this need not be true:
+ ;; the next line is not 'X dir' but just 'dir', so we
+ ;; actually need to parse that line, or the results will
+ ;; not contain dir!
+ ;; so we should merge lines 'X dir' with ' dir', but for now
+ ;; we just leave both in the results
+
+ ;; My attempt to merge the lines uses skip-double-external-dir-entry-name
+ ;; and externals-map
+ (setq skip-double-external-dir-entry-name (svn-match-string-no-properties 1))
+ ;; (message "Going to skip %s" skip-double-external-dir-entry-name)
+ nil)
+ ((looking-at "--- Changelist") ; skip svn changelist header lines
+ ;; See: http://svn.collab.net/repos/svn/trunk/notes/changelist-design.txt
+ nil)
+ (t
+ (setq svn-marks (buffer-substring (point) (+ (point) svn-marks-length))
+ svn-file-mark (elt svn-marks 0) ; 1st column - M,A,C,D,G,? etc
+ svn-property-mark (elt svn-marks 1) ; 2nd column - M,C (properties)
+ svn-wc-locked-mark (elt svn-marks 2) ; 3rd column - L or blank
+ svn-with-history-mark (elt svn-marks 3) ; 4th column - + or blank
+ svn-switched-mark (elt svn-marks 4) ; 5th column - S or blank
+ svn-repo-locked-mark (elt svn-marks 5)) ; 6th column - K,O,T,B or blank
+ (when svn-status-remote
+ (setq svn-update-mark (elt svn-marks 7))) ; 8th column - * or blank
+ (when (eq svn-property-mark ?\ ) (setq svn-property-mark nil))
+ (when (eq svn-wc-locked-mark ?\ ) (setq svn-wc-locked-mark nil))
+ (when (eq svn-with-history-mark ?\ ) (setq svn-with-history-mark nil))
+ (when (eq svn-switched-mark ?\ ) (setq svn-switched-mark nil))
+ (when (eq svn-update-mark ?\ ) (setq svn-update-mark nil))
+ (when (eq svn-repo-locked-mark ?\ ) (setq svn-repo-locked-mark nil))
+ (forward-char svn-marks-length)
+ (skip-chars-forward " ")
+ ;; (message "after marks: '%s'" (buffer-substring (point) (line-end-position)))
+ (cond
+ ((looking-at "\\([-?]\\|[0-9]+\\) +\\([-?]\\|[0-9]+\\) +\\([^ ]+\\) *\\(.+\\)$")
+ (setq local-rev (svn-parse-rev-num (match-string 1))
+ last-change-rev (svn-parse-rev-num (match-string 2))
+ author (match-string 3)
+ path (match-string 4)))
+ ((looking-at "\\([-?]\\|[0-9]+\\) +\\([^ ]+\\)$")
+ (setq local-rev (svn-parse-rev-num (match-string 1))
+ last-change-rev -1
+ author "?"
+ path (match-string 2)))
+ ((looking-at "\\(.*\\)")
+ (setq path (match-string 1)
+ local-rev -1
+ last-change-rev -1
+ author (if (eq svn-file-mark ?X) "" "?"))) ;clear author of svn:externals dirs
+ (t
+ (error "Unknown status line format")))
+ (unless path (setq path "."))
+ (setq dir (file-name-directory path))
+ (if (and (not svn-status-verbose) dir)
+ (let ((dirname (directory-file-name dir)))
+ (if (not (member dirname dir-set))
+ (setq dir-set (cons dirname dir-set)))))
+ (if (and skip-double-external-dir-entry-name (string= skip-double-external-dir-entry-name path))
+ ;; merge this entry to a previous saved one
+ (let ((info (gethash path externals-map)))
+ ;; (message "skip-double-external-dir-entry-name: %s - path: %s" skip-double-external-dir-entry-name path)
+ (if info
+ (progn
+ (svn-status-line-info->set-localrev info local-rev)
+ (svn-status-line-info->set-lastchangerev info last-change-rev)
+ (svn-status-line-info->set-author info author)
+ (svn-status-message 3 "merging entry for %s to %s" path info)
+ (setq skip-double-external-dir-entry-name nil))
+ (message "psvn: %s not handled correct, please report this case." path)))
+ (setq svn-status-info
+ (cons (svn-status-make-line-info path
+ (gethash path old-ui-information)
+ svn-file-mark
+ svn-property-mark
+ local-rev
+ last-change-rev
+ author
+ svn-update-mark
+ svn-wc-locked-mark
+ svn-with-history-mark
+ svn-switched-mark
+ svn-repo-locked-mark
+ nil) ;;psvn-extra-info
+ svn-status-info)))
+ (when (eq svn-file-mark ?X)
+ (svn-puthash (match-string 1) (car svn-status-info) externals-map)
+ (svn-status-message 3 "found external: %s %S" (match-string 1) (car svn-status-info)))
+ (setq revision-width (max revision-width
+ (length (number-to-string local-rev))
+ (length (number-to-string last-change-rev))))
+ (setq author-width (max author-width (length author)))))
+ (forward-line 1))
+ (unless svn-status-verbose
+ (setq svn-status-info (svn-status-make-dummy-dirs dir-set
+ old-ui-information)))
+ (setq svn-status-default-column
+ (+ 6 revision-width revision-width author-width
+ (if svn-status-short-mod-flag-p 3 0)))
+ (setq svn-status-line-format (format " %%c%%c%%c %%%ds %%%ds %%-%ds"
+ revision-width
+ revision-width
+ author-width))
+ (setq svn-status-info (nreverse svn-status-info))
+ (when svn-status-sort-status-buffer
+ (setq svn-status-info (sort svn-status-info 'svn-status-sort-predicate))))))
+
+;;(string-lessp "." "%") => nil
+;(svn-status-sort-predicate '(t t t ".") '(t t t "%")) => t
+(defun svn-status-sort-predicate (a b)
+ "Return t if A should appear before B in the `svn-status-buffer-name' buffer.
+A and B must be line-info's."
+ (string-lessp (concat (svn-status-line-info->full-path a) "/")
+ (concat (svn-status-line-info->full-path b) "/")))
+
+(defun svn-status-remove-temp-file-maybe ()
+ "Remove any (no longer required) temporary files created by psvn.el."
+ (when svn-status-temp-file-to-remove
+ (when (file-exists-p svn-status-temp-file-to-remove)
+ (delete-file svn-status-temp-file-to-remove))
+ (when (file-exists-p svn-status-temp-arg-file)
+ (delete-file svn-status-temp-arg-file))
+ (setq svn-status-temp-file-to-remove nil)))
+
+(defun svn-status-remove-control-M ()
+ "Remove ^M at end of line in the whole buffer."
+ (interactive)
+ (let ((buffer-read-only nil))
+ (save-match-data
+ (save-excursion
+ (goto-char (point-min))
+ (while (re-search-forward "\r$" (point-max) t)
+ (replace-match "" nil nil))))))
+
+(condition-case nil
+ ;;(easy-menu-add-item nil '("tools") ["SVN Status" svn-status t] "PCL-CVS")
+ (easy-menu-add-item nil '("tools") ["SVN Status" svn-status t])
+ (error (message "psvn: could not install menu")))
+
+(defvar svn-status-mode-map () "Keymap used in `svn-status-mode' buffers.")
+(put 'svn-status-mode-map 'risky-local-variable t) ;for Emacs 20.7
+(defvar svn-status-mode-mark-map ()
+ "Subkeymap used in `svn-status-mode' for mark commands.")
+(put 'svn-status-mode-mark-map 'risky-local-variable t) ;for Emacs 20.7
+(defvar svn-status-mode-property-map ()
+ "Subkeymap used in `svn-status-mode' for property commands.")
+(put 'svn-status-mode-property-map 'risky-local-variable t) ;for Emacs 20.7
+(defvar svn-status-mode-options-map ()
+ "Subkeymap used in `svn-status-mode' for option commands.")
+(put 'svn-status-mode-options-map 'risky-local-variable t) ;for Emacs 20.7
+(defvar svn-status-mode-trac-map ()
+ "Subkeymap used in `svn-status-mode' for trac issue tracker commands.")
+(put 'svn-status-mode-trac-map 'risky-local-variable t) ;for Emacs 20.7
+(defvar svn-status-mode-extension-map ()
+ "Subkeymap used in `svn-status-mode' for some seldom used commands.")
+(put 'svn-status-mode-extension-map 'risky-local-variable t) ;for Emacs 20.7
+(defvar svn-status-mode-branch-map ()
+ "Subkeymap used in `svn-status-mode' for branching commands.")
+(put 'svn-status-mode-extension-map 'risky-local-variable t) ;for Emacs 20.7
+
+(when (not svn-status-mode-map)
+ (setq svn-status-mode-map (make-sparse-keymap))
+ (suppress-keymap svn-status-mode-map)
+ ;; Don't use (kbd "<return>"); it's unreachable with GNU Emacs 21.3 on a TTY.
+ (define-key svn-status-mode-map (kbd "RET") 'svn-status-find-file-or-examine-directory)
+ (define-key svn-status-mode-map (kbd "<mouse-2>") 'svn-status-mouse-find-file-or-examine-directory)
+ (define-key svn-status-mode-map (kbd "^") 'svn-status-examine-parent)
+ (define-key svn-status-mode-map (kbd "s") 'svn-status-show-process-buffer)
+ (define-key svn-status-mode-map (kbd "h") 'svn-status-pop-to-partner-buffer)
+ (define-key svn-status-mode-map (kbd "f") 'svn-status-find-files)
+ (define-key svn-status-mode-map (kbd "o") 'svn-status-find-file-other-window)
+ (define-key svn-status-mode-map (kbd "C-o") 'svn-status-find-file-other-window-noselect)
+ (define-key svn-status-mode-map (kbd "v") 'svn-status-view-file-other-window)
+ (define-key svn-status-mode-map (kbd "e") 'svn-status-toggle-edit-cmd-flag)
+ (define-key svn-status-mode-map (kbd "g") 'svn-status-update)
+ (define-key svn-status-mode-map (kbd "M-s") 'svn-status-update) ;; PCL-CVS compatibility
+ (define-key svn-status-mode-map (kbd "q") 'svn-status-bury-buffer)
+ (define-key svn-status-mode-map (kbd "x") 'svn-status-redraw-status-buffer)
+ (define-key svn-status-mode-map (kbd "H") 'svn-status-use-history)
+ (define-key svn-status-mode-map (kbd "m") 'svn-status-set-user-mark)
+ (define-key svn-status-mode-map (kbd "u") 'svn-status-unset-user-mark)
+ ;; This matches a binding of `dired-unmark-all-files' in `dired-mode-map'
+ ;; of both GNU Emacs and XEmacs. It seems unreachable with XEmacs on
+ ;; TTY, but if that's a problem then its Dired needs fixing too.
+ ;; Or you could just use "*!".
+ (define-key svn-status-mode-map "\M-\C-?" 'svn-status-unset-all-usermarks)
+ ;; The key that normally deletes characters backwards should here
+ ;; instead unmark files backwards. In GNU Emacs, that would be (kbd
+ ;; "DEL") aka [?\177], but XEmacs treats those as [(delete)] and
+ ;; would bind a key that normally deletes forwards. [(backspace)]
+ ;; is unreachable with GNU Emacs on a tty. Try to recognize the
+ ;; dialect and act accordingly.
+ ;;
+ ;; XEmacs has a `delete-forward-p' function that checks the
+ ;; `delete-key-deletes-forward' option. We don't use those, for two
+ ;; reasons: psvn.el may be loaded before user customizations, and
+ ;; XEmacs allows simultaneous connections to multiple devices with
+ ;; different keyboards.
+ (define-key svn-status-mode-map
+ (if (member (kbd "DEL") '([(delete)] [delete]))
+ [(backspace)] ; XEmacs
+ (kbd "DEL")) ; GNU Emacs
+ 'svn-status-unset-user-mark-backwards)
+ (define-key svn-status-mode-map (kbd "$") 'svn-status-toggle-elide)
+ (define-key svn-status-mode-map (kbd "w") 'svn-status-copy-current-line-info)
+ (define-key svn-status-mode-map (kbd ".") 'svn-status-goto-root-or-return)
+ (define-key svn-status-mode-map (kbd "I") 'svn-status-parse-info)
+ (define-key svn-status-mode-map (kbd "V") 'svn-status-svnversion)
+ (define-key svn-status-mode-map (kbd "?") 'svn-status-toggle-hide-unknown)
+ (define-key svn-status-mode-map (kbd "_") 'svn-status-toggle-hide-unmodified)
+ (define-key svn-status-mode-map (kbd "a") 'svn-status-add-file)
+ (define-key svn-status-mode-map (kbd "A") 'svn-status-add-file-recursively)
+ (define-key svn-status-mode-map (kbd "+") 'svn-status-make-directory)
+ (define-key svn-status-mode-map (kbd "R") 'svn-status-mv)
+ (define-key svn-status-mode-map (kbd "C") 'svn-status-cp)
+ (define-key svn-status-mode-map (kbd "D") 'svn-status-rm)
+ (define-key svn-status-mode-map (kbd "c") 'svn-status-commit)
+ (define-key svn-status-mode-map (kbd "M-c") 'svn-status-cleanup)
+ (define-key svn-status-mode-map (kbd "k") 'svn-status-lock)
+ (define-key svn-status-mode-map (kbd "K") 'svn-status-unlock)
+ (define-key svn-status-mode-map (kbd "U") 'svn-status-update-cmd)
+ (define-key svn-status-mode-map (kbd "M-u") 'svn-status-update-cmd)
+ (define-key svn-status-mode-map (kbd "r") 'svn-status-revert)
+ (define-key svn-status-mode-map (kbd "l") 'svn-status-show-svn-log)
+ (define-key svn-status-mode-map (kbd "i") 'svn-status-info)
+ (define-key svn-status-mode-map (kbd "b") 'svn-status-blame)
+ (define-key svn-status-mode-map (kbd "=") 'svn-status-show-svn-diff)
+ ;; [(control ?=)] is unreachable on TTY, but you can use "*u" instead.
+ ;; (Is the "u" mnemonic for something?)
+ (define-key svn-status-mode-map (kbd "C-=") 'svn-status-show-svn-diff-for-marked-files)
+ (define-key svn-status-mode-map (kbd "~") 'svn-status-get-specific-revision)
+ (define-key svn-status-mode-map (kbd "E") 'svn-status-ediff-with-revision)
+
+ (define-key svn-status-mode-map (kbd "S g") 'svn-status-grep-files)
+ (define-key svn-status-mode-map (kbd "S s") 'svn-status-search-files)
+
+ (define-key svn-status-mode-map (kbd "n") 'svn-status-next-line)
+ (define-key svn-status-mode-map (kbd "p") 'svn-status-previous-line)
+ (define-key svn-status-mode-map (kbd "<down>") 'svn-status-next-line)
+ (define-key svn-status-mode-map (kbd "<up>") 'svn-status-previous-line)
+ (define-key svn-status-mode-map (kbd "C-x C-j") 'svn-status-dired-jump)
+ (define-key svn-status-mode-map [down-mouse-3] 'svn-status-popup-menu)
+ (setq svn-status-mode-mark-map (make-sparse-keymap))
+ (define-key svn-status-mode-map (kbd "*") svn-status-mode-mark-map)
+ (define-key svn-status-mode-mark-map (kbd "!") 'svn-status-unset-all-usermarks)
+ (define-key svn-status-mode-mark-map (kbd "?") 'svn-status-mark-unknown)
+ (define-key svn-status-mode-mark-map (kbd "A") 'svn-status-mark-added)
+ (define-key svn-status-mode-mark-map (kbd "M") 'svn-status-mark-modified)
+ (define-key svn-status-mode-mark-map (kbd "P") 'svn-status-mark-modified-properties)
+ (define-key svn-status-mode-mark-map (kbd "D") 'svn-status-mark-deleted)
+ (define-key svn-status-mode-mark-map (kbd "*") 'svn-status-mark-changed)
+ (define-key svn-status-mode-mark-map (kbd ".") 'svn-status-mark-by-file-ext)
+ (define-key svn-status-mode-mark-map (kbd "%") 'svn-status-mark-filename-regexp)
+ (define-key svn-status-mode-mark-map (kbd "u") 'svn-status-show-svn-diff-for-marked-files))
+(when (not svn-status-mode-property-map)
+ (setq svn-status-mode-property-map (make-sparse-keymap))
+ (define-key svn-status-mode-property-map (kbd "l") 'svn-status-property-list)
+ (define-key svn-status-mode-property-map (kbd "s") 'svn-status-property-set)
+ (define-key svn-status-mode-property-map (kbd "d") 'svn-status-property-delete)
+ (define-key svn-status-mode-property-map (kbd "e") 'svn-status-property-edit-one-entry)
+ (define-key svn-status-mode-property-map (kbd "i") 'svn-status-property-ignore-file)
+ (define-key svn-status-mode-property-map (kbd "I") 'svn-status-property-ignore-file-extension)
+ ;; XEmacs 21.4.15 on TTY (vt420) converts `C-i' to `TAB',
+ ;; which [(control ?i)] won't match. Handle it separately.
+ ;; On GNU Emacs, the following two forms bind the same key,
+ ;; reducing clutter in `where-is'.
+ (define-key svn-status-mode-property-map [(control ?i)] 'svn-status-property-edit-svn-ignore)
+ (define-key svn-status-mode-property-map (kbd "TAB") 'svn-status-property-edit-svn-ignore)
+ (define-key svn-status-mode-property-map (kbd "k") 'svn-status-property-set-keyword-list)
+ (define-key svn-status-mode-property-map (kbd "Ki") 'svn-status-property-set-keyword-id)
+ (define-key svn-status-mode-property-map (kbd "Kd") 'svn-status-property-set-keyword-date)
+ (define-key svn-status-mode-property-map (kbd "y") 'svn-status-property-set-eol-style)
+ (define-key svn-status-mode-property-map (kbd "x") 'svn-status-property-set-executable)
+ (define-key svn-status-mode-property-map (kbd "m") 'svn-status-property-set-mime-type)
+ ;; TODO: Why is `svn-status-select-line' in `svn-status-mode-property-map'?
+ (define-key svn-status-mode-property-map (kbd "RET") 'svn-status-select-line)
+ (define-key svn-status-mode-map (kbd "P") svn-status-mode-property-map))
+(when (not svn-status-mode-extension-map)
+ (setq svn-status-mode-extension-map (make-sparse-keymap))
+ (define-key svn-status-mode-extension-map (kbd "v") 'svn-status-resolved)
+ (define-key svn-status-mode-extension-map (kbd "X") 'svn-status-resolve-conflicts)
+ (define-key svn-status-mode-extension-map (kbd "e") 'svn-status-export)
+ (define-key svn-status-mode-map (kbd "X") svn-status-mode-extension-map))
+(when (not svn-status-mode-options-map)
+ (setq svn-status-mode-options-map (make-sparse-keymap))
+ (define-key svn-status-mode-options-map (kbd "s") 'svn-status-save-state)
+ (define-key svn-status-mode-options-map (kbd "l") 'svn-status-load-state)
+ (define-key svn-status-mode-options-map (kbd "x") 'svn-status-toggle-sort-status-buffer)
+ (define-key svn-status-mode-options-map (kbd "v") 'svn-status-toggle-svn-verbose-flag)
+ (define-key svn-status-mode-options-map (kbd "f") 'svn-status-toggle-display-full-path)
+ (define-key svn-status-mode-options-map (kbd "t") 'svn-status-set-trac-project-root)
+ (define-key svn-status-mode-options-map (kbd "n") 'svn-status-set-module-name)
+ (define-key svn-status-mode-options-map (kbd "c") 'svn-status-set-changelog-style)
+ (define-key svn-status-mode-options-map (kbd "b") 'svn-status-set-branch-list)
+ (define-key svn-status-mode-map (kbd "O") svn-status-mode-options-map))
+(when (not svn-status-mode-trac-map)
+ (setq svn-status-mode-trac-map (make-sparse-keymap))
+ (define-key svn-status-mode-trac-map (kbd "w") 'svn-trac-browse-wiki)
+ (define-key svn-status-mode-trac-map (kbd "t") 'svn-trac-browse-timeline)
+ (define-key svn-status-mode-trac-map (kbd "m") 'svn-trac-browse-roadmap)
+ (define-key svn-status-mode-trac-map (kbd "r") 'svn-trac-browse-report)
+ (define-key svn-status-mode-trac-map (kbd "s") 'svn-trac-browse-source)
+ (define-key svn-status-mode-trac-map (kbd "i") 'svn-trac-browse-ticket)
+ (define-key svn-status-mode-trac-map (kbd "c") 'svn-trac-browse-changeset)
+ (define-key svn-status-mode-map (kbd "T") svn-status-mode-trac-map))
+(when (not svn-status-mode-branch-map)
+ (setq svn-status-mode-branch-map (make-sparse-keymap))
+ (define-key svn-status-mode-branch-map (kbd "d") 'svn-branch-diff)
+ (define-key svn-status-mode-map (kbd "B") svn-status-mode-branch-map))
+
+(easy-menu-define svn-status-mode-menu svn-status-mode-map
+ "'svn-status-mode' menu"
+ '("SVN"
+ ["svn status" svn-status-update t]
+ ["svn update" svn-status-update-cmd t]
+ ["svn commit" svn-status-commit t]
+ ["svn log" svn-status-show-svn-log t]
+ ["svn info" svn-status-info t]
+ ["svn blame" svn-status-blame t]
+ ("Diff"
+ ["svn diff current file" svn-status-show-svn-diff t]
+ ["svn diff marked files" svn-status-show-svn-diff-for-marked-files t]
+ ["svn ediff current file" svn-status-ediff-with-revision t]
+ ["svn resolve conflicts" svn-status-resolve-conflicts]
+ )
+ ("Search"
+ ["Grep marked files" svn-status-grep-files t]
+ ["Search marked files" svn-status-search-files t]
+ )
+ ["svn cat ..." svn-status-get-specific-revision t]
+ ["svn add" svn-status-add-file t]
+ ["svn add recursively" svn-status-add-file-recursively t]
+ ["svn mkdir..." svn-status-make-directory t]
+ ["svn mv..." svn-status-mv t]
+ ["svn cp..." svn-status-cp t]
+ ["svn rm..." svn-status-rm t]
+ ["svn export..." svn-status-export t]
+ ["Up Directory" svn-status-examine-parent t]
+ ["Elide Directory" svn-status-toggle-elide t]
+ ["svn revert" svn-status-revert t]
+ ["svn resolved" svn-status-resolved t]
+ ["svn cleanup" svn-status-cleanup t]
+ ["svn lock" svn-status-lock t]
+ ["svn unlock" svn-status-unlock t]
+ ["Show Process Buffer" svn-status-show-process-buffer t]
+ ("Branch"
+ ["diff" svn-branch-diff t]
+ ["Set Branch list" svn-status-set-branch-list t]
+ )
+ ("Property"
+ ["svn proplist" svn-status-property-list t]
+ ["Set Multiple Properties..." svn-status-property-set t]
+ ["Edit One Property..." svn-status-property-edit-one-entry t]
+ ["svn propdel..." svn-status-property-delete t]
+ "---"
+ ["svn:ignore File..." svn-status-property-ignore-file t]
+ ["svn:ignore File Extension..." svn-status-property-ignore-file-extension t]
+ ["Edit svn:ignore Property" svn-status-property-edit-svn-ignore t]
+ "---"
+ ["Edit svn:keywords List" svn-status-property-set-keyword-list t]
+ ["Add/Remove Id to/from svn:keywords" svn-status-property-set-keyword-id t]
+ ["Add/Remove Date to/from svn:keywords" svn-status-property-set-keyword-date t]
+ "---"
+ ["Select svn:eol-style" svn-status-property-set-eol-style t]
+ ["Set svn:executable" svn-status-property-set-executable t]
+ ["Set svn:mime-type" svn-status-property-set-mime-type t]
+ )
+ ("Options"
+ ["Save Options" svn-status-save-state t]
+ ["Load Options" svn-status-load-state t]
+ ["Set Trac project root" svn-status-set-trac-project-root t]
+ ["Set Short module name" svn-status-set-module-name t]
+ ["Set Changelog style" svn-status-set-changelog-style t]
+ ["Set Branch list" svn-status-set-branch-list t]
+ ["Sort the *svn-status* buffer" svn-status-toggle-sort-status-buffer
+ :style toggle :selected svn-status-sort-status-buffer]
+ ["Use -v for svn status calls" svn-status-toggle-svn-verbose-flag
+ :style toggle :selected svn-status-verbose]
+ ["Display full path names" svn-status-toggle-display-full-path
+ :style toggle :selected svn-status-display-full-path]
+ )
+ ("Trac"
+ ["Browse wiki" svn-trac-browse-wiki t]
+ ["Browse timeline" svn-trac-browse-timeline t]
+ ["Browse roadmap" svn-trac-browse-roadmap t]
+ ["Browse source" svn-trac-browse-source t]
+ ["Browse report" svn-trac-browse-report t]
+ ["Browse ticket" svn-trac-browse-ticket t]
+ ["Browse changeset" svn-trac-browse-changeset t]
+ ["Set Trac project root" svn-status-set-trac-project-root t]
+ )
+ "---"
+ ["Edit Next SVN Cmd Line" svn-status-toggle-edit-cmd-flag t]
+ ["Work Directory History..." svn-status-use-history t]
+ ("Mark / Unmark"
+ ["Mark" svn-status-set-user-mark t]
+ ["Unmark" svn-status-unset-user-mark t]
+ ["Unmark all" svn-status-unset-all-usermarks t]
+ "---"
+ ["Mark/Unmark unknown" svn-status-mark-unknown t]
+ ["Mark/Unmark modified" svn-status-mark-modified t]
+ ["Mark/Unmark modified properties" svn-status-mark-modified-properties t]
+ ["Mark/Unmark added" svn-status-mark-added t]
+ ["Mark/Unmark deleted" svn-status-mark-deleted t]
+ ["Mark/Unmark modified/added/deleted" svn-status-mark-changed t]
+ ["Mark/Unmark filename by extension" svn-status-mark-by-file-ext t]
+ ["Mark/Unmark filename by regexp" svn-status-mark-filename-regexp t]
+ )
+ ["Hide Unknown" svn-status-toggle-hide-unknown
+ :style toggle :selected svn-status-hide-unknown]
+ ["Hide Unmodified" svn-status-toggle-hide-unmodified
+ :style toggle :selected svn-status-hide-unmodified]
+ ["Show Client versions" svn-status-version t]
+ ["Prepare bug report" svn-prepare-bug-report t]
+ ))
+
+(defvar svn-status-file-popup-menu-list
+ '(["open" svn-status-find-file-other-window t]
+ ["svn diff" svn-status-show-svn-diff t]
+ ["svn commit" svn-status-commit t]
+ ["svn log" svn-status-show-svn-log t]
+ ["svn blame" svn-status-blame t]
+ ["mark" svn-status-set-user-mark t]
+ ["unmark" svn-status-unset-user-mark t]
+ ["svn add" svn-status-add-file t]
+ ["svn add recursively" svn-status-add-file-recursively t]
+ ["svn mv..." svn-status-mv t]
+ ["svn rm..." svn-status-rm t]
+ ["svn lock" svn-status-lock t]
+ ["svn unlock" svn-status-unlock t]
+ ["svn info" svn-status-info t]
+ ) "A list of menu entries for `svn-status-popup-menu'")
+
+;; extend svn-status-file-popup-menu-list via:
+;; (add-to-list 'svn-status-file-popup-menu-list ["commit" svn-status-commit t])
+
+(defun svn-status-popup-menu (event)
+ "Display a file specific popup menu"
+ (interactive "e")
+ (mouse-set-point event)
+ (let* ((line-info (svn-status-get-line-information))
+ (name (svn-status-line-info->filename line-info)))
+ (when line-info
+ (easy-menu-define svn-status-actual-popup-menu nil nil
+ (append (list name) svn-status-file-popup-menu-list))
+ (svn-status-face-set-temporary-during-popup
+ 'svn-status-marked-popup-face (svn-point-at-bol) (svn-point-at-eol)
+ svn-status-actual-popup-menu))))
+
+(defun svn-status-face-set-temporary-during-popup (face begin end menu &optional prefix)
+ "Put FACE on BEGIN and END in the buffer during Popup MENU.
+PREFIX is passed to `popup-menu'."
+ (let (o)
+ (unwind-protect
+ (progn
+ (setq o (make-overlay begin end))
+ (overlay-put o 'face face)
+ (sit-for 0)
+ (popup-menu menu prefix))
+ (delete-overlay o))))
+
+(defun svn-status-mode ()
+ "Major mode used by psvn.el to display the output of \"svn status\".
+
+The Output has the following format:
+ FPH BASE CMTD Author em File
+F = Filemark
+P = Property mark
+H = History mark
+BASE = local base revision
+CMTD = last committed revision
+Author = author of change
+em = \"**\" or \"(Update Available)\" [see `svn-status-short-mod-flag-p']
+ if file can be updated
+File = path/filename
+
+The following keys are defined:
+\\{svn-status-mode-map}"
+ (interactive)
+ (kill-all-local-variables)
+
+ (use-local-map svn-status-mode-map)
+ (easy-menu-add svn-status-mode-menu)
+
+ (setq major-mode 'svn-status-mode)
+ (setq mode-name "svn-status")
+ (setq mode-line-process 'svn-status-mode-line-process)
+ (run-hooks 'svn-status-mode-hook)
+ (let ((view-read-only nil))
+ (toggle-read-only 1)))
+
+(defun svn-status-update-mode-line ()
+ (setq svn-status-mode-line-process
+ (concat svn-status-mode-line-process-edit-flag svn-status-mode-line-process-status))
+ (force-mode-line-update))
+
+(defun svn-status-bury-buffer (arg)
+ "Bury the buffers used by psvn.el
+Currently this is:
+ `svn-status-buffer-name'
+ `svn-process-buffer-name'
+ `svn-log-edit-buffer-name'
+ *svn-property-edit*
+ *svn-log*
+ *svn-info*
+When called with a prefix argument, ARG, switch back to the window configuration that was
+in use before `svn-status' was called."
+ (interactive "P")
+ (cond (arg
+ (when svn-status-initial-window-configuration
+ (set-window-configuration svn-status-initial-window-configuration)))
+ (t
+ (let ((bl `(,svn-log-edit-buffer-name "*svn-property-edit*" "*svn-log*" "*svn-info*" ,svn-process-buffer-name)))
+ (while bl
+ (when (get-buffer (car bl))
+ (bury-buffer (car bl)))
+ (setq bl (cdr bl)))
+ (when (string= (buffer-name) svn-status-buffer-name)
+ (bury-buffer))))))
+
+(defun svn-status-save-some-buffers (&optional tree)
+ "Save all buffers visiting a file in TREE.
+If TREE is not given, try `svn-status-base-dir' as TREE."
+ (interactive)
+ ;; (message "svn-status-save-some-buffers: tree1: %s" tree)
+ (let ((ok t)
+ (tree (or (svn-status-base-dir)
+ tree)))
+ ;; (message "svn-status-save-some-buffers: tree2: %s" tree)
+ (unless tree
+ (error "Not in a svn project tree"))
+ (dolist (buffer (buffer-list))
+ (with-current-buffer buffer
+ (when (buffer-modified-p)
+ (let ((file (buffer-file-name)))
+ (when file
+ (let ((root (svn-status-base-dir (file-name-directory file))))
+ ;; (message "svn-status-save-some-buffers: file: %s, root: %s" file root)
+ (when (and root
+ (string= root tree)
+ ;; buffer is modified and in the tree TREE.
+ (or (y-or-n-p (concat "Save buffer " (buffer-name) "? "))
+ (setq ok nil)))
+ (save-buffer))))))))
+ ok))
+
+(defun svn-status-find-files ()
+ "Open selected file(s) for editing.
+See `svn-status-marked-files' for what counts as selected."
+ (interactive)
+ (let ((fnames (mapcar 'svn-status-line-info->full-path (svn-status-marked-files))))
+ (mapc 'find-file fnames)))
+
+
+(defun svn-status-find-file-other-window ()
+ "Open the file in the other window for editing."
+ (interactive)
+ (svn-status-ensure-cursor-on-file)
+ (find-file-other-window (svn-status-line-info->filename
+ (svn-status-get-line-information))))
+
+(defun svn-status-find-file-other-window-noselect ()
+ "Open the file in the other window for editing, but don't select it."
+ (interactive)
+ (svn-status-ensure-cursor-on-file)
+ (display-buffer
+ (find-file-noselect (svn-status-line-info->filename
+ (svn-status-get-line-information)))))
+
+(defun svn-status-view-file-other-window ()
+ "Open the file in the other window for viewing."
+ (interactive)
+ (svn-status-ensure-cursor-on-file)
+ (view-file-other-window (svn-status-line-info->filename
+ (svn-status-get-line-information))))
+
+(defun svn-status-find-file-or-examine-directory ()
+ "If point is on a directory, run `svn-status' on that directory.
+Otherwise run `find-file'."
+ (interactive)
+ (svn-status-ensure-cursor-on-file)
+ (let ((line-info (svn-status-get-line-information)))
+ (if (svn-status-line-info->directory-p line-info)
+ (svn-status (svn-status-line-info->full-path line-info))
+ (find-file (svn-status-line-info->filename line-info)))))
+
+(defun svn-status-examine-parent ()
+ "Run `svn-status' on the parent of the current directory."
+ (interactive)
+ (svn-status (expand-file-name "../")))
+
+(defun svn-status-mouse-find-file-or-examine-directory (event)
+ "Move point to where EVENT occurred, and do `svn-status-find-file-or-examine-directory'
+EVENT could be \"mouse clicked\" or similar."
+ (interactive "e")
+ (mouse-set-point event)
+ (svn-status-find-file-or-examine-directory))
+
+(defun svn-status-line-info->ui-status (line-info)
+ "Return the ui-status structure of LINE-INFO.
+See `svn-status-make-ui-status' for information about the ui-status."
+ (nth 0 line-info))
+
+(defun svn-status-line-info->has-usermark (line-info) (nth 0 (nth 0 line-info)))
+(defun svn-status-line-info->user-elide (line-info) (nth 1 (nth 0 line-info)))
+
+(defun svn-status-line-info->filemark (line-info) (nth 1 line-info))
+(defun svn-status-line-info->propmark (line-info) (nth 2 line-info))
+(defun svn-status-line-info->filename (line-info) (nth 3 line-info))
+(defun svn-status-line-info->filename-nondirectory (line-info)
+ (file-name-nondirectory (svn-status-line-info->filename line-info)))
+(defun svn-status-line-info->localrev (line-info)
+ (if (>= (nth 4 line-info) 0)
+ (nth 4 line-info)
+ nil))
+(defun svn-status-line-info->lastchangerev (line-info)
+ "Return the last revision in which LINE-INFO was modified."
+ (let ((l (nth 5 line-info)))
+ (if (and l (>= l 0))
+ l
+ nil)))
+(defun svn-status-line-info->author (line-info)
+ "Return the last author that changed the item that is represented in LINE-INFO."
+ (nth 6 line-info))
+(defun svn-status-line-info->update-available (line-info)
+ "Return whether LINE-INFO is out of date.
+In other words, whether there is a newer version available in the
+repository than the working copy."
+ (nth 7 line-info))
+(defun svn-status-line-info->locked (line-info)
+ "Return whether LINE-INFO represents a locked file.
+This is column three of the `svn status' output.
+The result will be nil or \"L\".
+\(A file becomes locked when an operation is interupted; run \\[svn-status-cleanup]'
+to unlock it.\)"
+ (nth 8 line-info))
+(defun svn-status-line-info->historymark (line-info)
+ "Mark from column four of output from `svn status'.
+This will be nil unless the file is scheduled for addition with
+history, when it will be \"+\"."
+ (nth 9 line-info))
+(defun svn-status-line-info->switched (line-info)
+ "Return whether LINE-INFO is switched relative to its parent.
+This is column five of the output from `svn status'.
+The result will be nil or \"S\"."
+ (nth 10 line-info))
+(defun svn-status-line-info->repo-locked (line-info)
+ "Return whether LINE-INFO contains some locking information.
+This is column six of the output from `svn status'.
+The result will be \"K\", \"O\", \"T\", \"B\" or nil."
+ (nth 11 line-info))
+(defun svn-status-line-info->psvn-extra-info (line-info)
+ "Return a list of extra information for psvn associated with LINE-INFO.
+This list holds currently only one element:
+* The action after a commit or update."
+ (nth 12 line-info))
+
+(defun svn-status-line-info->is-visiblep (line-info)
+ "Return whether the line is visible or not"
+ (or (not (or (svn-status-line-info->hide-because-unknown line-info)
+ (svn-status-line-info->hide-because-unmodified line-info)
+ (svn-status-line-info->hide-because-custom-hide-function line-info)
+ (svn-status-line-info->hide-because-user-elide line-info)))
+ (svn-status-line-info->update-available line-info) ;; show the line, if an update is available
+ (svn-status-line-info->psvn-extra-info line-info) ;; show the line, if there is some extra info displayed on this line
+ ))
+
+(defun svn-status-line-info->hide-because-unknown (line-info)
+ (and svn-status-hide-unknown
+ (eq (svn-status-line-info->filemark line-info) ??)))
+
+(defun svn-status-line-info->hide-because-custom-hide-function (line-info)
+ (and svn-status-custom-hide-function
+ (apply svn-status-custom-hide-function (list line-info))))
+
+(defun svn-status-line-info->hide-because-unmodified (line-info)
+ ;;(message " %S %S %S %S - %s" svn-status-hide-unmodified (svn-status-line-info->propmark line-info) ?_
+ ;; (svn-status-line-info->filemark line-info) (svn-status-line-info->filename line-info))
+ (and svn-status-hide-unmodified
+ (and (or (eq (svn-status-line-info->filemark line-info) ?_)
+ (eq (svn-status-line-info->filemark line-info) ? ))
+ (or (eq (svn-status-line-info->propmark line-info) ?_)
+ (eq (svn-status-line-info->propmark line-info) ? )
+ (eq (svn-status-line-info->propmark line-info) nil)))))
+
+(defun svn-status-line-info->hide-because-user-elide (line-info)
+ (eq (svn-status-line-info->user-elide line-info) t))
+
+(defun svn-status-line-info->show-user-elide-continuation (line-info)
+ (eq (svn-status-line-info->user-elide line-info) 'directory))
+
+;; modify the line-info
+(defun svn-status-line-info->set-filemark (line-info value)
+ (setcar (nthcdr 1 line-info) value))
+
+(defun svn-status-line-info->set-propmark (line-info value)
+ (setcar (nthcdr 2 line-info) value))
+
+(defun svn-status-line-info->set-localrev (line-info value)
+ (setcar (nthcdr 4 line-info) value))
+
+(defun svn-status-line-info->set-author (line-info value)
+ (setcar (nthcdr 6 line-info) value))
+
+(defun svn-status-line-info->set-lastchangerev (line-info value)
+ (setcar (nthcdr 5 line-info) value))
+
+(defun svn-status-line-info->set-repo-locked (line-info value)
+ (setcar (nthcdr 11 line-info) value))
+
+(defun svn-status-line-info->set-psvn-extra-info (line-info value)
+ (setcar (nthcdr 12 line-info) value))
+
+(defun svn-status-copy-current-line-info (arg)
+ "Copy the current file name at point, using `svn-status-copy-filename-as-kill'.
+If no file is at point, copy everything starting from ':' to the end of line."
+ (interactive "P")
+ (if (svn-status-get-line-information)
+ (svn-status-copy-filename-as-kill arg)
+ (save-excursion
+ (goto-char (svn-point-at-bol))
+ (when (looking-at ".+?: *\\(.+\\)$")
+ (kill-new (svn-match-string-no-properties 1))
+ (message "Copied: %s" (svn-match-string-no-properties 1))))))
+
+(defun svn-status-copy-filename-as-kill (arg)
+ "Copy the actual file name to the kill-ring.
+When called with the prefix argument 0, use the full path name."
+ (interactive "P")
+ (let ((str (if (eq arg 0)
+ (svn-status-line-info->full-path (svn-status-get-line-information))
+ (svn-status-line-info->filename (svn-status-get-line-information)))))
+ (kill-new str)
+ (message "Copied %s" str)))
+
+(defun svn-status-get-child-directories (&optional dir)
+ "Return a list of subdirectories for DIR"
+ (interactive)
+ (let ((this-dir (concat (expand-file-name (or dir (svn-status-line-info->filename (svn-status-get-line-information)))) "/"))
+ (test-dir)
+ (sub-dir-list))
+ ;;(message "this-dir %S" this-dir)
+ (dolist (line-info svn-status-info)
+ (when (svn-status-line-info->directory-p line-info)
+ (setq test-dir (svn-status-line-info->full-path line-info))
+ (when (string= (file-name-directory test-dir) this-dir)
+ (add-to-list 'sub-dir-list (file-relative-name (svn-status-line-info->full-path line-info)) t))))
+ sub-dir-list))
+
+(defun svn-status-toggle-elide (arg)
+ "Toggle eliding of the current file or directory.
+When called with a prefix argument, toggle the hiding of all subdirectories for the current directory."
+ (interactive "P")
+ (if arg
+ (let ((cur-line (svn-status-line-info->filename (svn-status-get-line-information))))
+ (when (svn-status-line-info->user-elide (svn-status-get-line-information))
+ (svn-status-toggle-elide nil))
+ (dolist (dir-name (svn-status-get-child-directories))
+ (svn-status-goto-file-name dir-name)
+ (svn-status-toggle-elide nil))
+ (svn-status-goto-file-name cur-line))
+ (let ((st-info svn-status-info)
+ (fname)
+ (test (svn-status-line-info->filename (svn-status-get-line-information)))
+ (len-test)
+ (len-fname)
+ (new-elide-mark t)
+ (elide-mark))
+ (if (member test svn-status-elided-list)
+ (setq svn-status-elided-list (delete test svn-status-elided-list))
+ (add-to-list 'svn-status-elided-list test))
+ (when (string= test ".")
+ (setq test ""))
+ (setq len-test (length test))
+ (while st-info
+ (setq fname (svn-status-line-info->filename (car st-info)))
+ (setq len-fname (length fname))
+ (when (and (>= len-fname len-test)
+ (string= (substring fname 0 len-test) test))
+ (setq elide-mark new-elide-mark)
+ (when (or (string= fname ".")
+ (and (= len-fname len-test) (svn-status-line-info->directory-p (car st-info))))
+ (message "Elided directory %s and all its files." fname)
+ (setq new-elide-mark (not (svn-status-line-info->user-elide (car st-info))))
+ (setq elide-mark (if new-elide-mark 'directory nil)))
+ ;;(message "elide-mark: %S member: %S" elide-mark (member fname svn-status-elided-list))
+ (when (and (member fname svn-status-elided-list) (not elide-mark))
+ (setq svn-status-elided-list (delete fname svn-status-elided-list)))
+ (setcar (nthcdr 1 (svn-status-line-info->ui-status (car st-info))) elide-mark))
+ (setq st-info (cdr st-info))))
+ ;;(message "svn-status-elided-list: %S" svn-status-elided-list)
+ (svn-status-update-buffer)))
+
+(defun svn-status-apply-elide-list ()
+ "Elide files/directories according to `svn-status-elided-list'."
+ (interactive)
+ (let ((st-info svn-status-info)
+ (fname)
+ (len-fname)
+ (test)
+ (len-test)
+ (elided-list)
+ (elide-mark))
+ (while st-info
+ (setq fname (svn-status-line-info->filename (car st-info)))
+ (setq len-fname (length fname))
+ (setq elided-list svn-status-elided-list)
+ (setq elide-mark nil)
+ (while elided-list
+ (setq test (car elided-list))
+ (when (string= test ".")
+ (setq test ""))
+ (setq len-test (length test))
+ (when (and (>= len-fname len-test)
+ (string= (substring fname 0 len-test) test))
+ (setq elide-mark t)
+ (when (or (string= fname ".")
+ (and (= len-fname len-test) (svn-status-line-info->directory-p (car st-info))))
+ (setq elide-mark 'directory)))
+ (setq elided-list (cdr elided-list)))
+ ;;(message "fname: %s elide-mark: %S" fname elide-mark)
+ (setcar (nthcdr 1 (svn-status-line-info->ui-status (car st-info))) elide-mark)
+ (setq st-info (cdr st-info))))
+ (svn-status-update-buffer))
+
+(defun svn-status-update-with-command-list (cmd-list)
+ (save-excursion
+ (set-buffer svn-status-buffer-name)
+ (let ((st-info)
+ (found)
+ (action)
+ (fname (svn-status-line-info->filename (svn-status-get-line-information)))
+ (fname-pos (point))
+ (column (current-column)))
+ (setq cmd-list (sort cmd-list '(lambda (item1 item2) (string-lessp (car item1) (car item2)))))
+ (while cmd-list
+ (unless st-info (setq st-info svn-status-info))
+ ;;(message "%S" (caar cmd-list))
+ (setq found nil)
+ (while (and (not found) st-info)
+ (setq found (string= (caar cmd-list) (svn-status-line-info->filename (car st-info))))
+ ;;(message "found: %S" found)
+ (unless found (setq st-info (cdr st-info))))
+ (unless found
+ (svn-status-message 3 "psvn: continue to search for %s" (caar cmd-list))
+ (setq st-info svn-status-info)
+ (while (and (not found) st-info)
+ (setq found (string= (caar cmd-list) (svn-status-line-info->filename (car st-info))))
+ (unless found (setq st-info (cdr st-info)))))
+ (if found
+ ;;update the info line
+ (progn
+ (setq action (cadar cmd-list))
+ ;;(message "found %s, action: %S" (caar cmd-list) action)
+ (svn-status-annotate-status-buffer-entry action (car st-info)))
+ (svn-status-message 3 "psvn: did not find %s" (caar cmd-list)))
+ (setq cmd-list (cdr cmd-list)))
+ (if fname
+ (progn
+ (goto-char fname-pos)
+ (svn-status-goto-file-name fname)
+ (goto-char (+ column (svn-point-at-bol))))
+ (goto-char (+ (next-overlay-change (point-min)) svn-status-default-column))))))
+
+(defun svn-status-annotate-status-buffer-entry (action line-info)
+ (let ((tag-string))
+ (svn-status-goto-file-name (svn-status-line-info->filename line-info))
+ (when (and (member action '(committed added))
+ svn-status-commit-rev-number)
+ (svn-status-line-info->set-localrev line-info svn-status-commit-rev-number)
+ (svn-status-line-info->set-lastchangerev line-info svn-status-commit-rev-number))
+ (when svn-status-last-commit-author
+ (svn-status-line-info->set-author line-info svn-status-last-commit-author))
+ (svn-status-line-info->set-psvn-extra-info line-info (list action))
+ (cond ((equal action 'committed)
+ (setq tag-string " <committed>")
+ (when (member (svn-status-line-info->repo-locked line-info) '(?K))
+ (svn-status-line-info->set-repo-locked line-info nil)))
+ ((equal action 'added)
+ (setq tag-string " <added>"))
+ ((equal action 'deleted)
+ (setq tag-string " <deleted>"))
+ ((equal action 'replaced)
+ (setq tag-string " <replaced>"))
+ ((equal action 'updated)
+ (setq tag-string " <updated>"))
+ ((equal action 'updated-props)
+ (setq tag-string " <updated-props>"))
+ ((equal action 'conflicted)
+ (setq tag-string " <conflicted>")
+ (svn-status-line-info->set-filemark line-info ?C))
+ ((equal action 'merged)
+ (setq tag-string " <merged>"))
+ ((equal action 'propset)
+ ;;(setq tag-string " <propset>")
+ (svn-status-line-info->set-propmark line-info svn-status-file-modified-after-save-flag))
+ ((equal action 'added-wc)
+ (svn-status-line-info->set-filemark line-info ?A)
+ (svn-status-line-info->set-localrev line-info 0))
+ ((equal action 'deleted-wc)
+ (svn-status-line-info->set-filemark line-info ?D))
+ (t
+ (error "Unknown action '%s for %s" action (svn-status-line-info->filename line-info))))
+ (when (and tag-string (not (member action '(conflicted merged))))
+ (svn-status-line-info->set-filemark line-info ? )
+ (svn-status-line-info->set-propmark line-info ? ))
+ (let ((buffer-read-only nil))
+ (delete-region (svn-point-at-bol) (svn-point-at-eol))
+ (svn-insert-line-in-status-buffer line-info)
+ (backward-char 1)
+ (when tag-string
+ (insert tag-string))
+ (delete-char 1))))
+
+
+
+;; (svn-status-update-with-command-list '(("++ideas" committed) ("a.txt" committed) ("alf")))
+;; (svn-status-update-with-command-list (svn-status-parse-commit-output))
+
+(defun svn-status-parse-commit-output ()
+ "Parse the output of svn commit.
+Return a list that is suitable for `svn-status-update-with-command-list'"
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (let ((action)
+ (file-name)
+ (skip)
+ (result))
+ (goto-char (point-min))
+ (setq svn-status-commit-rev-number nil)
+ (setq skip nil) ; set to t whenever we find a line not about a committed file
+ (while (< (point) (point-max))
+ (cond ((= (svn-point-at-eol) (svn-point-at-bol)) ;skip blank lines
+ (setq skip t))
+ ((looking-at "Sending")
+ (setq action 'committed))
+ ((looking-at "Adding")
+ (setq action 'added))
+ ((looking-at "Deleting")
+ (setq action 'deleted))
+ ((looking-at "Replacing")
+ (setq action 'replaced))
+ ((looking-at "Transmitting file data")
+ (setq skip t))
+ ((looking-at "Committed revision \\([0-9]+\\)")
+ (setq svn-status-commit-rev-number
+ (string-to-number (svn-match-string-no-properties 1)))
+ (setq skip t))
+ (t ;; this should never be needed(?)
+ (setq action 'unknown)))
+ (unless skip ;found an interesting line
+ (forward-char 15)
+ (when svn-status-operated-on-dot
+ ;; when the commit used . as argument, delete the trailing directory
+ ;; from the svn output
+ (search-forward "/" nil t))
+ (setq file-name (buffer-substring-no-properties (point) (svn-point-at-eol)))
+ (unless svn-status-last-commit-author
+ (setq svn-status-last-commit-author (car (svn-status-info-for-path (expand-file-name (concat default-directory file-name))))))
+ (setq result (cons (list file-name action)
+ result))
+ (setq skip nil))
+ (forward-line 1))
+ result)))
+;;(svn-status-parse-commit-output)
+;;(svn-status-annotate-status-buffer-entry)
+
+(defun svn-status-parse-ar-output ()
+ "Parse the output of svn add|remove.
+Return a list that is suitable for `svn-status-update-with-command-list'"
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (let ((action)
+ (name)
+ (skip)
+ (result))
+ (goto-char (point-min))
+ (while (< (point) (point-max))
+ (cond ((= (svn-point-at-eol) (svn-point-at-bol)) ;skip blank lines
+ (setq skip t))
+ ((looking-at "A")
+ (setq action 'added-wc))
+ ((looking-at "D")
+ (setq action 'deleted-wc))
+ (t ;; this should never be needed(?)
+ (setq action 'unknown)))
+ (unless skip ;found an interesting line
+ (forward-char 10)
+ (setq name (buffer-substring-no-properties (point) (svn-point-at-eol)))
+ (setq result (cons (list name action)
+ result))
+ (setq skip nil))
+ (forward-line 1))
+ result)))
+;; (svn-status-parse-ar-output)
+;; (svn-status-update-with-command-list (svn-status-parse-ar-output))
+
+(defun svn-status-parse-update-output ()
+ "Parse the output of svn update.
+Return a list that is suitable for `svn-status-update-with-command-list'"
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (setq svn-status-update-rev-number nil)
+ (let ((action)
+ (name)
+ (skip)
+ (result))
+ (goto-char (point-min))
+ (while (< (point) (point-max))
+ (cond ((= (svn-point-at-eol) (svn-point-at-bol)) ;skip blank lines
+ (setq skip t))
+ ((looking-at "Updated to revision \\([0-9]+\\)")
+ (setq svn-status-update-rev-number
+ (list t (string-to-number (svn-match-string-no-properties 1))))
+ (setq skip t))
+ ((looking-at "At revision \\([0-9]+\\)")
+ (setq svn-status-update-rev-number
+ (list nil (string-to-number (svn-match-string-no-properties 1))))
+ (setq skip t))
+ ((looking-at "U")
+ (setq action 'updated))
+ ((looking-at "A")
+ (setq action 'added))
+ ((looking-at "D")
+ (setq skip t))
+ ;;(setq action 'deleted)) ;;deleted files are not displayed in the svn status output.
+ ((looking-at "C")
+ (setq action 'conflicted))
+ ((looking-at "G")
+ (setq action 'merged))
+
+ ((looking-at " U")
+ (setq action 'updated-props))
+
+ (t ;; this should never be needed(?)
+ (setq action (concat "parse-update: '"
+ (buffer-substring-no-properties (point) (+ 2 (point))) "'"))))
+ (unless skip ;found an interesting line
+ (forward-char 3)
+ (setq name (buffer-substring-no-properties (point) (svn-point-at-eol)))
+ (setq result (cons (list name action)
+ result))
+ (setq skip nil))
+ (forward-line 1))
+ result)))
+;; (svn-status-parse-update-output)
+;; (svn-status-update-with-command-list (svn-status-parse-update-output))
+
+(defun svn-status-parse-property-output ()
+ "Parse the output of svn propset.
+Return a list that is suitable for `svn-status-update-with-command-list'"
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (let ((result))
+ (dolist (line (split-string (buffer-substring-no-properties (point-min) (point-max)) "\n"))
+ (message "%s" line)
+ (when (string-match "property '\\(.+\\)' set on '\\(.+\\)'" line)
+ ;;(message "property %s - file %s" (match-string 1 line) (match-string 2 line))
+ (setq result (cons (list (match-string 2 line) 'propset) result))))
+ result)))
+
+;; (svn-status-parse-property-output)
+;; (svn-status-update-with-command-list (svn-status-parse-property-output))
+
+
+(defun svn-status-line-info->symlink-p (line-info)
+ "Return non-nil if LINE-INFO refers to a symlink, nil otherwise.
+The value is the name of the file to which it is linked. \(See
+`file-symlink-p'.\)
+
+On win32 systems this won't work, even though symlinks are supported
+by subversion on such systems."
+ ;; on win32 would need to see how svn does symlinks
+ (file-symlink-p (svn-status-line-info->filename line-info)))
+
+(defun svn-status-line-info->directory-p (line-info)
+ "Return t if LINE-INFO refers to a directory, nil otherwise.
+Symbolic links to directories count as directories (see `file-directory-p')."
+ (file-directory-p (svn-status-line-info->filename line-info)))
+
+(defun svn-status-line-info->full-path (line-info)
+ "Return the full path of the file represented by LINE-INFO."
+ (expand-file-name
+ (svn-status-line-info->filename line-info)))
+
+;;Not convinced that this is the fastest way, but...
+(defun svn-status-count-/ (string)
+ "Return number of \"/\"'s in STRING."
+ (let ((n 0)
+ (last 0))
+ (while (setq last (string-match "/" string (1+ last)))
+ (setq n (1+ n)))
+ n))
+
+(defun svn-insert-line-in-status-buffer (line-info)
+ "Format LINE-INFO and insert the result in the current buffer."
+ (let ((usermark (if (svn-status-line-info->has-usermark line-info) "*" " "))
+ (update-available (if (svn-status-line-info->update-available line-info)
+ (svn-add-face (if svn-status-short-mod-flag-p
+ "** "
+ " (Update Available)")
+ 'svn-status-update-available-face)
+ (if svn-status-short-mod-flag-p " " "")))
+ (filename ;; <indentation>file or /path/to/file
+ (concat
+ (if (or svn-status-display-full-path
+ svn-status-hide-unmodified)
+ (svn-add-face
+ (let ((dir-name (file-name-as-directory
+ (svn-status-line-info->directory-containing-line-info
+ line-info nil))))
+ (if (and (<= 2 (length dir-name))
+ (= ?. (aref dir-name 0))
+ (= ?/ (aref dir-name 1)))
+ (substring dir-name 2)
+ dir-name))
+ 'svn-status-directory-face)
+ ;; showing all files, so add indentation
+ (make-string (* 2 (svn-status-count-/
+ (svn-status-line-info->filename line-info)))
+ 32))
+ ;;symlinks get a different face
+ (let ((target (svn-status-line-info->symlink-p line-info)))
+ (if target
+ ;; name -> trget
+ ;; name gets symlink-face, target gets file/directory face
+ (concat
+ (svn-add-face (svn-status-line-info->filename-nondirectory line-info)
+ 'svn-status-symlink-face)
+ " -> "
+ (svn-status-choose-face-to-add
+ ;; TODO: could use different faces for
+ ;; unversioned targets and broken symlinks?
+ (svn-status-line-info->directory-p line-info)
+ target
+ 'svn-status-directory-face
+ 'svn-status-filename-face))
+ ;; else target is not a link
+ (svn-status-choose-face-to-add
+ (svn-status-line-info->directory-p line-info)
+ (svn-status-line-info->filename-nondirectory line-info)
+ 'svn-status-directory-face
+ 'svn-status-filename-face)))
+ ))
+ (elide-hint (if (svn-status-line-info->show-user-elide-continuation line-info) " ..." "")))
+ (svn-puthash (svn-status-line-info->filename line-info)
+ (point)
+ svn-status-filename-to-buffer-position-cache)
+ (insert (svn-status-maybe-add-face
+ (svn-status-line-info->has-usermark line-info)
+ (concat usermark
+ (format svn-status-line-format
+ (svn-status-line-info->filemark line-info)
+ (or (svn-status-line-info->propmark line-info) ? )
+ (or (svn-status-line-info->historymark line-info) ? )
+ (or (svn-status-line-info->localrev line-info) "")
+ (or (svn-status-line-info->lastchangerev line-info) "")
+ (svn-status-line-info->author line-info))
+ (when svn-status-short-mod-flag-p update-available)
+ filename
+ (unless svn-status-short-mod-flag-p update-available)
+ (svn-status-maybe-add-string (svn-status-line-info->locked line-info)
+ " [ LOCKED ]" 'svn-status-locked-face)
+ (svn-status-maybe-add-string (svn-status-line-info->repo-locked line-info)
+ (let ((flag (svn-status-line-info->repo-locked line-info)))
+ (cond ((eq flag ?K) " [ REPO-LOCK-HERE ]")
+ ((eq flag ?O) " [ REPO-LOCK-OTHER ]")
+ ((eq flag ?T) " [ REPO-LOCK-STOLEN ]")
+ ((eq flag ?B) " [ REPO-LOCK-BROKEN ]")
+ (t " [ REPO-LOCK-UNKNOWN ]")))
+ 'svn-status-locked-face)
+ (svn-status-maybe-add-string (svn-status-line-info->switched line-info)
+ " (switched)" 'svn-status-switched-face)
+ elide-hint)
+ 'svn-status-marked-face)
+ "\n")))
+
+(defun svn-status-redraw-status-buffer ()
+ "Redraw the `svn-status-buffer-name' buffer.
+Additionally clear the psvn-extra-info field in all line-info lists."
+ (interactive)
+ (dolist (line-info svn-status-info)
+ (svn-status-line-info->set-psvn-extra-info line-info nil))
+ (svn-status-update-buffer))
+
+(defun svn-status-update-buffer ()
+ "Update the `svn-status-buffer-name' buffer, using `svn-status-info'.
+ This function does not access the repository."
+ (interactive)
+ ;(message "buffer-name: %s" (buffer-name))
+ (unless (string= (buffer-name) svn-status-buffer-name)
+ (set-buffer svn-status-buffer-name))
+ (svn-status-mode)
+ (when svn-status-refresh-info
+ (when (eq svn-status-refresh-info 'once)
+ (setq svn-status-refresh-info nil))
+ (svn-status-parse-info t))
+ (let ((st-info svn-status-info)
+ (buffer-read-only nil)
+ (start-pos)
+ (overlay)
+ (unmodified-count 0) ;how many unmodified files are hidden
+ (unknown-count 0) ;how many unknown files are hidden
+ (custom-hide-count 0) ;how many files are hidden via svn-status-custom-hide-function
+ (marked-count 0) ;how many files are elided
+ (user-elide-count 0)
+ (first-line t)
+ (fname (svn-status-line-info->filename (svn-status-get-line-information)))
+ (fname-pos (point))
+ (window-line-pos (svn-status-window-line-position (get-buffer-window (current-buffer))))
+ (header-line-string)
+ (column (current-column)))
+ (delete-region (point-min) (point-max))
+ (insert "\n")
+ ;; Insert all files and directories
+ (while st-info
+ (setq start-pos (point))
+ (cond ((or (svn-status-line-info->has-usermark (car st-info)) first-line)
+ ;; Show a marked file and the "." always
+ (svn-insert-line-in-status-buffer (car st-info))
+ (setq first-line nil))
+ ((svn-status-line-info->update-available (car st-info))
+ (svn-insert-line-in-status-buffer (car st-info)))
+ ((and svn-status-custom-hide-function
+ (apply svn-status-custom-hide-function (list (car st-info))))
+ (setq custom-hide-count (1+ custom-hide-count)))
+ ((svn-status-line-info->hide-because-user-elide (car st-info))
+ (setq user-elide-count (1+ user-elide-count)))
+ ((svn-status-line-info->hide-because-unknown (car st-info))
+ (setq unknown-count (1+ unknown-count)))
+ ((svn-status-line-info->hide-because-unmodified (car st-info))
+ (setq unmodified-count (1+ unmodified-count)))
+ (t
+ (svn-insert-line-in-status-buffer (car st-info))))
+ (when (svn-status-line-info->has-usermark (car st-info))
+ (setq marked-count (+ marked-count 1)))
+ (setq overlay (make-overlay start-pos (point)))
+ (overlay-put overlay 'svn-info (car st-info))
+ (setq st-info (cdr st-info)))
+ ;; Insert status information at the buffer beginning
+ (goto-char (point-min))
+ (insert (format "svn status for directory %s%s\n"
+ default-directory
+ (if svn-status-head-revision (format " (status against revision: %s)"
+ svn-status-head-revision)
+ "")))
+ (when svn-status-module-name
+ (insert (format "Project name: %s\n" svn-status-module-name)))
+ (when svn-status-branch-list
+ (insert (format "Branches: %s\n" svn-status-branch-list)))
+ (when svn-status-base-info
+ (insert (concat "Repository Root: " (svn-status-base-info->repository-root) "\n"))
+ (insert (concat "Repository Url: " (svn-status-base-info->url) "\n")))
+ (when svn-status-hide-unknown
+ (insert
+ (format "%d Unknown file(s) are hidden - press `?' to toggle hiding\n"
+ unknown-count)))
+ (when svn-status-hide-unmodified
+ (insert
+ (format "%d Unmodified file(s) are hidden - press `_' to toggle hiding\n"
+ unmodified-count)))
+ (when (> custom-hide-count 0)
+ (insert
+ (format "%d file(s) are hidden via the svn-status-custom-hide-function\n"
+ custom-hide-count)))
+ (when (> user-elide-count 0)
+ (insert (format "%d file(s) elided\n" user-elide-count)))
+ (insert (format "%d file(s) marked\n" marked-count))
+ (setq header-line-string (concat (format svn-status-line-format
+ 70 80 72 "BASE" "CMTD" "Author")
+ (if svn-status-short-mod-flag-p "em " "")
+ "File"))
+ (cond ((eq svn-status-use-header-line t)
+ (setq header-line-format (concat " " header-line-string)))
+ ((eq svn-status-use-header-line 'inline)
+ (insert "\n " header-line-string "\n")))
+ (setq svn-start-of-file-list-line-number (+ (count-lines (point-min) (point)) 1))
+ (if fname
+ (progn
+ (goto-char fname-pos)
+ (svn-status-goto-file-name fname)
+ (goto-char (+ column (svn-point-at-bol)))
+ (when window-line-pos
+ (recenter window-line-pos)))
+ (goto-char (+ (next-overlay-change (point-min)) svn-status-default-column)))))
+
+(defun svn-status-parse-info (arg)
+ "Parse the svn info output for the base directory.
+Show the repository url after this call in the `svn-status-buffer-name' buffer.
+When called with the prefix argument 0, reset the information to nil.
+This hides the repository information again.
+
+When ARG is t, don't update the svn status buffer. This is useful for
+non-interactive use."
+ (interactive "P")
+ (if (eq arg 0)
+ (setq svn-status-base-info nil)
+ (let ((svn-process-buffer-name "*svn-info-output*"))
+ (when (get-buffer svn-process-buffer-name)
+ (kill-buffer svn-process-buffer-name))
+ (svn-run nil t 'parse-info "info" ".")
+ (svn-status-parse-info-result)))
+ (unless (eq arg t)
+ (svn-status-update-buffer)))
+
+(defun svn-status-parse-info-result ()
+ "Parse the result from the svn info command.
+Put the found values in `svn-status-base-info'."
+ (let ((url)
+ (repository-root)
+ (last-changed-author))
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (goto-char (point-min))
+ (let ((case-fold-search t))
+ (search-forward "url: ")
+ (setq url (buffer-substring-no-properties (point) (svn-point-at-eol)))
+ (when (search-forward "repository root: " nil t)
+ (setq repository-root (buffer-substring-no-properties (point) (svn-point-at-eol))))
+ (when (search-forward "last changed author: " nil t)
+ (setq last-changed-author (buffer-substring-no-properties (point) (svn-point-at-eol))))))
+ (setq svn-status-base-info `((url ,url) (repository-root ,repository-root) (last-changed-author ,last-changed-author)))))
+
+(defun svn-status-base-info->url ()
+ "Extract the url part from `svn-status-base-info'."
+ (if svn-status-base-info
+ (cadr (assoc 'url svn-status-base-info))
+ ""))
+
+(defun svn-status-base-info->repository-root ()
+ "Extract the repository-root part from `svn-status-base-info'."
+ (if svn-status-base-info
+ (cadr (assoc 'repository-root svn-status-base-info))
+ ""))
+
+(defun svn-status-checkout-prefix-path ()
+ "When only a part of the svn repository is checked out, return the file path for this checkout."
+ (interactive)
+ (svn-status-parse-info t)
+ (let ((root (svn-status-base-info->repository-root))
+ (url (svn-status-base-info->url))
+ (p)
+ (base-dir (svn-status-base-dir))
+ (wc-checkout-prefix))
+ (setq p (substring url (length root)))
+ (setq wc-checkout-prefix (file-relative-name default-directory base-dir))
+ (when (string= wc-checkout-prefix "./")
+ (setq wc-checkout-prefix ""))
+ ;; (message "svn-status-checkout-prefix-path: wc-checkout-prefix: '%s' p: '%s' base-dir: %s" wc-checkout-prefix p base-dir)
+ (setq p (substring p 0 (- (length p) (length wc-checkout-prefix))))
+ (when (interactive-p)
+ (message "svn-status-checkout-prefix-path: '%s'" p))
+ p))
+
+(defun svn-status-ls (path &optional synchron)
+ "Run svn ls PATH."
+ (interactive "sPath for svn ls: ")
+ (svn-run (not synchron) t 'ls "ls" path)
+ (when synchron
+ (split-string (with-current-buffer svn-process-buffer-name
+ (buffer-substring-no-properties (point-min) (point-max))))))
+
+(defun svn-status-ls-branches ()
+ "Show, which branches exist for the actual working copy.
+Note: this command assumes the proposed standard svn repository layout."
+ (interactive)
+ (svn-status-parse-info t)
+ (svn-status-ls (concat (svn-status-base-info->repository-root) "/branches")))
+
+(defun svn-status-ls-tags ()
+ "Show, which tags exist for the actual working copy.
+Note: this command assumes the proposed standard svn repository layout."
+ (interactive)
+ (svn-status-parse-info t)
+ (svn-status-ls (concat (svn-status-base-info->repository-root) "/tags")))
+
+(defun svn-status-toggle-edit-cmd-flag (&optional reset)
+ "Allow the user to edit the parameters for the next svn command.
+This command toggles between
+* editing the next command parameters (EditCmd)
+* editing all all command parameters (EditCmd#)
+* don't edit the command parameters ()
+The string in parentheses is shown in the status line to show the state."
+ (interactive)
+ (cond ((or reset (eq svn-status-edit-svn-command 'sticky))
+ (setq svn-status-edit-svn-command nil))
+ ((eq svn-status-edit-svn-command nil)
+ (setq svn-status-edit-svn-command t))
+ ((eq svn-status-edit-svn-command t)
+ (setq svn-status-edit-svn-command 'sticky)))
+ (cond ((eq svn-status-edit-svn-command t)
+ (setq svn-status-mode-line-process-edit-flag " EditCmd"))
+ ((eq svn-status-edit-svn-command 'sticky)
+ (setq svn-status-mode-line-process-edit-flag " EditCmd#"))
+ (t
+ (setq svn-status-mode-line-process-edit-flag "")))
+ (svn-status-update-mode-line))
+
+(defun svn-status-goto-root-or-return ()
+ "Bounce point between the root (\".\") and the current line."
+ (interactive)
+ (if (string= (svn-status-line-info->filename (svn-status-get-line-information)) ".")
+ (when svn-status-root-return-info
+ (svn-status-goto-file-name
+ (svn-status-line-info->filename svn-status-root-return-info)))
+ (setq svn-status-root-return-info (svn-status-get-line-information))
+ (svn-status-goto-file-name ".")))
+
+(defun svn-status-next-line (nr-of-lines)
+ "Go to the next line that holds a file information.
+When called with a prefix argument advance the given number of lines."
+ (interactive "p")
+ (while (progn
+ (forward-line nr-of-lines)
+ (and (not (eobp))
+ (not (svn-status-get-line-information)))))
+ (when (svn-status-get-line-information)
+ (goto-char (+ (svn-point-at-bol) svn-status-default-column))))
+
+(defun svn-status-previous-line (nr-of-lines)
+ "Go to the previous line that holds a file information.
+When called with a prefix argument go back the given number of lines."
+ (interactive "p")
+ (while (progn
+ (forward-line (- nr-of-lines))
+ (and (not (bobp))
+ (not (svn-status-get-line-information)))))
+ (when (svn-status-get-line-information)
+ (goto-char (+ (svn-point-at-bol) svn-status-default-column))))
+
+(defun svn-status-dired-jump ()
+ "Jump to a dired buffer, containing the file at point."
+ (interactive)
+ (let* ((line-info (svn-status-get-line-information))
+ (file-full-path (if line-info
+ (svn-status-line-info->full-path line-info)
+ default-directory)))
+ (let ((default-directory
+ (file-name-as-directory
+ (expand-file-name (if line-info
+ (svn-status-line-info->directory-containing-line-info line-info t)
+ default-directory)))))
+ (if (fboundp 'dired-jump-back) (dired-jump-back) (dired-jump))) ;; Xemacs uses dired-jump-back
+ (dired-goto-file file-full-path)))
+
+(defun svn-status-possibly-negate-meaning-of-arg (arg &optional command)
+ "Negate arg, if this-command is a member of svn-status-possibly-negate-meaning-of-arg."
+ (unless command
+ (setq command this-command))
+ (if (member command svn-status-negate-meaning-of-arg-commands)
+ (not arg)
+ arg))
+
+(defun svn-status-update (&optional arg)
+ "Run 'svn status -v'.
+When called with a prefix argument run 'svn status -vu'."
+ (interactive "P")
+ (unless (interactive-p)
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (setq svn-status-update-previous-process-output
+ (buffer-substring (point-min) (point-max)))))
+ (svn-status default-directory arg))
+
+(defun svn-status-get-line-information ()
+ "Find out about the file under point.
+The result may be parsed with the various `svn-status-line-info->...' functions."
+ (if (eq major-mode 'svn-status-mode)
+ (let ((svn-info nil))
+ (dolist (overlay (overlays-at (point)))
+ (setq svn-info (or svn-info
+ (overlay-get overlay 'svn-info))))
+ svn-info)
+ ;; different mode, means called not from the *svn-status* buffer
+ (if svn-status-get-line-information-for-file
+ (svn-status-make-line-info (if (eq svn-status-get-line-information-for-file 'relative)
+ (file-relative-name (buffer-file-name) (svn-status-base-dir))
+ (buffer-file-name)))
+ (svn-status-make-line-info "."))))
+
+
+(defun svn-status-get-file-list (use-marked-files)
+ "Get either the selected files or the file under point.
+USE-MARKED-FILES decides which we do.
+See `svn-status-marked-files' for what counts as selected."
+ (if use-marked-files
+ (svn-status-marked-files)
+ (list (svn-status-get-line-information))))
+
+(defun svn-status-get-file-list-names (use-marked-files)
+ (mapcar 'svn-status-line-info->filename (svn-status-get-file-list use-marked-files)))
+
+(defun svn-status-get-file-information ()
+ "Find out about the file under point.
+The result may be parsed with the various `svn-status-line-info->...' functions.
+When called from a *svn-status* buffer, do the same as `svn-status-get-line-information'.
+When called from a file buffer provide a structure that contains the filename."
+ (cond ((eq major-mode 'svn-status-mode)
+ (svn-status-get-line-information))
+ (t
+ ;; a fake strukture that contains the buffername for the current buffer
+ (svn-status-make-line-info (buffer-file-name (current-buffer))))))
+
+(defun svn-status-select-line ()
+ "Return information about the file under point.
+\(Only used for debugging\)"
+ (interactive)
+ (let ((info (svn-status-get-line-information)))
+ (if info
+ (message "%S hide-because-unknown: %S hide-because-unmodified: %S" info
+ (svn-status-line-info->hide-because-unknown info)
+ (svn-status-line-info->hide-because-unmodified info))
+ (message "No file on this line"))))
+
+(defun svn-status-ensure-cursor-on-file ()
+ "Raise an error unless point is on a valid file."
+ (unless (svn-status-get-line-information)
+ (error "No file on the current line")))
+
+(defun svn-status-directory-containing-point (allow-self)
+ "Find the (full path of) directory containing the file under point.
+
+If ALLOW-SELF and the file is a directory, return that directory,
+otherwise return the directory containing the file under point."
+ ;;the first `or' below is because s-s-g-l-i returns `nil' if
+ ;;point was outside the file list, but we need
+ ;;s-s-l-i->f to return a string to add to `default-directory'.
+ (let ((line-info (or (svn-status-get-line-information)
+ (svn-status-make-line-info))))
+ (file-name-as-directory
+ (expand-file-name
+ (svn-status-line-info->directory-containing-line-info line-info allow-self)))))
+
+(defun svn-status-line-info->directory-containing-line-info (line-info allow-self)
+ "Find the directory containing for LINE-INFO.
+
+If ALLOW-SELF is t and LINE-INFO refers to a directory then return the
+directory itself, in all other cases find the parent directory"
+ (if (and allow-self (svn-status-line-info->directory-p line-info))
+ (svn-status-line-info->filename line-info)
+ ;;The next `or' is because (file-name-directory "file") returns nil
+ (or (file-name-directory (svn-status-line-info->filename line-info))
+ ".")))
+
+(defun svn-status-set-user-mark (arg)
+ "Set a user mark on the current file or directory.
+If the cursor is on a file this file is marked and the cursor advances to the next line.
+If the cursor is on a directory all files in this directory are marked.
+
+If this function is called with a prefix argument, only the current line is
+marked, even if it is a directory."
+ (interactive "P")
+ (setq arg (svn-status-possibly-negate-meaning-of-arg arg 'svn-status-set-user-mark))
+ (let ((info (svn-status-get-line-information)))
+ (if info
+ (progn
+ (svn-status-apply-usermark t arg)
+ (svn-status-next-line 1))
+ (message "No file on this line - cannot set a mark"))))
+
+(defun svn-status-unset-user-mark (arg)
+ "Remove a user mark on the current file or directory.
+If the cursor is on a file, this file is unmarked and the cursor advances to the next line.
+If the cursor is on a directory, all files in this directory are unmarked.
+
+If this function is called with a prefix argument, only the current line is
+unmarked, even if is a directory."
+ (interactive "P")
+ (setq arg (svn-status-possibly-negate-meaning-of-arg arg 'svn-status-set-user-mark))
+ (let ((info (svn-status-get-line-information)))
+ (if info
+ (progn
+ (svn-status-apply-usermark nil arg)
+ (svn-status-next-line 1))
+ (message "No file on this line - cannot unset a mark"))))
+
+(defun svn-status-unset-user-mark-backwards ()
+ "Remove a user mark from the previous file.
+Then move to that line."
+ ;; This is consistent with `dired-unmark-backward' and
+ ;; `cvs-mode-unmark-up'.
+ (interactive)
+ (let ((info (save-excursion
+ (svn-status-next-line -1)
+ (svn-status-get-line-information))))
+ (if info
+ (progn
+ (svn-status-next-line -1)
+ (svn-status-apply-usermark nil t))
+ (message "No file on previous line - cannot unset a mark"))))
+
+(defun svn-status-apply-usermark (set-mark only-this-line)
+ "Do the work for the various marking/unmarking functions."
+ (let* ((st-info svn-status-info)
+ (mark-count 0)
+ (line-info (svn-status-get-line-information))
+ (file-name (svn-status-line-info->filename line-info))
+ (sub-file-regexp (if (file-directory-p file-name)
+ (concat "^" (regexp-quote
+ (file-name-as-directory file-name)))
+ nil))
+ (newcursorpos-fname)
+ (i-fname)
+ (first-line t)
+ (current-line svn-start-of-file-list-line-number))
+ (while st-info
+ (when (or (svn-status-line-info->is-visiblep (car st-info)) first-line)
+ (setq current-line (1+ current-line))
+ (setq first-line nil))
+ (setq i-fname (svn-status-line-info->filename (car st-info)))
+ (when (or (string= file-name i-fname)
+ (when sub-file-regexp
+ (string-match sub-file-regexp i-fname)))
+ (when (svn-status-line-info->is-visiblep (car st-info))
+ (when (or (not only-this-line) (string= file-name i-fname))
+ (setq newcursorpos-fname i-fname)
+ (unless (eq (car (svn-status-line-info->ui-status (car st-info))) set-mark)
+ (setcar (svn-status-line-info->ui-status (car st-info)) set-mark)
+ (setq mark-count (+ 1 mark-count))
+ (save-excursion
+ (let ((buffer-read-only nil))
+ (goto-line current-line)
+ (delete-region (svn-point-at-bol) (svn-point-at-eol))
+ (svn-insert-line-in-status-buffer (car st-info))
+ (delete-char 1)))
+ (message "%s %s" (if set-mark "Marked" "Unmarked") i-fname)))))
+ (setq st-info (cdr st-info)))
+ ;;(svn-status-update-buffer)
+ (svn-status-goto-file-name newcursorpos-fname)
+ (when (> mark-count 1)
+ (message "%s %d files" (if set-mark "Marked" "Unmarked") mark-count))))
+
+(defun svn-status-apply-usermark-checked (check-function set-mark)
+ "Mark or unmark files, whether a given function returns t.
+The function is called with the line information. Therefore the
+svn-status-line-info->* functions can be used in the check."
+ (let ((st-info svn-status-info)
+ (mark-count 0))
+ (while st-info
+ (when (apply check-function (list (car st-info)))
+ (unless (eq (svn-status-line-info->has-usermark (car st-info)) set-mark)
+ (setq mark-count (+ 1 mark-count))
+ (message "%s %s"
+ (if set-mark "Marked" "Unmarked")
+ (svn-status-line-info->filename (car st-info))))
+ (setcar (svn-status-line-info->ui-status (car st-info)) set-mark))
+ (setq st-info (cdr st-info)))
+ (svn-status-update-buffer)
+ (when (> mark-count 1)
+ (message "%s %d files" (if set-mark "Marked" "Unmarked") mark-count))))
+
+(defun svn-status-mark-unknown (arg)
+ "Mark all unknown files.
+These are the files marked with '?' in the `svn-status-buffer-name' buffer.
+If the function is called with a prefix arg, unmark all these files."
+ (interactive "P")
+ (svn-status-apply-usermark-checked
+ '(lambda (info) (eq (svn-status-line-info->filemark info) ??)) (not arg)))
+
+(defun svn-status-mark-added (arg)
+ "Mark all added files.
+These are the files marked with 'A' in the `svn-status-buffer-name' buffer.
+If the function is called with a prefix ARG, unmark all these files."
+ (interactive "P")
+ (svn-status-apply-usermark-checked
+ '(lambda (info) (eq (svn-status-line-info->filemark info) ?A)) (not arg)))
+
+(defun svn-status-mark-modified (arg)
+ "Mark all modified files.
+These are the files marked with 'M' in the `svn-status-buffer-name' buffer.
+Changed properties are considered.
+If the function is called with a prefix ARG, unmark all these files."
+ (interactive "P")
+ (svn-status-apply-usermark-checked
+ '(lambda (info) (or (eq (svn-status-line-info->filemark info) ?M)
+ (eq (svn-status-line-info->filemark info)
+ svn-status-file-modified-after-save-flag)
+ (eq (svn-status-line-info->propmark info) ?M)))
+ (not arg)))
+
+(defun svn-status-mark-modified-properties (arg)
+ "Mark all files and directories with modified properties.
+If the function is called with a prefix ARG, unmark all these entries."
+ (interactive "P")
+ (svn-status-apply-usermark-checked
+ '(lambda (info) (or (eq (svn-status-line-info->propmark info) ?M)))
+ (not arg)))
+
+(defun svn-status-mark-deleted (arg)
+ "Mark all files scheduled for deletion.
+These are the files marked with 'D' in the `svn-status-buffer-name' buffer.
+If the function is called with a prefix ARG, unmark all these files."
+ (interactive "P")
+ (svn-status-apply-usermark-checked
+ '(lambda (info) (eq (svn-status-line-info->filemark info) ?D)) (not arg)))
+
+(defun svn-status-mark-changed (arg)
+ "Mark all files that could be committed.
+This means we mark
+* all modified files
+* all files scheduled for addition
+* all files scheduled for deletion
+
+The last two categories include all copied and moved files.
+If called with a prefix ARG, unmark all such files."
+ (interactive "P")
+ (svn-status-mark-added arg)
+ (svn-status-mark-modified arg)
+ (svn-status-mark-deleted arg))
+
+(defun svn-status-unset-all-usermarks ()
+ (interactive)
+ (svn-status-apply-usermark-checked '(lambda (info) t) nil))
+
+(defvar svn-status-regexp-history nil
+ "History list of regular expressions used in svn status commands.")
+
+(defun svn-status-read-regexp (prompt)
+ (read-from-minibuffer prompt nil nil nil 'svn-status-regexp-history))
+
+(defun svn-status-mark-filename-regexp (regexp &optional unmark)
+ "Mark all files matching REGEXP.
+If the function is called with a prefix arg, unmark all these files."
+ (interactive
+ (list (svn-status-read-regexp (concat (if current-prefix-arg "Unmark" "Mark")
+ " files (regexp): "))
+ (if current-prefix-arg t nil)))
+ (svn-status-apply-usermark-checked
+ '(lambda (info) (string-match regexp (svn-status-line-info->filename-nondirectory info))) (not unmark)))
+
+(defun svn-status-mark-by-file-ext (ext &optional unmark)
+ "Mark all files matching the given file extension EXT.
+If the function is called with a prefix arg, unmark all these files."
+ (interactive
+ (list (read-string (concat (if current-prefix-arg "Unmark" "Mark")
+ " files with extensions: "))
+ (if current-prefix-arg t nil)))
+ (svn-status-apply-usermark-checked
+ '(lambda (info) (let ((case-fold-search nil))
+ (string-match (concat "\\." ext "$") (svn-status-line-info->filename-nondirectory info)))) (not unmark)))
+
+(defun svn-status-toggle-hide-unknown ()
+ (interactive)
+ (setq svn-status-hide-unknown (not svn-status-hide-unknown))
+ (svn-status-update-buffer))
+
+(defun svn-status-toggle-hide-unmodified ()
+ (interactive)
+ (setq svn-status-hide-unmodified (not svn-status-hide-unmodified))
+ (svn-status-update-buffer))
+
+(defun svn-status-get-file-name-buffer-position (name)
+ "Find the buffer position for a file.
+If the file is not found, return nil."
+ (let ((start-pos (let ((cached-pos (gethash name
+ svn-status-filename-to-buffer-position-cache)))
+ (when cached-pos
+ (goto-char (previous-overlay-change cached-pos)))
+ (point)))
+ (found))
+ ;; performance optimization: search from point to end of buffer
+ (while (and (not found) (< (point) (point-max)))
+ (goto-char (next-overlay-change (point)))
+ (when (string= name (svn-status-line-info->filename
+ (svn-status-get-line-information)))
+ (setq start-pos (+ (point) svn-status-default-column))
+ (setq found t)))
+ ;; search from buffer start to point
+ (goto-char (point-min))
+ (while (and (not found) (< (point) start-pos))
+ (goto-char (next-overlay-change (point)))
+ (when (string= name (svn-status-line-info->filename
+ (svn-status-get-line-information)))
+ (setq start-pos (+ (point) svn-status-default-column))
+ (setq found t)))
+ (and found start-pos)))
+
+(defun svn-status-goto-file-name (name)
+ "Move the cursor the the line that displays NAME."
+ (let ((pos (svn-status-get-file-name-buffer-position name)))
+ (if pos
+ (goto-char pos)
+ (svn-status-message 7 "Note: svn-status-goto-file-name: %s not found" name))))
+
+(defun svn-status-find-info-for-file-name (name)
+ (let* ((st-info svn-status-info)
+ (info))
+ (while st-info
+ (when (string= name (svn-status-line-info->filename (car st-info)))
+ (setq info (car st-info))
+ (setq st-info nil)) ; terminate loop
+ (setq st-info (cdr st-info)))
+ info))
+
+(defun svn-status-marked-files ()
+ "Return all files marked by `svn-status-set-user-mark',
+or (if no files were marked) the file under point."
+ (if (eq major-mode 'svn-status-mode)
+ (let* ((st-info svn-status-info)
+ (file-list))
+ (while st-info
+ (when (svn-status-line-info->has-usermark (car st-info))
+ (setq file-list (append file-list (list (car st-info)))))
+ (setq st-info (cdr st-info)))
+ (or file-list
+ (if (svn-status-get-line-information)
+ (list (svn-status-get-line-information))
+ nil)))
+ ;; different mode, means called not from the *svn-status* buffer
+ (if svn-status-get-line-information-for-file
+ (list (svn-status-make-line-info (if (eq svn-status-get-line-information-for-file 'relative)
+ (file-relative-name (buffer-file-name) (svn-status-base-dir))
+ (buffer-file-name))))
+ (list (svn-status-make-line-info ".")))))
+
+(defun svn-status-marked-file-names ()
+ (mapcar 'svn-status-line-info->filename (svn-status-marked-files)))
+
+(defun svn-status-some-files-marked-p ()
+ "Return non-nil iff a file has been marked by `svn-status-set-user-mark'.
+Unlike `svn-status-marked-files', this does not select the file under point
+if no files have been marked."
+ ;; `some' would be shorter but requires cl-seq at runtime.
+ ;; (Because it accepts both lists and vectors, it is difficult to inline.)
+ (loop for line-info in svn-status-info
+ thereis (svn-status-line-info->has-usermark line-info)))
+
+(defun svn-status-only-dirs-or-nothing-marked-p ()
+ "Return non-nil iff only dirs has been marked by `svn-status-set-user-mark'."
+ ;; `some' would be shorter but requires cl-seq at runtime.
+ ;; (Because it accepts both lists and vectors, it is difficult to inline.)
+ (loop for line-info in svn-status-info
+ thereis (and (not (svn-status-line-info->directory-p line-info))
+ (svn-status-line-info->has-usermark line-info))))
+
+(defun svn-status-ui-information-hash-table ()
+ (let ((st-info svn-status-info)
+ (svn-status-ui-information (make-hash-table :test 'equal)))
+ (while st-info
+ (svn-puthash (svn-status-line-info->filename (car st-info))
+ (svn-status-line-info->ui-status (car st-info))
+ svn-status-ui-information)
+ (setq st-info (cdr st-info)))
+ svn-status-ui-information))
+
+
+(defun svn-status-create-arg-file (file-name prefix file-info-list postfix)
+ (with-temp-file file-name
+ (insert prefix)
+ (let ((st-info file-info-list))
+ (while st-info
+ (insert (svn-status-line-info->filename (car st-info)))
+ (insert "\n")
+ (setq st-info (cdr st-info)))
+
+ (insert postfix))))
+
+(defun svn-status-show-process-buffer-internal (&optional scroll-to-top)
+ (let ((cur-buff (current-buffer)))
+ (unless svn-status-preserve-window-configuration
+ (when (string= (buffer-name) svn-status-buffer-name)
+ (delete-other-windows)))
+ (pop-to-buffer svn-process-buffer-name)
+ (svn-process-mode)
+ (when scroll-to-top
+ (goto-char (point-min)))
+ (pop-to-buffer cur-buff)))
+
+(defun svn-status-show-process-output (cmd &optional scroll-to-top)
+ "Display the result of a svn command.
+Consider svn-status-window-alist to choose the buffer name."
+ (let ((window-mode (cadr (assoc cmd svn-status-window-alist)))
+ (process-default-directory))
+ (cond ((eq window-mode nil) ;; use *svn-process* buffer
+ (setq svn-status-last-output-buffer-name svn-process-buffer-name))
+ ((eq window-mode t) ;; use *svn-info* buffer
+ (setq svn-status-last-output-buffer-name "*svn-info*"))
+ ((eq window-mode 'invisible) ;; don't display the buffer
+ (setq svn-status-last-output-buffer-name nil))
+ (t
+ (setq svn-status-last-output-buffer-name window-mode)))
+ (when svn-status-last-output-buffer-name
+ (if window-mode
+ (progn
+ (unless svn-status-preserve-window-configuration
+ (when (string= (buffer-name) svn-status-buffer-name)
+ (delete-other-windows)))
+ (pop-to-buffer svn-process-buffer-name)
+ (setq process-default-directory default-directory)
+ (switch-to-buffer (get-buffer-create svn-status-last-output-buffer-name))
+ (setq default-directory process-default-directory)
+ (let ((buffer-read-only nil))
+ (delete-region (point-min) (point-max))
+ (insert-buffer-substring svn-process-buffer-name)
+ (when scroll-to-top
+ (goto-char (point-min))))
+ (when (eq window-mode t) ;; *svn-info* buffer
+ (svn-info-mode))
+ (other-window 1))
+ (svn-status-show-process-buffer-internal scroll-to-top)))))
+
+(defun svn-status-svn-log-switches (arg)
+ (cond ((eq arg 0) '())
+ ((or (eq arg -1) (eq arg '-)) '("-q"))
+ (arg '("-v"))
+ (t svn-status-default-log-arguments)))
+
+(defun svn-status-show-svn-log (arg)
+ "Run `svn log' on selected files.
+The output is put into the *svn-log* buffer
+The optional prefix argument ARG determines which switches are passed to `svn log':
+ no prefix --- use whatever is in the list `svn-status-default-log-arguments'
+ prefix argument of -1: --- use the -q switch (quiet)
+ prefix argument of 0 --- use no arguments
+ other prefix arguments: --- use the -v switch (verbose)
+
+See `svn-status-marked-files' for what counts as selected."
+ (interactive "P")
+ (let ((switches (svn-status-svn-log-switches arg))
+ (svn-status-get-line-information-for-file t))
+ ;; (message "svn-status-show-svn-log %S" arg)
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-marked-files) "")
+ (svn-run t t 'log "log" "--targets" svn-status-temp-arg-file switches)))
+
+(defun svn-status-version ()
+ "Show the version numbers for psvn.el and the svn command line client.
+The version number of the client is cached in `svn-client-version'."
+ (interactive)
+ (let ((window-conf (current-window-configuration))
+ (version-string))
+ (if (or (interactive-p) (not svn-status-cached-version-string))
+ (progn
+ (svn-run nil t 'version "--version")
+ (when (interactive-p)
+ (svn-status-show-process-output 'info t))
+ (with-current-buffer svn-status-last-output-buffer-name
+ (goto-char (point-min))
+ (setq svn-client-version
+ (when (re-search-forward "svn, version \\([0-9\.]+\\) " nil t)
+ (mapcar 'string-to-number (split-string (match-string 1) "\\."))))
+ (let ((buffer-read-only nil))
+ (goto-char (point-min))
+ (insert (format "psvn.el revision: %s\n\n" svn-psvn-revision)))
+ (setq version-string (buffer-substring-no-properties (point-min) (point-max))))
+ (setq svn-status-cached-version-string version-string))
+ (setq version-string svn-status-cached-version-string)
+ (unless (interactive-p)
+ (set-window-configuration window-conf)
+ version-string))))
+
+(defun svn-status-info ()
+ "Run `svn info' on all selected files.
+See `svn-status-marked-files' for what counts as selected."
+ (interactive)
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-marked-files) "")
+ (svn-run t t 'info "info" "--targets" svn-status-temp-arg-file))
+
+(defun svn-status-info-for-path (path)
+ "Run svn info on the given PATH.
+Return some interesting parts of the resulting output.
+At the moment a list containing the last changed author is returned."
+ (let ((svn-process-buffer-name "*svn-info-output*")
+ (last-changed-author))
+ (svn-run nil t 'info "info" path)
+ (with-current-buffer svn-process-buffer-name
+ (goto-char (point-min))
+ (when (search-forward "last changed author: " nil t)
+ (setq last-changed-author (buffer-substring-no-properties (point) (svn-point-at-eol)))))
+ (svn-status-message 7 "last-changed-author for '%s': %s" path last-changed-author)
+ (list last-changed-author)))
+
+(defun svn-status-blame (revision)
+ "Run `svn blame' on the current file.
+When called with a prefix argument, ask the user for the REVISION to use.
+When called from a file buffer, go to the current line in the resulting blame output."
+ (interactive "P")
+ (when current-prefix-arg
+ (setq revision (svn-status-read-revision-string "Blame for version: " "BASE")))
+ (unless revision (setq revision "BASE"))
+ (setq svn-status-blame-file-name (svn-status-line-info->filename (svn-status-get-file-information)))
+ (svn-run t t 'blame "blame" svn-status-default-blame-arguments "-r" revision svn-status-blame-file-name))
+
+(defun svn-status-show-svn-diff (arg)
+ "Run `svn diff' on the current file.
+If the current file is a directory, compare it recursively.
+If there is a newer revision in the repository, the diff is done against HEAD,
+otherwise compare the working copy with BASE.
+If ARG then prompt for revision to diff against (unless arg is '-)
+When called with a negative prefix argument, do a non recursive diff."
+ (interactive "P")
+ (let ((non-recursive (or (and (numberp arg) (< arg 0)) (eq arg '-)))
+ (revision (if (and (not (eq arg '-)) arg) :ask :auto)))
+ (svn-status-ensure-cursor-on-file)
+ (svn-status-show-svn-diff-internal (list (svn-status-get-line-information)) (not non-recursive)
+ revision)))
+
+(defun svn-file-show-svn-diff (arg)
+ "Run `svn diff' on the current file.
+If there is a newer revision in the repository, the diff is done against HEAD,
+otherwise compare the working copy with BASE.
+If ARG then prompt for revision to diff against."
+ (interactive "P")
+ (svn-status-show-svn-diff-internal (list (svn-status-make-line-info buffer-file-name)) nil
+ (if arg :ask :auto)))
+
+(defun svn-status-show-svn-diff-for-marked-files (arg)
+ "Run `svn diff' on all selected files.
+If some files have been marked, compare those non-recursively;
+this is because marking a directory with \\[svn-status-set-user-mark]
+normally marks all of its files as well.
+If no files have been marked, compare recursively the file at point.
+If ARG then prompt for revision to diff against, else compare working copy with BASE."
+ (interactive "P")
+ (svn-status-show-svn-diff-internal (svn-status-marked-files)
+ (not (svn-status-some-files-marked-p))
+ (if arg :ask "BASE")))
+
+(defun svn-status-diff-show-changeset (rev &optional user-confirmation)
+ "Show the changeset for a given log entry.
+When called with a prefix argument, ask the user for the revision."
+ (let* ((upper-rev rev)
+ (lower-rev (number-to-string (- (string-to-number upper-rev) 1)))
+ (rev-arg (concat lower-rev ":" upper-rev)))
+ (when user-confirmation
+ (setq rev-arg (read-string "Revision for changeset: " rev-arg)))
+ (svn-run nil t 'diff "diff" (concat "-r" rev-arg))
+ (svn-status-activate-diff-mode)))
+
+(defun svn-status-show-svn-diff-internal (line-infos recursive revision)
+ ;; REVISION must be one of:
+ ;; - a string: whatever the -r option allows.
+ ;; - `:ask': asks the user to specify the revision, which then becomes
+ ;; saved in `minibuffer-history' rather than in `command-history'.
+ ;; - `:auto': use "HEAD" if an update is known to exist, "BASE" otherwise.
+ ;; In the future, `nil' might mean omit the -r option entirely;
+ ;; but that currently seems to imply "BASE", so we just use that.
+ (when (eq revision :ask)
+ (setq revision (svn-status-read-revision-string
+ "Diff with files for version: " "PREV")))
+
+ (setq svn-status-last-diff-options (list line-infos recursive revision))
+
+ (let ((clear-buf t)
+ (beginning nil))
+ (dolist (line-info line-infos)
+ (svn-run nil clear-buf 'diff "diff" svn-status-default-diff-arguments
+ "-r" (if (eq revision :auto)
+ (if (svn-status-line-info->update-available line-info)
+ "HEAD" "BASE")
+ revision)
+ (unless recursive "--non-recursive")
+ (svn-status-line-info->filename line-info))
+ (setq clear-buf nil)
+
+ ;; "svn diff --non-recursive" skips only subdirectories, not files.
+ ;; But a non-recursive diff via psvn should skip files too, because
+ ;; the user would have marked them if he wanted them to be compared.
+ ;; So we'll look for the "Index: foo" line that marks the first file
+ ;; in the diff output, and delete it and everything that follows.
+ ;; This is made more complicated by the fact that `svn-status-activate-diff-mode'
+ ;; expects the output to be left in the *svn-process* buffer.
+ (unless recursive
+ ;; Check `directory-p' relative to the `default-directory' of the
+ ;; "*svn-status*" buffer, not that of the svn-process-buffer-name buffer.
+ (let ((directory-p (svn-status-line-info->directory-p line-info)))
+ (with-current-buffer svn-process-buffer-name
+ (when directory-p
+ (goto-char (or beginning (point-min)))
+ (when (re-search-forward "^Index: " nil t)
+ (delete-region (match-beginning 0) (point-max))))
+ (goto-char (setq beginning (point-max))))))))
+ (svn-status-activate-diff-mode))
+
+(defun svn-status-diff-save-current-defun-as-kill ()
+ "Copy the function name for the change at point to the kill-ring.
+That function uses `add-log-current-defun'"
+ (interactive)
+ (let ((func-name (add-log-current-defun)))
+ (if func-name
+ (progn
+ (kill-new func-name)
+ (message "Copied %S" func-name))
+ (message "No current defun detected."))))
+
+(defun svn-status-diff-pop-to-commit-buffer ()
+ "Temporary switch to the `svn-status-buffer-name' buffer and start a commit from there."
+ (interactive)
+ (let ((window-conf (current-window-configuration)))
+ (svn-status-switch-to-status-buffer)
+ (svn-status-commit)
+ (set-window-configuration window-conf)
+ (setq svn-status-pre-commit-window-configuration window-conf)
+ (pop-to-buffer svn-log-edit-buffer-name)))
+
+(defun svn-status-activate-diff-mode ()
+ "Show the `svn-process-buffer-name' buffer, using the diff-mode."
+ (svn-status-show-process-output 'diff t)
+ (let ((working-directory default-directory))
+ (save-excursion
+ (set-buffer svn-status-last-output-buffer-name)
+ (setq default-directory working-directory)
+ (svn-status-diff-mode)
+ (setq buffer-read-only t))))
+
+(define-derived-mode svn-status-diff-mode fundamental-mode "svn-diff"
+ "Major mode to display svn diffs. Derives from `diff-mode'.
+
+Commands:
+\\{svn-status-diff-mode-map}
+"
+ (let ((diff-mode-shared-map (copy-keymap svn-status-diff-mode-map))
+ major-mode mode-name)
+ (diff-mode)
+ (set (make-local-variable 'revert-buffer-function) 'svn-status-diff-update)))
+
+(defun svn-status-diff-update (arg noconfirm)
+ "Rerun the last svn diff command and update the *svn-diff* buffer."
+ (interactive)
+ (svn-status-save-some-buffers)
+ (save-window-excursion
+ (apply 'svn-status-show-svn-diff-internal svn-status-last-diff-options)))
+
+(defun svn-status-show-process-buffer ()
+ "Show the content of the `svn-process-buffer-name' buffer"
+ (interactive)
+ (svn-status-show-process-output nil))
+
+(defun svn-status-pop-to-partner-buffer ()
+ "Pop to the `svn-status-partner-buffer' if that variable is set."
+ (interactive)
+ (when svn-status-partner-buffer
+ (let ((cur-buf (current-buffer)))
+ (pop-to-buffer svn-status-partner-buffer)
+ (setq svn-status-partner-buffer cur-buf))))
+
+(defun svn-status-pop-to-new-partner-buffer (buffer)
+ "Call `pop-to-buffer' and register the current buffer as partner buffer for BUFFER."
+ (let ((cur-buf (current-buffer)))
+ (pop-to-buffer buffer)
+ (setq svn-status-partner-buffer cur-buf)))
+
+(defun svn-status-add-file-recursively (arg)
+ "Run `svn add' on all selected files.
+When a directory is added, add files recursively.
+See `svn-status-marked-files' for what counts as selected.
+When this function is called with a prefix argument, use the actual file instead."
+ (interactive "P")
+ (message "adding: %S" (svn-status-get-file-list-names (not arg)))
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-get-file-list (not arg)) "")
+ (svn-run t t 'add "add" "--targets" svn-status-temp-arg-file))
+
+(defun svn-status-add-file (arg)
+ "Run `svn add' on all selected files.
+When a directory is added, don't add the files of the directory
+ (svn add --non-recursive <file-list> is called).
+See `svn-status-marked-files' for what counts as selected.
+When this function is called with a prefix argument, use the actual file instead."
+ (interactive "P")
+ (message "adding: %S" (svn-status-get-file-list-names (not arg)))
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-get-file-list (not arg)) "")
+ (svn-run t t 'add "add" "--non-recursive" "--targets" svn-status-temp-arg-file))
+
+(defun svn-status-lock (arg)
+ "Run `svn lock' on all selected files.
+See `svn-status-marked-files' for what counts as selected."
+ (interactive "P")
+ (message "locking: %S" (svn-status-get-file-list-names t))
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-get-file-list t) "")
+ (svn-run t t 'lock "lock" "--targets" svn-status-temp-arg-file))
+
+(defun svn-status-unlock (arg)
+ "Run `svn unlock' on all selected files.
+See `svn-status-marked-files' for what counts as selected."
+ (interactive "P")
+ (message "unlocking: %S" (svn-status-get-file-list-names t))
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-get-file-list t) "")
+ (svn-run t t 'unlock "unlock" "--targets" svn-status-temp-arg-file))
+
+(defun svn-status-make-directory (dir)
+ "Run `svn mkdir DIR'."
+ ;; TODO: Allow entering a URI interactively.
+ ;; Currently, `read-file-name' corrupts it.
+ (interactive (list (read-file-name "Make directory: "
+ (svn-status-directory-containing-point t))))
+ (unless (string-match "^[^:/]+://" dir) ; Is it a URI?
+ (setq dir (file-relative-name dir)))
+ (svn-run t t 'mkdir "mkdir" "--" dir))
+
+(defun svn-status-mv ()
+ "Prompt for a destination, and `svn mv' selected files there.
+See `svn-status-marked-files' for what counts as `selected'.
+
+If one file was selected then the destination DEST should be a
+filename to rename the selected file to, or a directory to move the
+file into; if multiple files were selected then DEST should be a
+directory to move the selected files into.
+
+The default DEST is the directory containing point.
+
+BUG: If we've marked some directory containging a file as well as the
+file itself, then we should just mv the directory, but this implementation
+doesn't check for that.
+SOLUTION: for each dir, umark all its contents (but not the dir
+itself) before running mv."
+ (interactive)
+ (svn-status-mv-cp "mv" "Rename" "Move" "mv"))
+
+(defun svn-status-cp ()
+ "See `svn-status-mv'"
+ (interactive)
+ (svn-status-mv-cp "cp" "Copy" "Copy" "cp"))
+
+(defun svn-status-mv-cp (command singleprompt manyprompt fallback)
+ "Run svn COMMAND on marked files, prompting for destination
+
+This function acts on `svn-status-marked-files': at the prompt the
+user can enter a new file name, or an existing directory: this is used as the argument for svn COMMAND.
+ COMMAND --- string saying what to do: \"mv\" or \"cp\"
+ SINGLEPROMPT --- string at start of prompt when one file marked
+ MANYPROMPT --- string at start of prompt when multiple files marked
+ FALLBACK --- If any marked file is unversioned, use this instead of 'svn COMMAND'"
+ (let* ((marked-files (svn-status-marked-files))
+ (num-of-files (length marked-files))
+ dest)
+ (if (= 1 num-of-files)
+ ;; one file to act on: new name, or directory to hold results
+ (setq dest (read-file-name
+ (format "%s %s to: " singleprompt
+ (svn-status-line-info->filename (car marked-files)))
+ (svn-status-directory-containing-point t)
+ (svn-status-line-info->full-path (car marked-files))))
+ ;;TODO: (when file-exists-p but-no-dir-p dest (error "%s already exists" dest))
+ ;;multiple files selected, so prompt for existing directory to mv them into.
+ (setq dest (svn-read-directory-name
+ (format "%s %d files to directory: " manyprompt num-of-files)
+ (svn-status-directory-containing-point t) nil t))
+ (unless (file-directory-p dest)
+ (error "%s is not a directory" dest)))
+ (when (string= dest "")
+ (error "No destination entered"))
+ (unless (string-match "^[^:/]+://" dest) ; Is it a URI?
+ (setq dest (file-relative-name dest)))
+
+ ;;do the move: svn mv only lets us move things once at a time, so
+ ;;we need to run svn mv once for each file (hence second arg to
+ ;;svn-run is nil.)
+
+ ;;TODO: before doing any moving, For every marked directory,
+ ;;ensure none of its contents are also marked, since we dont want
+ ;;to move both file *and* its parent...
+ ;; what about elided files? what if user marks a dir+contents, then presses `_' ?
+;; ;one solution:
+;; (dolist (original marked-files)
+;; (when (svn-status-line-info->directory-p original)
+;; ;; run svn-status-goto-file-name to move point to line of file
+;; ;; run svn-status-unset-user-mark to unmark dir+all contents
+;; ;; run svn-status-set-user-mark to remark dir
+;; ;; maybe check for local mods here, and unmark if user does't say --force?
+;; ))
+ (dolist (original marked-files)
+ (let ((original-name (svn-status-line-info->filename original))
+ (original-filemarks (svn-status-line-info->filemark original))
+ (original-propmarks (svn-status-line-info->propmark original))
+ (moved nil))
+ (cond
+ ((or (eq original-filemarks ?M) ;local mods: maybe do `svn mv --force'
+ (eq original-propmarks ?M)) ;local prop mods: maybe do `svn mv --force'
+ (if (yes-or-no-p
+ (format "%s has local modifications; use `--force' to really move it? " original-name))
+ (progn
+ (svn-status-run-mv-cp command original-name dest t)
+ (setq moved t))
+ (message "Not acting on %s" original-name)))
+ ((eq original-filemarks ??) ;original is unversioned: use fallback
+ (if (yes-or-no-p (format "%s is unversioned. Use `%s -i -- %s %s'? "
+ original-name fallback original-name dest))
+ ;; TODO: consider svn-call-process-function here also...
+ (progn (call-process fallback nil (get-buffer-create svn-process-buffer-name) nil
+ "-i" "--" original-name dest)
+ (setq moved t))
+ ;;new files created by fallback are not in *svn-status* now,
+ ;;TODO: so call (svn-status-update) here?
+ (message "Not acting on %s" original-name)))
+
+ ((eq original-filemarks ?A) ;;`A' (`svn add'ed, but not committed)
+ (message "Not acting on %s (commit it first)" original-name))
+
+ ((eq original-filemarks ? ) ;original is unmodified: can proceed
+ (svn-status-run-mv-cp command original-name dest)
+ (setq moved t))
+
+ ;;file has some other mark (eg conflicted)
+ (t
+ (if (yes-or-no-p
+ (format "The status of %s looks scary. Risk moving it anyway? "
+ original-name))
+ (progn
+ (svn-status-run-mv-cp command original-name dest)
+ (setq moved t))
+ (message "Not acting on %s" original-name))))
+ (when moved
+ (message "psvn: did '%s' from %s to %s" command original-name dest)
+ ;; Silently rename the visited file of any buffer visiting this file.
+ (when (get-file-buffer original-name)
+ (with-current-buffer (get-file-buffer original-name)
+ (set-visited-file-name dest nil t))))))
+ (svn-status-update)))
+
+(defun svn-status-run-mv-cp (command original destination &optional force)
+ "Actually run svn mv or svn cp.
+This is just to prevent duplication in `svn-status-prompt-and-act-on-files'"
+ (if force
+ (svn-run nil t (intern command) command "--force" "--" original destination)
+ (svn-run nil t (intern command) command "--" original destination))
+;;;TODO: use something like the following instead of calling svn-status-update
+;;; at the end of svn-status-mv-cp.
+;; (let ((output (svn-status-parse-ar-output))
+;; newfile
+;; buffer-read-only) ; otherwise insert-line-in-status-buffer fails
+;; (dolist (new-file output)
+;; (when (eq (cadr new-file) 'added-wc)
+;; ;; files with 'wc-added action do not exist in *svn-status*
+;; ;; buffer yet, so give each of them their own line-info
+;; ;; TODO: need to insert the new line-info in a sensible place, ie in the correct directory! [svn-status-filename-to-buffer-position-cache might help?]
+
+;; (svn-insert-line-in-status-buffer
+;; (svn-status-make-line-info (car new-file)))))
+;; (svn-status-update-with-command-list output))
+ )
+
+(defun svn-status-revert ()
+ "Run `svn revert' on all selected files.
+See `svn-status-marked-files' for what counts as selected."
+ (interactive)
+ (let* ((marked-files (svn-status-marked-files))
+ (num-of-files (length marked-files)))
+ (when (yes-or-no-p
+ (if (= 1 num-of-files)
+ (format "Revert %s? " (svn-status-line-info->filename (car marked-files)))
+ (format "Revert %d files? " num-of-files)))
+ (message "reverting: %S" (svn-status-marked-file-names))
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-marked-files) "")
+ (svn-run t t 'revert "revert" "--targets" svn-status-temp-arg-file))))
+
+(defun svn-status-rm (force)
+ "Run `svn rm' on all selected files.
+See `svn-status-marked-files' for what counts as selected.
+When called with a prefix argument add the command line switch --force.
+
+Forcing the deletion can also be used to delete files not under svn control."
+ (interactive "P")
+ (let* ((marked-files (svn-status-marked-files))
+ (num-of-files (length marked-files)))
+ (when (yes-or-no-p
+ (if (= 1 num-of-files)
+ (format "%sRemove %s? " (if force "Force " "") (svn-status-line-info->filename (car marked-files)))
+ (format "%sRemove %d files? " (if force "Force " "") num-of-files)))
+ (message "removing: %S" (svn-status-marked-file-names))
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-marked-files) "")
+ (if force
+ (save-excursion
+ (svn-run t t 'rm "rm" "--force" "--targets" svn-status-temp-arg-file)
+ (dolist (to-delete (svn-status-marked-files))
+ (when (eq (svn-status-line-info->filemark to-delete) ??)
+ (svn-status-goto-file-name (svn-status-line-info->filename to-delete))
+ (let ((buffer-read-only nil))
+ (delete-region (svn-point-at-bol) (+ 1 (svn-point-at-eol)))
+ (delete to-delete svn-status-info)))))
+ (svn-run t t 'rm "rm" "--targets" svn-status-temp-arg-file)))))
+
+(defun svn-status-update-cmd (arg)
+ "Run svn update.
+When called with a prefix argument, ask the user for the revision to update to.
+When called with a negative prefix argument, only update the selected files."
+ (interactive "P")
+ (let* ((selective-update (or (and (numberp arg) (< arg 0)) (eq arg '-)))
+ (rev (when arg (svn-status-read-revision-string
+ (if selective-update
+ (format "Selected entries: Run svn update -r ")
+ (format "Directory: %s: Run svn update -r " default-directory))
+ (if selective-update "HEAD" nil)))))
+ (if selective-update
+ (progn
+ (message "Running svn-update for %s" (svn-status-marked-file-names))
+ (svn-run t t 'update "update"
+ (when rev (list "-r" rev))
+ (list "--non-interactive")
+ (svn-status-marked-file-names)))
+ (message "Running svn-update for %s" default-directory)
+ (svn-run t t 'update "update"
+ (when rev (list "-r" rev))
+ (list "--non-interactive") (expand-file-name default-directory)))))
+
+(defun svn-status-commit ()
+ "Commit selected files.
+If some files have been marked, commit those non-recursively;
+this is because marking a directory with \\[svn-status-set-user-mark]
+normally marks all of its files as well.
+If no files have been marked, commit recursively the file at point."
+ (interactive)
+ (svn-status-save-some-buffers)
+ (let* ((selected-files (svn-status-marked-files)))
+ (setq svn-status-files-to-commit selected-files
+ svn-status-recursive-commit (not (svn-status-only-dirs-or-nothing-marked-p)))
+ (svn-log-edit-show-files-to-commit)
+ (svn-status-pop-to-commit-buffer)
+ (when svn-log-edit-insert-files-to-commit
+ (svn-log-edit-insert-files-to-commit))
+ (when svn-log-edit-show-diff-for-commit
+ (svn-log-edit-svn-diff nil))))
+
+(defun svn-status-pop-to-commit-buffer ()
+ "Pop to the svn commit buffer.
+If a saved log message exists in `svn-log-edit-file-name' insert it in the buffer."
+ (interactive)
+ (setq svn-status-pre-commit-window-configuration (current-window-configuration))
+ (let* ((use-existing-buffer (get-buffer svn-log-edit-buffer-name))
+ (commit-buffer (get-buffer-create svn-log-edit-buffer-name))
+ (dir default-directory)
+ (log-edit-file-name))
+ (pop-to-buffer commit-buffer)
+ (setq default-directory dir)
+ (setq log-edit-file-name (svn-log-edit-file-name))
+ (unless use-existing-buffer
+ (when (and log-edit-file-name (file-readable-p log-edit-file-name))
+ (insert-file-contents log-edit-file-name)))
+ (svn-log-edit-mode)))
+
+(defun svn-status-switch-to-status-buffer ()
+ "Switch to the `svn-status-buffer-name' buffer."
+ (interactive)
+ (switch-to-buffer svn-status-buffer-name))
+
+(defun svn-status-pop-to-status-buffer ()
+ "Pop to the `svn-status-buffer-name' buffer."
+ (interactive)
+ (pop-to-buffer svn-status-buffer-name))
+
+(defun svn-status-via-bookmark (bookmark)
+ "Allows a quick selection of a bookmark in `svn-bookmark-list'.
+Run `svn-status' on the selected bookmark."
+ (interactive
+ (list
+ (let ((completion-ignore-case t))
+ (funcall svn-status-completing-read-function "SVN status bookmark: " svn-bookmark-list))))
+ (unless bookmark
+ (error "No bookmark specified"))
+ (let ((directory (cdr (assoc bookmark svn-bookmark-list))))
+ (if (file-directory-p directory)
+ (svn-status directory)
+ (error "%s is not a directory" directory))))
+
+(defun svn-status-export ()
+ "Run `svn export' for the current working copy.
+Ask the user for the destination path.
+`svn-status-default-export-directory' is suggested as export directory."
+ (interactive)
+ (let* ((src default-directory)
+ (dir1-name (nth 1 (nreverse (split-string src "/"))))
+ (dest (read-file-name (format "Export %s to " src) (concat svn-status-default-export-directory dir1-name))))
+ (svn-run t t 'export "export" (expand-file-name src) (expand-file-name dest))
+ (message "svn-status-export %s %s" src dest)))
+
+(defun svn-status-cleanup (arg)
+ "Run `svn cleanup' on all selected files.
+See `svn-status-marked-files' for what counts as selected.
+When this function is called with a prefix argument, use the actual file instead."
+ (interactive "P")
+ (let ((file-names (svn-status-get-file-list-names (not arg))))
+ (if file-names
+ (progn
+ (message "svn-status-cleanup %S" file-names)
+ (svn-run t t 'cleanup (append (list "cleanup") file-names)))
+ (message "No valid file selected - No status cleanup possible"))))
+
+(defun svn-status-resolved ()
+ "Run `svn resolved' on all selected files.
+See `svn-status-marked-files' for what counts as selected."
+ (interactive)
+ (let* ((marked-files (svn-status-marked-files))
+ (num-of-files (length marked-files)))
+ (when (yes-or-no-p
+ (if (= 1 num-of-files)
+ (format "Resolve %s? " (svn-status-line-info->filename (car marked-files)))
+ (format "Resolve %d files? " num-of-files)))
+ (message "resolving: %S" (svn-status-marked-file-names))
+ (svn-status-create-arg-file svn-status-temp-arg-file "" (svn-status-marked-files) "")
+ (svn-run t t 'resolved "resolved" "--targets" svn-status-temp-arg-file))))
+
+
+(defun svn-status-svnversion ()
+ "Run svnversion on the directory that contains the file at point."
+ (interactive)
+ (svn-status-ensure-cursor-on-file)
+ (let ((simple-path (svn-status-line-info->filename (svn-status-get-line-information)))
+ (full-path (svn-status-line-info->full-path (svn-status-get-line-information)))
+ (version))
+ (unless (file-directory-p simple-path)
+ (setq simple-path (or (file-name-directory simple-path) "."))
+ (setq full-path (file-name-directory full-path)))
+ (setq version (shell-command-to-string (concat "svnversion -n " full-path)))
+ (message "svnversion for '%s': %s" simple-path version)
+ version))
+
+;; --------------------------------------------------------------------------------
+;; Update the `svn-status-buffer-name' buffer, when a file is saved
+;; --------------------------------------------------------------------------------
+
+(defvar svn-status-file-modified-after-save-flag ?m
+ "Flag shown whenever a file is modified and saved in Emacs.
+The flag is shown in the `svn-status-buffer-name' buffer.
+Recommended values are ?m or ?M.")
+(defun svn-status-after-save-hook ()
+ "Set a modified indication, when a file is saved from a svn working copy."
+ (let* ((svn-dir (car-safe svn-status-directory-history))
+ (svn-dir (when svn-dir (expand-file-name svn-dir)))
+ (file-dir (file-name-directory (buffer-file-name)))
+ (svn-dir-len (length (or svn-dir "")))
+ (file-dir-len (length file-dir))
+ (file-name))
+ (when (and (get-buffer svn-status-buffer-name)
+ svn-dir
+ (>= file-dir-len svn-dir-len)
+ (string= (substring file-dir 0 svn-dir-len) svn-dir))
+ (setq file-name (substring (buffer-file-name) svn-dir-len))
+ ;;(message "In svn-status directory %S" file-name)
+ (let ((st-info svn-status-info)
+ (i-fname))
+ (while st-info
+ (setq i-fname (svn-status-line-info->filename (car st-info)))
+ ;;(message "i-fname=%S" i-fname)
+ (when (and (string= file-name i-fname)
+ (not (eq (svn-status-line-info->filemark (car st-info)) ??)))
+ (svn-status-line-info->set-filemark (car st-info)
+ svn-status-file-modified-after-save-flag)
+ (save-window-excursion
+ (set-buffer svn-status-buffer-name)
+ (save-excursion
+ (let ((buffer-read-only nil)
+ (pos (svn-status-get-file-name-buffer-position i-fname)))
+ (if pos
+ (progn
+ (goto-char pos)
+ (delete-region (svn-point-at-bol) (svn-point-at-eol))
+ (svn-insert-line-in-status-buffer (car st-info))
+ (delete-char 1))
+ (svn-status-message 3 "psvn: file %s not found, updating %s buffer content..."
+ i-fname svn-status-buffer-name)
+ (svn-status-update-buffer))))))
+ (setq st-info (cdr st-info))))))
+ nil)
+
+(add-hook 'after-save-hook 'svn-status-after-save-hook)
+
+;; --------------------------------------------------------------------------------
+;; vc-svn integration
+;; --------------------------------------------------------------------------------
+(defvar svn-status-state-mark-modeline t) ; modeline mark display or not
+(defvar svn-status-state-mark-tooltip nil) ; modeline tooltip display
+
+(defun svn-status-state-mark-modeline-dot (color)
+ (propertize " "
+ 'help-echo 'svn-status-state-mark-tooltip
+ 'display
+ `(image :type xpm
+ :data ,(format "/* XPM */
+static char * data[] = {
+\"18 13 3 1\",
+\" c None\",
+\"+ c #000000\",
+\". c %s\",
+\" \",
+\" +++++ \",
+\" +.....+ \",
+\" +.......+ \",
+\" +.........+ \",
+\" +.........+ \",
+\" +.........+ \",
+\" +.........+ \",
+\" +.........+ \",
+\" +.......+ \",
+\" +.....+ \",
+\" +++++ \",
+\" \"};"
+ color)
+ :ascent center)))
+
+(defun svn-status-install-state-mark-modeline (color)
+ (push `(svn-status-state-mark-modeline
+ ,(svn-status-state-mark-modeline-dot color))
+ mode-line-format)
+ (force-mode-line-update t))
+
+(defun svn-status-uninstall-state-mark-modeline ()
+ (setq mode-line-format
+ (remove-if #'(lambda (mode) (eq (car-safe mode)
+ 'svn-status-state-mark-modeline))
+ mode-line-format))
+ (force-mode-line-update t))
+
+(defun svn-status-update-state-mark-tooltip (tooltip)
+ (setq svn-status-state-mark-tooltip tooltip))
+
+(defun svn-status-update-state-mark (color)
+ (svn-status-uninstall-state-mark-modeline)
+ (svn-status-install-state-mark-modeline color))
+
+(defsubst svn-status-in-vc-mode? ()
+ "Is vc-svn active?"
+ (and vc-mode (string-match "^ SVN" (svn-substring-no-properties vc-mode))))
+
+(when svn-status-fancy-file-state-in-modeline
+ (defadvice vc-find-file-hook (after svn-status-vc-svn-find-file-hook activate)
+ "vc-find-file-hook advice for synchronizing psvn with vc-svn interface"
+ (when (svn-status-in-vc-mode?) (svn-status-update-modeline)))
+
+ (defadvice vc-after-save (after svn-status-vc-svn-after-save activate)
+ "vc-after-save advice for synchronizing psvn when saving buffer"
+ (when (svn-status-in-vc-mode?) (svn-status-update-modeline)))
+
+ (defadvice ediff-refresh-mode-lines
+ (around svn-modeline-ediff-fixup activate compile)
+ "Fixup svn file status in the modeline when using ediff"
+ (ediff-with-current-buffer ediff-buffer-A
+ (svn-status-uninstall-state-mark-modeline))
+ (ediff-with-current-buffer ediff-buffer-B
+ (svn-status-uninstall-state-mark-modeline))
+ ad-do-it
+ (ediff-with-current-buffer ediff-buffer-A
+ (svn-status-update-modeline))
+ (ediff-with-current-buffer ediff-buffer-B
+ (svn-status-update-modeline))))
+
+(defun svn-status-update-modeline ()
+ "Update modeline state dot mark properly"
+ (when (and buffer-file-name (svn-status-in-vc-mode?))
+ (svn-status-update-state-mark
+ (svn-status-interprete-state-mode-color
+ (vc-svn-state buffer-file-name)))))
+
+(defsubst svn-status-interprete-state-mode-color (stat)
+ "Interpret vc-svn-state symbol to mode line color"
+ (case stat
+ ('edited "tomato" )
+ ('up-to-date "GreenYellow" )
+ ;; what is missing here??
+ ;; ('unknown "gray" )
+ ;; ('added "blue" )
+ ;; ('deleted "red" )
+ ;; ('unmerged "purple" )
+ (t "red")))
+
+;; --------------------------------------------------------------------------------
+;; Getting older revisions
+;; --------------------------------------------------------------------------------
+
+(defun svn-status-get-specific-revision (arg)
+ "Retrieve older revisions.
+The older revisions are stored in backup files named F.~REVISION~.
+
+When the function is called without a prefix argument: get all marked files.
+With a prefix argument: get only the actual file."
+ (interactive "P")
+ (svn-status-get-specific-revision-internal
+ (svn-status-get-file-list (not arg)) :ask t))
+
+(defun svn-status-get-specific-revision-internal (line-infos revision handle-relative-svn-status-dir)
+ "Retrieve older revisions of files.
+LINE-INFOS is a list of line-info structures (see
+`svn-status-get-line-information').
+REVISION is one of:
+- a string: whatever the -r option allows.
+- `:ask': asks the user to specify the revision, which then becomes
+ saved in `minibuffer-history' rather than in `command-history'.
+- `:auto': Use \"HEAD\" if an update is known to exist, \"BASE\" otherwise.
+
+After the call, `svn-status-get-revision-file-info' will be an alist
+\((WORKING-FILE-NAME . RETRIEVED-REVISION-FILE-NAME) ...). These file
+names are relative to the directory where `svn-status' was run."
+ ;; In `svn-status-show-svn-diff-internal', there is a comment
+ ;; that REVISION `nil' might mean omitting the -r option entirely.
+ ;; That doesn't seem like a good idea with svn cat.
+
+ ;; (message "svn-status-get-specific-revision-internal: %S %S" line-infos revision)
+
+ (when (eq revision :ask)
+ (setq revision (svn-status-read-revision-string
+ "Get files for version: " "PREV")))
+
+ (let ((count (length line-infos)))
+ (if (= count 1)
+ (let ((line-info (car line-infos)))
+ (message "Getting revision %s of %s"
+ (if (eq revision :auto)
+ (if (svn-status-line-info->update-available line-info)
+ "HEAD" "BASE")
+ revision)
+ (svn-status-line-info->filename line-info)))
+ ;; We could compute "Getting HEAD of 8 files and BASE of 11 files"
+ ;; but that'd be more bloat than it's worth.
+ (message "Getting revision %s of %d files"
+ (if (eq revision :auto) "HEAD or BASE" revision)
+ count)))
+
+ (let ((svn-status-get-specific-revision-file-info '()))
+ (dolist (line-info line-infos)
+ (let* ((revision (if (eq revision :auto)
+ (if (svn-status-line-info->update-available line-info)
+ "HEAD" "BASE")
+ revision)) ;must be a string by this point
+ (file-name (svn-status-line-info->filename line-info))
+ ;; If REVISION is e.g. "HEAD", should we find out the actual
+ ;; revision number and save "foo.~123~" rather than "foo.~HEAD~"?
+ ;; OTOH, `auto-mode-alist' already ignores ".~HEAD~" suffixes,
+ ;; and if users often want to know the revision numbers of such
+ ;; files, they can use svn:keywords.
+ (file-name-with-revision (concat (file-name-nondirectory file-name) ".~" revision "~"))
+ (default-directory (concat (svn-status-base-dir)
+ (if handle-relative-svn-status-dir
+ (file-relative-name default-directory (svn-status-base-dir))
+ "")
+ (file-name-directory file-name))))
+ ;; `add-to-list' would unnecessarily check for duplicates.
+ (push (cons file-name (concat (file-name-directory file-name) file-name-with-revision))
+ svn-status-get-specific-revision-file-info)
+ (svn-status-message 3 "svn-status-get-specific-revision-internal: file: %s, default-directory: %s"
+ file-name default-directory)
+ (svn-status-message 3 "svn-status-get-specific-revision-internal: file-name-with-revision: %s %S"
+ file-name-with-revision (file-exists-p file-name-with-revision))
+ (save-excursion
+ (if (or (not (file-exists-p file-name-with-revision)) ;; file does not exist
+ (not (string= (number-to-string (string-to-number revision)) revision))) ;; revision is not a number
+ (progn
+ (message "Getting revision %s of %s, target: %s" revision file-name
+ (expand-file-name(concat default-directory file-name-with-revision)))
+ (let ((content
+ (with-temp-buffer
+ (if (string= revision "BASE")
+ (insert-file-contents (concat (svn-wc-adm-dir-name)
+ "/text-base/"
+ (file-name-nondirectory file-name)
+ ".svn-base"))
+ (progn
+ (svn-run nil t 'cat "cat" "-r" revision
+ (concat default-directory (file-name-nondirectory file-name)))
+ ;;todo: error processing
+ ;;svn: Filesystem has no item
+ ;;svn: file not found: revision `15', path `/trunk/file.txt'
+ (insert-buffer-substring svn-process-buffer-name)))
+ (buffer-string))))
+ (find-file file-name-with-revision)
+ (setq buffer-read-only nil)
+ (erase-buffer) ;Widen, because we'll save the whole buffer.
+ (insert content)
+ (goto-char (point-min))
+ (let ((write-file-functions nil)
+ (require-final-newline nil))
+ (save-buffer))))
+ (find-file file-name-with-revision)))))
+ ;;(message "default-directory: %s revision-file-info: %S" default-directory svn-status-get-specific-revision-file-info)
+ (nreverse svn-status-get-specific-revision-file-info)))
+
+(defun svn-status-ediff-with-revision (arg)
+ "Run ediff on the current file with a different revision.
+If there is a newer revision in the repository, the diff is done against HEAD,
+otherwise compare the working copy with BASE.
+If ARG then prompt for revision to diff against."
+ (interactive "P")
+ (let* ((svn-status-get-specific-revision-file-info
+ (svn-status-get-specific-revision-internal
+ (list (svn-status-make-line-info
+ (file-relative-name
+ (svn-status-line-info->full-path (svn-status-get-line-information))
+ (svn-status-base-dir))
+ nil nil nil nil nil nil
+ (svn-status-line-info->update-available (svn-status-get-line-information))))
+ (if arg :ask :auto)
+ nil))
+ (ediff-after-quit-destination-buffer (current-buffer))
+ (default-directory (svn-status-base-dir))
+ (my-buffer (find-file-noselect (caar svn-status-get-specific-revision-file-info)))
+ (base-buff (find-file-noselect (cdar svn-status-get-specific-revision-file-info)))
+ (svn-transient-buffers (list my-buffer base-buff))
+ (startup-hook '(svn-ediff-startup-hook)))
+ (ediff-buffers base-buff my-buffer startup-hook)))
+
+(defun svn-ediff-startup-hook ()
+ ;; (message "svn-ediff-startup-hook: ediff-after-quit-hook-internal: %S" ediff-after-quit-hook-internal)
+ (add-hook 'ediff-after-quit-hook-internal
+ `(lambda ()
+ (svn-ediff-exit-hook
+ ',ediff-after-quit-destination-buffer ',svn-transient-buffers))
+ nil 'local))
+
+(defun svn-ediff-exit-hook (svn-buf tmp-bufs)
+ ;; (message "svn-ediff-exit-hook: svn-buf: %s, tmp-bufs: %s" svn-buf tmp-bufs)
+ ;; kill the temp buffers (and their associated windows)
+ (dolist (tb tmp-bufs)
+ (when (and tb (buffer-live-p tb) (not (buffer-modified-p tb)))
+ (let* ((win (get-buffer-window tb t))
+ (file-name (buffer-file-name tb))
+ (is-temp-file (numberp (string-match "~\\([0-9]+\\|BASE\\)~" file-name))))
+ ;; (message "svn-ediff-exit-hook - is-temp-file: %s, temp-buf:: %s - %s " is-temp-file (current-buffer) file-name)
+ (when (and win (> (count-windows) 1)
+ (delete-window win)))
+ (kill-buffer tb)
+ (when (and is-temp-file svn-status-ediff-delete-temporary-files)
+ (when (or (eq svn-status-ediff-delete-temporary-files t)
+ (y-or-n-p (format "Delete File '%s' ? " file-name)))
+ (delete-file file-name))))))
+ ;; switch back to the *svn* buffer
+ (when (and svn-buf (buffer-live-p svn-buf)
+ (not (get-buffer-window svn-buf t)))
+ (ignore-errors (switch-to-buffer svn-buf))))
+
+
+(defun svn-status-read-revision-string (prompt &optional default-value)
+ "Prompt the user for a svn revision number."
+ (interactive)
+ (read-string prompt default-value))
+
+(defun svn-file-show-svn-ediff (arg)
+ "Run ediff on the current file with a previous revision.
+If ARG then prompt for revision to diff against."
+ (interactive "P")
+ (let ((svn-status-get-line-information-for-file 'relative)
+ (default-directory (svn-status-base-dir)))
+ (svn-status-ediff-with-revision arg)))
+
+;; --------------------------------------------------------------------------------
+;; SVN process handling
+;; --------------------------------------------------------------------------------
+
+(defun svn-process-kill ()
+ "Kill the current running svn process."
+ (interactive)
+ (let ((process (get-process "svn")))
+ (if process
+ (delete-process process)
+ (message "No running svn process"))))
+
+(defun svn-process-send-string (string &optional send-passwd)
+ "Send a string to the running svn process.
+This is useful, if the running svn process asks the user a question.
+Note: use C-q C-j to send a line termination character."
+ (interactive "sSend string to svn process: ")
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (goto-char (point-max))
+ (let ((buffer-read-only nil))
+ (insert (if send-passwd (make-string (length string) ?.) string)))
+ (set-marker (process-mark (get-process "svn")) (point)))
+ (process-send-string "svn" string))
+
+(defun svn-process-send-string-and-newline (string &optional send-passwd)
+ "Send a string to the running svn process.
+Just call `svn-process-send-string' with STRING and an end of line termination.
+When called with a prefix argument, read the data from user as password."
+ (interactive (let* ((use-passwd current-prefix-arg)
+ (s (if use-passwd
+ (read-passwd "Send secret line to svn process: ")
+ (read-string "Send line to svn process: "))))
+ (list s use-passwd)))
+ (svn-process-send-string (concat string "\n") send-passwd))
+
+;; --------------------------------------------------------------------------------
+;; Search interface
+;; --------------------------------------------------------------------------------
+
+(defun svn-status-grep-files (regexp)
+ "Run grep on selected file(s).
+See `svn-status-marked-files' for what counts as selected."
+ (interactive "sGrep files for: ")
+ (unless grep-command
+ (grep-compute-defaults))
+ (let ((default-directory (svn-status-base-dir)))
+ (grep (format "%s %s %s" grep-command (shell-quote-argument regexp)
+ (mapconcat 'identity (svn-status-marked-file-names) " ")))))
+
+(defun svn-status-search-files (search-string)
+ "Search selected file(s) for a fixed SEARCH-STRING.
+See `svn-status-marked-files' for what counts as selected."
+ (interactive "sSearch files for: ")
+ (svn-status-grep-files (regexp-quote search-string)))
+
+;; --------------------------------------------------------------------------------
+;; Property List stuff
+;; --------------------------------------------------------------------------------
+
+(defun svn-status-property-list ()
+ (interactive)
+ (let ((file-names (svn-status-marked-file-names)))
+ (if file-names
+ (progn
+ (svn-run t t 'proplist (append (list "proplist" "-v") file-names)))
+ (message "No valid file selected - No property listing possible"))))
+
+(defun svn-status-proplist-start ()
+ (svn-status-ensure-cursor-on-file)
+ (svn-run t t 'proplist-parse "proplist" (svn-status-line-info->filename
+ (svn-status-get-line-information))))
+(defun svn-status-property-edit-one-entry (arg)
+ "Edit a property.
+When called with a prefix argument, it is possible to enter a new property."
+ (interactive "P")
+ (setq svn-status-property-edit-must-match-flag (not arg))
+ (svn-status-proplist-start))
+
+(defun svn-status-property-set ()
+ (interactive)
+ (setq svn-status-property-edit-must-match-flag nil)
+ (svn-status-proplist-start))
+
+(defun svn-status-property-delete ()
+ (interactive)
+ (setq svn-status-property-edit-must-match-flag t)
+ (svn-status-proplist-start))
+
+(defun svn-status-property-parse-property-names ()
+ ;(svn-status-show-process-buffer-internal t)
+ (message "svn-status-property-parse-property-names")
+ (let ((pl)
+ (prop-name)
+ (prop-value))
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (goto-char (point-min))
+ (forward-line 1)
+ (while (looking-at " \\(.+\\)")
+ (setq pl (append pl (list (match-string 1))))
+ (forward-line 1)))
+ ;(cond last-command: svn-status-property-set, svn-status-property-edit-one-entry
+ (cond ((eq last-command 'svn-status-property-edit-one-entry)
+ ;;(message "svn-status-property-edit-one-entry")
+ (setq prop-name
+ (completing-read "Set Property - Name: " (mapcar 'list pl)
+ nil svn-status-property-edit-must-match-flag))
+ (unless (string= prop-name "")
+ (save-excursion
+ (set-buffer svn-status-buffer-name)
+ (svn-status-property-edit (list (svn-status-get-line-information))
+ prop-name))))
+ ((eq last-command 'svn-status-property-set)
+ (message "svn-status-property-set")
+ (setq prop-name
+ (completing-read "Set Property - Name: " (mapcar 'list pl) nil nil))
+ (setq prop-value (read-from-minibuffer "Property value: "))
+ (unless (string= prop-name "")
+ (save-excursion
+ (set-buffer svn-status-buffer-name)
+ (message "Setting property %s := %s for %S" prop-name prop-value
+ (svn-status-marked-file-names))
+ (let ((file-names (svn-status-marked-file-names)))
+ (when file-names
+ (svn-run nil t 'propset
+ (append (list "propset" prop-name prop-value) file-names))
+ )
+ )
+ (message "propset finished.")
+ )))
+ ((eq last-command 'svn-status-property-delete)
+ (setq prop-name
+ (completing-read "Delete Property - Name: " (mapcar 'list pl) nil t))
+ (unless (string= prop-name "")
+ (save-excursion
+ (set-buffer svn-status-buffer-name)
+ (let ((file-names (svn-status-marked-file-names)))
+ (when file-names
+ (message "Going to delete prop %s for %s" prop-name file-names)
+ (svn-run t t 'propdel
+ (append (list "propdel" prop-name) file-names))))))))))
+
+(defun svn-status-property-edit (file-info-list prop-name &optional new-prop-value remove-values)
+ (let* ((commit-buffer (get-buffer-create "*svn-property-edit*"))
+ (dir default-directory)
+ ;; now only one file is implemented ...
+ (file-name (svn-status-line-info->filename (car file-info-list)))
+ (prop-value))
+ (message "Edit property %s for file %s" prop-name file-name)
+ (svn-run nil t 'propget-parse "propget" prop-name file-name)
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (setq prop-value (if (> (point-max) 1)
+ (buffer-substring (point-min) (- (point-max) 1))
+ "")))
+ (setq svn-status-propedit-property-name prop-name)
+ (setq svn-status-propedit-file-list file-info-list)
+ (setq svn-status-pre-propedit-window-configuration (current-window-configuration))
+ (pop-to-buffer commit-buffer)
+ ;; If the buffer has been narrowed, `svn-prop-edit-done' will use
+ ;; only the accessible part. So we need not erase the rest here.
+ (delete-region (point-min) (point-max))
+ (setq default-directory dir)
+ (insert prop-value)
+ (svn-status-remove-control-M)
+ (when new-prop-value
+ (when (listp new-prop-value)
+ (if remove-values
+ (message "Remove prop values %S " new-prop-value)
+ (message "Adding new prop values %S " new-prop-value))
+ (while new-prop-value
+ (goto-char (point-min))
+ (if (re-search-forward (concat "^" (regexp-quote (car new-prop-value)) "$") nil t)
+ (when remove-values
+ (kill-whole-line 1))
+ (unless remove-values
+ (goto-char (point-max))
+ (when (> (current-column) 0) (insert "\n"))
+ (insert (car new-prop-value))))
+ (setq new-prop-value (cdr new-prop-value)))))
+ (svn-prop-edit-mode)))
+
+(defun svn-status-property-set-property (file-info-list prop-name prop-value)
+ "Set a property on a given file list."
+ (save-excursion
+ (set-buffer (get-buffer-create "*svn-property-edit*"))
+ ;; If the buffer has been narrowed, `svn-prop-edit-do-it' will use
+ ;; only the accessible part. So we need not erase the rest here.
+ (delete-region (point-min) (point-max))
+ (insert prop-value))
+ (setq svn-status-propedit-file-list (svn-status-marked-files))
+ (setq svn-status-propedit-property-name prop-name)
+ (svn-prop-edit-do-it nil)
+ (svn-status-update))
+
+
+(defun svn-status-get-directory (line-info)
+ (let* ((file-name (svn-status-line-info->filename line-info))
+ (file-dir (file-name-directory file-name)))
+ ;;(message "file-dir: %S" file-dir)
+ (if file-dir
+ (substring file-dir 0 (- (length file-dir) 1))
+ ".")))
+
+(defun svn-status-get-file-list-per-directory (files)
+ ;;(message "%S" files)
+ (let ((dir-list nil)
+ (i files)
+ (j)
+ (dir))
+ (while i
+ (setq dir (svn-status-get-directory (car i)))
+ (setq j (assoc dir dir-list))
+ (if j
+ (progn
+ ;;(message "dir already present %S %s" j dir)
+ (setcdr j (append (cdr j) (list (car i)))))
+ (setq dir-list (append dir-list (list (list dir (car i))))))
+ (setq i (cdr i)))
+ ;;(message "svn-status-get-file-list-per-directory: %S" dir-list)
+ dir-list))
+
+(defun svn-status-property-ignore-file ()
+ (interactive)
+ (let ((d-list (svn-status-get-file-list-per-directory (svn-status-marked-files)))
+ (dir)
+ (f-info)
+ (ext-list))
+ (while d-list
+ (setq dir (caar d-list))
+ (setq f-info (cdar d-list))
+ (setq ext-list (mapcar '(lambda (i)
+ (svn-status-line-info->filename-nondirectory i)) f-info))
+ ;;(message "ignore in dir %s: %S" dir f-info)
+ (save-window-excursion
+ (when (y-or-n-p (format "Ignore %S for %s? " ext-list dir))
+ (svn-status-property-edit
+ (list (svn-status-find-info-for-file-name dir)) "svn:ignore" ext-list)
+ (svn-prop-edit-do-it nil))) ; synchronous
+ (setq d-list (cdr d-list)))
+ (svn-status-update)))
+
+(defun svn-status-property-ignore-file-extension ()
+ (interactive)
+ (let ((d-list (svn-status-get-file-list-per-directory (svn-status-marked-files)))
+ (dir)
+ (f-info)
+ (ext-list))
+ (while d-list
+ (setq dir (caar d-list))
+ (setq f-info (cdar d-list))
+ ;;(message "ignore in dir %s: %S" dir f-info)
+ (setq ext-list nil)
+ (while f-info
+ (add-to-list 'ext-list (concat "*."
+ (file-name-extension
+ (svn-status-line-info->filename (car f-info)))))
+ (setq f-info (cdr f-info)))
+ ;;(message "%S" ext-list)
+ (save-window-excursion
+ (when (y-or-n-p (format "Ignore %S for %s? " ext-list dir))
+ (svn-status-property-edit
+ (list (svn-status-find-info-for-file-name dir)) "svn:ignore"
+ ext-list)
+ (svn-prop-edit-do-it nil)))
+ (setq d-list (cdr d-list)))
+ (svn-status-update)))
+
+(defun svn-status-property-edit-svn-ignore ()
+ (interactive)
+ (let* ((line-info (svn-status-get-line-information))
+ (dir (if (svn-status-line-info->directory-p line-info)
+ (svn-status-line-info->filename line-info)
+ (svn-status-get-directory line-info))))
+ (svn-status-property-edit
+ (list (svn-status-find-info-for-file-name dir)) "svn:ignore")
+ (message "Edit svn:ignore on %s" dir)))
+
+
+(defun svn-status-property-set-keyword-list ()
+ "Edit the svn:keywords property on the marked files."
+ (interactive)
+ ;;(message "Set svn:keywords for %S" (svn-status-marked-file-names))
+ (svn-status-property-edit (svn-status-marked-files) "svn:keywords"))
+
+(defun svn-status-property-set-keyword-id (arg)
+ "Set/Remove Id from the svn:keywords property.
+Normally Id is added to the svn:keywords property.
+
+When called with the prefix arg -, remove Id from the svn:keywords property."
+ (interactive "P")
+ (svn-status-property-edit (svn-status-marked-files) "svn:keywords" '("Id") (eq arg '-))
+ (svn-prop-edit-do-it nil))
+
+(defun svn-status-property-set-keyword-date (arg)
+ "Set/Remove Date from the svn:keywords property.
+Normally Date is added to the svn:keywords property.
+
+When called with the prefix arg -, remove Date from the svn:keywords property."
+ (interactive "P")
+ (svn-status-property-edit (svn-status-marked-files) "svn:keywords" '("Date") (eq arg '-))
+ (svn-prop-edit-do-it nil))
+
+
+(defun svn-status-property-set-eol-style ()
+ "Edit the svn:eol-style property on the marked files."
+ (interactive)
+ (svn-status-property-set-property
+ (svn-status-marked-files) "svn:eol-style"
+ (completing-read "Set svn:eol-style for the marked files: "
+ (mapcar 'list '("native" "CRLF" "LF" "CR"))
+ nil t)))
+
+(defun svn-status-property-set-executable ()
+ "Set the svn:executable property on the marked files."
+ (interactive)
+ (svn-status-property-set-property (svn-status-marked-files) "svn:executable" "*"))
+
+(defun svn-status-property-set-mime-type ()
+ "Set the svn:mime-type property on the marked files."
+ (interactive)
+ (require 'mailcap nil t)
+ (let ((completion-ignore-case t)
+ (mime-types (when (fboundp 'mailcap-mime-types)
+ (mailcap-mime-types))))
+ (svn-status-property-set-property
+ (svn-status-marked-files) "svn:mime-type"
+ (funcall svn-status-completing-read-function "Set svn:mime-type for the marked files: "
+ (mapcar (lambda (x) (cons x x)) ; for Emacs 21
+ (sort mime-types 'string<))))))
+
+;; --------------------------------------------------------------------------------
+;; svn-prop-edit-mode:
+;; --------------------------------------------------------------------------------
+
+(defvar svn-prop-edit-mode-map () "Keymap used in `svn-prop-edit-mode' buffers.")
+(put 'svn-prop-edit-mode-map 'risky-local-variable t) ;for Emacs 20.7
+
+(when (not svn-prop-edit-mode-map)
+ (setq svn-prop-edit-mode-map (make-sparse-keymap))
+ (define-key svn-prop-edit-mode-map [(control ?c) (control ?c)] 'svn-prop-edit-done)
+ (define-key svn-prop-edit-mode-map [(control ?c) (control ?d)] 'svn-prop-edit-svn-diff)
+ (define-key svn-prop-edit-mode-map [(control ?c) (control ?s)] 'svn-prop-edit-svn-status)
+ (define-key svn-prop-edit-mode-map [(control ?c) (control ?l)] 'svn-prop-edit-svn-log)
+ (define-key svn-prop-edit-mode-map [(control ?c) (control ?q)] 'svn-prop-edit-abort))
+
+(easy-menu-define svn-prop-edit-mode-menu svn-prop-edit-mode-map
+"'svn-prop-edit-mode' menu"
+ '("SVN-PropEdit"
+ ["Commit" svn-prop-edit-done t]
+ ["Show Diff" svn-prop-edit-svn-diff t]
+ ["Show Status" svn-prop-edit-svn-status t]
+ ["Show Log" svn-prop-edit-svn-log t]
+ ["Abort" svn-prop-edit-abort t]))
+
+(defun svn-prop-edit-mode ()
+ "Major Mode to edit file properties of files under svn control.
+Commands:
+\\{svn-prop-edit-mode-map}"
+ (interactive)
+ (kill-all-local-variables)
+ (use-local-map svn-prop-edit-mode-map)
+ (easy-menu-add svn-prop-edit-mode-menu)
+ (setq major-mode 'svn-prop-edit-mode)
+ (setq mode-name "svn-prop-edit"))
+
+(defun svn-prop-edit-abort ()
+ (interactive)
+ (bury-buffer)
+ (set-window-configuration svn-status-pre-propedit-window-configuration))
+
+(defun svn-prop-edit-done ()
+ (interactive)
+ (svn-prop-edit-do-it t))
+
+(defun svn-prop-edit-do-it (async)
+ "Run svn propset `svn-status-propedit-property-name' with the content of the
+*svn-property-edit* buffer."
+ (message "svn propset %s on %s"
+ svn-status-propedit-property-name
+ (mapcar 'svn-status-line-info->filename svn-status-propedit-file-list))
+ (save-excursion
+ (set-buffer (get-buffer "*svn-property-edit*"))
+ (when (fboundp 'set-buffer-file-coding-system)
+ (set-buffer-file-coding-system svn-status-svn-file-coding-system nil))
+ (setq svn-status-temp-file-to-remove
+ (concat svn-status-temp-dir "svn-prop-edit.txt" svn-temp-suffix))
+ (write-region (point-min) (point-max) svn-status-temp-file-to-remove nil 1))
+ (when svn-status-propedit-file-list ; there are files to change properties
+ (svn-status-create-arg-file svn-status-temp-arg-file ""
+ svn-status-propedit-file-list "")
+ (setq svn-status-propedit-file-list nil)
+ (svn-run async t 'propset "propset"
+ svn-status-propedit-property-name
+ "--targets" svn-status-temp-arg-file
+ (when (eq svn-status-svn-file-coding-system 'utf-8)
+ '("--encoding" "UTF-8"))
+ "-F" (concat svn-status-temp-dir "svn-prop-edit.txt" svn-temp-suffix))
+ (unless async (svn-status-remove-temp-file-maybe)))
+ (when svn-status-pre-propedit-window-configuration
+ (set-window-configuration svn-status-pre-propedit-window-configuration)))
+
+(defun svn-prop-edit-svn-diff (arg)
+ (interactive "P")
+ (set-buffer svn-status-buffer-name)
+ ;; Because propedit is not recursive in our use, neither is this diff.
+ (svn-status-show-svn-diff-internal svn-status-propedit-file-list nil
+ (if arg :ask "BASE")))
+
+(defun svn-prop-edit-svn-log (arg)
+ (interactive "P")
+ (set-buffer svn-status-buffer-name)
+ (svn-status-show-svn-log arg))
+
+(defun svn-prop-edit-svn-status ()
+ (interactive)
+ (pop-to-buffer svn-status-buffer-name)
+ (other-window 1))
+
+;; --------------------------------------------------------------------------------
+;; svn-log-edit-mode:
+;; --------------------------------------------------------------------------------
+
+(defvar svn-log-edit-mode-map () "Keymap used in `svn-log-edit-mode' buffers.")
+(put 'svn-log-edit-mode-map 'risky-local-variable t) ;for Emacs 20.7
+
+(defvar svn-log-edit-mode-menu) ;really defined with `easy-menu-define' below.
+
+(defun svn-log-edit-common-setup ()
+ (set (make-local-variable 'paragraph-start) svn-log-edit-paragraph-start)
+ (set (make-local-variable 'paragraph-separate) svn-log-edit-paragraph-separate))
+
+(if svn-log-edit-use-log-edit-mode
+ (define-derived-mode svn-log-edit-mode log-edit-mode "svn-log-edit"
+ "Wrapper around `log-edit-mode' for psvn.el"
+ (easy-menu-add svn-log-edit-mode-menu)
+ (setq svn-log-edit-update-log-entry nil)
+ (set (make-local-variable 'log-edit-callback) 'svn-log-edit-done)
+ (set (make-local-variable 'log-edit-listfun) 'svn-log-edit-files-to-commit)
+ (set (make-local-variable 'log-edit-initial-files) (log-edit-files))
+ (svn-log-edit-common-setup)
+ (message "Press %s when you are done editing."
+ (substitute-command-keys "\\[log-edit-done]"))
+ )
+ (defun svn-log-edit-mode ()
+ "Major Mode to edit svn log messages.
+Commands:
+\\{svn-log-edit-mode-map}"
+ (interactive)
+ (kill-all-local-variables)
+ (use-local-map svn-log-edit-mode-map)
+ (easy-menu-add svn-log-edit-mode-menu)
+ (setq major-mode 'svn-log-edit-mode)
+ (setq mode-name "svn-log-edit")
+ (setq svn-log-edit-update-log-entry nil)
+ (svn-log-edit-common-setup)
+ (run-hooks 'svn-log-edit-mode-hook)))
+
+(when (not svn-log-edit-mode-map)
+ (setq svn-log-edit-mode-map (make-sparse-keymap))
+ (unless svn-log-edit-use-log-edit-mode
+ (define-key svn-log-edit-mode-map (kbd "C-c C-c") 'svn-log-edit-done))
+ (define-key svn-log-edit-mode-map (kbd "C-c C-d") 'svn-log-edit-svn-diff)
+ (define-key svn-log-edit-mode-map (kbd "C-c C-s") 'svn-log-edit-save-message)
+ (define-key svn-log-edit-mode-map (kbd "C-c C-i") 'svn-log-edit-svn-status)
+ (define-key svn-log-edit-mode-map (kbd "C-c C-l") 'svn-log-edit-svn-log)
+ (define-key svn-log-edit-mode-map (kbd "C-c C-?") 'svn-log-edit-show-files-to-commit)
+ (define-key svn-log-edit-mode-map (kbd "C-c C-z") 'svn-log-edit-erase-edit-buffer)
+ (define-key svn-log-edit-mode-map (kbd "C-c C-q") 'svn-log-edit-abort))
+
+(easy-menu-define svn-log-edit-mode-menu svn-log-edit-mode-map
+"'svn-log-edit-mode' menu"
+ '("SVN-Log"
+ ["Save to disk" svn-log-edit-save-message t]
+ ["Commit" svn-log-edit-done t]
+ ["Show Diff" svn-log-edit-svn-diff t]
+ ["Show Status" svn-log-edit-svn-status t]
+ ["Show Log" svn-log-edit-svn-log t]
+ ["Show files to commit" svn-log-edit-show-files-to-commit t]
+ ["Erase buffer" svn-log-edit-erase-edit-buffer]
+ ["Abort" svn-log-edit-abort t]))
+(put 'svn-log-edit-mode-menu 'risky-local-variable t)
+
+(defun svn-log-edit-abort ()
+ (interactive)
+ (bury-buffer)
+ (set-window-configuration svn-status-pre-commit-window-configuration))
+
+(defun svn-log-edit-done ()
+ "Finish editing the log message and run svn commit."
+ (interactive)
+ (svn-status-save-some-buffers)
+ (save-excursion
+ (set-buffer (get-buffer svn-log-edit-buffer-name))
+ (when svn-log-edit-insert-files-to-commit
+ (svn-log-edit-remove-comment-lines))
+ (when (fboundp 'set-buffer-file-coding-system)
+ (set-buffer-file-coding-system svn-status-svn-file-coding-system nil))
+ (when (or svn-log-edit-update-log-entry svn-status-files-to-commit)
+ (setq svn-status-temp-file-to-remove
+ (concat svn-status-temp-dir "svn-log-edit.txt" svn-temp-suffix))
+ (write-region (point-min) (point-max) svn-status-temp-file-to-remove nil 1))
+ (bury-buffer))
+ (if svn-log-edit-update-log-entry
+ (when (y-or-n-p "Update the log entry? ")
+ ;; svn propset svn:log --revprop -r11672 -F file
+ (svn-run nil t 'propset "propset" "svn:log" "--revprop"
+ (concat "-r" svn-log-edit-update-log-entry)
+ "-F" svn-status-temp-file-to-remove)
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (message "%s" (buffer-substring (point-min) (- (point-max) 1)))))
+ (when svn-status-files-to-commit ; there are files to commit
+ (setq svn-status-operated-on-dot
+ (and (= 1 (length svn-status-files-to-commit))
+ (string= "." (svn-status-line-info->filename (car svn-status-files-to-commit)))))
+ (svn-status-create-arg-file svn-status-temp-arg-file ""
+ svn-status-files-to-commit "")
+ (svn-run t t 'commit "commit"
+ (unless svn-status-recursive-commit "--non-recursive")
+ "--targets" svn-status-temp-arg-file
+ "-F" svn-status-temp-file-to-remove
+ (when (eq svn-status-svn-file-coding-system 'utf-8)
+ '("--encoding" "UTF-8"))
+ svn-status-default-commit-arguments))
+ (set-window-configuration svn-status-pre-commit-window-configuration)
+ (message "svn-log editing done")))
+
+(defun svn-log-edit-svn-diff (arg)
+ "Show the diff we are about to commit.
+If ARG then show diff between some other version of the selected files."
+ (interactive "P")
+ (set-buffer svn-status-buffer-name) ; TODO: is this necessary?
+ ;; This call is very much like `svn-status-show-svn-diff-for-marked-files'
+ ;; but uses commit-specific variables instead of the current marks.
+ (svn-status-show-svn-diff-internal svn-status-files-to-commit
+ svn-status-recursive-commit
+ (if arg :ask "BASE")))
+
+(defun svn-log-edit-svn-log (arg)
+ (interactive "P")
+ (set-buffer svn-status-buffer-name)
+ (svn-status-show-svn-log arg))
+
+(defun svn-log-edit-svn-status ()
+ (interactive)
+ (pop-to-buffer svn-status-buffer-name)
+ (other-window 1))
+
+(defun svn-log-edit-files-to-commit ()
+ (mapcar 'svn-status-line-info->filename svn-status-files-to-commit))
+
+(defun svn-log-edit-show-files-to-commit ()
+ (interactive)
+ (message "Files to commit%s: %S"
+ (if svn-status-recursive-commit " recursively" "")
+ (svn-log-edit-files-to-commit)))
+
+(defun svn-log-edit-save-message ()
+ "Save the current log message to the file `svn-log-edit-file-name'."
+ (interactive)
+ (let ((log-edit-file-name (svn-log-edit-file-name)))
+ (if (string= buffer-file-name log-edit-file-name)
+ (save-buffer)
+ (write-region (point-min) (point-max) log-edit-file-name))))
+
+(defun svn-log-edit-erase-edit-buffer ()
+ "Delete everything in the `svn-log-edit-buffer-name' buffer."
+ (interactive)
+ (set-buffer svn-log-edit-buffer-name)
+ (erase-buffer))
+
+(defun svn-log-edit-insert-files-to-commit ()
+ (interactive)
+ (svn-log-edit-remove-comment-lines)
+ (let ((buf-size (- (point-max) (point-min))))
+ (save-excursion
+ (goto-char (point-min))
+ (insert "## Lines starting with '## ' will be removed from the log message.\n")
+ (insert "## File(s) to commit"
+ (if svn-status-recursive-commit " recursively" "") ":\n")
+ (let ((file-list svn-status-files-to-commit))
+ (while file-list
+ (insert (concat "## " (svn-status-line-info->filename (car file-list)) "\n"))
+ (setq file-list (cdr file-list)))))
+ (when (= 0 buf-size)
+ (goto-char (point-max)))))
+
+(defun svn-log-edit-remove-comment-lines ()
+ (interactive)
+ (save-excursion
+ (goto-char (point-min))
+ (flush-lines "^## .*")))
+
+(defun svn-file-add-to-changelog (prefix-arg)
+ "Create a changelog entry for the function at point.
+The variable `svn-status-changelog-style' allows to select the used changlog style"
+ (interactive "P")
+ (cond ((eq svn-status-changelog-style 'changelog)
+ (svn-file-add-to-log-changelog-style prefix-arg))
+ ((eq svn-status-changelog-style 'svn-dev)
+ (svn-file-add-to-log-svn-dev-style prefix-arg))
+ ((fboundp svn-status-changelog-style)
+ (funcall svn-status-changelog-style prefix-arg))
+ (t
+ (error "Invalid setting for `svn-status-changelog-style'"))))
+
+(defun svn-file-add-to-log-changelog-style (curdir)
+ "Create a changelog entry for the function at point.
+`add-change-log-entry-other-window' creates the header information.
+If CURDIR, save the log file in the current directory, otherwise in the base directory of this working copy."
+ (interactive "P")
+ (add-change-log-entry-other-window nil (svn-log-edit-file-name curdir))
+ (svn-log-edit-mode))
+
+;; taken from svn-dev.el: svn-log-path-derive
+(defun svn-dev-log-path-derive (path)
+ "Derive a relative directory path for absolute PATH, for a log entry."
+ (save-match-data
+ (let ((base (file-name-nondirectory path))
+ (chop-spot (string-match
+ "\\(code/\\)\\|\\(src/\\)\\|\\(projects/\\)"
+ path)))
+ (if chop-spot
+ (progn
+ (setq path (substring path (match-end 0)))
+ ;; Kluge for Subversion developers.
+ (if (string-match "subversion/" path)
+ (substring path (+ (match-beginning 0) 11))
+ path))
+ (string-match (expand-file-name "~/") path)
+ (substring path (match-end 0))))))
+
+;; taken from svn-dev.el: svn-log-message
+(defun svn-file-add-to-log-svn-dev-style (prefix-arg)
+ "Add to an in-progress log message, based on context around point.
+If PREFIX-ARG is negative, then use basenames only in
+log messages, otherwise use full paths. The current defun name is
+always used.
+
+If PREFIX-ARG is a list (e.g. by using C-u), save the log file in
+the current directory, otherwise in the base directory of this
+working copy.
+
+If the log message already contains material about this defun, then put
+point there, so adding to that material is easy.
+
+Else if the log message already contains material about this file, put
+point there, and push onto the kill ring the defun name with log
+message dressing around it, plus the raw defun name, so yank and
+yank-next are both useful.
+
+Else if there is no material about this defun nor file anywhere in the
+log message, then put point at the end of the message and insert a new
+entry for file with defun.
+"
+ (interactive "P")
+ (let* ((short-file-names (and (numberp prefix-arg) (< prefix-arg 0)))
+ (curdir (listp prefix-arg))
+ (this-file (if short-file-names
+ (file-name-nondirectory buffer-file-name)
+ (svn-dev-log-path-derive buffer-file-name)))
+ (this-defun (or (add-log-current-defun)
+ (save-excursion
+ (save-match-data
+ (if (eq major-mode 'c-mode)
+ (progn
+ (if (fboundp 'c-beginning-of-statement-1)
+ (c-beginning-of-statement-1)
+ (c-beginning-of-statement))
+ (search-forward "(" nil t)
+ (forward-char -1)
+ (forward-sexp -1)
+ (buffer-substring
+ (point)
+ (progn (forward-sexp 1) (point)))))))))
+ (log-file (svn-log-edit-file-name curdir)))
+ (find-file log-file)
+ (goto-char (point-min))
+ ;; Strip text properties from strings
+ (set-text-properties 0 (length this-file) nil this-file)
+ (set-text-properties 0 (length this-defun) nil this-defun)
+ ;; If log message for defun already in progress, add to it
+ (if (and
+ this-defun ;; we have a defun to work with
+ (search-forward this-defun nil t) ;; it's in the log msg already
+ (save-excursion ;; and it's about the same file
+ (save-match-data
+ (if (re-search-backward ; Ick, I want a real filename regexp!
+ "^\\*\\s-+\\([a-zA-Z0-9-_.@=+^$/%!?(){}<>]+\\)" nil t)
+ (string-equal (match-string 1) this-file)
+ t))))
+ (if (re-search-forward ":" nil t)
+ (if (looking-at " ") (forward-char 1)))
+ ;; Else no log message for this defun in progress...
+ (goto-char (point-min))
+ ;; But if log message for file already in progress, add to it.
+ (if (search-forward this-file nil t)
+ (progn
+ (if this-defun (progn
+ (kill-new (format "(%s): " this-defun))
+ (kill-new this-defun)))
+ (search-forward ")" nil t)
+ (if (looking-at " ") (forward-char 1)))
+ ;; Found neither defun nor its file, so create new entry.
+ (goto-char (point-max))
+ (if (not (bolp)) (insert "\n"))
+ (insert (format "\n* %s (%s): " this-file (or this-defun "")))
+ ;; Finally, if no derived defun, put point where the user can
+ ;; type it themselves.
+ (if (not this-defun) (forward-char -3))))))
+
+;; --------------------------------------------------------------------------------
+;; svn-log-view-mode:
+;; --------------------------------------------------------------------------------
+
+(defvar svn-log-view-mode-map () "Keymap used in `svn-log-view-mode' buffers.")
+(put 'svn-log-view-mode-map 'risky-local-variable t) ;for Emacs 20.7
+
+(when (not svn-log-view-mode-map)
+ (setq svn-log-view-mode-map (make-sparse-keymap))
+ (suppress-keymap svn-log-view-mode-map)
+ (define-key svn-log-view-mode-map (kbd "p") 'svn-log-view-prev)
+ (define-key svn-log-view-mode-map (kbd "n") 'svn-log-view-next)
+ (define-key svn-log-view-mode-map (kbd "~") 'svn-log-get-specific-revision)
+ (define-key svn-log-view-mode-map (kbd "E") 'svn-log-ediff-specific-revision)
+ (define-key svn-log-view-mode-map (kbd "=") 'svn-log-view-diff)
+ (define-key svn-log-view-mode-map (kbd "TAB") 'svn-log-next-link)
+ (define-key svn-log-view-mode-map [backtab] 'svn-log-prev-link)
+ (define-key svn-log-view-mode-map (kbd "RET") 'svn-log-find-file-at-point)
+ (define-key svn-log-view-mode-map (kbd "e") 'svn-log-edit-log-entry)
+ (define-key svn-log-view-mode-map (kbd "q") 'bury-buffer))
+
+(defvar svn-log-view-popup-menu-map ()
+ "Keymap used to show popup menu in `svn-log-view-mode' buffers.")
+(put 'svn-log-view-popup-menu-map 'risky-local-variable t) ;for Emacs 20.7
+(when (not svn-log-view-popup-menu-map)
+ (setq svn-log-view-popup-menu-map (make-sparse-keymap))
+ (suppress-keymap svn-log-view-popup-menu-map)
+ (define-key svn-log-view-popup-menu-map [down-mouse-3] 'svn-log-view-popup-menu))
+
+(easy-menu-define svn-log-view-mode-menu svn-log-view-mode-map
+"'svn-log-view-mode' menu"
+ '("SVN-LogView"
+ ["Show Changeset" svn-log-view-diff t]
+ ["Ediff file at point" svn-log-ediff-specific-revision t]
+ ["Find file at point" svn-log-find-file-at-point t]
+ ["Get older revision for file at point" svn-log-get-specific-revision t]
+ ["Edit log message" svn-log-edit-log-entry t]))
+
+(defun svn-log-view-popup-menu (event)
+ (interactive "e")
+ (mouse-set-point event)
+ (let* ((rev (svn-log-revision-at-point)))
+ (when rev
+ (svn-status-face-set-temporary-during-popup
+ 'svn-status-marked-popup-face (svn-point-at-bol) (svn-point-at-eol)
+ svn-log-view-mode-menu))))
+
+(defvar svn-log-view-font-lock-basic-keywords
+ '(("^r[0-9]+ .+" (0 `(face font-lock-keyword-face
+ mouse-face highlight
+ keymap ,svn-log-view-popup-menu-map))))
+ "Basic keywords in `svn-log-view-mode'.")
+(put 'svn-log-view-font-basic-lock-keywords 'risky-local-variable t) ;for Emacs 20.7
+
+(defvar svn-log-view-font-lock-keywords)
+(define-derived-mode svn-log-view-mode fundamental-mode "svn-log-view"
+ "Major Mode to show the output from svn log.
+Commands:
+\\{svn-log-view-mode-map}
+"
+ (use-local-map svn-log-view-mode-map)
+ (easy-menu-add svn-log-view-mode-menu)
+ (set (make-local-variable 'svn-log-view-font-lock-keywords) svn-log-view-font-lock-basic-keywords)
+ (dolist (lh svn-log-link-handlers)
+ (add-to-list 'svn-log-view-font-lock-keywords (gethash lh svn-log-registered-link-handlers)))
+ (set (make-local-variable 'font-lock-defaults) '(svn-log-view-font-lock-keywords t)))
+
+(defun svn-log-view-next ()
+ (interactive)
+ (when (re-search-forward "^r[0-9]+" nil t)
+ (beginning-of-line 2)
+ (unless (looking-at "Changed paths:")
+ (beginning-of-line 1))))
+
+(defun svn-log-view-prev ()
+ (interactive)
+ (when (re-search-backward "^r[0-9]+" nil t 2)
+ (beginning-of-line 2)
+ (unless (looking-at "Changed paths:")
+ (beginning-of-line 1))))
+
+(defun svn-log-revision-at-point ()
+ (save-excursion
+ (end-of-line)
+ (re-search-backward "^r\\([0-9]+\\)")
+ (svn-match-string-no-properties 1)))
+
+(defun svn-log-file-name-at-point (respect-checkout-prefix-path)
+ (let ((full-file-name)
+ (file-name)
+ (checkout-prefix-path (if respect-checkout-prefix-path
+ (url-unhex-string
+ (svn-status-checkout-prefix-path))
+ "")))
+ (save-excursion
+ (beginning-of-line)
+ (when (looking-at " [MA] /\\(.+\\)$")
+ (setq full-file-name (svn-match-string-no-properties 1))))
+ (when (string= checkout-prefix-path "")
+ (setq checkout-prefix-path "/"))
+ (if (null full-file-name)
+ (progn
+ (message "No file at point")
+ nil)
+ (setq file-name
+ (if (eq (string-match (regexp-quote (substring checkout-prefix-path 1)) full-file-name) 0)
+ (substring full-file-name (- (length checkout-prefix-path) (if (string= checkout-prefix-path "/") 1 0)))
+ full-file-name))
+ ;; (message "svn-log-file-name-at-point %s prefix: '%s', full-file-name: %s" file-name checkout-prefix-path full-file-name)
+ file-name)))
+
+(defun svn-log-find-file-at-point ()
+ (interactive)
+ (let ((file-name (svn-log-file-name-at-point t)))
+ (when file-name
+ (let ((default-directory (svn-status-base-dir)))
+ ;;(message "svn-log-file-name-at-point: %s, default-directory: %s" file-name default-directory)
+ (find-file file-name)))))
+
+(defun svn-log-next-link ()
+ "Jump to the next external link in this buffer"
+ (interactive)
+ (let ((start-pos (if (get-text-property (point) 'link-handler)
+ (next-single-property-change (point) 'link-handler)
+ (point))))
+ (goto-char (or (next-single-property-change start-pos 'link-handler) (point)))))
+
+(defun svn-log-prev-link ()
+ "Jump to the previous external link in this buffer"
+ (interactive)
+ (let ((start-pos (if (get-text-property (point) 'link-handler)
+ (previous-single-property-change (point) 'link-handler)
+ (point))))
+ (goto-char (or (previous-single-property-change (or start-pos (point)) 'link-handler) (point)))))
+
+(defun svn-log-view-diff (arg)
+ "Show the changeset for a given log entry.
+When called with a prefix argument, ask the user for the revision."
+ (interactive "P")
+ (svn-status-diff-show-changeset (svn-log-revision-at-point) arg))
+
+(defun svn-log-get-specific-revision ()
+ "Get an older revision of the file at point via svn cat."
+ (interactive)
+ ;; (message "%S" (svn-status-make-line-info (svn-log-file-name-at-point t)))
+ (let ((default-directory (svn-status-base-dir)))
+ (svn-status-get-specific-revision-internal
+ (list (svn-status-make-line-info (svn-log-file-name-at-point t)))
+ (svn-log-revision-at-point)
+ nil)))
+
+(defun svn-log-ediff-specific-revision ()
+ "Call ediff for the file at point to view a changeset"
+ (interactive)
+ ;; (message "svn-log-ediff-specific-revision: %s" (svn-log-file-name-at-point t))
+ (let* ((cur-buf (current-buffer))
+ (upper-rev (svn-log-revision-at-point))
+ (lower-rev (number-to-string (- (string-to-number upper-rev) 1)))
+ (file-name (svn-log-file-name-at-point t))
+ (default-directory (svn-status-base-dir))
+ (upper-rev-file-name (when file-name
+ (cdar (svn-status-get-specific-revision-internal
+ (list (svn-status-make-line-info file-name)) upper-rev nil))))
+ (lower-rev-file-name (when file-name
+ (cdar (svn-status-get-specific-revision-internal
+ (list (svn-status-make-line-info file-name)) lower-rev nil)))))
+ ;;(message "%S %S" upper-rev-file-name lower-rev-file-name)
+ (if file-name
+ (let* ((ediff-after-quit-destination-buffer cur-buf)
+ (newer-buffer (find-file-noselect upper-rev-file-name))
+ (base-buff (find-file-noselect lower-rev-file-name))
+ (svn-transient-buffers (list base-buff newer-buffer))
+ (startup-hook '(svn-ediff-startup-hook)))
+ (ediff-buffers base-buff newer-buffer startup-hook))
+ (message "No file at point"))))
+
+(defun svn-log-edit-log-entry ()
+ "Edit the given log entry."
+ (interactive)
+ (let ((rev (svn-log-revision-at-point))
+ (log-message))
+ (svn-run nil t 'propget-parse "propget" "--revprop" (concat "-r" rev) "svn:log")
+ (save-excursion
+ (set-buffer svn-process-buffer-name)
+ (setq log-message (if (> (point-max) 1)
+ (buffer-substring (point-min) (- (point-max) 1))
+ "")))
+ (svn-status-pop-to-commit-buffer)
+ ;; If the buffer has been narrowed, `svn-log-edit-done' will use
+ ;; only the accessible part. So we need not erase the rest here.
+ (delete-region (point-min) (point-max))
+ (insert log-message)
+ (goto-char (point-min))
+ (setq svn-log-edit-update-log-entry rev)))
+
+
+;; allow additional hyperlinks in log view buffers
+(defvar svn-log-link-keymap ()
+ "Keymap used to resolve links `svn-log-view-mode' buffers.")
+(put 'svn-log-link-keymap 'risky-local-variable t) ;for Emacs 20.7
+(when (not svn-log-link-keymap)
+ (setq svn-log-link-keymap (make-sparse-keymap))
+ (suppress-keymap svn-log-link-keymap)
+ (define-key svn-log-link-keymap [mouse-2] 'svn-log-resolve-mouse-link)
+ (define-key svn-log-link-keymap (kbd "RET") 'svn-log-resolve-link))
+
+(defun svn-log-resolve-mouse-link (event)
+ (interactive "e")
+ (mouse-set-point event)
+ (svn-log-resolve-link))
+
+(defun svn-log-resolve-link ()
+ (interactive)
+ (let* ((point-adjustment (if (not (get-text-property (- (point) 1) 'link-handler)) 1
+ (if (not (get-text-property (+ (point) 1) 'link-handler)) -1 0)))
+ (link-name (buffer-substring-no-properties (previous-single-property-change (+ (point) point-adjustment) 'link-handler)
+ (next-single-property-change (+ (point) point-adjustment) 'link-handler))))
+ ;; (message "svn-log-resolve-link '%s'" link-name)
+ (funcall (get-text-property (point) 'link-handler) link-name)))
+
+(defun svn-log-register-link-handler (handler-id link-regexp handler-function)
+ "Register a link handler for external links in *svn-log* buffers
+HANDLER-ID is a symbolic name for this handler. The link handler is active when HANDLER-ID
+is registered in `svn-log-link-handlers'.
+LINK-REGEXP specifies a regular expression that matches the external link.
+HANDLER-FUNCTION is called with the match of LINK-REGEXP when the user clicks at the external link."
+ (let ((font-lock-desc (list link-regexp '(0 `(face font-lock-function-name-face
+ mouse-face highlight
+ link-handler invalid-handler-function
+ keymap ,svn-log-link-keymap)))))
+ ;; no idea, how to use handler-function in invalid-handler-function above, so set it here
+ (setcar (nthcdr 5 (nth 1 (nth 1 (nth 1 font-lock-desc)))) handler-function)
+ (svn-puthash handler-id font-lock-desc svn-log-registered-link-handlers)))
+
+;; example: add support for ditrack links and handle them via svn-log-resolve-ditrack
+;;(svn-log-register-link-handler 'ditrack-issue "i#[0-9]+" 'svn-log-resolve-ditrack)
+;;(defun svn-log-resolve-ditrack (link-name)
+;; (interactive)
+;; (message "svn-log-resolve-ditrack %s" link-name))
+
+
+(defun svn-log-resolve-trac-ticket-short (link-name)
+ "Show the trac ticket specified by LINK-NAME via `svn-trac-browse-ticket'."
+ (interactive)
+ (let ((ticket-nr (string-to-number (svn-substring-no-properties link-name 1))))
+ (svn-trac-browse-ticket ticket-nr)))
+
+;; register the out of the box provided link handlers
+(svn-log-register-link-handler 'trac-ticket-short "#[0-9]+" 'svn-log-resolve-trac-ticket-short)
+
+;; the actually used link handlers are specified in svn-log-link-handlers
+
+;; --------------------------------------------------------------------------------
+;; svn-info-mode
+;; --------------------------------------------------------------------------------
+(defvar svn-info-mode-map () "Keymap used in `svn-info-mode' buffers.")
+(put 'svn-info-mode-map 'risky-local-variable t) ;for Emacs 20.7
+
+(when (not svn-info-mode-map)
+ (setq svn-info-mode-map (make-sparse-keymap))
+ (define-key svn-info-mode-map [?s] 'svn-status-pop-to-status-buffer)
+ (define-key svn-info-mode-map (kbd "h") 'svn-status-pop-to-partner-buffer)
+ (define-key svn-info-mode-map (kbd "n") 'next-line)
+ (define-key svn-info-mode-map (kbd "p") 'previous-line)
+ (define-key svn-info-mode-map (kbd "RET") 'svn-info-show-context)
+ (define-key svn-info-mode-map [?q] 'bury-buffer))
+
+(defun svn-info-mode ()
+ "Major Mode to view informative output from svn."
+ (interactive)
+ (kill-all-local-variables)
+ (use-local-map svn-info-mode-map)
+ (setq major-mode 'svn-info-mode)
+ (setq mode-name "svn-info")
+ (toggle-read-only 1))
+
+(defun svn-info-show-context ()
+ "Show the context for a line in the info buffer.
+Currently is the output from the svn update command known."
+ (interactive)
+ (cond ((save-excursion
+ (goto-char (point-max))
+ (forward-line -1)
+ (beginning-of-line)
+ (looking-at "Updated to revision"))
+ ;; svn-info contains info from an svn update
+ (let ((cur-pos (point))
+ (file-name (buffer-substring-no-properties
+ (progn (beginning-of-line) (re-search-forward ".. +") (point))
+ (line-end-position)))
+ (pos))
+ (when (eq system-type 'windows-nt)
+ (setq file-name (replace-regexp-in-string "\\\\" "/" file-name)))
+ (goto-char cur-pos)
+ (with-current-buffer svn-status-buffer-name
+ (setq pos (svn-status-get-file-name-buffer-position file-name)))
+ (when pos
+ (svn-status-pop-to-new-partner-buffer svn-status-buffer-name)
+ (goto-char pos))))))
+
+;; --------------------------------------------------------------------------------
+;; svn blame minor mode
+;; --------------------------------------------------------------------------------
+
+(unless (assq 'svn-blame-mode minor-mode-alist)
+ (setq minor-mode-alist
+ (cons (list 'svn-blame-mode " SvnBlame")
+ minor-mode-alist)))
+
+(defvar svn-blame-mode-map () "Keymap used in `svn-blame-mode' buffers.")
+(put 'svn-blame-mode-map 'risky-local-variable t) ;for Emacs 20.7
+
+(when (not svn-blame-mode-map)
+ (setq svn-blame-mode-map (make-sparse-keymap))
+ (define-key svn-blame-mode-map [?s] 'svn-status-pop-to-status-buffer)
+ (define-key svn-blame-mode-map (kbd "n") 'next-line)
+ (define-key svn-blame-mode-map (kbd "p") 'previous-line)
+ (define-key svn-blame-mode-map (kbd "RET") 'svn-blame-open-source-file)
+ (define-key svn-blame-mode-map (kbd "a") 'svn-blame-highlight-author)
+ (define-key svn-blame-mode-map (kbd "r") 'svn-blame-highlight-revision)
+ (define-key svn-blame-mode-map (kbd "=") 'svn-blame-show-changeset)
+ (define-key svn-blame-mode-map (kbd "l") 'svn-blame-show-log)
+ (define-key svn-blame-mode-map [?q] 'bury-buffer))
+
+(easy-menu-define svn-blame-mode-menu svn-blame-mode-map
+"svn blame minor mode menu"
+ '("SvnBlame"
+ ["Jump to source location" svn-blame-open-source-file t]
+ ["Show changeset" svn-blame-show-changeset t]
+ ["Show log" svn-blame-show-log t]
+ ["Highlight by author" svn-blame-highlight-author t]
+ ["Highlight by revision" svn-blame-highlight-revision t]))
+
+(or (assq 'svn-blame-mode minor-mode-map-alist)
+ (setq minor-mode-map-alist
+ (cons (cons 'svn-blame-mode svn-blame-mode-map) minor-mode-map-alist)))
+
+(make-variable-buffer-local 'svn-blame-mode)
+
+(defun svn-blame-mode (&optional arg)
+ "Toggle svn blame minor mode.
+With ARG, turn svn blame minor mode on if ARG is positive, off otherwise.
+
+Note: This mode does not yet work on XEmacs...
+It is probably because the revisions are in 'before-string properties of overlays
+
+Key bindings:
+\\{svn-blame-mode-map}"
+ (interactive "P")
+ (setq svn-blame-mode (if (null arg)
+ (not svn-blame-mode)
+ (> (prefix-numeric-value arg) 0)))
+ (if svn-blame-mode
+ (progn
+ (easy-menu-add svn-blame-mode-menu)
+ (toggle-read-only 1))
+ (easy-menu-remove svn-blame-mode-menu))
+ (force-mode-line-update))
+
+(defun svn-status-activate-blame-mode ()
+ "Activate the svn blame minor in the current buffer.
+The current buffer must contain a valid output from svn blame"
+ (save-excursion
+ (goto-char (point-min))
+ (let ((buffer-read-only nil)
+ (line (svn-line-number-at-pos))
+ (limit (point-max))
+ (info-end-col (save-excursion (forward-word 2) (+ (current-column) 1)))
+ (s)
+ ov)
+ ;; remove the old overlays (only for testing)
+ ;; (dolist (ov (overlays-in (point) limit))
+ ;; (when (overlay-get ov 'svn-blame-line-info)
+ ;; (delete-overlay ov)))
+ (while (and (not (eobp)) (< (point) limit))
+ (setq ov (make-overlay (point) (point)))
+ (overlay-put ov 'svn-blame-line-info t)
+ (setq s (buffer-substring-no-properties (svn-point-at-bol) (+ (svn-point-at-bol) info-end-col)))
+ (overlay-put ov 'before-string (propertize s 'face 'svn-status-blame-rev-number-face))
+ (overlay-put ov 'rev-info (delete "" (split-string s " ")))
+ (delete-region (svn-point-at-bol) (+ (svn-point-at-bol) info-end-col))
+ (forward-line)
+ (setq line (1+ line)))))
+ (let* ((buf-name (format "*svn-blame: %s*" (file-relative-name svn-status-blame-file-name)))
+ (buffer (get-buffer buf-name)))
+ (when buffer
+ (kill-buffer buffer))
+ (rename-buffer buf-name))
+ ;; use the correct mode for the displayed blame output
+ (let ((buffer-file-name svn-status-blame-file-name))
+ (normal-mode)
+ (set (make-local-variable 'svn-status-blame-file-name) svn-status-blame-file-name))
+ (font-lock-fontify-buffer)
+ (svn-blame-mode 1))
+
+(defun svn-blame-open-source-file ()
+ "Jump to the source file location for the current position in the svn blame buffer"
+ (interactive)
+ (let ((src-line-number (svn-line-number-at-pos))
+ (src-line-col (current-column)))
+ (find-file-other-window svn-status-blame-file-name)
+ (goto-line src-line-number)
+ (forward-char src-line-col)))
+
+(defun svn-blame-rev-at-point ()
+ (let ((rev))
+ (dolist (ov (overlays-in (svn-point-at-bol) (line-end-position)))
+ (when (overlay-get ov 'svn-blame-line-info)
+ (setq rev (car (overlay-get ov 'rev-info)))))
+ rev))
+
+(defun svn-blame-show-changeset (arg)
+ "Show a diff for the revision at point.
+When called with a prefix argument, allow the user to edit the revision."
+ (interactive "P")
+ (svn-status-diff-show-changeset (svn-blame-rev-at-point) arg))
+
+(defun svn-blame-show-log (arg)
+ "Show the log for the revision at point.
+The output is put into the *svn-log* buffer
+The optional prefix argument ARG determines which switches are passed to `svn log':
+ no prefix --- use whatever is in the list `svn-status-default-log-arguments'
+ prefix argument of -1: --- use the -q switch (quiet)
+ prefix argument of 0 --- use no arguments
+ other prefix arguments: --- use the -v switch (verbose)"
+ (interactive "P")
+ (let ((switches (svn-status-svn-log-switches arg))
+ (rev (svn-blame-rev-at-point)))
+ (svn-run t t 'log "log" "--revision" rev switches)))
+
+(defun svn-blame-highlight-line-maybe (compare-func)
+ (let ((reference-value)
+ (is-highlighted)
+ (consider-this-line)
+ (hl-ov))
+ (dolist (ov (overlays-in (svn-point-at-bol) (line-end-position)))
+ (when (overlay-get ov 'svn-blame-line-info)
+ (setq reference-value (funcall compare-func ov)))
+ (when (overlay-get ov 'svn-blame-highlighted)
+ (setq is-highlighted t)))
+ (save-excursion
+ (goto-char (point-min))
+ (while (not (eobp))
+ (setq consider-this-line nil)
+ (dolist (ov (overlays-in (svn-point-at-bol) (line-end-position)))
+ (when (overlay-get ov 'svn-blame-line-info)
+ (when (string= reference-value (funcall compare-func ov))
+ (setq consider-this-line t))))
+ (when consider-this-line
+ (dolist (ov (overlays-in (svn-point-at-bol) (line-end-position)))
+ (when (and (overlay-get ov 'svn-blame-highlighted) is-highlighted)
+ (delete-overlay ov))
+ (unless is-highlighted
+ (setq hl-ov (make-overlay (svn-point-at-bol) (line-end-position)))
+ (overlay-put hl-ov 'svn-blame-highlighted t)
+ (overlay-put hl-ov 'face 'svn-status-blame-highlight-face))))
+ (forward-line)))))
+
+(defun svn-blame-highlight-author-field (ov)
+ (cadr (overlay-get ov 'rev-info)))
+
+(defun svn-blame-highlight-author ()
+ "(Un)Highlight all lines with the same author."
+ (interactive)
+ (svn-blame-highlight-line-maybe 'svn-blame-highlight-author-field))
+
+(defun svn-blame-highlight-revision-field (ov)
+ (car (overlay-get ov 'rev-info)))
+
+(defun svn-blame-highlight-revision ()
+ "(Un)Highlight all lines with the same revision."
+ (interactive)
+ (svn-blame-highlight-line-maybe 'svn-blame-highlight-revision-field))
+
+;; --------------------------------------------------------------------------------
+;; svn-process-mode
+;; --------------------------------------------------------------------------------
+(defvar svn-process-mode-map () "Keymap used in `svn-process-mode' buffers.")
+(put 'svn-process-mode-map 'risky-local-variable t) ;for Emacs 20.7
+
+(when (not svn-process-mode-map)
+ (setq svn-process-mode-map (make-sparse-keymap))
+ (define-key svn-process-mode-map (kbd "RET") 'svn-process-send-string-and-newline)
+ (define-key svn-process-mode-map [?s] 'svn-process-send-string)
+ (define-key svn-process-mode-map [?q] 'bury-buffer))
+
+(easy-menu-define svn-process-mode-menu svn-process-mode-map
+"'svn-process-mode' menu"
+ '("SvnProcess"
+ ["Send line to process" svn-process-send-string-and-newline t]
+ ["Send raw string to process" svn-process-send-string t]
+ ["Bury process buffer" bury-buffer t]))
+
+(defun svn-process-mode ()
+ "Major Mode to view process output from svn.
+
+You can send a new line terminated string to the process via \\[svn-process-send-string-and-newline]
+You can send raw data to the process via \\[svn-process-send-string]."
+ (interactive)
+ (kill-all-local-variables)
+ (use-local-map svn-process-mode-map)
+ (easy-menu-add svn-log-view-mode-menu)
+ (setq major-mode 'svn-process-mode)
+ (setq mode-name "svn-process"))
+
+;; --------------------------------------------------------------------------------
+;; svn status persistent options
+;; --------------------------------------------------------------------------------
+
+(defun svn-status-repo-for-path (directory)
+ "Find the repository root for DIRECTORY."
+ (let ((old-process-default-dir))
+ (with-current-buffer (get-buffer-create svn-process-buffer-name)
+ (setq old-process-default-dir default-directory)
+ (setq default-directory directory)) ;; update the default-directory for the *svn-process* buffer
+ (svn-run nil t 'parse-info "info" ".")
+ (with-current-buffer svn-process-buffer-name
+ ;; (message "svn-status-repo-for-path: %s: default-directory: %s directory: %s old-process-default-dir: %s" svn-process-buffer-name default-directory directory old-process-default-dir)
+ (setq default-directory old-process-default-dir)
+ (goto-char (point-min))
+ (let ((case-fold-search t))
+ (if (search-forward "repository root: " nil t)
+ (buffer-substring-no-properties (point) (svn-point-at-eol))
+ (when (search-forward "repository uuid: " nil t)
+ (message "psvn.el: Detected an old svn working copy in '%s'. Please check it out again to get a 'Repository Root' entry in the svn info output."
+ default-directory)
+ (concat "Svn Repo UUID: " (buffer-substring-no-properties (point) (svn-point-at-eol)))))))))
+
+(defun svn-status-base-dir (&optional start-directory)
+ "Find the svn root directory for the current working copy.
+Return nil, if not in a svn working copy."
+ (let* ((start-dir (expand-file-name (or start-directory default-directory)))
+ (base-dir (gethash start-dir svn-status-base-dir-cache 'not-found)))
+ ;;(message "svn-status-base-dir: %S %S" start-dir base-dir)
+ (if (not (eq base-dir 'not-found))
+ base-dir
+ ;; (message "calculating base-dir for %s" start-dir)
+ (unless svn-client-version
+ (svn-status-version))
+ (let* ((base-dir start-dir)
+ (repository-root (svn-status-repo-for-path base-dir))
+ (dot-svn-dir (concat base-dir (svn-wc-adm-dir-name)))
+ (in-tree (and repository-root (file-exists-p dot-svn-dir)))
+ (dir-below (expand-file-name base-dir)))
+ ;; (message "repository-root: %s start-dir: %s" repository-root start-dir)
+ (if (and (<= (car svn-client-version) 1) (< (cadr svn-client-version) 3))
+ (setq base-dir (svn-status-base-dir-for-ancient-svn-client start-dir)) ;; svn version < 1.3
+ (while (when (and dir-below (file-exists-p dot-svn-dir))
+ (setq base-dir (file-name-directory dot-svn-dir))
+ (string-match "\\(.+/\\).+/" dir-below)
+ (setq dir-below
+ (and (string-match "\\(.*/\\)[^/]+/" dir-below)
+ (match-string 1 dir-below)))
+ ;; (message "base-dir: %s, dir-below: %s, dot-svn-dir: %s in-tree: %s" base-dir dir-below dot-svn-dir in-tree)
+ (when dir-below
+ (if (string= (svn-status-repo-for-path dir-below) repository-root)
+ (setq dot-svn-dir (concat dir-below (svn-wc-adm-dir-name)))
+ (setq dir-below nil)))))
+ (setq base-dir (and in-tree base-dir)))
+ (svn-puthash start-dir base-dir svn-status-base-dir-cache)
+ (svn-status-message 7 "svn-status-base-dir %s => %s" start-dir base-dir)
+ base-dir))))
+
+(defun svn-status-base-dir-for-ancient-svn-client (&optional start-directory)
+ "Find the svn root directory for the current working copy.
+Return nil, if not in a svn working copy.
+This function is used for svn clients version 1.2 and below."
+ (let* ((base-dir (expand-file-name (or start-directory default-directory)))
+ (dot-svn-dir (concat base-dir (svn-wc-adm-dir-name)))
+ (in-tree (file-exists-p dot-svn-dir))
+ (dir-below (expand-file-name default-directory)))
+ (while (when (and dir-below (file-exists-p dot-svn-dir))
+ (setq base-dir (file-name-directory dot-svn-dir))
+ (string-match "\\(.+/\\).+/" dir-below)
+ (setq dir-below
+ (and (string-match "\\(.*/\\)[^/]+/" dir-below)
+ (match-string 1 dir-below)))
+ (setq dot-svn-dir (concat dir-below (svn-wc-adm-dir-name)))))
+ (and in-tree base-dir)))
+
+(defun svn-status-save-state ()
+ "Save psvn persistent options for this working copy to a file."
+ (interactive)
+ (let ((buf (find-file (concat (svn-status-base-dir) "++psvn.state"))))
+ (erase-buffer) ;Widen, because we'll save the whole buffer.
+ ;; TO CHECK: why is svn-status-options a global variable??
+ (setq svn-status-options
+ (list
+ (list "svn-trac-project-root" svn-trac-project-root)
+ (list "sort-status-buffer" svn-status-sort-status-buffer)
+ (list "elide-list" svn-status-elided-list)
+ (list "module-name" svn-status-module-name)
+ (list "branch-list" svn-status-branch-list)
+ (list "changelog-style" svn-status-changelog-style)
+ ))
+ (insert (pp-to-string svn-status-options))
+ (save-buffer)
+ (kill-buffer buf)))
+
+(defun svn-status-load-state (&optional no-error)
+ "Load psvn persistent options for this working copy from a file."
+ (interactive)
+ (let ((file (concat (svn-status-base-dir) "++psvn.state")))
+ (if (file-readable-p file)
+ (with-temp-buffer
+ (insert-file-contents file)
+ (setq svn-status-options (read (current-buffer)))
+ (setq svn-status-sort-status-buffer
+ (nth 1 (assoc "sort-status-buffer" svn-status-options)))
+ (setq svn-trac-project-root
+ (nth 1 (assoc "svn-trac-project-root" svn-status-options)))
+ (setq svn-status-elided-list
+ (nth 1 (assoc "elide-list" svn-status-options)))
+ (setq svn-status-module-name
+ (nth 1 (assoc "module-name" svn-status-options)))
+ (setq svn-status-branch-list
+ (nth 1 (assoc "branch-list" svn-status-options)))
+ (setq svn-status-changelog-style
+ (nth 1 (assoc "changelog-style" svn-status-options)))
+ (when (and (interactive-p) svn-status-elided-list (svn-status-apply-elide-list)))
+ (message "psvn.el: loaded %s" file))
+ (if no-error
+ (setq svn-trac-project-root nil
+ svn-status-elided-list nil
+ svn-status-module-name nil
+ svn-status-branch-list nil
+ svn-status-changelog-style 'changelog)
+ (error "psvn.el: %s is not readable." file)))))
+
+(defun svn-status-toggle-sort-status-buffer ()
+ "Toggle sorting of the *svn-status* buffer.
+
+If you turn off sorting, you can speed up \\[svn-status]. However,
+the buffer is not correctly sorted then. This function will be
+removed again, when a faster parsing and display routine for
+`svn-status' is available."
+ (interactive)
+ (setq svn-status-sort-status-buffer (not svn-status-sort-status-buffer))
+ (message "The %s buffer will %sbe sorted." svn-status-buffer-name
+ (if svn-status-sort-status-buffer "" "not ")))
+
+(defun svn-status-toggle-svn-verbose-flag ()
+ "Toggle `svn-status-verbose'. "
+ (interactive)
+ (setq svn-status-verbose (not svn-status-verbose))
+ (message "svn status calls will %suse the -v flag." (if svn-status-verbose "" "not ")))
+
+(defun svn-status-toggle-display-full-path ()
+ "Toggle displaying the full path in the `svn-status-buffer-name' buffer"
+ (interactive)
+ (setq svn-status-display-full-path (not svn-status-display-full-path))
+ (message "The %s buffer will%s use full path names." svn-status-buffer-name
+ (if svn-status-display-full-path "" " not"))
+ (svn-status-update-buffer))
+
+(defun svn-status-set-trac-project-root ()
+ (interactive)
+ (setq svn-trac-project-root
+ (read-string "Trac project root (e.g.: http://projects.edgewall.com/trac/): "
+ svn-trac-project-root))
+ (when (yes-or-no-p "Save the new setting for svn-trac-project-root to disk? ")
+ (svn-status-save-state)))
+
+(defun svn-status-set-module-name ()
+ "Interactively set `svn-status-module-name'."
+ (interactive)
+ (setq svn-status-module-name
+ (read-string "Short Unit Name (e.g.: MyProject): "
+ svn-status-module-name))
+ (when (yes-or-no-p "Save the new setting for svn-status-module-name to disk? ")
+ (svn-status-save-state)))
+
+(defun svn-status-set-changelog-style ()
+ "Interactively set `svn-status-changelog-style'."
+ (interactive)
+ (setq svn-status-changelog-style
+ (intern (funcall svn-status-completing-read-function "svn-status on directory: " '("changelog" "svn-dev" "other"))))
+ (when (string= svn-status-changelog-style 'other)
+ (setq svn-status-changelog-style (car (find-function-read))))
+ (when (yes-or-no-p "Save the new setting for svn-status-changelog-style to disk? ")
+ (svn-status-save-state)))
+
+(defun svn-status-set-branch-list ()
+ "Interactively set `svn-status-branch-list'."
+ (interactive)
+ (setq svn-status-branch-list
+ (split-string (read-string "Branch list: "
+ (mapconcat 'identity svn-status-branch-list " "))))
+ (when (yes-or-no-p "Save the new setting for svn-status-branch-list to disk? ")
+ (svn-status-save-state)))
+
+(defun svn-browse-url (url)
+ "Call `browse-url', using `svn-browse-url-function'."
+ (let ((browse-url-browser-function (or svn-browse-url-function
+ browse-url-browser-function)))
+ (browse-url url)))
+
+;; --------------------------------------------------------------------------------
+;; svn status trac integration
+;; --------------------------------------------------------------------------------
+(defun svn-trac-browse-wiki ()
+ "Open the trac wiki view for the current svn repository."
+ (interactive)
+ (unless svn-trac-project-root
+ (svn-status-set-trac-project-root))
+ (svn-browse-url (concat svn-trac-project-root "wiki")))
+
+(defun svn-trac-browse-timeline ()
+ "Open the trac timeline view for the current svn repository."
+ (interactive)
+ (unless svn-trac-project-root
+ (svn-status-set-trac-project-root))
+ (svn-browse-url (concat svn-trac-project-root "timeline")))
+
+(defun svn-trac-browse-roadmap ()
+ "Open the trac roadmap view for the current svn repository."
+ (interactive)
+ (unless svn-trac-project-root
+ (svn-status-set-trac-project-root))
+ (svn-browse-url (concat svn-trac-project-root "roadmap")))
+
+(defun svn-trac-browse-source ()
+ "Open the trac source browser for the current svn repository."
+ (interactive)
+ (unless svn-trac-project-root
+ (svn-status-set-trac-project-root))
+ (svn-browse-url (concat svn-trac-project-root "browser")))
+
+(defun svn-trac-browse-report (arg)
+ "Open the trac report view for the current svn repository.
+When called with a prefix argument, display the given report number."
+ (interactive "P")
+ (unless svn-trac-project-root
+ (svn-status-set-trac-project-root))
+ (svn-browse-url (concat svn-trac-project-root "report" (if (numberp arg) (format "/%s" arg) ""))))
+
+(defun svn-trac-browse-changeset (changeset-nr)
+ "Show a changeset in the trac issue tracker."
+ (interactive (list (read-number "Browse changeset number: " (number-at-point))))
+ (unless svn-trac-project-root
+ (svn-status-set-trac-project-root))
+ (svn-browse-url (concat svn-trac-project-root "changeset/" (number-to-string changeset-nr))))
+
+(defun svn-trac-browse-ticket (ticket-nr)
+ "Show a ticket in the trac issue tracker."
+ (interactive (list (read-number "Browse ticket number: " (number-at-point))))
+ (unless svn-trac-project-root
+ (svn-status-set-trac-project-root))
+ (svn-browse-url (concat svn-trac-project-root "ticket/" (number-to-string ticket-nr))))
+
+;;;------------------------------------------------------------
+;;; resolve conflicts using ediff
+;;;------------------------------------------------------------
+(defun svn-resolve-conflicts-ediff (&optional name-A name-B)
+ "Invoke ediff to resolve conflicts in the current buffer.
+The conflicts must be marked with rcsmerge conflict markers."
+ (interactive)
+ (let* ((found nil)
+ (file-name (file-name-nondirectory buffer-file-name))
+ (your-buffer (generate-new-buffer
+ (concat "*" file-name
+ " " (or name-A "WORKFILE") "*")))
+ (other-buffer (generate-new-buffer
+ (concat "*" file-name
+ " " (or name-B "CHECKED-IN") "*")))
+ (result-buffer (current-buffer)))
+ (save-excursion
+ (set-buffer your-buffer)
+ (erase-buffer)
+ (insert-buffer-substring result-buffer)
+ (goto-char (point-min))
+ (while (re-search-forward "^<<<<<<< .\\(mine\\|working\\)\n" nil t)
+ (setq found t)
+ (replace-match "")
+ (if (not (re-search-forward "^=======\n" nil t))
+ (error "Malformed conflict marker"))
+ (replace-match "")
+ (let ((start (point)))
+ (if (not (re-search-forward "^>>>>>>> .\\(r[0-9]+\\|merge.*\\)\n" nil t))
+ (error "Malformed conflict marker"))
+ (delete-region start (point))))
+ (if (not found)
+ (progn
+ (kill-buffer your-buffer)
+ (kill-buffer other-buffer)
+ (error "No conflict markers found")))
+ (set-buffer other-buffer)
+ (erase-buffer)
+ (insert-buffer-substring result-buffer)
+ (goto-char (point-min))
+ (while (re-search-forward "^<<<<<<< .\\(mine\\|working\\)\n" nil t)
+ (let ((start (match-beginning 0)))
+ (if (not (re-search-forward "^=======\n" nil t))
+ (error "Malformed conflict marker"))
+ (delete-region start (point))
+ (if (not (re-search-forward "^>>>>>>> .\\(r[0-9]+\\|merge.*\\)\n" nil t))
+ (error "Malformed conflict marker"))
+ (replace-match "")))
+ (let ((config (current-window-configuration))
+ (ediff-default-variant 'default-B))
+
+ ;; Fire up ediff.
+
+ (set-buffer (ediff-merge-buffers your-buffer other-buffer))
+
+ ;; Ediff is now set up, and we are in the control buffer.
+ ;; Do a few further adjustments and take precautions for exit.
+
+ (make-local-variable 'svn-ediff-windows)
+ (setq svn-ediff-windows config)
+ (make-local-variable 'svn-ediff-result)
+ (setq svn-ediff-result result-buffer)
+ (make-local-variable 'ediff-quit-hook)
+ (setq ediff-quit-hook
+ (lambda ()
+ (let ((buffer-A ediff-buffer-A)
+ (buffer-B ediff-buffer-B)
+ (buffer-C ediff-buffer-C)
+ (result svn-ediff-result)
+ (windows svn-ediff-windows))
+ (ediff-cleanup-mess)
+ (set-buffer result)
+ (erase-buffer)
+ (insert-buffer-substring buffer-C)
+ (kill-buffer buffer-A)
+ (kill-buffer buffer-B)
+ (kill-buffer buffer-C)
+ (set-window-configuration windows)
+ (message "Conflict resolution finished; you may save the buffer"))))
+ (message "Please resolve conflicts now; exit ediff when done")
+ nil))))
+
+(defun svn-resolve-conflicts (filename)
+ (let ((buff (find-file-noselect filename)))
+ (if buff
+ (progn (switch-to-buffer buff)
+ (svn-resolve-conflicts-ediff))
+ (error "can not open file %s" filename))))
+
+(defun svn-status-resolve-conflicts ()
+ "Resolve conflict in the selected file"
+ (interactive)
+ (let ((file-info (svn-status-get-line-information)))
+ (or (and file-info
+ (= ?C (svn-status-line-info->filemark file-info))
+ (svn-resolve-conflicts
+ (svn-status-line-info->full-path file-info)))
+ (error "can not resolve conflicts at this point"))))
+
+
+;; --------------------------------------------------------------------------------
+;; Working with branches
+;; --------------------------------------------------------------------------------
+
+(defun svn-branch-select (&optional prompt)
+ "Select a branch interactively from `svn-status-branch-list'"
+ (interactive)
+ (unless prompt
+ (setq prompt "Select branch: "))
+ (let* ((branch (funcall svn-status-completing-read-function prompt svn-status-branch-list))
+ (directory)
+ (base-url))
+ (when (string-match "#\\(1#\\)?\\(.+\\)" branch)
+ (setq directory (match-string 2 branch))
+ (setq base-url (concat (svn-status-base-info->repository-root) "/" directory))
+ (save-match-data
+ (svn-status-parse-info t))
+ (if (eq (length (match-string 1 branch)) 0)
+ (setq branch base-url)
+ (let ((svn-status-branch-list (svn-status-ls base-url t)))
+ (setq branch (concat (svn-status-base-info->repository-root) "/"
+ directory "/"
+ (svn-branch-select (format "Select branch from '%s': " directory)))))))
+ branch))
+
+(defun svn-branch-diff (branch1 branch2)
+ "Show the diff between two svn repository urls.
+When called interactively, use `svn-branch-select' to choose two branches from `svn-status-branch-list'."
+ (interactive
+ (let* ((branch1 (svn-branch-select "svn diff branch1: "))
+ (branch2 (svn-branch-select (format "svn diff %s against: " branch1))))
+ (list branch1 branch2)))
+ (svn-run t t 'diff "diff" svn-status-default-diff-arguments branch1 branch2))
+
+;; --------------------------------------------------------------------------------
+;; svnadmin interface
+;; --------------------------------------------------------------------------------
+(defun svn-admin-create (dir)
+ "Run svnadmin create DIR."
+ (interactive (list (expand-file-name
+ (svn-read-directory-name "Create a svn repository at: "
+ svn-admin-default-create-directory nil nil))))
+ (shell-command-to-string (concat "svnadmin create " dir))
+ (setq svn-admin-last-repository-dir (concat "file://" dir))
+ (message "Svn repository created at %s" dir)
+ (run-hooks 'svn-admin-create-hook))
+
+;; - Import an empty directory
+;; cd to an empty directory
+;; svn import -m "Initial import" . file:///home/stefan/svn_repos/WaldiConfig/trunk
+(defun svn-admin-create-trunk-directory ()
+ "Import an empty trunk directory to `svn-admin-last-repository-dir'.
+Set `svn-admin-last-repository-dir' to the new created trunk url."
+ (interactive)
+ (let ((empty-temp-dir-name (make-temp-name svn-status-temp-dir)))
+ (make-directory empty-temp-dir-name t)
+ (setq svn-admin-last-repository-dir (concat svn-admin-last-repository-dir "/trunk"))
+ (svn-run nil t 'import "import" "-m" "Created trunk directory"
+ empty-temp-dir-name svn-admin-last-repository-dir)
+ (delete-directory empty-temp-dir-name)))
+
+(defun svn-admin-start-import ()
+ "Start to import the current working directory in a subversion repository.
+The user is asked to perform the following two steps:
+1. Create a local repository
+2. Add a trunk directory to that repository
+
+After that step the empty base directory (either the root directory or
+the trunk directory of the selected repository) is checked out in the current
+working directory."
+ (interactive)
+ (if (y-or-n-p "Create local repository? ")
+ (progn
+ (call-interactively 'svn-admin-create)
+ (when (y-or-n-p "Add a trunk directory? ")
+ (svn-admin-create-trunk-directory)))
+ (setq svn-admin-last-repository-dir (read-string "Repository Url: ")))
+ (svn-checkout svn-admin-last-repository-dir "."))
+
+;; --------------------------------------------------------------------------------
+;; svn status profiling
+;; --------------------------------------------------------------------------------
+;;; Note about profiling psvn:
+;; (load-library "elp")
+;; M-x elp-reset-all
+;; (elp-instrument-package "svn-")
+;; M-x svn-status
+;; M-x elp-results
+
+(defun svn-status-elp-init ()
+ (interactive)
+ (require 'elp)
+ (elp-reset-all)
+ (elp-instrument-package "svn-")
+ (message "Run the desired svn command (e.g. M-x svn-status), then use M-x elp-results."))
+
+(defun svn-status-last-commands (&optional string-prefix)
+ "Return a string with the last executed svn commands"
+ (interactive)
+ (unless string-prefix
+ (setq string-prefix ""))
+ (with-output-to-string
+ (dolist (e (ring-elements svn-last-cmd-ring))
+ (princ (format "%s%s: svn %s <%s>\n" string-prefix (nth 0 e) (mapconcat 'concat (nth 1 e) " ") (nth 2 e))))))
+
+;; --------------------------------------------------------------------------------
+;; reporting bugs
+;; --------------------------------------------------------------------------------
+(defun svn-insert-indented-lines (text)
+ "Helper function to insert TEXT, indented by two characters."
+ (dolist (line (split-string text "\n"))
+ (insert (format " %s\n" line))))
+
+(defun svn-prepare-bug-report ()
+ "Create the buffer *psvn-bug-report*. This buffer can be useful to debug problems with psvn.el"
+ (interactive)
+ (let* ((last-output-buffer-name (or svn-status-last-output-buffer-name svn-process-buffer-name))
+ (last-svn-cmd-output (with-current-buffer last-output-buffer-name
+ (buffer-substring-no-properties (point-min) (point-max)))))
+ (switch-to-buffer "*psvn-bug-report*")
+ (delete-region (point-min) (point-max))
+ (insert "This buffer holds some debug informations for psvn.el\n")
+ (insert "Please enter a description of the observed and the wanted behaviour\n")
+ (insert "and send it to the author (stefan at xsteve.at) to allow easier debugging\n\n")
+ (insert "Revisions:\n")
+ (svn-insert-indented-lines (svn-status-version))
+ (insert "Language environment:\n")
+ (dolist (elem (svn-process-environment))
+ (when (member (car (split-string elem "=")) '("LC_MESSAGES" "LC_ALL" "LANG"))
+ (insert (format " %s\n" elem))))
+ (insert "\nLast svn commands:\n")
+ (svn-insert-indented-lines (svn-status-last-commands))
+ (insert (format "\nContent of the <%s> buffer:\n" last-output-buffer-name))
+ (svn-insert-indented-lines last-svn-cmd-output)
+ (goto-char (point-min))))
+
+;; --------------------------------------------------------------------------------
+;; Make it easier to reload psvn, if a distribution has an older version
+;; Just add the following to your .emacs:
+;; (svn-prepare-for-reload)
+;; (load "/path/to/psvn.el")
+
+;; Note the above will only work, if the loaded psvn.el has already the
+;; function svn-prepare-for-reload
+;; If this is not the case, do the following:
+;; (load "/path/to/psvn.el");;make svn-prepare-for-reload available
+;; (svn-prepare-for-reload)
+;; (load "/path/to/psvn.el");; update the keybindings
+;; --------------------------------------------------------------------------------
+
+(defvar svn-prepare-for-reload-dont-touch-list '() "A list of variables that should not be touched by `svn-prepare-for-reload'")
+(defvar svn-prepare-for-reload-variables-list '(svn-global-keymap svn-status-diff-mode-map svn-global-trac-map svn-status-mode-map
+ svn-status-mode-property-map svn-status-mode-extension-map
+ svn-status-mode-options-map svn-status-mode-trac-map svn-status-mode-branch-map
+ svn-log-edit-mode-map svn-log-view-mode-map
+ svn-log-view-popup-menu-map svn-info-mode-map svn-blame-mode-map svn-process-mode-map)
+ "A list of variables that should be set to nil via M-x `svn-prepare-for-reload'")
+(defun svn-prepare-for-reload ()
+ "This function resets some psvn.el variables to nil.
+It makes reloading a newer version of psvn.el easier, if for example the used
+GNU/Linux distribution uses an older version.
+
+The variables specified in `svn-prepare-for-reload-variables-list' will be reseted by this function.
+
+A variable will keep its value, if it is specified in `svn-prepare-for-reload-dont-touch-list'."
+ (interactive)
+ (dolist (var svn-prepare-for-reload-variables-list)
+ (unless (member var svn-prepare-for-reload-dont-touch-list)
+ (message (format "Resetting value of %s to nil" var)))
+ (set var nil)))
+
+(provide 'psvn)
+
+;; Local Variables:
+;; indent-tabs-mode: nil
+;; End:
+;;; psvn.el ends here
diff --git a/emacs/pycomplete.el b/emacs/pycomplete.el
new file mode 100644
index 0000000..2fb9bd6
--- /dev/null
+++ b/emacs/pycomplete.el
@@ -0,0 +1,36 @@
+;;; Complete symbols at point using Pymacs.
+
+;;; See pycomplete.py for the Python side of things and a short description
+;;; of what to expect.
+
+(require 'pymacs)
+(require 'python-mode)
+
+(pymacs-load "pycomplete")
+
+(defun py-complete ()
+ (interactive)
+ (let ((pymacs-forget-mutability t))
+ (insert (pycomplete-pycomplete (py-symbol-near-point)
+ (py-find-global-imports)))))
+
+(defun py-find-global-imports ()
+ (save-excursion
+ (let (first-class-or-def imports)
+ (goto-char (point-min))
+ (setq first-class-or-def
+ (re-search-forward "^ *\\(def\\|class\\) " nil t))
+ (goto-char (point-min))
+ (setq imports nil)
+ (while (re-search-forward
+ "^\\(import \\|from \\([A-Za-z_][A-Za-z_0-9]*\\) import \\).*"
+ nil t)
+ (setq imports (append imports
+ (list (buffer-substring
+ (match-beginning 0)
+ (match-end 0))))))
+ imports)))
+
+(define-key py-mode-map "\M-\C-i" 'py-complete)
+
+(provide 'pycomplete)
diff --git a/emacs/pycomplete.py b/emacs/pycomplete.py
new file mode 100644
index 0000000..a39b37b
--- /dev/null
+++ b/emacs/pycomplete.py
@@ -0,0 +1,97 @@
+from __future__ import print_function
+
+"""
+Python dot expression completion using Pymacs.
+
+This almost certainly needs work, but if you add
+
+ (require 'pycomplete)
+
+to your .xemacs/init.el file (untried w/ GNU Emacs so far) and have Pymacs
+installed, when you hit M-TAB it will try to complete the dot expression
+before point. For example, given this import at the top of the file:
+
+ import time
+
+typing "time.cl" then hitting M-TAB should complete "time.clock".
+
+This is unlikely to be done the way Emacs completion ought to be done, but
+it's a start. Perhaps someone with more Emacs mojo can take this stuff and
+do it right.
+
+See pycomplete.el for the Emacs Lisp side of things.
+"""
+
+import sys
+import os.path
+
+try:
+ x = set
+except NameError:
+ from sets import Set as set
+else:
+ del x
+
+def get_all_completions(s, imports=None):
+ """Return contextual completion of s (string of >= zero chars).
+
+ If given, imports is a list of import statements to be executed first.
+ """
+ locald = {}
+ if imports is not None:
+ for stmt in imports:
+ try:
+ exec stmt in globals(), locald
+ except TypeError:
+ raise TypeError, "invalid type: %s" % stmt
+
+ dots = s.split(".")
+ if not s or len(dots) == 1:
+ keys = set()
+ keys.update(locald.keys())
+ keys.update(globals().keys())
+ import __builtin__
+ keys.update(dir(__builtin__))
+ keys = list(keys)
+ keys.sort()
+ if s:
+ return [k for k in keys if k.startswith(s)]
+ else:
+ return keys
+
+ sym = None
+ for i in range(1, len(dots)):
+ s = ".".join(dots[:i])
+ try:
+ sym = eval(s, globals(), locald)
+ except NameError:
+ try:
+ sym = __import__(s, globals(), locald, [])
+ except ImportError:
+ return []
+ if sym is not None:
+ s = dots[-1]
+ return [k for k in dir(sym) if k.startswith(s)]
+
+def pycomplete(s, imports=None):
+ completions = get_all_completions(s, imports)
+ dots = s.split(".")
+ return os.path.commonprefix([k[len(dots[-1]):] for k in completions])
+
+if __name__ == "__main__":
+ print("<empty> ->", pycomplete(""))
+ print("sys.get ->", pycomplete("sys.get"))
+ print("sy ->", pycomplete("sy"))
+ print("sy (sys in context) ->", pycomplete("sy", imports=["import sys"]))
+ print("foo. ->", pycomplete("foo."))
+ print("Enc (email * imported) ->",)
+ print(pycomplete("Enc", imports=["from email import *"]))
+ print("E (email * imported) ->",)
+ print(pycomplete("E", imports=["from email import *"]))
+
+ print("Enc ->", pycomplete("Enc"))
+ print("E ->", pycomplete("E"))
+
+# Local Variables :
+# pymacs-auto-reload : t
+# End :
diff --git a/emacs/python-mode.el b/emacs/python-mode.el
new file mode 100644
index 0000000..b51b992
--- /dev/null
+++ b/emacs/python-mode.el
@@ -0,0 +1,3922 @@
+;;; python-mode.el --- Major mode for editing Python programs
+
+;; Copyright (C) 1992,1993,1994 Tim Peters
+
+;; Author: 2003-2006 http://sf.net/projects/python-mode
+;; 1995-2002 Barry A. Warsaw
+;; 1992-1994 Tim Peters
+;; Maintainer: python-mode at python.org
+;; Created: Feb 1992
+;; Keywords: python languages oop
+
+(defconst py-version "$Revision: 4.78 $"
+ "`python-mode' version number.")
+
+;; This software is provided as-is, without express or implied
+;; warranty. Permission to use, copy, modify, distribute or sell this
+;; software, without fee, for any purpose and by any individual or
+;; organization, is hereby granted, provided that the above copyright
+;; notice and this paragraph appear in all copies.
+
+;;; Commentary:
+
+;; This is a major mode for editing Python programs. It was developed by Tim
+;; Peters after an original idea by Michael A. Guravage. Tim subsequently
+;; left the net and in 1995, Barry Warsaw inherited the mode. Tim's now back
+;; but disavows all responsibility for the mode. In fact, we suspect he
+;; doesn't even use Emacs any more. In 2003, python-mode.el was moved to its
+;; own SourceForge project apart from the Python project, and now is
+;; maintained by the volunteers at the python-mode at python.org mailing list.
+
+;; pdbtrack support contributed by Ken Manheimer, April 2001. Skip Montanaro
+;; has also contributed significantly to python-mode's development.
+
+;; Please use the SourceForge Python project to submit bugs or
+;; patches:
+;;
+;; http://sourceforge.net/projects/python
+
+;; INSTALLATION:
+
+;; To install, just drop this file into a directory on your load-path and
+;; byte-compile it. To set up Emacs to automatically edit files ending in
+;; ".py" using python-mode add the following to your ~/.emacs file (GNU
+;; Emacs) or ~/.xemacs/init.el file (XEmacs):
+;; (setq auto-mode-alist (cons '("\\.py$" . python-mode) auto-mode-alist))
+;; (setq interpreter-mode-alist (cons '("python" . python-mode)
+;; interpreter-mode-alist))
+;; (autoload 'python-mode "python-mode" "Python editing mode." t)
+;;
+;; In XEmacs syntax highlighting should be enabled automatically. In GNU
+;; Emacs you may have to add these lines to your ~/.emacs file:
+;; (global-font-lock-mode t)
+;; (setq font-lock-maximum-decoration t)
+
+;; FOR MORE INFORMATION:
+
+;; There is some information on python-mode.el at
+
+;; http://www.python.org/emacs/python-mode/
+;;
+;; It does contain links to other packages that you might find useful,
+;; such as pdb interfaces, OO-Browser links, etc.
+
+;; BUG REPORTING:
+
+;; As mentioned above, please use the SourceForge Python project for
+;; submitting bug reports or patches. The old recommendation, to use
+;; C-c C-b will still work, but those reports have a higher chance of
+;; getting buried in my mailbox. Please include a complete, but
+;; concise code sample and a recipe for reproducing the bug. Send
+;; suggestions and other comments to python-mode at python.org.
+
+;; When in a Python mode buffer, do a C-h m for more help. It's
+;; doubtful that a texinfo manual would be very useful, but if you
+;; want to contribute one, I'll certainly accept it!
+
+;;; Code:
+
+(require 'comint)
+(require 'custom)
+(require 'cl)
+(require 'compile)
+(require 'ansi-color)
+
+
+;; user definable variables
+;; vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
+
+(defgroup python nil
+ "Support for the Python programming language, <http://www.python.org/>"
+ :group 'languages
+ :prefix "py-")
+
+(defcustom py-tab-always-indent t
+ "*Non-nil means TAB in Python mode should always reindent the current line,
+regardless of where in the line point is when the TAB command is used."
+ :type 'boolean
+ :group 'python)
+
+(defcustom py-python-command "python"
+ "*Shell command used to start Python interpreter."
+ :type 'string
+ :group 'python)
+
+(make-obsolete-variable 'py-jpython-command 'py-jython-command)
+(defcustom py-jython-command "jython"
+ "*Shell command used to start the Jython interpreter."
+ :type 'string
+ :group 'python
+ :tag "Jython Command")
+
+(defcustom py-default-interpreter 'cpython
+ "*Which Python interpreter is used by default.
+The value for this variable can be either `cpython' or `jython'.
+
+When the value is `cpython', the variables `py-python-command' and
+`py-python-command-args' are consulted to determine the interpreter
+and arguments to use.
+
+When the value is `jython', the variables `py-jython-command' and
+`py-jython-command-args' are consulted to determine the interpreter
+and arguments to use.
+
+Note that this variable is consulted only the first time that a Python
+mode buffer is visited during an Emacs session. After that, use
+\\[py-toggle-shells] to change the interpreter shell."
+ :type '(choice (const :tag "Python (a.k.a. CPython)" cpython)
+ (const :tag "Jython" jython))
+ :group 'python)
+
+(defcustom py-python-command-args '("-i")
+ "*List of string arguments to be used when starting a Python shell."
+ :type '(repeat string)
+ :group 'python)
+
+(make-obsolete-variable 'py-jpython-command-args 'py-jython-command-args)
+(defcustom py-jython-command-args '("-i")
+ "*List of string arguments to be used when starting a Jython shell."
+ :type '(repeat string)
+ :group 'python
+ :tag "Jython Command Args")
+
+(defcustom py-indent-offset 4
+ "*Amount of offset per level of indentation.
+`\\[py-guess-indent-offset]' can usually guess a good value when
+you're editing someone else's Python code."
+ :type 'integer
+ :group 'python)
+
+(defcustom py-continuation-offset 4
+ "*Additional amount of offset to give for some continuation lines.
+Continuation lines are those that immediately follow a backslash
+terminated line. Only those continuation lines for a block opening
+statement are given this extra offset."
+ :type 'integer
+ :group 'python)
+
+(defcustom py-smart-indentation t
+ "*Should `python-mode' try to automagically set some indentation variables?
+When this variable is non-nil, two things happen when a buffer is set
+to `python-mode':
+
+ 1. `py-indent-offset' is guessed from existing code in the buffer.
+ Only guessed values between 2 and 8 are considered. If a valid
+ guess can't be made (perhaps because you are visiting a new
+ file), then the value in `py-indent-offset' is used.
+
+ 2. `indent-tabs-mode' is turned off if `py-indent-offset' does not
+ equal `tab-width' (`indent-tabs-mode' is never turned on by
+ Python mode). This means that for newly written code, tabs are
+ only inserted in indentation if one tab is one indentation
+ level, otherwise only spaces are used.
+
+Note that both these settings occur *after* `python-mode-hook' is run,
+so if you want to defeat the automagic configuration, you must also
+set `py-smart-indentation' to nil in your `python-mode-hook'."
+ :type 'boolean
+ :group 'python)
+
+(defcustom py-align-multiline-strings-p t
+ "*Flag describing how multi-line triple quoted strings are aligned.
+When this flag is non-nil, continuation lines are lined up under the
+preceding line's indentation. When this flag is nil, continuation
+lines are aligned to column zero."
+ :type '(choice (const :tag "Align under preceding line" t)
+ (const :tag "Align to column zero" nil))
+ :group 'python)
+
+(defcustom py-block-comment-prefix "##"
+ "*String used by \\[comment-region] to comment out a block of code.
+This should follow the convention for non-indenting comment lines so
+that the indentation commands won't get confused (i.e., the string
+should be of the form `#x...' where `x' is not a blank or a tab, and
+`...' is arbitrary). However, this string should not end in whitespace."
+ :type 'string
+ :group 'python)
+
+(defcustom py-honor-comment-indentation t
+ "*Controls how comment lines influence subsequent indentation.
+
+When nil, all comment lines are skipped for indentation purposes, and
+if possible, a faster algorithm is used (i.e. X/Emacs 19 and beyond).
+
+When t, lines that begin with a single `#' are a hint to subsequent
+line indentation. If the previous line is such a comment line (as
+opposed to one that starts with `py-block-comment-prefix'), then its
+indentation is used as a hint for this line's indentation. Lines that
+begin with `py-block-comment-prefix' are ignored for indentation
+purposes.
+
+When not nil or t, comment lines that begin with a single `#' are used
+as indentation hints, unless the comment character is in column zero."
+ :type '(choice
+ (const :tag "Skip all comment lines (fast)" nil)
+ (const :tag "Single # `sets' indentation for next line" t)
+ (const :tag "Single # `sets' indentation except at column zero"
+ other)
+ )
+ :group 'python)
+
+(defcustom py-temp-directory
+ (let ((ok '(lambda (x)
+ (and x
+ (setq x (expand-file-name x)) ; always true
+ (file-directory-p x)
+ (file-writable-p x)
+ x))))
+ (or (funcall ok (getenv "TMPDIR"))
+ (funcall ok "/usr/tmp")
+ (funcall ok "/tmp")
+ (funcall ok "/var/tmp")
+ (funcall ok ".")
+ (error
+ "Couldn't find a usable temp directory -- set `py-temp-directory'")))
+ "*Directory used for temporary files created by a *Python* process.
+By default, the first directory from this list that exists and that you
+can write into: the value (if any) of the environment variable TMPDIR,
+/usr/tmp, /tmp, /var/tmp, or the current directory."
+ :type 'string
+ :group 'python)
+
+(defcustom py-beep-if-tab-change t
+ "*Ring the bell if `tab-width' is changed.
+If a comment of the form
+
+ \t# vi:set tabsize=<number>:
+
+is found before the first code line when the file is entered, and the
+current value of (the general Emacs variable) `tab-width' does not
+equal <number>, `tab-width' is set to <number>, a message saying so is
+displayed in the echo area, and if `py-beep-if-tab-change' is non-nil
+the Emacs bell is also rung as a warning."
+ :type 'boolean
+ :group 'python)
+
+(defcustom py-jump-on-exception t
+ "*Jump to innermost exception frame in *Python Output* buffer.
+When this variable is non-nil and an exception occurs when running
+Python code synchronously in a subprocess, jump immediately to the
+source code of the innermost traceback frame."
+ :type 'boolean
+ :group 'python)
+
+(defcustom py-ask-about-save t
+ "If not nil, ask about which buffers to save before executing some code.
+Otherwise, all modified buffers are saved without asking."
+ :type 'boolean
+ :group 'python)
+
+(defcustom py-backspace-function 'backward-delete-char-untabify
+ "*Function called by `py-electric-backspace' when deleting backwards."
+ :type 'function
+ :group 'python)
+
+(defcustom py-delete-function 'delete-char
+ "*Function called by `py-electric-delete' when deleting forwards."
+ :type 'function
+ :group 'python)
+
+(defcustom py-imenu-show-method-args-p nil
+ "*Controls echoing of arguments of functions & methods in the Imenu buffer.
+When non-nil, arguments are printed."
+ :type 'boolean
+ :group 'python)
+(make-variable-buffer-local 'py-indent-offset)
+
+(defcustom py-pdbtrack-do-tracking-p t
+ "*Controls whether the pdbtrack feature is enabled or not.
+When non-nil, pdbtrack is enabled in all comint-based buffers,
+e.g. shell buffers and the *Python* buffer. When using pdb to debug a
+Python program, pdbtrack notices the pdb prompt and displays the
+source file and line that the program is stopped at, much the same way
+as gud-mode does for debugging C programs with gdb."
+ :type 'boolean
+ :group 'python)
+(make-variable-buffer-local 'py-pdbtrack-do-tracking-p)
+
+(defcustom py-pdbtrack-minor-mode-string " PDB"
+ "*String to use in the minor mode list when pdbtrack is enabled."
+ :type 'string
+ :group 'python)
+
+(defcustom py-import-check-point-max
+ 20000
+ "Maximum number of characters to search for a Java-ish import statement.
+When `python-mode' tries to calculate the shell to use (either a
+CPython or a Jython shell), it looks at the so-called `shebang' line
+-- i.e. #! line. If that's not available, it looks at some of the
+file heading imports to see if they look Java-like."
+ :type 'integer
+ :group 'python
+ )
+
+(make-obsolete-variable 'py-jpython-packages 'py-jython-packages)
+(defcustom py-jython-packages
+ '("java" "javax" "org" "com")
+ "Imported packages that imply `jython-mode'."
+ :type '(repeat string)
+ :group 'python)
+
+;; Not customizable
+(defvar py-master-file nil
+ "If non-nil, execute the named file instead of the buffer's file.
+The intent is to allow you to set this variable in the file's local
+variable section, e.g.:
+
+ # Local Variables:
+ # py-master-file: \"master.py\"
+ # End:
+
+so that typing \\[py-execute-buffer] in that buffer executes the named
+master file instead of the buffer's file. If the file name has a
+relative path, the value of variable `default-directory' for the
+buffer is prepended to come up with a file name.")
+(make-variable-buffer-local 'py-master-file)
+
+(defcustom py-pychecker-command "pychecker"
+ "*Shell command used to run Pychecker."
+ :type 'string
+ :group 'python
+ :tag "Pychecker Command")
+
+(defcustom py-pychecker-command-args '("--stdlib")
+ "*List of string arguments to be passed to pychecker."
+ :type '(repeat string)
+ :group 'python
+ :tag "Pychecker Command Args")
+
+(defvar py-shell-alist
+ '(("jython" . 'jython)
+ ("python" . 'cpython))
+ "*Alist of interpreters and python shells. Used by `py-choose-shell'
+to select the appropriate python interpreter mode for a file.")
+
+(defcustom py-shell-input-prompt-1-regexp "^>>> "
+ "*A regular expression to match the input prompt of the shell."
+ :type 'string
+ :group 'python)
+
+(defcustom py-shell-input-prompt-2-regexp "^[.][.][.] "
+ "*A regular expression to match the input prompt of the shell after the
+ first line of input."
+ :type 'string
+ :group 'python)
+
+(defcustom py-shell-switch-buffers-on-execute t
+ "*Controls switching to the Python buffer where commands are
+ executed. When non-nil the buffer switches to the Python buffer, if
+ not no switching occurs."
+ :type 'boolean
+ :group 'python)
+
+
+;; ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+;; NO USER DEFINABLE VARIABLES BEYOND THIS POINT
+
+(defvar py-line-number-offset 0
+ "When an exception occurs as a result of py-execute-region, a
+subsequent py-up-exception needs the line number where the region
+started, in order to jump to the correct file line. This variable is
+set in py-execute-region and used in py-jump-to-exception.")
+
+(defconst py-emacs-features
+ (let (features)
+ features)
+ "A list of features extant in the Emacs you are using.
+There are many flavors of Emacs out there, with different levels of
+support for features needed by `python-mode'.")
+
+;; Face for None, True, False, self, and Ellipsis
+(defvar py-pseudo-keyword-face 'py-pseudo-keyword-face
+ "Face for pseudo keywords in Python mode, like self, True, False, Ellipsis.")
+(make-face 'py-pseudo-keyword-face)
+
+;; PEP 318 decorators
+(defvar py-decorators-face 'py-decorators-face
+ "Face method decorators.")
+(make-face 'py-decorators-face)
+
+;; Face for builtins
+(defvar py-builtins-face 'py-builtins-face
+ "Face for builtins like TypeError, object, open, and exec.")
+(make-face 'py-builtins-face)
+
+;; XXX, TODO, and FIXME comments and such
+(defvar py-XXX-tag-face 'py-XXX-tag-face
+ "Face for XXX, TODO, and FIXME tags")
+(make-face 'py-XXX-tag-face)
+
+(defun py-font-lock-mode-hook ()
+ (or (face-differs-from-default-p 'py-pseudo-keyword-face)
+ (copy-face 'font-lock-keyword-face 'py-pseudo-keyword-face))
+ (or (face-differs-from-default-p 'py-builtins-face)
+ (copy-face 'font-lock-keyword-face 'py-builtins-face))
+ (or (face-differs-from-default-p 'py-decorators-face)
+ (copy-face 'py-pseudo-keyword-face 'py-decorators-face))
+ (or (face-differs-from-default-p 'py-XXX-tag-face)
+ (copy-face 'font-lock-comment-face 'py-XXX-tag-face))
+ )
+(add-hook 'font-lock-mode-hook 'py-font-lock-mode-hook)
+
+(defvar python-font-lock-keywords
+ (let ((kw1 (mapconcat 'identity
+ '("and" "assert" "break" "class"
+ "continue" "def" "del" "elif"
+ "else" "except" "exec" "for"
+ "from" "global" "if" "import"
+ "in" "is" "lambda" "not"
+ "or" "pass" "print" "raise"
+ "return" "while" "yield"
+ )
+ "\\|"))
+ (kw2 (mapconcat 'identity
+ '("else:" "except:" "finally:" "try:")
+ "\\|"))
+ (kw3 (mapconcat 'identity
+ ;; Don't include True, False, None, or
+ ;; Ellipsis in this list, since they are
+ ;; already defined as pseudo keywords.
+ '("__debug__"
+ "__import__" "__name__" "abs" "apply" "basestring"
+ "bool" "buffer" "callable" "chr" "classmethod"
+ "cmp" "coerce" "compile" "complex" "copyright"
+ "delattr" "dict" "dir" "divmod"
+ "enumerate" "eval" "execfile" "exit" "file"
+ "filter" "float" "getattr" "globals" "hasattr"
+ "hash" "hex" "id" "input" "int" "intern"
+ "isinstance" "issubclass" "iter" "len" "license"
+ "list" "locals" "long" "map" "max" "min" "object"
+ "oct" "open" "ord" "pow" "property" "range"
+ "raw_input" "reduce" "reload" "repr" "round"
+ "setattr" "slice" "staticmethod" "str" "sum"
+ "super" "tuple" "type" "unichr" "unicode" "vars"
+ "xrange" "zip")
+ "\\|"))
+ (kw4 (mapconcat 'identity
+ ;; Exceptions and warnings
+ '("ArithmeticError" "AssertionError"
+ "AttributeError" "DeprecationWarning" "EOFError"
+ "EnvironmentError" "Exception"
+ "FloatingPointError" "FutureWarning" "IOError"
+ "ImportError" "IndentationError" "IndexError"
+ "KeyError" "KeyboardInterrupt" "LookupError"
+ "MemoryError" "NameError" "NotImplemented"
+ "NotImplementedError" "OSError" "OverflowError"
+ "OverflowWarning" "PendingDeprecationWarning"
+ "ReferenceError" "RuntimeError" "RuntimeWarning"
+ "StandardError" "StopIteration" "SyntaxError"
+ "SyntaxWarning" "SystemError" "SystemExit"
+ "TabError" "TypeError" "UnboundLocalError"
+ "UnicodeDecodeError" "UnicodeEncodeError"
+ "UnicodeError" "UnicodeTranslateError"
+ "UserWarning" "ValueError" "Warning"
+ "ZeroDivisionError")
+ "\\|"))
+ )
+ (list
+ '("^[ \t]*\\(@.+\\)" 1 'py-decorators-face)
+ ;; keywords
+ (cons (concat "\\<\\(" kw1 "\\)\\>[ \n\t(]") 1)
+ ;; builtins when they don't appear as object attributes
+ (list (concat "\\([^. \t]\\|^\\)[ \t]*\\<\\(" kw3 "\\)\\>[ \n\t(]") 2
+ 'py-builtins-face)
+ ;; block introducing keywords with immediately following colons.
+ ;; Yes "except" is in both lists.
+ (cons (concat "\\<\\(" kw2 "\\)[ \n\t(]") 1)
+ ;; Exceptions
+ (list (concat "\\<\\(" kw4 "\\)[ \n\t:,(]") 1 'py-builtins-face)
+ ;; `as' but only in "import foo as bar"
+ '("[ \t]*\\(\\<from\\>.*\\)?\\<import\\>.*\\<\\(as\\)\\>" . 2)
+
+ ;; classes
+ '("\\<class[ \t]+\\([a-zA-Z_]+[a-zA-Z0-9_]*\\)" 1 font-lock-type-face)
+ ;; functions
+ '("\\<def[ \t]+\\([a-zA-Z_]+[a-zA-Z0-9_]*\\)"
+ 1 font-lock-function-name-face)
+ ;; pseudo-keywords
+ '("\\<\\(self\\|None\\|True\\|False\\|Ellipsis\\)\\>"
+ 1 py-pseudo-keyword-face)
+ ;; XXX, TODO, and FIXME tags
+ '("XXX\\|TODO\\|FIXME" 0 py-XXX-tag-face t)
+ ))
+ "Additional expressions to highlight in Python mode.")
+(put 'python-mode 'font-lock-defaults '(python-font-lock-keywords))
+
+;; have to bind py-file-queue before installing the kill-emacs-hook
+(defvar py-file-queue nil
+ "Queue of Python temp files awaiting execution.
+Currently-active file is at the head of the list.")
+
+(defvar py-pdbtrack-is-tracking-p nil)
+
+(defvar py-pychecker-history nil)
+
+
+
+;; Constants
+
+(defconst py-stringlit-re
+ (concat
+ ;; These fail if backslash-quote ends the string (not worth
+ ;; fixing?). They precede the short versions so that the first two
+ ;; quotes don't look like an empty short string.
+ ;;
+ ;; (maybe raw), long single quoted triple quoted strings (SQTQ),
+ ;; with potential embedded single quotes
+ "[rR]?'''[^']*\\(\\('[^']\\|''[^']\\)[^']*\\)*'''"
+ "\\|"
+ ;; (maybe raw), long double quoted triple quoted strings (DQTQ),
+ ;; with potential embedded double quotes
+ "[rR]?\"\"\"[^\"]*\\(\\(\"[^\"]\\|\"\"[^\"]\\)[^\"]*\\)*\"\"\""
+ "\\|"
+ "[rR]?'\\([^'\n\\]\\|\\\\.\\)*'" ; single-quoted
+ "\\|" ; or
+ "[rR]?\"\\([^\"\n\\]\\|\\\\.\\)*\"" ; double-quoted
+ )
+ "Regular expression matching a Python string literal.")
+
+(defconst py-continued-re
+ ;; This is tricky because a trailing backslash does not mean
+ ;; continuation if it's in a comment
+ (concat
+ "\\(" "[^#'\"\n\\]" "\\|" py-stringlit-re "\\)*"
+ "\\\\$")
+ "Regular expression matching Python backslash continuation lines.")
+
+(defconst py-blank-or-comment-re "[ \t]*\\($\\|#\\)"
+ "Regular expression matching a blank or comment line.")
+
+(defconst py-outdent-re
+ (concat "\\(" (mapconcat 'identity
+ '("else:"
+ "except\\(\\s +.*\\)?:"
+ "finally:"
+ "elif\\s +.*:")
+ "\\|")
+ "\\)")
+ "Regular expression matching statements to be dedented one level.")
+
+(defconst py-block-closing-keywords-re
+ "\\(return\\|raise\\|break\\|continue\\|pass\\)"
+ "Regular expression matching keywords which typically close a block.")
+
+(defconst py-no-outdent-re
+ (concat
+ "\\("
+ (mapconcat 'identity
+ (list "try:"
+ "except\\(\\s +.*\\)?:"
+ "while\\s +.*:"
+ "for\\s +.*:"
+ "if\\s +.*:"
+ "elif\\s +.*:"
+ (concat py-block-closing-keywords-re "[ \t\n]")
+ )
+ "\\|")
+ "\\)")
+ "Regular expression matching lines not to dedent after.")
+
+(defvar py-traceback-line-re
+ "[ \t]+File \"\\([^\"]+\\)\", line \\([0-9]+\\)"
+ "Regular expression that describes tracebacks.")
+
+;; pdbtrack constants
+(defconst py-pdbtrack-stack-entry-regexp
+; "^> \\([^(]+\\)(\\([0-9]+\\))\\([?a-zA-Z0-9_]+\\)()"
+ "^> \\(.*\\)(\\([0-9]+\\))\\([?a-zA-Z0-9_]+\\)()"
+ "Regular expression pdbtrack uses to find a stack trace entry.")
+
+(defconst py-pdbtrack-input-prompt "\n[(<]*[Pp]db[>)]+ "
+ "Regular expression pdbtrack uses to recognize a pdb prompt.")
+
+(defconst py-pdbtrack-track-range 10000
+ "Max number of characters from end of buffer to search for stack entry.")
+
+
+
+;; Major mode boilerplate
+
+;; define a mode-specific abbrev table for those who use such things
+(defvar python-mode-abbrev-table nil
+ "Abbrev table in use in `python-mode' buffers.")
+(define-abbrev-table 'python-mode-abbrev-table nil)
+
+(defvar python-mode-hook nil
+ "*Hook called by `python-mode'.")
+
+(make-obsolete-variable 'jpython-mode-hook 'jython-mode-hook)
+(defvar jython-mode-hook nil
+ "*Hook called by `jython-mode'. `jython-mode' also calls
+`python-mode-hook'.")
+
+(defvar py-shell-hook nil
+ "*Hook called by `py-shell'.")
+
+;; In previous version of python-mode.el, the hook was incorrectly
+;; called py-mode-hook, and was not defvar'd. Deprecate its use.
+(and (fboundp 'make-obsolete-variable)
+ (make-obsolete-variable 'py-mode-hook 'python-mode-hook))
+
+(defvar py-mode-map ()
+ "Keymap used in `python-mode' buffers.")
+(if py-mode-map
+ nil
+ (setq py-mode-map (make-sparse-keymap))
+ ;; electric keys
+ (define-key py-mode-map ":" 'py-electric-colon)
+ ;; indentation level modifiers
+ (define-key py-mode-map "\C-c\C-l" 'py-shift-region-left)
+ (define-key py-mode-map "\C-c\C-r" 'py-shift-region-right)
+ (define-key py-mode-map "\C-c<" 'py-shift-region-left)
+ (define-key py-mode-map "\C-c>" 'py-shift-region-right)
+ ;; subprocess commands
+ (define-key py-mode-map "\C-c\C-c" 'py-execute-buffer)
+ (define-key py-mode-map "\C-c\C-m" 'py-execute-import-or-reload)
+ (define-key py-mode-map "\C-c\C-s" 'py-execute-string)
+ (define-key py-mode-map "\C-c|" 'py-execute-region)
+ (define-key py-mode-map "\e\C-x" 'py-execute-def-or-class)
+ (define-key py-mode-map "\C-c!" 'py-shell)
+ (define-key py-mode-map "\C-c\C-t" 'py-toggle-shells)
+ ;; Caution! Enter here at your own risk. We are trying to support
+ ;; several behaviors and it gets disgusting. :-( This logic ripped
+ ;; largely from CC Mode.
+ ;;
+ ;; In XEmacs 19, Emacs 19, and Emacs 20, we use this to bind
+ ;; backwards deletion behavior to DEL, which both Delete and
+ ;; Backspace get translated to. There's no way to separate this
+ ;; behavior in a clean way, so deal with it! Besides, it's been
+ ;; this way since the dawn of time.
+ (if (not (boundp 'delete-key-deletes-forward))
+ (define-key py-mode-map "\177" 'py-electric-backspace)
+ ;; However, XEmacs 20 actually achieved enlightenment. It is
+ ;; possible to sanely define both backward and forward deletion
+ ;; behavior under X separately (TTYs are forever beyond hope, but
+ ;; who cares? XEmacs 20 does the right thing with these too).
+ (define-key py-mode-map [delete] 'py-electric-delete)
+ (define-key py-mode-map [backspace] 'py-electric-backspace))
+ ;; Separate M-BS from C-M-h. The former should remain
+ ;; backward-kill-word.
+ (define-key py-mode-map [(control meta h)] 'py-mark-def-or-class)
+ (define-key py-mode-map "\C-c\C-k" 'py-mark-block)
+ ;; Miscellaneous
+ (define-key py-mode-map "\C-c:" 'py-guess-indent-offset)
+ (define-key py-mode-map "\C-c\t" 'py-indent-region)
+ (define-key py-mode-map "\C-c\C-d" 'py-pdbtrack-toggle-stack-tracking)
+ (define-key py-mode-map "\C-c\C-n" 'py-next-statement)
+ (define-key py-mode-map "\C-c\C-p" 'py-previous-statement)
+ (define-key py-mode-map "\C-c\C-u" 'py-goto-block-up)
+ (define-key py-mode-map "\C-c#" 'py-comment-region)
+ (define-key py-mode-map "\C-c?" 'py-describe-mode)
+ (define-key py-mode-map "\C-c\C-h" 'py-help-at-point)
+ (define-key py-mode-map "\e\C-a" 'py-beginning-of-def-or-class)
+ (define-key py-mode-map "\e\C-e" 'py-end-of-def-or-class)
+ (define-key py-mode-map "\C-c-" 'py-up-exception)
+ (define-key py-mode-map "\C-c=" 'py-down-exception)
+ ;; stuff that is `standard' but doesn't interface well with
+ ;; python-mode, which forces us to rebind to special commands
+ (define-key py-mode-map "\C-xnd" 'py-narrow-to-defun)
+ ;; information
+ (define-key py-mode-map "\C-c\C-b" 'py-submit-bug-report)
+ (define-key py-mode-map "\C-c\C-v" 'py-version)
+ (define-key py-mode-map "\C-c\C-w" 'py-pychecker-run)
+ ;; shadow global bindings for newline-and-indent w/ the py- version.
+ ;; BAW - this is extremely bad form, but I'm not going to change it
+ ;; for now.
+ (mapcar #'(lambda (key)
+ (define-key py-mode-map key 'py-newline-and-indent))
+ (where-is-internal 'newline-and-indent))
+ ;; Force RET to be py-newline-and-indent even if it didn't get
+ ;; mapped by the above code. motivation: Emacs' default binding for
+ ;; RET is `newline' and C-j is `newline-and-indent'. Most Pythoneers
+ ;; expect RET to do a `py-newline-and-indent' and any Emacsers who
+ ;; dislike this are probably knowledgeable enough to do a rebind.
+ ;; However, we do *not* change C-j since many Emacsers have already
+ ;; swapped RET and C-j and they don't want C-j bound to `newline' to
+ ;; change.
+ (define-key py-mode-map "\C-m" 'py-newline-and-indent)
+ )
+
+(defvar py-mode-output-map nil
+ "Keymap used in *Python Output* buffers.")
+(if py-mode-output-map
+ nil
+ (setq py-mode-output-map (make-sparse-keymap))
+ (define-key py-mode-output-map [button2] 'py-mouseto-exception)
+ (define-key py-mode-output-map "\C-c\C-c" 'py-goto-exception)
+ ;; TBD: Disable all self-inserting keys. This is bogus, we should
+ ;; really implement this as *Python Output* buffer being read-only
+ (mapcar #' (lambda (key)
+ (define-key py-mode-output-map key
+ #'(lambda () (interactive) (beep))))
+ (where-is-internal 'self-insert-command))
+ )
+
+(defvar py-shell-map nil
+ "Keymap used in *Python* shell buffers.")
+(if py-shell-map
+ nil
+ (setq py-shell-map (copy-keymap comint-mode-map))
+ (define-key py-shell-map [tab] 'tab-to-tab-stop)
+ (define-key py-shell-map "\C-c-" 'py-up-exception)
+ (define-key py-shell-map "\C-c=" 'py-down-exception)
+ )
+
+(defvar py-mode-syntax-table nil
+ "Syntax table used in `python-mode' buffers.")
+(when (not py-mode-syntax-table)
+ (setq py-mode-syntax-table (make-syntax-table))
+ (modify-syntax-entry ?\( "()" py-mode-syntax-table)
+ (modify-syntax-entry ?\) ")(" py-mode-syntax-table)
+ (modify-syntax-entry ?\[ "(]" py-mode-syntax-table)
+ (modify-syntax-entry ?\] ")[" py-mode-syntax-table)
+ (modify-syntax-entry ?\{ "(}" py-mode-syntax-table)
+ (modify-syntax-entry ?\} "){" py-mode-syntax-table)
+ ;; Add operator symbols misassigned in the std table
+ (modify-syntax-entry ?\$ "." py-mode-syntax-table)
+ (modify-syntax-entry ?\% "." py-mode-syntax-table)
+ (modify-syntax-entry ?\& "." py-mode-syntax-table)
+ (modify-syntax-entry ?\* "." py-mode-syntax-table)
+ (modify-syntax-entry ?\+ "." py-mode-syntax-table)
+ (modify-syntax-entry ?\- "." py-mode-syntax-table)
+ (modify-syntax-entry ?\/ "." py-mode-syntax-table)
+ (modify-syntax-entry ?\< "." py-mode-syntax-table)
+ (modify-syntax-entry ?\= "." py-mode-syntax-table)
+ (modify-syntax-entry ?\> "." py-mode-syntax-table)
+ (modify-syntax-entry ?\| "." py-mode-syntax-table)
+ ;; For historical reasons, underscore is word class instead of
+ ;; symbol class. GNU conventions say it should be symbol class, but
+ ;; there's a natural conflict between what major mode authors want
+ ;; and what users expect from `forward-word' and `backward-word'.
+ ;; Guido and I have hashed this out and have decided to keep
+ ;; underscore in word class. If you're tempted to change it, try
+ ;; binding M-f and M-b to py-forward-into-nomenclature and
+ ;; py-backward-into-nomenclature instead. This doesn't help in all
+ ;; situations where you'd want the different behavior
+ ;; (e.g. backward-kill-word).
+ (modify-syntax-entry ?\_ "w" py-mode-syntax-table)
+ ;; Both single quote and double quote are string delimiters
+ (modify-syntax-entry ?\' "\"" py-mode-syntax-table)
+ (modify-syntax-entry ?\" "\"" py-mode-syntax-table)
+ ;; backquote is open and close paren
+ (modify-syntax-entry ?\` "$" py-mode-syntax-table)
+ ;; comment delimiters
+ (modify-syntax-entry ?\# "<" py-mode-syntax-table)
+ (modify-syntax-entry ?\n ">" py-mode-syntax-table)
+ )
+
+;; An auxiliary syntax table which places underscore and dot in the
+;; symbol class for simplicity
+(defvar py-dotted-expression-syntax-table nil
+ "Syntax table used to identify Python dotted expressions.")
+(when (not py-dotted-expression-syntax-table)
+ (setq py-dotted-expression-syntax-table
+ (copy-syntax-table py-mode-syntax-table))
+ (modify-syntax-entry ?_ "_" py-dotted-expression-syntax-table)
+ (modify-syntax-entry ?. "_" py-dotted-expression-syntax-table))
+
+
+
+;; Utilities
+(defmacro py-safe (&rest body)
+ "Safely execute BODY, return nil if an error occurred."
+ (` (condition-case nil
+ (progn (,@ body))
+ (error nil))))
+
+(defsubst py-keep-region-active ()
+ "Keep the region active in XEmacs."
+ ;; Ignore byte-compiler warnings you might see. Also note that
+ ;; FSF's Emacs 19 does it differently; its policy doesn't require us
+ ;; to take explicit action.
+ (and (boundp 'zmacs-region-stays)
+ (setq zmacs-region-stays t)))
+
+(defsubst py-point (position)
+ "Returns the value of point at certain commonly referenced POSITIONs.
+POSITION can be one of the following symbols:
+
+ bol -- beginning of line
+ eol -- end of line
+ bod -- beginning of def or class
+ eod -- end of def or class
+ bob -- beginning of buffer
+ eob -- end of buffer
+ boi -- back to indentation
+ bos -- beginning of statement
+
+This function does not modify point or mark."
+ (let ((here (point)))
+ (cond
+ ((eq position 'bol) (beginning-of-line))
+ ((eq position 'eol) (end-of-line))
+ ((eq position 'bod) (py-beginning-of-def-or-class 'either))
+ ((eq position 'eod) (py-end-of-def-or-class 'either))
+ ;; Kind of funny, I know, but useful for py-up-exception.
+ ((eq position 'bob) (beginning-of-buffer))
+ ((eq position 'eob) (end-of-buffer))
+ ((eq position 'boi) (back-to-indentation))
+ ((eq position 'bos) (py-goto-initial-line))
+ (t (error "Unknown buffer position requested: %s" position))
+ )
+ (prog1
+ (point)
+ (goto-char here))))
+
+(defsubst py-highlight-line (from to file line)
+ (cond
+ ((fboundp 'make-extent)
+ ;; XEmacs
+ (let ((e (make-extent from to)))
+ (set-extent-property e 'mouse-face 'highlight)
+ (set-extent-property e 'py-exc-info (cons file line))
+ (set-extent-property e 'keymap py-mode-output-map)))
+ (t
+ ;; Emacs -- Please port this!
+ )
+ ))
+
+(defun py-in-literal (&optional lim)
+ "Return non-nil if point is in a Python literal (a comment or string).
+Optional argument LIM indicates the beginning of the containing form,
+i.e. the limit on how far back to scan."
+ ;; This is the version used for non-XEmacs, which has a nicer
+ ;; interface.
+ ;;
+ ;; WARNING: Watch out for infinite recursion.
+ (let* ((lim (or lim (py-point 'bod)))
+ (state (parse-partial-sexp lim (point))))
+ (cond
+ ((nth 3 state) 'string)
+ ((nth 4 state) 'comment)
+ (t nil))))
+
+;; XEmacs has a built-in function that should make this much quicker.
+;; In this case, lim is ignored
+(defun py-fast-in-literal (&optional lim)
+ "Fast version of `py-in-literal', used only by XEmacs.
+Optional LIM is ignored."
+ ;; don't have to worry about context == 'block-comment
+ (buffer-syntactic-context))
+
+(if (fboundp 'buffer-syntactic-context)
+ (defalias 'py-in-literal 'py-fast-in-literal))
+
+
+
+;; Menu definitions, only relevent if you have the easymenu.el package
+;; (standard in the latest Emacs 19 and XEmacs 19 distributions).
+(defvar py-menu nil
+ "Menu for Python Mode.
+This menu will get created automatically if you have the `easymenu'
+package. Note that the latest X/Emacs releases contain this package.")
+
+(and (py-safe (require 'easymenu) t)
+ (easy-menu-define
+ py-menu py-mode-map "Python Mode menu"
+ '("Python"
+ ["Comment Out Region" py-comment-region (mark)]
+ ["Uncomment Region" (py-comment-region (point) (mark) '(4)) (mark)]
+ "-"
+ ["Mark current block" py-mark-block t]
+ ["Mark current def" py-mark-def-or-class t]
+ ["Mark current class" (py-mark-def-or-class t) t]
+ "-"
+ ["Shift region left" py-shift-region-left (mark)]
+ ["Shift region right" py-shift-region-right (mark)]
+ "-"
+ ["Import/reload file" py-execute-import-or-reload t]
+ ["Execute buffer" py-execute-buffer t]
+ ["Execute region" py-execute-region (mark)]
+ ["Execute def or class" py-execute-def-or-class (mark)]
+ ["Execute string" py-execute-string t]
+ ["Start interpreter..." py-shell t]
+ "-"
+ ["Go to start of block" py-goto-block-up t]
+ ["Go to start of class" (py-beginning-of-def-or-class t) t]
+ ["Move to end of class" (py-end-of-def-or-class t) t]
+ ["Move to start of def" py-beginning-of-def-or-class t]
+ ["Move to end of def" py-end-of-def-or-class t]
+ "-"
+ ["Describe mode" py-describe-mode t]
+ )))
+
+
+
+;; Imenu definitions
+(defvar py-imenu-class-regexp
+ (concat ; <<classes>>
+ "\\(" ;
+ "^[ \t]*" ; newline and maybe whitespace
+ "\\(class[ \t]+[a-zA-Z0-9_]+\\)" ; class name
+ ; possibly multiple superclasses
+ "\\([ \t]*\\((\\([a-zA-Z0-9_,. \t\n]\\)*)\\)?\\)"
+ "[ \t]*:" ; and the final :
+ "\\)" ; >>classes<<
+ )
+ "Regexp for Python classes for use with the Imenu package."
+ )
+
+(defvar py-imenu-method-regexp
+ (concat ; <<methods and functions>>
+ "\\(" ;
+ "^[ \t]*" ; new line and maybe whitespace
+ "\\(def[ \t]+" ; function definitions start with def
+ "\\([a-zA-Z0-9_]+\\)" ; name is here
+ ; function arguments...
+;; "[ \t]*(\\([-+/a-zA-Z0-9_=,\* \t\n.()\"'#]*\\))"
+ "[ \t]*(\\([^:#]*\\))"
+ "\\)" ; end of def
+ "[ \t]*:" ; and then the :
+ "\\)" ; >>methods and functions<<
+ )
+ "Regexp for Python methods/functions for use with the Imenu package."
+ )
+
+(defvar py-imenu-method-no-arg-parens '(2 8)
+ "Indices into groups of the Python regexp for use with Imenu.
+
+Using these values will result in smaller Imenu lists, as arguments to
+functions are not listed.
+
+See the variable `py-imenu-show-method-args-p' for more
+information.")
+
+(defvar py-imenu-method-arg-parens '(2 7)
+ "Indices into groups of the Python regexp for use with imenu.
+Using these values will result in large Imenu lists, as arguments to
+functions are listed.
+
+See the variable `py-imenu-show-method-args-p' for more
+information.")
+
+;; Note that in this format, this variable can still be used with the
+;; imenu--generic-function. Otherwise, there is no real reason to have
+;; it.
+(defvar py-imenu-generic-expression
+ (cons
+ (concat
+ py-imenu-class-regexp
+ "\\|" ; or...
+ py-imenu-method-regexp
+ )
+ py-imenu-method-no-arg-parens)
+ "Generic Python expression which may be used directly with Imenu.
+Used by setting the variable `imenu-generic-expression' to this value.
+Also, see the function \\[py-imenu-create-index] for a better
+alternative for finding the index.")
+
+;; These next two variables are used when searching for the Python
+;; class/definitions. Just saving some time in accessing the
+;; generic-python-expression, really.
+(defvar py-imenu-generic-regexp nil)
+(defvar py-imenu-generic-parens nil)
+
+
+(defun py-imenu-create-index-function ()
+ "Python interface function for the Imenu package.
+Finds all Python classes and functions/methods. Calls function
+\\[py-imenu-create-index-engine]. See that function for the details
+of how this works."
+ (setq py-imenu-generic-regexp (car py-imenu-generic-expression)
+ py-imenu-generic-parens (if py-imenu-show-method-args-p
+ py-imenu-method-arg-parens
+ py-imenu-method-no-arg-parens))
+ (goto-char (point-min))
+ ;; Warning: When the buffer has no classes or functions, this will
+ ;; return nil, which seems proper according to the Imenu API, but
+ ;; causes an error in the XEmacs port of Imenu. Sigh.
+ (py-imenu-create-index-engine nil))
+
+(defun py-imenu-create-index-engine (&optional start-indent)
+ "Function for finding Imenu definitions in Python.
+
+Finds all definitions (classes, methods, or functions) in a Python
+file for the Imenu package.
+
+Returns a possibly nested alist of the form
+
+ (INDEX-NAME . INDEX-POSITION)
+
+The second element of the alist may be an alist, producing a nested
+list as in
+
+ (INDEX-NAME . INDEX-ALIST)
+
+This function should not be called directly, as it calls itself
+recursively and requires some setup. Rather this is the engine for
+the function \\[py-imenu-create-index-function].
+
+It works recursively by looking for all definitions at the current
+indention level. When it finds one, it adds it to the alist. If it
+finds a definition at a greater indentation level, it removes the
+previous definition from the alist. In its place it adds all
+definitions found at the next indentation level. When it finds a
+definition that is less indented then the current level, it returns
+the alist it has created thus far.
+
+The optional argument START-INDENT indicates the starting indentation
+at which to continue looking for Python classes, methods, or
+functions. If this is not supplied, the function uses the indentation
+of the first definition found."
+ (let (index-alist
+ sub-method-alist
+ looking-p
+ def-name prev-name
+ cur-indent def-pos
+ (class-paren (first py-imenu-generic-parens))
+ (def-paren (second py-imenu-generic-parens)))
+ (setq looking-p
+ (re-search-forward py-imenu-generic-regexp (point-max) t))
+ (while looking-p
+ (save-excursion
+ ;; used to set def-name to this value but generic-extract-name
+ ;; is new to imenu-1.14. this way it still works with
+ ;; imenu-1.11
+ ;;(imenu--generic-extract-name py-imenu-generic-parens))
+ (let ((cur-paren (if (match-beginning class-paren)
+ class-paren def-paren)))
+ (setq def-name
+ (buffer-substring-no-properties (match-beginning cur-paren)
+ (match-end cur-paren))))
+ (save-match-data
+ (py-beginning-of-def-or-class 'either))
+ (beginning-of-line)
+ (setq cur-indent (current-indentation)))
+ ;; HACK: want to go to the next correct definition location. We
+ ;; explicitly list them here but it would be better to have them
+ ;; in a list.
+ (setq def-pos
+ (or (match-beginning class-paren)
+ (match-beginning def-paren)))
+ ;; if we don't have a starting indent level, take this one
+ (or start-indent
+ (setq start-indent cur-indent))
+ ;; if we don't have class name yet, take this one
+ (or prev-name
+ (setq prev-name def-name))
+ ;; what level is the next definition on? must be same, deeper
+ ;; or shallower indentation
+ (cond
+ ;; Skip code in comments and strings
+ ((py-in-literal))
+ ;; at the same indent level, add it to the list...
+ ((= start-indent cur-indent)
+ (push (cons def-name def-pos) index-alist))
+ ;; deeper indented expression, recurse
+ ((< start-indent cur-indent)
+ ;; the point is currently on the expression we're supposed to
+ ;; start on, so go back to the last expression. The recursive
+ ;; call will find this place again and add it to the correct
+ ;; list
+ (re-search-backward py-imenu-generic-regexp (point-min) 'move)
+ (setq sub-method-alist (py-imenu-create-index-engine cur-indent))
+ (if sub-method-alist
+ ;; we put the last element on the index-alist on the start
+ ;; of the submethod alist so the user can still get to it.
+ (let ((save-elmt (pop index-alist)))
+ (push (cons prev-name
+ (cons save-elmt sub-method-alist))
+ index-alist))))
+ ;; found less indented expression, we're done.
+ (t
+ (setq looking-p nil)
+ (re-search-backward py-imenu-generic-regexp (point-min) t)))
+ ;; end-cond
+ (setq prev-name def-name)
+ (and looking-p
+ (setq looking-p
+ (re-search-forward py-imenu-generic-regexp
+ (point-max) 'move))))
+ (nreverse index-alist)))
+
+
+
+(defun py-choose-shell-by-shebang ()
+ "Choose CPython or Jython mode by looking at #! on the first line.
+Returns the appropriate mode function.
+Used by `py-choose-shell', and similar to but distinct from
+`set-auto-mode', though it uses `auto-mode-interpreter-regexp' (if available)."
+ ;; look for an interpreter specified in the first line
+ ;; similar to set-auto-mode (files.el)
+ (let* ((re (if (boundp 'auto-mode-interpreter-regexp)
+ auto-mode-interpreter-regexp
+ ;; stolen from Emacs 21.2
+ "#![ \t]?\\([^ \t\n]*/bin/env[ \t]\\)?\\([^ \t\n]+\\)"))
+ (interpreter (save-excursion
+ (goto-char (point-min))
+ (if (looking-at re)
+ (match-string 2)
+ "")))
+ elt)
+ ;; Map interpreter name to a mode.
+ (setq elt (assoc (file-name-nondirectory interpreter)
+ py-shell-alist))
+ (and elt (caddr elt))))
+
+
+
+(defun py-choose-shell-by-import ()
+ "Choose CPython or Jython mode based imports.
+If a file imports any packages in `py-jython-packages', within
+`py-import-check-point-max' characters from the start of the file,
+return `jython', otherwise return nil."
+ (let (mode)
+ (save-excursion
+ (goto-char (point-min))
+ (while (and (not mode)
+ (search-forward-regexp
+ "^\\(\\(from\\)\\|\\(import\\)\\) \\([^ \t\n.]+\\)"
+ py-import-check-point-max t))
+ (setq mode (and (member (match-string 4) py-jython-packages)
+ 'jython
+ ))))
+ mode))
+
+
+(defun py-choose-shell ()
+ "Choose CPython or Jython mode. Returns the appropriate mode function.
+This does the following:
+ - look for an interpreter with `py-choose-shell-by-shebang'
+ - examine imports using `py-choose-shell-by-import'
+ - default to the variable `py-default-interpreter'"
+ (interactive)
+ (or (py-choose-shell-by-shebang)
+ (py-choose-shell-by-import)
+ py-default-interpreter
+; 'cpython ;; don't use to py-default-interpreter, because default
+; ;; is only way to choose CPython
+ ))
+
+
+;;;###autoload
+(defun python-mode ()
+ "Major mode for editing Python files.
+To submit a problem report, enter `\\[py-submit-bug-report]' from a
+`python-mode' buffer. Do `\\[py-describe-mode]' for detailed
+documentation. To see what version of `python-mode' you are running,
+enter `\\[py-version]'.
+
+This mode knows about Python indentation, tokens, comments and
+continuation lines. Paragraphs are separated by blank lines only.
+
+COMMANDS
+\\{py-mode-map}
+VARIABLES
+
+py-indent-offset\t\tindentation increment
+py-block-comment-prefix\t\tcomment string used by `comment-region'
+py-python-command\t\tshell command to invoke Python interpreter
+py-temp-directory\t\tdirectory used for temp files (if needed)
+py-beep-if-tab-change\t\tring the bell if `tab-width' is changed"
+ (interactive)
+ ;; set up local variables
+ (kill-all-local-variables)
+ (make-local-variable 'font-lock-defaults)
+ (make-local-variable 'paragraph-separate)
+ (make-local-variable 'paragraph-start)
+ (make-local-variable 'require-final-newline)
+ (make-local-variable 'comment-start)
+ (make-local-variable 'comment-end)
+ (make-local-variable 'comment-start-skip)
+ (make-local-variable 'comment-column)
+ (make-local-variable 'comment-indent-function)
+ (make-local-variable 'indent-region-function)
+ (make-local-variable 'indent-line-function)
+ (make-local-variable 'add-log-current-defun-function)
+ (make-local-variable 'fill-paragraph-function)
+ ;;
+ (set-syntax-table py-mode-syntax-table)
+ (setq major-mode 'python-mode
+ mode-name "Python"
+ local-abbrev-table python-mode-abbrev-table
+ font-lock-defaults '(python-font-lock-keywords)
+ paragraph-separate "^[ \t]*$"
+ paragraph-start "^[ \t]*$"
+ require-final-newline t
+ comment-start "# "
+ comment-end ""
+ comment-start-skip "# *"
+ comment-column 40
+ comment-indent-function 'py-comment-indent-function
+ indent-region-function 'py-indent-region
+ indent-line-function 'py-indent-line
+ ;; tell add-log.el how to find the current function/method/variable
+ add-log-current-defun-function 'py-current-defun
+
+ fill-paragraph-function 'py-fill-paragraph
+ )
+ (use-local-map py-mode-map)
+ ;; add the menu
+ (if py-menu
+ (easy-menu-add py-menu))
+ ;; Emacs 19 requires this
+ (if (boundp 'comment-multi-line)
+ (setq comment-multi-line nil))
+ ;; Install Imenu if available
+ (when (py-safe (require 'imenu))
+ (setq imenu-create-index-function #'py-imenu-create-index-function)
+ (setq imenu-generic-expression py-imenu-generic-expression)
+ (if (fboundp 'imenu-add-to-menubar)
+ (imenu-add-to-menubar (format "%s-%s" "IM" mode-name)))
+ )
+ ;; Run the mode hook. Note that py-mode-hook is deprecated.
+ (if python-mode-hook
+ (run-hooks 'python-mode-hook)
+ (run-hooks 'py-mode-hook))
+ ;; Now do the automagical guessing
+ (if py-smart-indentation
+ (let ((offset py-indent-offset))
+ ;; It's okay if this fails to guess a good value
+ (if (and (py-safe (py-guess-indent-offset))
+ (<= py-indent-offset 8)
+ (>= py-indent-offset 2))
+ (setq offset py-indent-offset))
+ (setq py-indent-offset offset)
+ ;; Only turn indent-tabs-mode off if tab-width !=
+ ;; py-indent-offset. Never turn it on, because the user must
+ ;; have explicitly turned it off.
+ (if (/= tab-width py-indent-offset)
+ (setq indent-tabs-mode nil))
+ ))
+ ;; Set the default shell if not already set
+ (when (null py-which-shell)
+ (py-toggle-shells (py-choose-shell))))
+
+
+(make-obsolete 'jpython-mode 'jython-mode)
+(defun jython-mode ()
+ "Major mode for editing Jython/Jython files.
+This is a simple wrapper around `python-mode'.
+It runs `jython-mode-hook' then calls `python-mode.'
+It is added to `interpreter-mode-alist' and `py-choose-shell'.
+"
+ (interactive)
+ (python-mode)
+ (py-toggle-shells 'jython)
+ (when jython-mode-hook
+ (run-hooks 'jython-mode-hook)))
+
+
+;; It's handy to add recognition of Python files to the
+;; interpreter-mode-alist and to auto-mode-alist. With the former, we
+;; can specify different `derived-modes' based on the #! line, but
+;; with the latter, we can't. So we just won't add them if they're
+;; already added.
+;;;###autoload
+(let ((modes '(("jython" . jython-mode)
+ ("python" . python-mode))))
+ (while modes
+ (when (not (assoc (car modes) interpreter-mode-alist))
+ (push (car modes) interpreter-mode-alist))
+ (setq modes (cdr modes))))
+;;;###autoload
+(when (not (or (rassq 'python-mode auto-mode-alist)
+ (rassq 'jython-mode auto-mode-alist)))
+ (push '("\\.py$" . python-mode) auto-mode-alist))
+
+
+
+;; electric characters
+(defun py-outdent-p ()
+ "Returns non-nil if the current line should dedent one level."
+ (save-excursion
+ (and (progn (back-to-indentation)
+ (looking-at py-outdent-re))
+ ;; short circuit infloop on illegal construct
+ (not (bobp))
+ (progn (forward-line -1)
+ (py-goto-initial-line)
+ (back-to-indentation)
+ (while (or (looking-at py-blank-or-comment-re)
+ (bobp))
+ (backward-to-indentation 1))
+ (not (looking-at py-no-outdent-re)))
+ )))
+
+(defun py-electric-colon (arg)
+ "Insert a colon.
+In certain cases the line is dedented appropriately. If a numeric
+argument ARG is provided, that many colons are inserted
+non-electrically. Electric behavior is inhibited inside a string or
+comment."
+ (interactive "*P")
+ (self-insert-command (prefix-numeric-value arg))
+ ;; are we in a string or comment?
+ (if (save-excursion
+ (let ((pps (parse-partial-sexp (save-excursion
+ (py-beginning-of-def-or-class)
+ (point))
+ (point))))
+ (not (or (nth 3 pps) (nth 4 pps)))))
+ (save-excursion
+ (let ((here (point))
+ (outdent 0)
+ (indent (py-compute-indentation t)))
+ (if (and (not arg)
+ (py-outdent-p)
+ (= indent (save-excursion
+ (py-next-statement -1)
+ (py-compute-indentation t)))
+ )
+ (setq outdent py-indent-offset))
+ ;; Don't indent, only dedent. This assumes that any lines
+ ;; that are already dedented relative to
+ ;; py-compute-indentation were put there on purpose. It's
+ ;; highly annoying to have `:' indent for you. Use TAB, C-c
+ ;; C-l or C-c C-r to adjust. TBD: Is there a better way to
+ ;; determine this???
+ (if (< (current-indentation) indent) nil
+ (goto-char here)
+ (beginning-of-line)
+ (delete-horizontal-space)
+ (indent-to (- indent outdent))
+ )))))
+
+
+;; Python subprocess utilities and filters
+(defun py-execute-file (proc filename)
+ "Send to Python interpreter process PROC \"execfile('FILENAME')\".
+Make that process's buffer visible and force display. Also make
+comint believe the user typed this string so that
+`kill-output-from-shell' does The Right Thing."
+ (let ((curbuf (current-buffer))
+ (procbuf (process-buffer proc))
+; (comint-scroll-to-bottom-on-output t)
+ (msg (format "## working on region in file %s...\n" filename))
+ ;; add some comment, so that we can filter it out of history
+ (cmd (format "execfile(r'%s') # PYTHON-MODE\n" filename)))
+ (unwind-protect
+ (save-excursion
+ (set-buffer procbuf)
+ (goto-char (point-max))
+ (move-marker (process-mark proc) (point))
+ (funcall (process-filter proc) proc msg))
+ (set-buffer curbuf))
+ (process-send-string proc cmd)))
+
+(defun py-comint-output-filter-function (string)
+ "Watch output for Python prompt and exec next file waiting in queue.
+This function is appropriate for `comint-output-filter-functions'."
+ ;;remove ansi terminal escape sequences from string, not sure why they are
+ ;;still around...
+ (setq string (ansi-color-filter-apply string))
+ (when (and (string-match py-shell-input-prompt-1-regexp string)
+ py-file-queue)
+ (if py-shell-switch-buffers-on-execute
+ (pop-to-buffer (current-buffer)))
+ (py-safe (delete-file (car py-file-queue)))
+ (setq py-file-queue (cdr py-file-queue))
+ (if py-file-queue
+ (let ((pyproc (get-buffer-process (current-buffer))))
+ (py-execute-file pyproc (car py-file-queue))))
+ ))
+
+(defun py-pdbtrack-overlay-arrow (activation)
+ "Activate or de arrow at beginning-of-line in current buffer."
+ ;; This was derived/simplified from edebug-overlay-arrow
+ (cond (activation
+ (setq overlay-arrow-position (make-marker))
+ (setq overlay-arrow-string "=>")
+ (set-marker overlay-arrow-position (py-point 'bol) (current-buffer))
+ (setq py-pdbtrack-is-tracking-p t))
+ (overlay-arrow-position
+ (setq overlay-arrow-position nil)
+ (setq py-pdbtrack-is-tracking-p nil))
+ ))
+
+(defun py-pdbtrack-track-stack-file (text)
+ "Show the file indicated by the pdb stack entry line, in a separate window.
+
+Activity is disabled if the buffer-local variable
+`py-pdbtrack-do-tracking-p' is nil.
+
+We depend on the pdb input prompt matching `py-pdbtrack-input-prompt'
+at the beginning of the line.
+
+If the traceback target file path is invalid, we look for the most
+recently visited python-mode buffer which either has the name of the
+current function \(or class) or which defines the function \(or
+class). This is to provide for remote scripts, eg, Zope's 'Script
+(Python)' - put a _copy_ of the script in a buffer named for the
+script, and set to python-mode, and pdbtrack will find it.)"
+ ;; Instead of trying to piece things together from partial text
+ ;; (which can be almost useless depending on Emacs version), we
+ ;; monitor to the point where we have the next pdb prompt, and then
+ ;; check all text from comint-last-input-end to process-mark.
+ ;;
+ ;; Also, we're very conservative about clearing the overlay arrow,
+ ;; to minimize residue. This means, for instance, that executing
+ ;; other pdb commands wipe out the highlight. You can always do a
+ ;; 'where' (aka 'w') command to reveal the overlay arrow.
+ (let* ((origbuf (current-buffer))
+ (currproc (get-buffer-process origbuf)))
+
+ (if (not (and currproc py-pdbtrack-do-tracking-p))
+ (py-pdbtrack-overlay-arrow nil)
+
+ (let* ((procmark (process-mark currproc))
+ (block (buffer-substring (max comint-last-input-end
+ (- procmark
+ py-pdbtrack-track-range))
+ procmark))
+ target target_fname target_lineno target_buffer)
+
+ (if (not (string-match (concat py-pdbtrack-input-prompt "$") block))
+ (py-pdbtrack-overlay-arrow nil)
+
+ (setq target (py-pdbtrack-get-source-buffer block))
+
+ (if (stringp target)
+ (message "pdbtrack: %s" target)
+
+ (setq target_lineno (car target))
+ (setq target_buffer (cadr target))
+ (setq target_fname (buffer-file-name target_buffer))
+ (switch-to-buffer-other-window target_buffer)
+ (goto-line target_lineno)
+ (message "pdbtrack: line %s, file %s" target_lineno target_fname)
+ (py-pdbtrack-overlay-arrow t)
+ (pop-to-buffer origbuf t)
+
+ )))))
+ )
+
+(defun py-pdbtrack-get-source-buffer (block)
+ "Return line number and buffer of code indicated by block's traceback text.
+
+We look first to visit the file indicated in the trace.
+
+Failing that, we look for the most recently visited python-mode buffer
+with the same name or having the named function.
+
+If we're unable find the source code we return a string describing the
+problem as best as we can determine."
+
+ (if (not (string-match py-pdbtrack-stack-entry-regexp block))
+
+ "Traceback cue not found"
+
+ (let* ((filename (match-string 1 block))
+ (lineno (string-to-int (match-string 2 block)))
+ (funcname (match-string 3 block))
+ funcbuffer)
+
+ (cond ((file-exists-p filename)
+ (list lineno (find-file-noselect filename)))
+
+ ((setq funcbuffer (py-pdbtrack-grub-for-buffer funcname lineno))
+ (if (string-match "/Script (Python)$" filename)
+ ;; Add in number of lines for leading '##' comments:
+ (setq lineno
+ (+ lineno
+ (save-excursion
+ (set-buffer funcbuffer)
+ (count-lines
+ (point-min)
+ (max (point-min)
+ (string-match "^\\([^#]\\|#[^#]\\|#$\\)"
+ (buffer-substring (point-min)
+ (point-max)))
+ ))))))
+ (list lineno funcbuffer))
+
+ ((= (elt filename 0) ?\<)
+ (format "(Non-file source: '%s')" filename))
+
+ (t (format "Not found: %s(), %s" funcname filename)))
+ )
+ )
+ )
+
+(defun py-pdbtrack-grub-for-buffer (funcname lineno)
+ "Find most recent buffer itself named or having function funcname.
+
+We walk the buffer-list history for python-mode buffers that are
+named for funcname or define a function funcname."
+ (let ((buffers (buffer-list))
+ buf
+ got)
+ (while (and buffers (not got))
+ (setq buf (car buffers)
+ buffers (cdr buffers))
+ (if (and (save-excursion (set-buffer buf)
+ (string= major-mode "python-mode"))
+ (or (string-match funcname (buffer-name buf))
+ (string-match (concat "^\\s-*\\(def\\|class\\)\\s-+"
+ funcname "\\s-*(")
+ (save-excursion
+ (set-buffer buf)
+ (buffer-substring (point-min)
+ (point-max))))))
+ (setq got buf)))
+ got))
+
+(defun py-postprocess-output-buffer (buf)
+ "Highlight exceptions found in BUF.
+If an exception occurred return t, otherwise return nil. BUF must exist."
+ (let (line file bol err-p)
+ (save-excursion
+ (set-buffer buf)
+ (beginning-of-buffer)
+ (while (re-search-forward py-traceback-line-re nil t)
+ (setq file (match-string 1)
+ line (string-to-int (match-string 2))
+ bol (py-point 'bol))
+ (py-highlight-line bol (py-point 'eol) file line)))
+ (when (and py-jump-on-exception line)
+ (beep)
+ (py-jump-to-exception file line)
+ (setq err-p t))
+ err-p))
+
+
+
+;;; Subprocess commands
+
+;; only used when (memq 'broken-temp-names py-emacs-features)
+(defvar py-serial-number 0)
+(defvar py-exception-buffer nil)
+(defconst py-output-buffer "*Python Output*")
+(make-variable-buffer-local 'py-output-buffer)
+
+;; for toggling between CPython and Jython
+(defvar py-which-shell nil)
+(defvar py-which-args py-python-command-args)
+(defvar py-which-bufname "Python")
+(make-variable-buffer-local 'py-which-shell)
+(make-variable-buffer-local 'py-which-args)
+(make-variable-buffer-local 'py-which-bufname)
+
+(defun py-toggle-shells (arg)
+ "Toggles between the CPython and Jython shells.
+
+With positive argument ARG (interactively \\[universal-argument]),
+uses the CPython shell, with negative ARG uses the Jython shell, and
+with a zero argument, toggles the shell.
+
+Programmatically, ARG can also be one of the symbols `cpython' or
+`jython', equivalent to positive arg and negative arg respectively."
+ (interactive "P")
+ ;; default is to toggle
+ (if (null arg)
+ (setq arg 0))
+ ;; preprocess arg
+ (cond
+ ((equal arg 0)
+ ;; toggle
+ (if (string-equal py-which-bufname "Python")
+ (setq arg -1)
+ (setq arg 1)))
+ ((equal arg 'cpython) (setq arg 1))
+ ((equal arg 'jython) (setq arg -1)))
+ (let (msg)
+ (cond
+ ((< 0 arg)
+ ;; set to CPython
+ (setq py-which-shell py-python-command
+ py-which-args py-python-command-args
+ py-which-bufname "Python"
+ msg "CPython")
+ (if (string-equal py-which-bufname "Jython")
+ (setq mode-name "Python")))
+ ((> 0 arg)
+ (setq py-which-shell py-jython-command
+ py-which-args py-jython-command-args
+ py-which-bufname "Jython"
+ msg "Jython")
+ (if (string-equal py-which-bufname "Python")
+ (setq mode-name "Jython")))
+ )
+ (message "Using the %s shell" msg)
+ (setq py-output-buffer (format "*%s Output*" py-which-bufname))))
+
+;;;###autoload
+(defun py-shell (&optional argprompt)
+ "Start an interactive Python interpreter in another window.
+This is like Shell mode, except that Python is running in the window
+instead of a shell. See the `Interactive Shell' and `Shell Mode'
+sections of the Emacs manual for details, especially for the key
+bindings active in the `*Python*' buffer.
+
+With optional \\[universal-argument], the user is prompted for the
+flags to pass to the Python interpreter. This has no effect when this
+command is used to switch to an existing process, only when a new
+process is started. If you use this, you will probably want to ensure
+that the current arguments are retained (they will be included in the
+prompt). This argument is ignored when this function is called
+programmatically, or when running in Emacs 19.34 or older.
+
+Note: You can toggle between using the CPython interpreter and the
+Jython interpreter by hitting \\[py-toggle-shells]. This toggles
+buffer local variables which control whether all your subshell
+interactions happen to the `*Jython*' or `*Python*' buffers (the
+latter is the name used for the CPython buffer).
+
+Warning: Don't use an interactive Python if you change sys.ps1 or
+sys.ps2 from their default values, or if you're running code that
+prints `>>> ' or `... ' at the start of a line. `python-mode' can't
+distinguish your output from Python's output, and assumes that `>>> '
+at the start of a line is a prompt from Python. Similarly, the Emacs
+Shell mode code assumes that both `>>> ' and `... ' at the start of a
+line are Python prompts. Bad things can happen if you fool either
+mode.
+
+Warning: If you do any editing *in* the process buffer *while* the
+buffer is accepting output from Python, do NOT attempt to `undo' the
+changes. Some of the output (nowhere near the parts you changed!) may
+be lost if you do. This appears to be an Emacs bug, an unfortunate
+interaction between undo and process filters; the same problem exists in
+non-Python process buffers using the default (Emacs-supplied) process
+filter."
+ (interactive "P")
+ ;; Set the default shell if not already set
+ (when (null py-which-shell)
+ (py-toggle-shells py-default-interpreter))
+ (let ((args py-which-args))
+ (when (and argprompt
+ (interactive-p)
+ (fboundp 'split-string))
+ ;; TBD: Perhaps force "-i" in the final list?
+ (setq args (split-string
+ (read-string (concat py-which-bufname
+ " arguments: ")
+ (concat
+ (mapconcat 'identity py-which-args " ") " ")
+ ))))
+ (if (not (equal (buffer-name) "*Python*"))
+ (switch-to-buffer-other-window
+ (apply 'make-comint py-which-bufname py-which-shell nil args))
+ (apply 'make-comint py-which-bufname py-which-shell nil args))
+ (make-local-variable 'comint-prompt-regexp)
+ (setq comint-prompt-regexp (concat py-shell-input-prompt-1-regexp "\\|"
+ py-shell-input-prompt-2-regexp "\\|"
+ "^([Pp]db) "))
+ (add-hook 'comint-output-filter-functions
+ 'py-comint-output-filter-function)
+ ;; pdbtrack
+ (add-hook 'comint-output-filter-functions 'py-pdbtrack-track-stack-file)
+ (setq py-pdbtrack-do-tracking-p t)
+ (set-syntax-table py-mode-syntax-table)
+ (use-local-map py-shell-map)
+ (run-hooks 'py-shell-hook)
+ ))
+
+(defun py-clear-queue ()
+ "Clear the queue of temporary files waiting to execute."
+ (interactive)
+ (let ((n (length py-file-queue)))
+ (mapcar 'delete-file py-file-queue)
+ (setq py-file-queue nil)
+ (message "%d pending files de-queued." n)))
+
+
+(defun py-execute-region (start end &optional async)
+ "Execute the region in a Python interpreter.
+
+The region is first copied into a temporary file (in the directory
+`py-temp-directory'). If there is no Python interpreter shell
+running, this file is executed synchronously using
+`shell-command-on-region'. If the program is long running, use
+\\[universal-argument] to run the command asynchronously in its own
+buffer.
+
+When this function is used programmatically, arguments START and END
+specify the region to execute, and optional third argument ASYNC, if
+non-nil, specifies to run the command asynchronously in its own
+buffer.
+
+If the Python interpreter shell is running, the region is execfile()'d
+in that shell. If you try to execute regions too quickly,
+`python-mode' will queue them up and execute them one at a time when
+it sees a `>>> ' prompt from Python. Each time this happens, the
+process buffer is popped into a window (if it's not already in some
+window) so you can see it, and a comment of the form
+
+ \t## working on region in file <name>...
+
+is inserted at the end. See also the command `py-clear-queue'."
+ (interactive "r\nP")
+ ;; Skip ahead to the first non-blank line
+ (let* ((proc (get-process py-which-bufname))
+ (temp (if (memq 'broken-temp-names py-emacs-features)
+ (let
+ ((sn py-serial-number)
+ (pid (and (fboundp 'emacs-pid) (emacs-pid))))
+ (setq py-serial-number (1+ py-serial-number))
+ (if pid
+ (format "python-%d-%d" sn pid)
+ (format "python-%d" sn)))
+ (make-temp-name "python-")))
+ (file (concat (expand-file-name temp py-temp-directory) ".py"))
+ (cur (current-buffer))
+ (buf (get-buffer-create file))
+ shell)
+ ;; Write the contents of the buffer, watching out for indented regions.
+ (save-excursion
+ (goto-char start)
+ (beginning-of-line)
+ (while (and (looking-at "\\s *$")
+ (< (point) end))
+ (forward-line 1))
+ (setq start (point))
+ (or (< start end)
+ (error "Region is empty"))
+ (setq py-line-number-offset (count-lines 1 start))
+ (let ((needs-if (/= (py-point 'bol) (py-point 'boi))))
+ (set-buffer buf)
+ (python-mode)
+ (when needs-if
+ (insert "if 1:\n")
+ (setq py-line-number-offset (- py-line-number-offset 1)))
+ (insert-buffer-substring cur start end)
+ ;; Set the shell either to the #! line command, or to the
+ ;; py-which-shell buffer local variable.
+ (setq shell (or (py-choose-shell-by-shebang)
+ (py-choose-shell-by-import)
+ py-which-shell))))
+ (cond
+ ;; always run the code in its own asynchronous subprocess
+ (async
+ ;; User explicitly wants this to run in its own async subprocess
+ (save-excursion
+ (set-buffer buf)
+ (write-region (point-min) (point-max) file nil 'nomsg))
+ (let* ((buf (generate-new-buffer-name py-output-buffer))
+ ;; TBD: a horrible hack, but why create new Custom variables?
+ (arg (if (string-equal py-which-bufname "Python")
+ "-u" "")))
+ (start-process py-which-bufname buf shell arg file)
+ (pop-to-buffer buf)
+ (py-postprocess-output-buffer buf)
+ ;; TBD: clean up the temporary file!
+ ))
+ ;; if the Python interpreter shell is running, queue it up for
+ ;; execution there.
+ (proc
+ ;; use the existing python shell
+ (save-excursion
+ (set-buffer buf)
+ (write-region (point-min) (point-max) file nil 'nomsg))
+ (if (not py-file-queue)
+ (py-execute-file proc file)
+ (message "File %s queued for execution" file))
+ (setq py-file-queue (append py-file-queue (list file)))
+ (setq py-exception-buffer (cons file (current-buffer))))
+ (t
+ ;; TBD: a horrible hack, but why create new Custom variables?
+ (let ((cmd (concat py-which-shell (if (string-equal py-which-bufname
+ "Jython")
+ " -" ""))))
+ ;; otherwise either run it synchronously in a subprocess
+ (save-excursion
+ (set-buffer buf)
+ (shell-command-on-region (point-min) (point-max)
+ cmd py-output-buffer))
+ ;; shell-command-on-region kills the output buffer if it never
+ ;; existed and there's no output from the command
+ (if (not (get-buffer py-output-buffer))
+ (message "No output.")
+ (setq py-exception-buffer (current-buffer))
+ (let ((err-p (py-postprocess-output-buffer py-output-buffer)))
+ (pop-to-buffer py-output-buffer)
+ (if err-p
+ (pop-to-buffer py-exception-buffer)))
+ ))
+ ))
+ ;; Clean up after ourselves.
+ (kill-buffer buf)))
+
+
+;; Code execution commands
+(defun py-execute-buffer (&optional async)
+ "Send the contents of the buffer to a Python interpreter.
+If the file local variable `py-master-file' is non-nil, execute the
+named file instead of the buffer's file.
+
+If there is a *Python* process buffer it is used. If a clipping
+restriction is in effect, only the accessible portion of the buffer is
+sent. A trailing newline will be supplied if needed.
+
+See the `\\[py-execute-region]' docs for an account of some
+subtleties, including the use of the optional ASYNC argument."
+ (interactive "P")
+ (let ((old-buffer (current-buffer)))
+ (if py-master-file
+ (let* ((filename (expand-file-name py-master-file))
+ (buffer (or (get-file-buffer filename)
+ (find-file-noselect filename))))
+ (set-buffer buffer)))
+ (py-execute-region (point-min) (point-max) async)
+ (pop-to-buffer old-buffer)))
+
+(defun py-execute-import-or-reload (&optional async)
+ "Import the current buffer's file in a Python interpreter.
+
+If the file has already been imported, then do reload instead to get
+the latest version.
+
+If the file's name does not end in \".py\", then do execfile instead.
+
+If the current buffer is not visiting a file, do `py-execute-buffer'
+instead.
+
+If the file local variable `py-master-file' is non-nil, import or
+reload the named file instead of the buffer's file. The file may be
+saved based on the value of `py-execute-import-or-reload-save-p'.
+
+See the `\\[py-execute-region]' docs for an account of some
+subtleties, including the use of the optional ASYNC argument.
+
+This may be preferable to `\\[py-execute-buffer]' because:
+
+ - Definitions stay in their module rather than appearing at top
+ level, where they would clutter the global namespace and not affect
+ uses of qualified names (MODULE.NAME).
+
+ - The Python debugger gets line number information about the functions."
+ (interactive "P")
+ ;; Check file local variable py-master-file
+ (if py-master-file
+ (let* ((filename (expand-file-name py-master-file))
+ (buffer (or (get-file-buffer filename)
+ (find-file-noselect filename))))
+ (set-buffer buffer)))
+ (let ((file (buffer-file-name (current-buffer))))
+ (if file
+ (progn
+ ;; Maybe save some buffers
+ (save-some-buffers (not py-ask-about-save) nil)
+ (py-execute-string
+ (if (string-match "\\.py$" file)
+ (let ((f (file-name-sans-extension
+ (file-name-nondirectory file))))
+ (format "if globals().has_key('%s'):\n reload(%s)\nelse:\n import %s\n"
+ f f f))
+ (format "execfile(r'%s')\n" file))
+ async))
+ ;; else
+ (py-execute-buffer async))))
+
+
+(defun py-execute-def-or-class (&optional async)
+ "Send the current function or class definition to a Python interpreter.
+
+If there is a *Python* process buffer it is used.
+
+See the `\\[py-execute-region]' docs for an account of some
+subtleties, including the use of the optional ASYNC argument."
+ (interactive "P")
+ (save-excursion
+ (py-mark-def-or-class)
+ ;; mark is before point
+ (py-execute-region (mark) (point) async)))
+
+
+(defun py-execute-string (string &optional async)
+ "Send the argument STRING to a Python interpreter.
+
+If there is a *Python* process buffer it is used.
+
+See the `\\[py-execute-region]' docs for an account of some
+subtleties, including the use of the optional ASYNC argument."
+ (interactive "sExecute Python command: ")
+ (save-excursion
+ (set-buffer (get-buffer-create
+ (generate-new-buffer-name " *Python Command*")))
+ (insert string)
+ (py-execute-region (point-min) (point-max) async)))
+
+
+
+(defun py-jump-to-exception (file line)
+ "Jump to the Python code in FILE at LINE."
+ (let ((buffer (cond ((string-equal file "<stdin>")
+ (if (consp py-exception-buffer)
+ (cdr py-exception-buffer)
+ py-exception-buffer))
+ ((and (consp py-exception-buffer)
+ (string-equal file (car py-exception-buffer)))
+ (cdr py-exception-buffer))
+ ((py-safe (find-file-noselect file)))
+ ;; could not figure out what file the exception
+ ;; is pointing to, so prompt for it
+ (t (find-file (read-file-name "Exception file: "
+ nil
+ file t))))))
+ ;; Fiddle about with line number
+ (setq line (+ py-line-number-offset line))
+
+ (pop-to-buffer buffer)
+ ;; Force Python mode
+ (if (not (eq major-mode 'python-mode))
+ (python-mode))
+ (goto-line line)
+ (message "Jumping to exception in file %s on line %d" file line)))
+
+(defun py-mouseto-exception (event)
+ "Jump to the code which caused the Python exception at EVENT.
+EVENT is usually a mouse click."
+ (interactive "e")
+ (cond
+ ((fboundp 'event-point)
+ ;; XEmacs
+ (let* ((point (event-point event))
+ (buffer (event-buffer event))
+ (e (and point buffer (extent-at point buffer 'py-exc-info)))
+ (info (and e (extent-property e 'py-exc-info))))
+ (message "Event point: %d, info: %s" point info)
+ (and info
+ (py-jump-to-exception (car info) (cdr info)))
+ ))
+ ;; Emacs -- Please port this!
+ ))
+
+(defun py-goto-exception ()
+ "Go to the line indicated by the traceback."
+ (interactive)
+ (let (file line)
+ (save-excursion
+ (beginning-of-line)
+ (if (looking-at py-traceback-line-re)
+ (setq file (match-string 1)
+ line (string-to-int (match-string 2)))))
+ (if (not file)
+ (error "Not on a traceback line"))
+ (py-jump-to-exception file line)))
+
+(defun py-find-next-exception (start buffer searchdir errwhere)
+ "Find the next Python exception and jump to the code that caused it.
+START is the buffer position in BUFFER from which to begin searching
+for an exception. SEARCHDIR is a function, either
+`re-search-backward' or `re-search-forward' indicating the direction
+to search. ERRWHERE is used in an error message if the limit (top or
+bottom) of the trackback stack is encountered."
+ (let (file line)
+ (save-excursion
+ (set-buffer buffer)
+ (goto-char (py-point start))
+ (if (funcall searchdir py-traceback-line-re nil t)
+ (setq file (match-string 1)
+ line (string-to-int (match-string 2)))))
+ (if (and file line)
+ (py-jump-to-exception file line)
+ (error "%s of traceback" errwhere))))
+
+(defun py-down-exception (&optional bottom)
+ "Go to the next line down in the traceback.
+With \\[univeral-argument] (programmatically, optional argument
+BOTTOM), jump to the bottom (innermost) exception in the exception
+stack."
+ (interactive "P")
+ (let* ((proc (get-process "Python"))
+ (buffer (if proc "*Python*" py-output-buffer)))
+ (if bottom
+ (py-find-next-exception 'eob buffer 're-search-backward "Bottom")
+ (py-find-next-exception 'eol buffer 're-search-forward "Bottom"))))
+
+(defun py-up-exception (&optional top)
+ "Go to the previous line up in the traceback.
+With \\[universal-argument] (programmatically, optional argument TOP)
+jump to the top (outermost) exception in the exception stack."
+ (interactive "P")
+ (let* ((proc (get-process "Python"))
+ (buffer (if proc "*Python*" py-output-buffer)))
+ (if top
+ (py-find-next-exception 'bob buffer 're-search-forward "Top")
+ (py-find-next-exception 'bol buffer 're-search-backward "Top"))))
+
+
+;; Electric deletion
+(defun py-electric-backspace (arg)
+ "Delete preceding character or levels of indentation.
+Deletion is performed by calling the function in `py-backspace-function'
+with a single argument (the number of characters to delete).
+
+If point is at the leftmost column, delete the preceding newline.
+
+Otherwise, if point is at the leftmost non-whitespace character of a
+line that is neither a continuation line nor a non-indenting comment
+line, or if point is at the end of a blank line, this command reduces
+the indentation to match that of the line that opened the current
+block of code. The line that opened the block is displayed in the
+echo area to help you keep track of where you are. With
+\\[universal-argument] dedents that many blocks (but not past column
+zero).
+
+Otherwise the preceding character is deleted, converting a tab to
+spaces if needed so that only a single column position is deleted.
+\\[universal-argument] specifies how many characters to delete;
+default is 1.
+
+When used programmatically, argument ARG specifies the number of
+blocks to dedent, or the number of characters to delete, as indicated
+above."
+ (interactive "*p")
+ (if (or (/= (current-indentation) (current-column))
+ (bolp)
+ (py-continuation-line-p)
+; (not py-honor-comment-indentation)
+; (looking-at "#[^ \t\n]") ; non-indenting #
+ )
+ (funcall py-backspace-function arg)
+ ;; else indent the same as the colon line that opened the block
+ ;; force non-blank so py-goto-block-up doesn't ignore it
+ (insert-char ?* 1)
+ (backward-char)
+ (let ((base-indent 0) ; indentation of base line
+ (base-text "") ; and text of base line
+ (base-found-p nil))
+ (save-excursion
+ (while (< 0 arg)
+ (condition-case nil ; in case no enclosing block
+ (progn
+ (py-goto-block-up 'no-mark)
+ (setq base-indent (current-indentation)
+ base-text (py-suck-up-leading-text)
+ base-found-p t))
+ (error nil))
+ (setq arg (1- arg))))
+ (delete-char 1) ; toss the dummy character
+ (delete-horizontal-space)
+ (indent-to base-indent)
+ (if base-found-p
+ (message "Closes block: %s" base-text)))))
+
+
+(defun py-electric-delete (arg)
+ "Delete preceding or following character or levels of whitespace.
+
+The behavior of this function depends on the variable
+`delete-key-deletes-forward'. If this variable is nil (or does not
+exist, as in older Emacsen and non-XEmacs versions), then this
+function behaves identically to \\[c-electric-backspace].
+
+If `delete-key-deletes-forward' is non-nil and is supported in your
+Emacs, then deletion occurs in the forward direction, by calling the
+function in `py-delete-function'.
+
+\\[universal-argument] (programmatically, argument ARG) specifies the
+number of characters to delete (default is 1)."
+ (interactive "*p")
+ (if (or (and (fboundp 'delete-forward-p) ;XEmacs 21
+ (delete-forward-p))
+ (and (boundp 'delete-key-deletes-forward) ;XEmacs 20
+ delete-key-deletes-forward))
+ (funcall py-delete-function arg)
+ (py-electric-backspace arg)))
+
+;; required for pending-del and delsel modes
+(put 'py-electric-colon 'delete-selection t) ;delsel
+(put 'py-electric-colon 'pending-delete t) ;pending-del
+(put 'py-electric-backspace 'delete-selection 'supersede) ;delsel
+(put 'py-electric-backspace 'pending-delete 'supersede) ;pending-del
+(put 'py-electric-delete 'delete-selection 'supersede) ;delsel
+(put 'py-electric-delete 'pending-delete 'supersede) ;pending-del
+
+
+
+(defun py-indent-line (&optional arg)
+ "Fix the indentation of the current line according to Python rules.
+With \\[universal-argument] (programmatically, the optional argument
+ARG non-nil), ignore dedenting rules for block closing statements
+(e.g. return, raise, break, continue, pass)
+
+This function is normally bound to `indent-line-function' so
+\\[indent-for-tab-command] will call it."
+ (interactive "P")
+ (let* ((ci (current-indentation))
+ (move-to-indentation-p (<= (current-column) ci))
+ (need (py-compute-indentation (not arg)))
+ (cc (current-column)))
+ ;; dedent out a level if previous command was the same unless we're in
+ ;; column 1
+ (if (and (equal last-command this-command)
+ (/= cc 0))
+ (progn
+ (beginning-of-line)
+ (delete-horizontal-space)
+ (indent-to (* (/ (- cc 1) py-indent-offset) py-indent-offset)))
+ (progn
+ ;; see if we need to dedent
+ (if (py-outdent-p)
+ (setq need (- need py-indent-offset)))
+ (if (or py-tab-always-indent
+ move-to-indentation-p)
+ (progn (if (/= ci need)
+ (save-excursion
+ (beginning-of-line)
+ (delete-horizontal-space)
+ (indent-to need)))
+ (if move-to-indentation-p (back-to-indentation)))
+ (insert-tab))))))
+
+(defun py-newline-and-indent ()
+ "Strives to act like the Emacs `newline-and-indent'.
+This is just `strives to' because correct indentation can't be computed
+from scratch for Python code. In general, deletes the whitespace before
+point, inserts a newline, and takes an educated guess as to how you want
+the new line indented."
+ (interactive)
+ (let ((ci (current-indentation)))
+ (if (< ci (current-column)) ; if point beyond indentation
+ (newline-and-indent)
+ ;; else try to act like newline-and-indent "normally" acts
+ (beginning-of-line)
+ (insert-char ?\n 1)
+ (move-to-column ci))))
+
+(defun py-compute-indentation (honor-block-close-p)
+ "Compute Python indentation.
+When HONOR-BLOCK-CLOSE-P is non-nil, statements such as `return',
+`raise', `break', `continue', and `pass' force one level of
+dedenting."
+ (save-excursion
+ (beginning-of-line)
+ (let* ((bod (py-point 'bod))
+ (pps (parse-partial-sexp bod (point)))
+ (boipps (parse-partial-sexp bod (py-point 'boi)))
+ placeholder)
+ (cond
+ ;; are we inside a multi-line string or comment?
+ ((or (and (nth 3 pps) (nth 3 boipps))
+ (and (nth 4 pps) (nth 4 boipps)))
+ (save-excursion
+ (if (not py-align-multiline-strings-p) 0
+ ;; skip back over blank & non-indenting comment lines
+ ;; note: will skip a blank or non-indenting comment line
+ ;; that happens to be a continuation line too
+ (re-search-backward "^[ \t]*\\([^ \t\n#]\\|#[ \t\n]\\)" nil 'move)
+ (back-to-indentation)
+ (current-column))))
+ ;; are we on a continuation line?
+ ((py-continuation-line-p)
+ (let ((startpos (point))
+ (open-bracket-pos (py-nesting-level))
+ endpos searching found state)
+ (if open-bracket-pos
+ (progn
+ ;; align with first item in list; else a normal
+ ;; indent beyond the line with the open bracket
+ (goto-char (1+ open-bracket-pos)) ; just beyond bracket
+ ;; is the first list item on the same line?
+ (skip-chars-forward " \t")
+ (if (null (memq (following-char) '(?\n ?# ?\\)))
+ ; yes, so line up with it
+ (current-column)
+ ;; first list item on another line, or doesn't exist yet
+ (forward-line 1)
+ (while (and (< (point) startpos)
+ (looking-at "[ \t]*[#\n\\\\]")) ; skip noise
+ (forward-line 1))
+ (if (and (< (point) startpos)
+ (/= startpos
+ (save-excursion
+ (goto-char (1+ open-bracket-pos))
+ (forward-comment (point-max))
+ (point))))
+ ;; again mimic the first list item
+ (current-indentation)
+ ;; else they're about to enter the first item
+ (goto-char open-bracket-pos)
+ (setq placeholder (point))
+ (py-goto-initial-line)
+ (py-goto-beginning-of-tqs
+ (save-excursion (nth 3 (parse-partial-sexp
+ placeholder (point)))))
+ (+ (current-indentation) py-indent-offset))))
+
+ ;; else on backslash continuation line
+ (forward-line -1)
+ (if (py-continuation-line-p) ; on at least 3rd line in block
+ (current-indentation) ; so just continue the pattern
+ ;; else started on 2nd line in block, so indent more.
+ ;; if base line is an assignment with a start on a RHS,
+ ;; indent to 2 beyond the leftmost "="; else skip first
+ ;; chunk of non-whitespace characters on base line, + 1 more
+ ;; column
+ (end-of-line)
+ (setq endpos (point)
+ searching t)
+ (back-to-indentation)
+ (setq startpos (point))
+ ;; look at all "=" from left to right, stopping at first
+ ;; one not nested in a list or string
+ (while searching
+ (skip-chars-forward "^=" endpos)
+ (if (= (point) endpos)
+ (setq searching nil)
+ (forward-char 1)
+ (setq state (parse-partial-sexp startpos (point)))
+ (if (and (zerop (car state)) ; not in a bracket
+ (null (nth 3 state))) ; & not in a string
+ (progn
+ (setq searching nil) ; done searching in any case
+ (setq found
+ (not (or
+ (eq (following-char) ?=)
+ (memq (char-after (- (point) 2))
+ '(?< ?> ?!)))))))))
+ (if (or (not found) ; not an assignment
+ (looking-at "[ \t]*\\\\")) ; <=><spaces><backslash>
+ (progn
+ (goto-char startpos)
+ (skip-chars-forward "^ \t\n")))
+ ;; if this is a continuation for a block opening
+ ;; statement, add some extra offset.
+ (+ (current-column) (if (py-statement-opens-block-p)
+ py-continuation-offset 0)
+ 1)
+ ))))
+
+ ;; not on a continuation line
+ ((bobp) (current-indentation))
+
+ ;; Dfn: "Indenting comment line". A line containing only a
+ ;; comment, but which is treated like a statement for
+ ;; indentation calculation purposes. Such lines are only
+ ;; treated specially by the mode; they are not treated
+ ;; specially by the Python interpreter.
+
+ ;; The rules for indenting comment lines are a line where:
+ ;; - the first non-whitespace character is `#', and
+ ;; - the character following the `#' is whitespace, and
+ ;; - the line is dedented with respect to (i.e. to the left
+ ;; of) the indentation of the preceding non-blank line.
+
+ ;; The first non-blank line following an indenting comment
+ ;; line is given the same amount of indentation as the
+ ;; indenting comment line.
+
+ ;; All other comment-only lines are ignored for indentation
+ ;; purposes.
+
+ ;; Are we looking at a comment-only line which is *not* an
+ ;; indenting comment line? If so, we assume that it's been
+ ;; placed at the desired indentation, so leave it alone.
+ ;; Indenting comment lines are aligned as statements down
+ ;; below.
+ ((and (looking-at "[ \t]*#[^ \t\n]")
+ ;; NOTE: this test will not be performed in older Emacsen
+ (fboundp 'forward-comment)
+ (<= (current-indentation)
+ (save-excursion
+ (forward-comment (- (point-max)))
+ (current-indentation))))
+ (current-indentation))
+
+ ;; else indentation based on that of the statement that
+ ;; precedes us; use the first line of that statement to
+ ;; establish the base, in case the user forced a non-std
+ ;; indentation for the continuation lines (if any)
+ (t
+ ;; skip back over blank & non-indenting comment lines note:
+ ;; will skip a blank or non-indenting comment line that
+ ;; happens to be a continuation line too. use fast Emacs 19
+ ;; function if it's there.
+ (if (and (eq py-honor-comment-indentation nil)
+ (fboundp 'forward-comment))
+ (forward-comment (- (point-max)))
+ (let ((prefix-re (concat py-block-comment-prefix "[ \t]*"))
+ done)
+ (while (not done)
+ (re-search-backward "^[ \t]*\\([^ \t\n#]\\|#\\)" nil 'move)
+ (setq done (or (bobp)
+ (and (eq py-honor-comment-indentation t)
+ (save-excursion
+ (back-to-indentation)
+ (not (looking-at prefix-re))
+ ))
+ (and (not (eq py-honor-comment-indentation t))
+ (save-excursion
+ (back-to-indentation)
+ (and (not (looking-at prefix-re))
+ (or (looking-at "[^#]")
+ (not (zerop (current-column)))
+ ))
+ ))
+ ))
+ )))
+ ;; if we landed inside a string, go to the beginning of that
+ ;; string. this handles triple quoted, multi-line spanning
+ ;; strings.
+ (py-goto-beginning-of-tqs (nth 3 (parse-partial-sexp bod (point))))
+ ;; now skip backward over continued lines
+ (setq placeholder (point))
+ (py-goto-initial-line)
+ ;; we may *now* have landed in a TQS, so find the beginning of
+ ;; this string.
+ (py-goto-beginning-of-tqs
+ (save-excursion (nth 3 (parse-partial-sexp
+ placeholder (point)))))
+ (+ (current-indentation)
+ (if (py-statement-opens-block-p)
+ py-indent-offset
+ (if (and honor-block-close-p (py-statement-closes-block-p))
+ (- py-indent-offset)
+ 0)))
+ )))))
+
+(defun py-guess-indent-offset (&optional global)
+ "Guess a good value for, and change, `py-indent-offset'.
+
+By default, make a buffer-local copy of `py-indent-offset' with the
+new value, so that other Python buffers are not affected. With
+\\[universal-argument] (programmatically, optional argument GLOBAL),
+change the global value of `py-indent-offset'. This affects all
+Python buffers (that don't have their own buffer-local copy), both
+those currently existing and those created later in the Emacs session.
+
+Some people use a different value for `py-indent-offset' than you use.
+There's no excuse for such foolishness, but sometimes you have to deal
+with their ugly code anyway. This function examines the file and sets
+`py-indent-offset' to what it thinks it was when they created the
+mess.
+
+Specifically, it searches forward from the statement containing point,
+looking for a line that opens a block of code. `py-indent-offset' is
+set to the difference in indentation between that line and the Python
+statement following it. If the search doesn't succeed going forward,
+it's tried again going backward."
+ (interactive "P") ; raw prefix arg
+ (let (new-value
+ (start (point))
+ (restart (point))
+ (found nil)
+ colon-indent)
+ (py-goto-initial-line)
+ (while (not (or found (eobp)))
+ (when (and (re-search-forward ":[ \t]*\\($\\|[#\\]\\)" nil 'move)
+ (not (py-in-literal restart)))
+ (setq restart (point))
+ (py-goto-initial-line)
+ (if (py-statement-opens-block-p)
+ (setq found t)
+ (goto-char restart))))
+ (unless found
+ (goto-char start)
+ (py-goto-initial-line)
+ (while (not (or found (bobp)))
+ (setq found (and
+ (re-search-backward ":[ \t]*\\($\\|[#\\]\\)" nil 'move)
+ (or (py-goto-initial-line) t) ; always true -- side effect
+ (py-statement-opens-block-p)))))
+ (setq colon-indent (current-indentation)
+ found (and found (zerop (py-next-statement 1)))
+ new-value (- (current-indentation) colon-indent))
+ (goto-char start)
+ (if (not found)
+ (error "Sorry, couldn't guess a value for py-indent-offset")
+ (funcall (if global 'kill-local-variable 'make-local-variable)
+ 'py-indent-offset)
+ (setq py-indent-offset new-value)
+ (or noninteractive
+ (message "%s value of py-indent-offset set to %d"
+ (if global "Global" "Local")
+ py-indent-offset)))
+ ))
+
+(defun py-comment-indent-function ()
+ "Python version of `comment-indent-function'."
+ ;; This is required when filladapt is turned off. Without it, when
+ ;; filladapt is not used, comments which start in column zero
+ ;; cascade one character to the right
+ (save-excursion
+ (beginning-of-line)
+ (let ((eol (py-point 'eol)))
+ (and comment-start-skip
+ (re-search-forward comment-start-skip eol t)
+ (setq eol (match-beginning 0)))
+ (goto-char eol)
+ (skip-chars-backward " \t")
+ (max comment-column (+ (current-column) (if (bolp) 0 1)))
+ )))
+
+(defun py-narrow-to-defun (&optional class)
+ "Make text outside current defun invisible.
+The defun visible is the one that contains point or follows point.
+Optional CLASS is passed directly to `py-beginning-of-def-or-class'."
+ (interactive "P")
+ (save-excursion
+ (widen)
+ (py-end-of-def-or-class class)
+ (let ((end (point)))
+ (py-beginning-of-def-or-class class)
+ (narrow-to-region (point) end))))
+
+
+(defun py-shift-region (start end count)
+ "Indent lines from START to END by COUNT spaces."
+ (save-excursion
+ (goto-char end)
+ (beginning-of-line)
+ (setq end (point))
+ (goto-char start)
+ (beginning-of-line)
+ (setq start (point))
+ (indent-rigidly start end count)))
+
+(defun py-shift-region-left (start end &optional count)
+ "Shift region of Python code to the left.
+The lines from the line containing the start of the current region up
+to (but not including) the line containing the end of the region are
+shifted to the left, by `py-indent-offset' columns.
+
+If a prefix argument is given, the region is instead shifted by that
+many columns. With no active region, dedent only the current line.
+You cannot dedent the region if any line is already at column zero."
+ (interactive
+ (let ((p (point))
+ (m (mark))
+ (arg current-prefix-arg))
+ (if m
+ (list (min p m) (max p m) arg)
+ (list p (save-excursion (forward-line 1) (point)) arg))))
+ ;; if any line is at column zero, don't shift the region
+ (save-excursion
+ (goto-char start)
+ (while (< (point) end)
+ (back-to-indentation)
+ (if (and (zerop (current-column))
+ (not (looking-at "\\s *$")))
+ (error "Region is at left edge"))
+ (forward-line 1)))
+ (py-shift-region start end (- (prefix-numeric-value
+ (or count py-indent-offset))))
+ (py-keep-region-active))
+
+(defun py-shift-region-right (start end &optional count)
+ "Shift region of Python code to the right.
+The lines from the line containing the start of the current region up
+to (but not including) the line containing the end of the region are
+shifted to the right, by `py-indent-offset' columns.
+
+If a prefix argument is given, the region is instead shifted by that
+many columns. With no active region, indent only the current line."
+ (interactive
+ (let ((p (point))
+ (m (mark))
+ (arg current-prefix-arg))
+ (if m
+ (list (min p m) (max p m) arg)
+ (list p (save-excursion (forward-line 1) (point)) arg))))
+ (py-shift-region start end (prefix-numeric-value
+ (or count py-indent-offset)))
+ (py-keep-region-active))
+
+(defun py-indent-region (start end &optional indent-offset)
+ "Reindent a region of Python code.
+
+The lines from the line containing the start of the current region up
+to (but not including) the line containing the end of the region are
+reindented. If the first line of the region has a non-whitespace
+character in the first column, the first line is left alone and the
+rest of the region is reindented with respect to it. Else the entire
+region is reindented with respect to the (closest code or indenting
+comment) statement immediately preceding the region.
+
+This is useful when code blocks are moved or yanked, when enclosing
+control structures are introduced or removed, or to reformat code
+using a new value for the indentation offset.
+
+If a numeric prefix argument is given, it will be used as the value of
+the indentation offset. Else the value of `py-indent-offset' will be
+used.
+
+Warning: The region must be consistently indented before this function
+is called! This function does not compute proper indentation from
+scratch (that's impossible in Python), it merely adjusts the existing
+indentation to be correct in context.
+
+Warning: This function really has no idea what to do with
+non-indenting comment lines, and shifts them as if they were indenting
+comment lines. Fixing this appears to require telepathy.
+
+Special cases: whitespace is deleted from blank lines; continuation
+lines are shifted by the same amount their initial line was shifted,
+in order to preserve their relative indentation with respect to their
+initial line; and comment lines beginning in column 1 are ignored."
+ (interactive "*r\nP") ; region; raw prefix arg
+ (save-excursion
+ (goto-char end) (beginning-of-line) (setq end (point-marker))
+ (goto-char start) (beginning-of-line)
+ (let ((py-indent-offset (prefix-numeric-value
+ (or indent-offset py-indent-offset)))
+ (indents '(-1)) ; stack of active indent levels
+ (target-column 0) ; column to which to indent
+ (base-shifted-by 0) ; amount last base line was shifted
+ (indent-base (if (looking-at "[ \t\n]")
+ (py-compute-indentation t)
+ 0))
+ ci)
+ (while (< (point) end)
+ (setq ci (current-indentation))
+ ;; figure out appropriate target column
+ (cond
+ ((or (eq (following-char) ?#) ; comment in column 1
+ (looking-at "[ \t]*$")) ; entirely blank
+ (setq target-column 0))
+ ((py-continuation-line-p) ; shift relative to base line
+ (setq target-column (+ ci base-shifted-by)))
+ (t ; new base line
+ (if (> ci (car indents)) ; going deeper; push it
+ (setq indents (cons ci indents))
+ ;; else we should have seen this indent before
+ (setq indents (memq ci indents)) ; pop deeper indents
+ (if (null indents)
+ (error "Bad indentation in region, at line %d"
+ (save-restriction
+ (widen)
+ (1+ (count-lines 1 (point)))))))
+ (setq target-column (+ indent-base
+ (* py-indent-offset
+ (- (length indents) 2))))
+ (setq base-shifted-by (- target-column ci))))
+ ;; shift as needed
+ (if (/= ci target-column)
+ (progn
+ (delete-horizontal-space)
+ (indent-to target-column)))
+ (forward-line 1))))
+ (set-marker end nil))
+
+(defun py-comment-region (beg end &optional arg)
+ "Like `comment-region' but uses double hash (`#') comment starter."
+ (interactive "r\nP")
+ (let ((comment-start py-block-comment-prefix))
+ (comment-region beg end arg)))
+
+
+;; Functions for moving point
+(defun py-previous-statement (count)
+ "Go to the start of the COUNTth preceding Python statement.
+By default, goes to the previous statement. If there is no such
+statement, goes to the first statement. Return count of statements
+left to move. `Statements' do not include blank, comment, or
+continuation lines."
+ (interactive "p") ; numeric prefix arg
+ (if (< count 0) (py-next-statement (- count))
+ (py-goto-initial-line)
+ (let (start)
+ (while (and
+ (setq start (point)) ; always true -- side effect
+ (> count 0)
+ (zerop (forward-line -1))
+ (py-goto-statement-at-or-above))
+ (setq count (1- count)))
+ (if (> count 0) (goto-char start)))
+ count))
+
+(defun py-next-statement (count)
+ "Go to the start of next Python statement.
+If the statement at point is the i'th Python statement, goes to the
+start of statement i+COUNT. If there is no such statement, goes to the
+last statement. Returns count of statements left to move. `Statements'
+do not include blank, comment, or continuation lines."
+ (interactive "p") ; numeric prefix arg
+ (if (< count 0) (py-previous-statement (- count))
+ (beginning-of-line)
+ (let (start)
+ (while (and
+ (setq start (point)) ; always true -- side effect
+ (> count 0)
+ (py-goto-statement-below))
+ (setq count (1- count)))
+ (if (> count 0) (goto-char start)))
+ count))
+
+(defun py-goto-block-up (&optional nomark)
+ "Move up to start of current block.
+Go to the statement that starts the smallest enclosing block; roughly
+speaking, this will be the closest preceding statement that ends with a
+colon and is indented less than the statement you started on. If
+successful, also sets the mark to the starting point.
+
+`\\[py-mark-block]' can be used afterward to mark the whole code
+block, if desired.
+
+If called from a program, the mark will not be set if optional argument
+NOMARK is not nil."
+ (interactive)
+ (let ((start (point))
+ (found nil)
+ initial-indent)
+ (py-goto-initial-line)
+ ;; if on blank or non-indenting comment line, use the preceding stmt
+ (if (looking-at "[ \t]*\\($\\|#[^ \t\n]\\)")
+ (progn
+ (py-goto-statement-at-or-above)
+ (setq found (py-statement-opens-block-p))))
+ ;; search back for colon line indented less
+ (setq initial-indent (current-indentation))
+ (if (zerop initial-indent)
+ ;; force fast exit
+ (goto-char (point-min)))
+ (while (not (or found (bobp)))
+ (setq found
+ (and
+ (re-search-backward ":[ \t]*\\($\\|[#\\]\\)" nil 'move)
+ (or (py-goto-initial-line) t) ; always true -- side effect
+ (< (current-indentation) initial-indent)
+ (py-statement-opens-block-p))))
+ (if found
+ (progn
+ (or nomark (push-mark start))
+ (back-to-indentation))
+ (goto-char start)
+ (error "Enclosing block not found"))))
+
+(defun py-beginning-of-def-or-class (&optional class count)
+ "Move point to start of `def' or `class'.
+
+Searches back for the closest preceding `def'. If you supply a prefix
+arg, looks for a `class' instead. The docs below assume the `def'
+case; just substitute `class' for `def' for the other case.
+Programmatically, if CLASS is `either', then moves to either `class'
+or `def'.
+
+When second optional argument is given programmatically, move to the
+COUNTth start of `def'.
+
+If point is in a `def' statement already, and after the `d', simply
+moves point to the start of the statement.
+
+Otherwise (i.e. when point is not in a `def' statement, or at or
+before the `d' of a `def' statement), searches for the closest
+preceding `def' statement, and leaves point at its start. If no such
+statement can be found, leaves point at the start of the buffer.
+
+Returns t iff a `def' statement is found by these rules.
+
+Note that doing this command repeatedly will take you closer to the
+start of the buffer each time.
+
+To mark the current `def', see `\\[py-mark-def-or-class]'."
+ (interactive "P") ; raw prefix arg
+ (setq count (or count 1))
+ (let ((at-or-before-p (<= (current-column) (current-indentation)))
+ (start-of-line (goto-char (py-point 'bol)))
+ (start-of-stmt (goto-char (py-point 'bos)))
+ (start-re (cond ((eq class 'either) "^[ \t]*\\(class\\|def\\)\\>")
+ (class "^[ \t]*class\\>")
+ (t "^[ \t]*def\\>")))
+ )
+ ;; searching backward
+ (if (and (< 0 count)
+ (or (/= start-of-stmt start-of-line)
+ (not at-or-before-p)))
+ (end-of-line))
+ ;; search forward
+ (if (and (> 0 count)
+ (zerop (current-column))
+ (looking-at start-re))
+ (end-of-line))
+ (if (re-search-backward start-re nil 'move count)
+ (goto-char (match-beginning 0)))))
+
+;; Backwards compatibility
+(defalias 'beginning-of-python-def-or-class 'py-beginning-of-def-or-class)
+
+(defun py-end-of-def-or-class (&optional class count)
+ "Move point beyond end of `def' or `class' body.
+
+By default, looks for an appropriate `def'. If you supply a prefix
+arg, looks for a `class' instead. The docs below assume the `def'
+case; just substitute `class' for `def' for the other case.
+Programmatically, if CLASS is `either', then moves to either `class'
+or `def'.
+
+When second optional argument is given programmatically, move to the
+COUNTth end of `def'.
+
+If point is in a `def' statement already, this is the `def' we use.
+
+Else, if the `def' found by `\\[py-beginning-of-def-or-class]'
+contains the statement you started on, that's the `def' we use.
+
+Otherwise, we search forward for the closest following `def', and use that.
+
+If a `def' can be found by these rules, point is moved to the start of
+the line immediately following the `def' block, and the position of the
+start of the `def' is returned.
+
+Else point is moved to the end of the buffer, and nil is returned.
+
+Note that doing this command repeatedly will take you closer to the
+end of the buffer each time.
+
+To mark the current `def', see `\\[py-mark-def-or-class]'."
+ (interactive "P") ; raw prefix arg
+ (if (and count (/= count 1))
+ (py-beginning-of-def-or-class (- 1 count)))
+ (let ((start (progn (py-goto-initial-line) (point)))
+ (which (cond ((eq class 'either) "\\(class\\|def\\)")
+ (class "class")
+ (t "def")))
+ (state 'not-found))
+ ;; move point to start of appropriate def/class
+ (if (looking-at (concat "[ \t]*" which "\\>")) ; already on one
+ (setq state 'at-beginning)
+ ;; else see if py-beginning-of-def-or-class hits container
+ (if (and (py-beginning-of-def-or-class class)
+ (progn (py-goto-beyond-block)
+ (> (point) start)))
+ (setq state 'at-end)
+ ;; else search forward
+ (goto-char start)
+ (if (re-search-forward (concat "^[ \t]*" which "\\>") nil 'move)
+ (progn (setq state 'at-beginning)
+ (beginning-of-line)))))
+ (cond
+ ((eq state 'at-beginning) (py-goto-beyond-block) t)
+ ((eq state 'at-end) t)
+ ((eq state 'not-found) nil)
+ (t (error "Internal error in `py-end-of-def-or-class'")))))
+
+;; Backwards compabitility
+(defalias 'end-of-python-def-or-class 'py-end-of-def-or-class)
+
+
+;; Functions for marking regions
+(defun py-mark-block (&optional extend just-move)
+ "Mark following block of lines. With prefix arg, mark structure.
+Easier to use than explain. It sets the region to an `interesting'
+block of succeeding lines. If point is on a blank line, it goes down to
+the next non-blank line. That will be the start of the region. The end
+of the region depends on the kind of line at the start:
+
+ - If a comment, the region will include all succeeding comment lines up
+ to (but not including) the next non-comment line (if any).
+
+ - Else if a prefix arg is given, and the line begins one of these
+ structures:
+
+ if elif else try except finally for while def class
+
+ the region will be set to the body of the structure, including
+ following blocks that `belong' to it, but excluding trailing blank
+ and comment lines. E.g., if on a `try' statement, the `try' block
+ and all (if any) of the following `except' and `finally' blocks
+ that belong to the `try' structure will be in the region. Ditto
+ for if/elif/else, for/else and while/else structures, and (a bit
+ degenerate, since they're always one-block structures) def and
+ class blocks.
+
+ - Else if no prefix argument is given, and the line begins a Python
+ block (see list above), and the block is not a `one-liner' (i.e.,
+ the statement ends with a colon, not with code), the region will
+ include all succeeding lines up to (but not including) the next
+ code statement (if any) that's indented no more than the starting
+ line, except that trailing blank and comment lines are excluded.
+ E.g., if the starting line begins a multi-statement `def'
+ structure, the region will be set to the full function definition,
+ but without any trailing `noise' lines.
+
+ - Else the region will include all succeeding lines up to (but not
+ including) the next blank line, or code or indenting-comment line
+ indented strictly less than the starting line. Trailing indenting
+ comment lines are included in this case, but not trailing blank
+ lines.
+
+A msg identifying the location of the mark is displayed in the echo
+area; or do `\\[exchange-point-and-mark]' to flip down to the end.
+
+If called from a program, optional argument EXTEND plays the role of
+the prefix arg, and if optional argument JUST-MOVE is not nil, just
+moves to the end of the block (& does not set mark or display a msg)."
+ (interactive "P") ; raw prefix arg
+ (py-goto-initial-line)
+ ;; skip over blank lines
+ (while (and
+ (looking-at "[ \t]*$") ; while blank line
+ (not (eobp))) ; & somewhere to go
+ (forward-line 1))
+ (if (eobp)
+ (error "Hit end of buffer without finding a non-blank stmt"))
+ (let ((initial-pos (point))
+ (initial-indent (current-indentation))
+ last-pos ; position of last stmt in region
+ (followers
+ '((if elif else) (elif elif else) (else)
+ (try except finally) (except except) (finally)
+ (for else) (while else)
+ (def) (class) ) )
+ first-symbol next-symbol)
+
+ (cond
+ ;; if comment line, suck up the following comment lines
+ ((looking-at "[ \t]*#")
+ (re-search-forward "^[ \t]*[^ \t#]" nil 'move) ; look for non-comment
+ (re-search-backward "^[ \t]*#") ; and back to last comment in block
+ (setq last-pos (point)))
+
+ ;; else if line is a block line and EXTEND given, suck up
+ ;; the whole structure
+ ((and extend
+ (setq first-symbol (py-suck-up-first-keyword) )
+ (assq first-symbol followers))
+ (while (and
+ (or (py-goto-beyond-block) t) ; side effect
+ (forward-line -1) ; side effect
+ (setq last-pos (point)) ; side effect
+ (py-goto-statement-below)
+ (= (current-indentation) initial-indent)
+ (setq next-symbol (py-suck-up-first-keyword))
+ (memq next-symbol (cdr (assq first-symbol followers))))
+ (setq first-symbol next-symbol)))
+
+ ;; else if line *opens* a block, search for next stmt indented <=
+ ((py-statement-opens-block-p)
+ (while (and
+ (setq last-pos (point)) ; always true -- side effect
+ (py-goto-statement-below)
+ (> (current-indentation) initial-indent)
+ )))
+
+ ;; else plain code line; stop at next blank line, or stmt or
+ ;; indenting comment line indented <
+ (t
+ (while (and
+ (setq last-pos (point)) ; always true -- side effect
+ (or (py-goto-beyond-final-line) t)
+ (not (looking-at "[ \t]*$")) ; stop at blank line
+ (or
+ (>= (current-indentation) initial-indent)
+ (looking-at "[ \t]*#[^ \t\n]"))) ; ignore non-indenting #
+ nil)))
+
+ ;; skip to end of last stmt
+ (goto-char last-pos)
+ (py-goto-beyond-final-line)
+
+ ;; set mark & display
+ (if just-move
+ () ; just return
+ (push-mark (point) 'no-msg)
+ (forward-line -1)
+ (message "Mark set after: %s" (py-suck-up-leading-text))
+ (goto-char initial-pos))))
+
+(defun py-mark-def-or-class (&optional class)
+ "Set region to body of def (or class, with prefix arg) enclosing point.
+Pushes the current mark, then point, on the mark ring (all language
+modes do this, but although it's handy it's never documented ...).
+
+In most Emacs language modes, this function bears at least a
+hallucinogenic resemblance to `\\[py-end-of-def-or-class]' and
+`\\[py-beginning-of-def-or-class]'.
+
+And in earlier versions of Python mode, all 3 were tightly connected.
+Turned out that was more confusing than useful: the `goto start' and
+`goto end' commands are usually used to search through a file, and
+people expect them to act a lot like `search backward' and `search
+forward' string-search commands. But because Python `def' and `class'
+can nest to arbitrary levels, finding the smallest def containing
+point cannot be done via a simple backward search: the def containing
+point may not be the closest preceding def, or even the closest
+preceding def that's indented less. The fancy algorithm required is
+appropriate for the usual uses of this `mark' command, but not for the
+`goto' variations.
+
+So the def marked by this command may not be the one either of the
+`goto' commands find: If point is on a blank or non-indenting comment
+line, moves back to start of the closest preceding code statement or
+indenting comment line. If this is a `def' statement, that's the def
+we use. Else searches for the smallest enclosing `def' block and uses
+that. Else signals an error.
+
+When an enclosing def is found: The mark is left immediately beyond
+the last line of the def block. Point is left at the start of the
+def, except that: if the def is preceded by a number of comment lines
+followed by (at most) one optional blank line, point is left at the
+start of the comments; else if the def is preceded by a blank line,
+point is left at its start.
+
+The intent is to mark the containing def/class and its associated
+documentation, to make moving and duplicating functions and classes
+pleasant."
+ (interactive "P") ; raw prefix arg
+ (let ((start (point))
+ (which (cond ((eq class 'either) "\\(class\\|def\\)")
+ (class "class")
+ (t "def"))))
+ (push-mark start)
+ (if (not (py-go-up-tree-to-keyword which))
+ (progn (goto-char start)
+ (error "Enclosing %s not found"
+ (if (eq class 'either)
+ "def or class"
+ which)))
+ ;; else enclosing def/class found
+ (setq start (point))
+ (py-goto-beyond-block)
+ (push-mark (point))
+ (goto-char start)
+ (if (zerop (forward-line -1)) ; if there is a preceding line
+ (progn
+ (if (looking-at "[ \t]*$") ; it's blank
+ (setq start (point)) ; so reset start point
+ (goto-char start)) ; else try again
+ (if (zerop (forward-line -1))
+ (if (looking-at "[ \t]*#") ; a comment
+ ;; look back for non-comment line
+ ;; tricky: note that the regexp matches a blank
+ ;; line, cuz \n is in the 2nd character class
+ (and
+ (re-search-backward "^[ \t]*[^ \t#]" nil 'move)
+ (forward-line 1))
+ ;; no comment, so go back
+ (goto-char start)))))))
+ (exchange-point-and-mark)
+ (py-keep-region-active))
+
+;; ripped from cc-mode
+(defun py-forward-into-nomenclature (&optional arg)
+ "Move forward to end of a nomenclature section or word.
+With \\[universal-argument] (programmatically, optional argument ARG),
+do it that many times.
+
+A `nomenclature' is a fancy way of saying AWordWithMixedCaseNotUnderscores."
+ (interactive "p")
+ (let ((case-fold-search nil))
+ (if (> arg 0)
+ (re-search-forward
+ "\\(\\W\\|[_]\\)*\\([A-Z]*[a-z0-9]*\\)"
+ (point-max) t arg)
+ (while (and (< arg 0)
+ (re-search-backward
+ "\\(\\W\\|[a-z0-9]\\)[A-Z]+\\|\\(\\W\\|[_]\\)\\w+"
+ (point-min) 0))
+ (forward-char 1)
+ (setq arg (1+ arg)))))
+ (py-keep-region-active))
+
+(defun py-backward-into-nomenclature (&optional arg)
+ "Move backward to beginning of a nomenclature section or word.
+With optional ARG, move that many times. If ARG is negative, move
+forward.
+
+A `nomenclature' is a fancy way of saying AWordWithMixedCaseNotUnderscores."
+ (interactive "p")
+ (py-forward-into-nomenclature (- arg))
+ (py-keep-region-active))
+
+
+
+;; pdbtrack functions
+(defun py-pdbtrack-toggle-stack-tracking (arg)
+ (interactive "P")
+ (if (not (get-buffer-process (current-buffer)))
+ (error "No process associated with buffer '%s'" (current-buffer)))
+ ;; missing or 0 is toggle, >0 turn on, <0 turn off
+ (if (or (not arg)
+ (zerop (setq arg (prefix-numeric-value arg))))
+ (setq py-pdbtrack-do-tracking-p (not py-pdbtrack-do-tracking-p))
+ (setq py-pdbtrack-do-tracking-p (> arg 0)))
+ (message "%sabled Python's pdbtrack"
+ (if py-pdbtrack-do-tracking-p "En" "Dis")))
+
+(defun turn-on-pdbtrack ()
+ (interactive)
+ (py-pdbtrack-toggle-stack-tracking 1))
+
+(defun turn-off-pdbtrack ()
+ (interactive)
+ (py-pdbtrack-toggle-stack-tracking 0))
+
+
+
+;; Pychecker
+
+;; hack for FSF Emacs
+(unless (fboundp 'read-shell-command)
+ (defalias 'read-shell-command 'read-string))
+
+(defun py-pychecker-run (command)
+ "*Run pychecker (default on the file currently visited)."
+ (interactive
+ (let ((default
+ (format "%s %s %s" py-pychecker-command
+ (mapconcat 'identity py-pychecker-command-args " ")
+ (buffer-file-name)))
+ (last (when py-pychecker-history
+ (let* ((lastcmd (car py-pychecker-history))
+ (cmd (cdr (reverse (split-string lastcmd))))
+ (newcmd (reverse (cons (buffer-file-name) cmd))))
+ (mapconcat 'identity newcmd " ")))))
+
+ (list
+ (if (fboundp 'read-shell-command)
+ (read-shell-command "Run pychecker like this: "
+ (if last
+ last
+ default)
+ 'py-pychecker-history)
+ (read-string "Run pychecker like this: "
+ (if last
+ last
+ default)
+ 'py-pychecker-history))
+ )))
+ (save-some-buffers (not py-ask-about-save) nil)
+ (compile-internal command "No more errors"))
+
+
+
+;; pydoc commands. The guts of this function is stolen from XEmacs's
+;; symbol-near-point, but without the useless regexp-quote call on the
+;; results, nor the interactive bit. Also, we've added the temporary
+;; syntax table setting, which Skip originally had broken out into a
+;; separate function. Note that Emacs doesn't have the original
+;; function.
+(defun py-symbol-near-point ()
+ "Return the first textual item to the nearest point."
+ ;; alg stolen from etag.el
+ (save-excursion
+ (with-syntax-table py-dotted-expression-syntax-table
+ (if (or (bobp) (not (memq (char-syntax (char-before)) '(?w ?_))))
+ (while (not (looking-at "\\sw\\|\\s_\\|\\'"))
+ (forward-char 1)))
+ (while (looking-at "\\sw\\|\\s_")
+ (forward-char 1))
+ (if (re-search-backward "\\sw\\|\\s_" nil t)
+ (progn (forward-char 1)
+ (buffer-substring (point)
+ (progn (forward-sexp -1)
+ (while (looking-at "\\s'")
+ (forward-char 1))
+ (point))))
+ nil))))
+
+(defun py-help-at-point ()
+ "Get help from Python based on the symbol nearest point."
+ (interactive)
+ (let* ((sym (py-symbol-near-point))
+ (base (substring sym 0 (or (search "." sym :from-end t) 0)))
+ cmd)
+ (if (not (equal base ""))
+ (setq cmd (concat "import " base "\n")))
+ (setq cmd (concat "import pydoc\n"
+ cmd
+ "try: pydoc.help('" sym "')\n"
+ "except: print 'No help available on:', \"" sym "\""))
+ (message cmd)
+ (py-execute-string cmd)
+ (set-buffer "*Python Output*")
+ ;; BAW: Should we really be leaving the output buffer in help-mode?
+ (help-mode)))
+
+
+
+;; Documentation functions
+
+;; dump the long form of the mode blurb; does the usual doc escapes,
+;; plus lines of the form ^[vc]:name$ to suck variable & command docs
+;; out of the right places, along with the keys they're on & current
+;; values
+(defun py-dump-help-string (str)
+ (with-output-to-temp-buffer "*Help*"
+ (let ((locals (buffer-local-variables))
+ funckind funcname func funcdoc
+ (start 0) mstart end
+ keys )
+ (while (string-match "^%\\([vc]\\):\\(.+\\)\n" str start)
+ (setq mstart (match-beginning 0) end (match-end 0)
+ funckind (substring str (match-beginning 1) (match-end 1))
+ funcname (substring str (match-beginning 2) (match-end 2))
+ func (intern funcname))
+ (princ (substitute-command-keys (substring str start mstart)))
+ (cond
+ ((equal funckind "c") ; command
+ (setq funcdoc (documentation func)
+ keys (concat
+ "Key(s): "
+ (mapconcat 'key-description
+ (where-is-internal func py-mode-map)
+ ", "))))
+ ((equal funckind "v") ; variable
+ (setq funcdoc (documentation-property func 'variable-documentation)
+ keys (if (assq func locals)
+ (concat
+ "Local/Global values: "
+ (prin1-to-string (symbol-value func))
+ " / "
+ (prin1-to-string (default-value func)))
+ (concat
+ "Value: "
+ (prin1-to-string (symbol-value func))))))
+ (t ; unexpected
+ (error "Error in py-dump-help-string, tag `%s'" funckind)))
+ (princ (format "\n-> %s:\t%s\t%s\n\n"
+ (if (equal funckind "c") "Command" "Variable")
+ funcname keys))
+ (princ funcdoc)
+ (terpri)
+ (setq start end))
+ (princ (substitute-command-keys (substring str start))))
+ (print-help-return-message)))
+
+(defun py-describe-mode ()
+ "Dump long form of Python-mode docs."
+ (interactive)
+ (py-dump-help-string "Major mode for editing Python files.
+Knows about Python indentation, tokens, comments and continuation lines.
+Paragraphs are separated by blank lines only.
+
+Major sections below begin with the string `@'; specific function and
+variable docs begin with `->'.
+
+ at EXECUTING PYTHON CODE
+
+\\[py-execute-import-or-reload]\timports or reloads the file in the Python interpreter
+\\[py-execute-buffer]\tsends the entire buffer to the Python interpreter
+\\[py-execute-region]\tsends the current region
+\\[py-execute-def-or-class]\tsends the current function or class definition
+\\[py-execute-string]\tsends an arbitrary string
+\\[py-shell]\tstarts a Python interpreter window; this will be used by
+\tsubsequent Python execution commands
+%c:py-execute-import-or-reload
+%c:py-execute-buffer
+%c:py-execute-region
+%c:py-execute-def-or-class
+%c:py-execute-string
+%c:py-shell
+
+ at VARIABLES
+
+py-indent-offset\tindentation increment
+py-block-comment-prefix\tcomment string used by comment-region
+
+py-python-command\tshell command to invoke Python interpreter
+py-temp-directory\tdirectory used for temp files (if needed)
+
+py-beep-if-tab-change\tring the bell if tab-width is changed
+%v:py-indent-offset
+%v:py-block-comment-prefix
+%v:py-python-command
+%v:py-temp-directory
+%v:py-beep-if-tab-change
+
+ at KINDS OF LINES
+
+Each physical line in the file is either a `continuation line' (the
+preceding line ends with a backslash that's not part of a comment, or
+the paren/bracket/brace nesting level at the start of the line is
+non-zero, or both) or an `initial line' (everything else).
+
+An initial line is in turn a `blank line' (contains nothing except
+possibly blanks or tabs), a `comment line' (leftmost non-blank
+character is `#'), or a `code line' (everything else).
+
+Comment Lines
+
+Although all comment lines are treated alike by Python, Python mode
+recognizes two kinds that act differently with respect to indentation.
+
+An `indenting comment line' is a comment line with a blank, tab or
+nothing after the initial `#'. The indentation commands (see below)
+treat these exactly as if they were code lines: a line following an
+indenting comment line will be indented like the comment line. All
+other comment lines (those with a non-whitespace character immediately
+following the initial `#') are `non-indenting comment lines', and
+their indentation is ignored by the indentation commands.
+
+Indenting comment lines are by far the usual case, and should be used
+whenever possible. Non-indenting comment lines are useful in cases
+like these:
+
+\ta = b # a very wordy single-line comment that ends up being
+\t #... continued onto another line
+
+\tif a == b:
+##\t\tprint 'panic!' # old code we've `commented out'
+\t\treturn a
+
+Since the `#...' and `##' comment lines have a non-whitespace
+character following the initial `#', Python mode ignores them when
+computing the proper indentation for the next line.
+
+Continuation Lines and Statements
+
+The Python-mode commands generally work on statements instead of on
+individual lines, where a `statement' is a comment or blank line, or a
+code line and all of its following continuation lines (if any)
+considered as a single logical unit. The commands in this mode
+generally (when it makes sense) automatically move to the start of the
+statement containing point, even if point happens to be in the middle
+of some continuation line.
+
+
+ at INDENTATION
+
+Primarily for entering new code:
+\t\\[indent-for-tab-command]\t indent line appropriately
+\t\\[py-newline-and-indent]\t insert newline, then indent
+\t\\[py-electric-backspace]\t reduce indentation, or delete single character
+
+Primarily for reindenting existing code:
+\t\\[py-guess-indent-offset]\t guess py-indent-offset from file content; change locally
+\t\\[universal-argument] \\[py-guess-indent-offset]\t ditto, but change globally
+
+\t\\[py-indent-region]\t reindent region to match its context
+\t\\[py-shift-region-left]\t shift region left by py-indent-offset
+\t\\[py-shift-region-right]\t shift region right by py-indent-offset
+
+Unlike most programming languages, Python uses indentation, and only
+indentation, to specify block structure. Hence the indentation supplied
+automatically by Python-mode is just an educated guess: only you know
+the block structure you intend, so only you can supply correct
+indentation.
+
+The \\[indent-for-tab-command] and \\[py-newline-and-indent] keys try to suggest plausible indentation, based on
+the indentation of preceding statements. E.g., assuming
+py-indent-offset is 4, after you enter
+\tif a > 0: \\[py-newline-and-indent]
+the cursor will be moved to the position of the `_' (_ is not a
+character in the file, it's just used here to indicate the location of
+the cursor):
+\tif a > 0:
+\t _
+If you then enter `c = d' \\[py-newline-and-indent], the cursor will move
+to
+\tif a > 0:
+\t c = d
+\t _
+Python-mode cannot know whether that's what you intended, or whether
+\tif a > 0:
+\t c = d
+\t_
+was your intent. In general, Python-mode either reproduces the
+indentation of the (closest code or indenting-comment) preceding
+statement, or adds an extra py-indent-offset blanks if the preceding
+statement has `:' as its last significant (non-whitespace and non-
+comment) character. If the suggested indentation is too much, use
+\\[py-electric-backspace] to reduce it.
+
+Continuation lines are given extra indentation. If you don't like the
+suggested indentation, change it to something you do like, and Python-
+mode will strive to indent later lines of the statement in the same way.
+
+If a line is a continuation line by virtue of being in an unclosed
+paren/bracket/brace structure (`list', for short), the suggested
+indentation depends on whether the current line contains the first item
+in the list. If it does, it's indented py-indent-offset columns beyond
+the indentation of the line containing the open bracket. If you don't
+like that, change it by hand. The remaining items in the list will mimic
+whatever indentation you give to the first item.
+
+If a line is a continuation line because the line preceding it ends with
+a backslash, the third and following lines of the statement inherit their
+indentation from the line preceding them. The indentation of the second
+line in the statement depends on the form of the first (base) line: if
+the base line is an assignment statement with anything more interesting
+than the backslash following the leftmost assigning `=', the second line
+is indented two columns beyond that `='. Else it's indented to two
+columns beyond the leftmost solid chunk of non-whitespace characters on
+the base line.
+
+Warning: indent-region should not normally be used! It calls \\[indent-for-tab-command]
+repeatedly, and as explained above, \\[indent-for-tab-command] can't guess the block
+structure you intend.
+%c:indent-for-tab-command
+%c:py-newline-and-indent
+%c:py-electric-backspace
+
+
+The next function may be handy when editing code you didn't write:
+%c:py-guess-indent-offset
+
+
+The remaining `indent' functions apply to a region of Python code. They
+assume the block structure (equals indentation, in Python) of the region
+is correct, and alter the indentation in various ways while preserving
+the block structure:
+%c:py-indent-region
+%c:py-shift-region-left
+%c:py-shift-region-right
+
+ at MARKING & MANIPULATING REGIONS OF CODE
+
+\\[py-mark-block]\t mark block of lines
+\\[py-mark-def-or-class]\t mark smallest enclosing def
+\\[universal-argument] \\[py-mark-def-or-class]\t mark smallest enclosing class
+\\[comment-region]\t comment out region of code
+\\[universal-argument] \\[comment-region]\t uncomment region of code
+%c:py-mark-block
+%c:py-mark-def-or-class
+%c:comment-region
+
+ at MOVING POINT
+
+\\[py-previous-statement]\t move to statement preceding point
+\\[py-next-statement]\t move to statement following point
+\\[py-goto-block-up]\t move up to start of current block
+\\[py-beginning-of-def-or-class]\t move to start of def
+\\[universal-argument] \\[py-beginning-of-def-or-class]\t move to start of class
+\\[py-end-of-def-or-class]\t move to end of def
+\\[universal-argument] \\[py-end-of-def-or-class]\t move to end of class
+
+The first two move to one statement beyond the statement that contains
+point. A numeric prefix argument tells them to move that many
+statements instead. Blank lines, comment lines, and continuation lines
+do not count as `statements' for these commands. So, e.g., you can go
+to the first code statement in a file by entering
+\t\\[beginning-of-buffer]\t to move to the top of the file
+\t\\[py-next-statement]\t to skip over initial comments and blank lines
+Or do `\\[py-previous-statement]' with a huge prefix argument.
+%c:py-previous-statement
+%c:py-next-statement
+%c:py-goto-block-up
+%c:py-beginning-of-def-or-class
+%c:py-end-of-def-or-class
+
+ at LITTLE-KNOWN EMACS COMMANDS PARTICULARLY USEFUL IN PYTHON MODE
+
+`\\[indent-new-comment-line]' is handy for entering a multi-line comment.
+
+`\\[set-selective-display]' with a `small' prefix arg is ideally suited for viewing the
+overall class and def structure of a module.
+
+`\\[back-to-indentation]' moves point to a line's first non-blank character.
+
+`\\[indent-relative]' is handy for creating odd indentation.
+
+ at OTHER EMACS HINTS
+
+If you don't like the default value of a variable, change its value to
+whatever you do like by putting a `setq' line in your .emacs file.
+E.g., to set the indentation increment to 4, put this line in your
+.emacs:
+\t(setq py-indent-offset 4)
+To see the value of a variable, do `\\[describe-variable]' and enter the variable
+name at the prompt.
+
+When entering a key sequence like `C-c C-n', it is not necessary to
+release the CONTROL key after doing the `C-c' part -- it suffices to
+press the CONTROL key, press and release `c' (while still holding down
+CONTROL), press and release `n' (while still holding down CONTROL), &
+then release CONTROL.
+
+Entering Python mode calls with no arguments the value of the variable
+`python-mode-hook', if that value exists and is not nil; for backward
+compatibility it also tries `py-mode-hook'; see the `Hooks' section of
+the Elisp manual for details.
+
+Obscure: When python-mode is first loaded, it looks for all bindings
+to newline-and-indent in the global keymap, and shadows them with
+local bindings to py-newline-and-indent."))
+
+(require 'info-look)
+;; The info-look package does not always provide this function (it
+;; appears this is the case with XEmacs 21.1)
+(when (fboundp 'info-lookup-maybe-add-help)
+ (info-lookup-maybe-add-help
+ :mode 'python-mode
+ :regexp "[a-zA-Z0-9_]+"
+ :doc-spec '(("(python-lib)Module Index")
+ ("(python-lib)Class-Exception-Object Index")
+ ("(python-lib)Function-Method-Variable Index")
+ ("(python-lib)Miscellaneous Index")))
+ )
+
+
+;; Helper functions
+(defvar py-parse-state-re
+ (concat
+ "^[ \t]*\\(elif\\|else\\|while\\|def\\|class\\)\\>"
+ "\\|"
+ "^[^ #\t\n]"))
+
+(defun py-parse-state ()
+ "Return the parse state at point (see `parse-partial-sexp' docs)."
+ (save-excursion
+ (let ((here (point))
+ pps done)
+ (while (not done)
+ ;; back up to the first preceding line (if any; else start of
+ ;; buffer) that begins with a popular Python keyword, or a
+ ;; non- whitespace and non-comment character. These are good
+ ;; places to start parsing to see whether where we started is
+ ;; at a non-zero nesting level. It may be slow for people who
+ ;; write huge code blocks or huge lists ... tough beans.
+ (re-search-backward py-parse-state-re nil 'move)
+ (beginning-of-line)
+ ;; In XEmacs, we have a much better way to test for whether
+ ;; we're in a triple-quoted string or not. Emacs does not
+ ;; have this built-in function, which is its loss because
+ ;; without scanning from the beginning of the buffer, there's
+ ;; no accurate way to determine this otherwise.
+ (save-excursion (setq pps (parse-partial-sexp (point) here)))
+ ;; make sure we don't land inside a triple-quoted string
+ (setq done (or (not (nth 3 pps))
+ (bobp)))
+ ;; Just go ahead and short circuit the test back to the
+ ;; beginning of the buffer. This will be slow, but not
+ ;; nearly as slow as looping through many
+ ;; re-search-backwards.
+ (if (not done)
+ (goto-char (point-min))))
+ pps)))
+
+(defun py-nesting-level ()
+ "Return the buffer position of the last unclosed enclosing list.
+If nesting level is zero, return nil."
+ (let ((status (py-parse-state)))
+ (if (zerop (car status))
+ nil ; not in a nest
+ (car (cdr status))))) ; char# of open bracket
+
+(defun py-backslash-continuation-line-p ()
+ "Return t iff preceding line ends with backslash that is not in a comment."
+ (save-excursion
+ (beginning-of-line)
+ (and
+ ;; use a cheap test first to avoid the regexp if possible
+ ;; use 'eq' because char-after may return nil
+ (eq (char-after (- (point) 2)) ?\\ )
+ ;; make sure; since eq test passed, there is a preceding line
+ (forward-line -1) ; always true -- side effect
+ (looking-at py-continued-re))))
+
+(defun py-continuation-line-p ()
+ "Return t iff current line is a continuation line."
+ (save-excursion
+ (beginning-of-line)
+ (or (py-backslash-continuation-line-p)
+ (py-nesting-level))))
+
+(defun py-goto-beginning-of-tqs (delim)
+ "Go to the beginning of the triple quoted string we find ourselves in.
+DELIM is the TQS string delimiter character we're searching backwards
+for."
+ (let ((skip (and delim (make-string 1 delim)))
+ (continue t))
+ (when skip
+ (save-excursion
+ (while continue
+ (py-safe (search-backward skip))
+ (setq continue (and (not (bobp))
+ (= (char-before) ?\\))))
+ (if (and (= (char-before) delim)
+ (= (char-before (1- (point))) delim))
+ (setq skip (make-string 3 delim))))
+ ;; we're looking at a triple-quoted string
+ (py-safe (search-backward skip)))))
+
+(defun py-goto-initial-line ()
+ "Go to the initial line of the current statement.
+Usually this is the line we're on, but if we're on the 2nd or
+following lines of a continuation block, we need to go up to the first
+line of the block."
+ ;; Tricky: We want to avoid quadratic-time behavior for long
+ ;; continued blocks, whether of the backslash or open-bracket
+ ;; varieties, or a mix of the two. The following manages to do that
+ ;; in the usual cases.
+ ;;
+ ;; Also, if we're sitting inside a triple quoted string, this will
+ ;; drop us at the line that begins the string.
+ (let (open-bracket-pos)
+ (while (py-continuation-line-p)
+ (beginning-of-line)
+ (if (py-backslash-continuation-line-p)
+ (while (py-backslash-continuation-line-p)
+ (forward-line -1))
+ ;; else zip out of nested brackets/braces/parens
+ (while (setq open-bracket-pos (py-nesting-level))
+ (goto-char open-bracket-pos)))))
+ (beginning-of-line))
+
+(defun py-goto-beyond-final-line ()
+ "Go to the point just beyond the fine line of the current statement.
+Usually this is the start of the next line, but if this is a
+multi-line statement we need to skip over the continuation lines."
+ ;; Tricky: Again we need to be clever to avoid quadratic time
+ ;; behavior.
+ ;;
+ ;; XXX: Not quite the right solution, but deals with multi-line doc
+ ;; strings
+ (if (looking-at (concat "[ \t]*\\(" py-stringlit-re "\\)"))
+ (goto-char (match-end 0)))
+ ;;
+ (forward-line 1)
+ (let (state)
+ (while (and (py-continuation-line-p)
+ (not (eobp)))
+ ;; skip over the backslash flavor
+ (while (and (py-backslash-continuation-line-p)
+ (not (eobp)))
+ (forward-line 1))
+ ;; if in nest, zip to the end of the nest
+ (setq state (py-parse-state))
+ (if (and (not (zerop (car state)))
+ (not (eobp)))
+ (progn
+ (parse-partial-sexp (point) (point-max) 0 nil state)
+ (forward-line 1))))))
+
+(defun py-statement-opens-block-p ()
+ "Return t iff the current statement opens a block.
+I.e., iff it ends with a colon that is not in a comment. Point should
+be at the start of a statement."
+ (save-excursion
+ (let ((start (point))
+ (finish (progn (py-goto-beyond-final-line) (1- (point))))
+ (searching t)
+ (answer nil)
+ state)
+ (goto-char start)
+ (while searching
+ ;; look for a colon with nothing after it except whitespace, and
+ ;; maybe a comment
+ (if (re-search-forward ":\\([ \t]\\|\\\\\n\\)*\\(#.*\\)?$"
+ finish t)
+ (if (eq (point) finish) ; note: no `else' clause; just
+ ; keep searching if we're not at
+ ; the end yet
+ ;; sure looks like it opens a block -- but it might
+ ;; be in a comment
+ (progn
+ (setq searching nil) ; search is done either way
+ (setq state (parse-partial-sexp start
+ (match-beginning 0)))
+ (setq answer (not (nth 4 state)))))
+ ;; search failed: couldn't find another interesting colon
+ (setq searching nil)))
+ answer)))
+
+(defun py-statement-closes-block-p ()
+ "Return t iff the current statement closes a block.
+I.e., if the line starts with `return', `raise', `break', `continue',
+and `pass'. This doesn't catch embedded statements."
+ (let ((here (point)))
+ (py-goto-initial-line)
+ (back-to-indentation)
+ (prog1
+ (looking-at (concat py-block-closing-keywords-re "\\>"))
+ (goto-char here))))
+
+(defun py-goto-beyond-block ()
+ "Go to point just beyond the final line of block begun by the current line.
+This is the same as where `py-goto-beyond-final-line' goes unless
+we're on colon line, in which case we go to the end of the block.
+Assumes point is at the beginning of the line."
+ (if (py-statement-opens-block-p)
+ (py-mark-block nil 'just-move)
+ (py-goto-beyond-final-line)))
+
+(defun py-goto-statement-at-or-above ()
+ "Go to the start of the first statement at or preceding point.
+Return t if there is such a statement, otherwise nil. `Statement'
+does not include blank lines, comments, or continuation lines."
+ (py-goto-initial-line)
+ (if (looking-at py-blank-or-comment-re)
+ ;; skip back over blank & comment lines
+ ;; note: will skip a blank or comment line that happens to be
+ ;; a continuation line too
+ (if (re-search-backward "^[ \t]*[^ \t#\n]" nil t)
+ (progn (py-goto-initial-line) t)
+ nil)
+ t))
+
+(defun py-goto-statement-below ()
+ "Go to start of the first statement following the statement containing point.
+Return t if there is such a statement, otherwise nil. `Statement'
+does not include blank lines, comments, or continuation lines."
+ (beginning-of-line)
+ (let ((start (point)))
+ (py-goto-beyond-final-line)
+ (while (and
+ (or (looking-at py-blank-or-comment-re)
+ (py-in-literal))
+ (not (eobp)))
+ (forward-line 1))
+ (if (eobp)
+ (progn (goto-char start) nil)
+ t)))
+
+(defun py-go-up-tree-to-keyword (key)
+ "Go to begining of statement starting with KEY, at or preceding point.
+
+KEY is a regular expression describing a Python keyword. Skip blank
+lines and non-indenting comments. If the statement found starts with
+KEY, then stop, otherwise go back to first enclosing block starting
+with KEY. If successful, leave point at the start of the KEY line and
+return t. Otherwise, leave point at an undefined place and return nil."
+ ;; skip blanks and non-indenting #
+ (py-goto-initial-line)
+ (while (and
+ (looking-at "[ \t]*\\($\\|#[^ \t\n]\\)")
+ (zerop (forward-line -1))) ; go back
+ nil)
+ (py-goto-initial-line)
+ (let* ((re (concat "[ \t]*" key "\\>"))
+ (case-fold-search nil) ; let* so looking-at sees this
+ (found (looking-at re))
+ (dead nil))
+ (while (not (or found dead))
+ (condition-case nil ; in case no enclosing block
+ (py-goto-block-up 'no-mark)
+ (error (setq dead t)))
+ (or dead (setq found (looking-at re))))
+ (beginning-of-line)
+ found))
+
+(defun py-suck-up-leading-text ()
+ "Return string in buffer from start of indentation to end of line.
+Prefix with \"...\" if leading whitespace was skipped."
+ (save-excursion
+ (back-to-indentation)
+ (concat
+ (if (bolp) "" "...")
+ (buffer-substring (point) (progn (end-of-line) (point))))))
+
+(defun py-suck-up-first-keyword ()
+ "Return first keyword on the line as a Lisp symbol.
+`Keyword' is defined (essentially) as the regular expression
+([a-z]+). Returns nil if none was found."
+ (let ((case-fold-search nil))
+ (if (looking-at "[ \t]*\\([a-z]+\\)\\>")
+ (intern (buffer-substring (match-beginning 1) (match-end 1)))
+ nil)))
+
+(defun py-current-defun ()
+ "Python value for `add-log-current-defun-function'.
+This tells add-log.el how to find the current function/method/variable."
+ (save-excursion
+
+ ;; Move back to start of the current statement.
+
+ (py-goto-initial-line)
+ (back-to-indentation)
+ (while (and (or (looking-at py-blank-or-comment-re)
+ (py-in-literal))
+ (not (bobp)))
+ (backward-to-indentation 1))
+ (py-goto-initial-line)
+
+ (let ((scopes "")
+ (sep "")
+ dead assignment)
+
+ ;; Check for an assignment. If this assignment exists inside a
+ ;; def, it will be overwritten inside the while loop. If it
+ ;; exists at top lever or inside a class, it will be preserved.
+
+ (when (looking-at "[ \t]*\\([a-zA-Z0-9_]+\\)[ \t]*=")
+ (setq scopes (buffer-substring (match-beginning 1) (match-end 1)))
+ (setq assignment t)
+ (setq sep "."))
+
+ ;; Prepend the name of each outer socpe (def or class).
+
+ (while (not dead)
+ (if (and (py-go-up-tree-to-keyword "\\(class\\|def\\)")
+ (looking-at
+ "[ \t]*\\(class\\|def\\)[ \t]*\\([a-zA-Z0-9_]+\\)[ \t]*"))
+ (let ((name (buffer-substring (match-beginning 2) (match-end 2))))
+ (if (and assignment (looking-at "[ \t]*def"))
+ (setq scopes name)
+ (setq scopes (concat name sep scopes))
+ (setq sep "."))))
+ (setq assignment nil)
+ (condition-case nil ; Terminate nicely at top level.
+ (py-goto-block-up 'no-mark)
+ (error (setq dead t))))
+ (if (string= scopes "")
+ nil
+ scopes))))
+
+
+
+(defconst py-help-address "python-mode at python.org"
+ "Address accepting submission of bug reports.")
+
+(defun py-version ()
+ "Echo the current version of `python-mode' in the minibuffer."
+ (interactive)
+ (message "Using `python-mode' version %s" py-version)
+ (py-keep-region-active))
+
+;; only works under Emacs 19
+;(eval-when-compile
+; (require 'reporter))
+
+(defun py-submit-bug-report (enhancement-p)
+ "Submit via mail a bug report on `python-mode'.
+With \\[universal-argument] (programmatically, argument ENHANCEMENT-P
+non-nil) just submit an enhancement request."
+ (interactive
+ (list (not (y-or-n-p
+ "Is this a bug report (hit `n' to send other comments)? "))))
+ (let ((reporter-prompt-for-summary-p (if enhancement-p
+ "(Very) brief summary: "
+ t)))
+ (require 'reporter)
+ (reporter-submit-bug-report
+ py-help-address ;address
+ (concat "python-mode " py-version) ;pkgname
+ ;; varlist
+ (if enhancement-p nil
+ '(py-python-command
+ py-indent-offset
+ py-block-comment-prefix
+ py-temp-directory
+ py-beep-if-tab-change))
+ nil ;pre-hooks
+ nil ;post-hooks
+ "Dear Barry,") ;salutation
+ (if enhancement-p nil
+ (set-mark (point))
+ (insert
+"Please replace this text with a sufficiently large code sample\n\
+and an exact recipe so that I can reproduce your problem. Failure\n\
+to do so may mean a greater delay in fixing your bug.\n\n")
+ (exchange-point-and-mark)
+ (py-keep-region-active))))
+
+
+(defun py-kill-emacs-hook ()
+ "Delete files in `py-file-queue'.
+These are Python temporary files awaiting execution."
+ (mapcar #'(lambda (filename)
+ (py-safe (delete-file filename)))
+ py-file-queue))
+
+;; arrange to kill temp files when Emacs exists
+(add-hook 'kill-emacs-hook 'py-kill-emacs-hook)
+(add-hook 'comint-output-filter-functions 'py-pdbtrack-track-stack-file)
+
+;; Add a designator to the minor mode strings
+(or (assq 'py-pdbtrack-is-tracking-p minor-mode-alist)
+ (push '(py-pdbtrack-is-tracking-p py-pdbtrack-minor-mode-string)
+ minor-mode-alist))
+
+
+
+;;; paragraph and string filling code from Bernhard Herzog
+;;; see http://mail.python.org/pipermail/python-list/2002-May/103189.html
+
+(defun py-fill-comment (&optional justify)
+ "Fill the comment paragraph around point"
+ (let (;; Non-nil if the current line contains a comment.
+ has-comment
+
+ ;; If has-comment, the appropriate fill-prefix for the comment.
+ comment-fill-prefix)
+
+ ;; Figure out what kind of comment we are looking at.
+ (save-excursion
+ (beginning-of-line)
+ (cond
+ ;; A line with nothing but a comment on it?
+ ((looking-at "[ \t]*#[# \t]*")
+ (setq has-comment t
+ comment-fill-prefix (buffer-substring (match-beginning 0)
+ (match-end 0))))
+
+ ;; A line with some code, followed by a comment? Remember that the hash
+ ;; which starts the comment shouldn't be part of a string or character.
+ ((progn
+ (while (not (looking-at "#\\|$"))
+ (skip-chars-forward "^#\n\"'\\")
+ (cond
+ ((eq (char-after (point)) ?\\) (forward-char 2))
+ ((memq (char-after (point)) '(?\" ?')) (forward-sexp 1))))
+ (looking-at "#+[\t ]*"))
+ (setq has-comment t)
+ (setq comment-fill-prefix
+ (concat (make-string (current-column) ? )
+ (buffer-substring (match-beginning 0) (match-end 0)))))))
+
+ (if (not has-comment)
+ (fill-paragraph justify)
+
+ ;; Narrow to include only the comment, and then fill the region.
+ (save-restriction
+ (narrow-to-region
+
+ ;; Find the first line we should include in the region to fill.
+ (save-excursion
+ (while (and (zerop (forward-line -1))
+ (looking-at "^[ \t]*#")))
+
+ ;; We may have gone to far. Go forward again.
+ (or (looking-at "^[ \t]*#")
+ (forward-line 1))
+ (point))
+
+ ;; Find the beginning of the first line past the region to fill.
+ (save-excursion
+ (while (progn (forward-line 1)
+ (looking-at "^[ \t]*#")))
+ (point)))
+
+ ;; Lines with only hashes on them can be paragraph boundaries.
+ (let ((paragraph-start (concat paragraph-start "\\|[ \t#]*$"))
+ (paragraph-separate (concat paragraph-separate "\\|[ \t#]*$"))
+ (fill-prefix comment-fill-prefix))
+ ;;(message "paragraph-start %S paragraph-separate %S"
+ ;;paragraph-start paragraph-separate)
+ (fill-paragraph justify))))
+ t))
+
+
+(defun py-fill-string (start &optional justify)
+ "Fill the paragraph around (point) in the string starting at start"
+ ;; basic strategy: narrow to the string and call the default
+ ;; implementation
+ (let (;; the start of the string's contents
+ string-start
+ ;; the end of the string's contents
+ string-end
+ ;; length of the string's delimiter
+ delim-length
+ ;; The string delimiter
+ delim
+ )
+
+ (save-excursion
+ (goto-char start)
+ (if (looking-at "\\('''\\|\"\"\"\\|'\\|\"\\)\\\\?\n?")
+ (setq string-start (match-end 0)
+ delim-length (- (match-end 1) (match-beginning 1))
+ delim (buffer-substring-no-properties (match-beginning 1)
+ (match-end 1)))
+ (error "The parameter start is not the beginning of a python string"))
+
+ ;; if the string is the first token on a line and doesn't start with
+ ;; a newline, fill as if the string starts at the beginning of the
+ ;; line. this helps with one line docstrings
+ (save-excursion
+ (beginning-of-line)
+ (and (/= (char-before string-start) ?\n)
+ (looking-at (concat "[ \t]*" delim))
+ (setq string-start (point))))
+
+ (forward-sexp (if (= delim-length 3) 2 1))
+
+ ;; with both triple quoted strings and single/double quoted strings
+ ;; we're now directly behind the first char of the end delimiter
+ ;; (this doesn't work correctly when the triple quoted string
+ ;; contains the quote mark itself). The end of the string's contents
+ ;; is one less than point
+ (setq string-end (1- (point))))
+
+ ;; Narrow to the string's contents and fill the current paragraph
+ (save-restriction
+ (narrow-to-region string-start string-end)
+ (let ((ends-with-newline (= (char-before (point-max)) ?\n)))
+ (fill-paragraph justify)
+ (if (and (not ends-with-newline)
+ (= (char-before (point-max)) ?\n))
+ ;; the default fill-paragraph implementation has inserted a
+ ;; newline at the end. Remove it again.
+ (save-excursion
+ (goto-char (point-max))
+ (delete-char -1)))))
+
+ ;; return t to indicate that we've done our work
+ t))
+
+(defun py-fill-paragraph (&optional justify)
+ "Like \\[fill-paragraph], but handle Python comments and strings.
+If any of the current line is a comment, fill the comment or the
+paragraph of it that point is in, preserving the comment's indentation
+and initial `#'s.
+If point is inside a string, narrow to that string and fill.
+"
+ (interactive "P")
+ ;; fill-paragraph will narrow incorrectly
+ (save-restriction
+ (widen)
+ (let* ((bod (py-point 'bod))
+ (pps (parse-partial-sexp bod (point))))
+ (cond
+ ;; are we inside a comment or on a line with only whitespace before
+ ;; the comment start?
+ ((or (nth 4 pps)
+ (save-excursion (beginning-of-line) (looking-at "[ \t]*#")))
+ (py-fill-comment justify))
+ ;; are we inside a string?
+ ((nth 3 pps)
+ (py-fill-string (nth 8 pps)))
+ ;; are we at the opening quote of a string, or in the indentation?
+ ((save-excursion
+ (forward-word 1)
+ (eq (py-in-literal) 'string))
+ (save-excursion
+ (py-fill-string (py-point 'boi))))
+ ;; are we at or after the closing quote of a string?
+ ((save-excursion
+ (backward-word 1)
+ (eq (py-in-literal) 'string))
+ (save-excursion
+ (py-fill-string (py-point 'boi))))
+ ;; otherwise use the default
+ (t
+ (fill-paragraph justify))))))
+
+
+
+(provide 'python-mode)
+;;; python-mode.el ends here
diff --git a/emacs/rst-mode.el b/emacs/rst-mode.el
new file mode 100644
index 0000000..cb3c201
--- /dev/null
+++ b/emacs/rst-mode.el
@@ -0,0 +1,698 @@
+;;; rst-mode.el --- Mode for viewing and editing reStructuredText-documents.
+
+;; Copyright 2003 Stefan Merten <smerten at oekonux.de>
+;;
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 2 of the License, or
+;; (at your option) any later version.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with this program; if not, write to the Free Software
+;; Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+;;; Commentary:
+
+;; This package provides support for documents marked up using the
+;; reStructuredText format
+;; [http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html].
+;; Support includes font locking as well as some convenience functions
+;; for editing.
+
+;; The package is based on `text-mode' and inherits some things from it.
+;; Particularly `text-mode-hook' is run before `rst-mode-hook'.
+
+;; Add the following lines to your `.emacs' file:
+;;
+;; (autoload 'rst-mode "rst-mode" "mode for editing reStructuredText documents" t)
+;; (setq auto-mode-alist
+;; (append '(("\\.rst$" . rst-mode)
+;; ("\\.rest$" . rst-mode)) auto-mode-alist))
+;;
+;; If you are using `.txt' as a standard extension for reST files as
+;; http://docutils.sourceforge.net/FAQ.html#what-s-the-standard-filename-extension-for-a-restructuredtext-file
+;; suggests you may use one of the `Local Variables in Files' mechanism Emacs
+;; provides to set the major mode automatically. For instance you may use
+;;
+;; .. -*- mode: rst -*-
+;;
+;; in the very first line of your file. However, because this is a major
+;; security breach you or your administrator may have chosen to switch that
+;; feature off. See `Local Variables in Files' in the Emacs documentation for a
+;; more complete discussion.
+
+;;; Code:
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;; Customization:
+
+(defgroup rst nil "Support for reStructuredText documents"
+ :group 'wp
+ :version "21.1"
+ :link '(url-link "http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html"))
+
+(defcustom rst-mode-hook nil
+ "Hook run when Rst Mode is turned on. The hook for Text Mode is run before
+ this one."
+ :group 'rst
+ :type '(hook))
+
+(defcustom rst-mode-lazy t
+ "*If non-nil Rst Mode font-locks comment, literal blocks, and section titles
+correctly. Because this is really slow it switches on Lazy Lock Mode
+automatically. You may increase Lazy Lock Defer Time for reasonable results.
+
+If nil comments and literal blocks are font-locked only on the line they start.
+
+The value of this variable is used when Rst Mode is turned on."
+ :group 'rst
+ :type '(boolean))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defgroup rst-faces nil "Faces used in Rst Mode"
+ :group 'rst
+ :group 'faces
+ :version "21.1")
+
+(defcustom rst-block-face 'font-lock-keyword-face
+ "All syntax marking up a special block"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-external-face 'font-lock-type-face
+ "Field names and interpreted text"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-definition-face 'font-lock-function-name-face
+ "All other defining constructs"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-directive-face
+ ;; XEmacs compatibility
+ (if (boundp 'font-lock-builtin-face)
+ 'font-lock-builtin-face
+ 'font-lock-preprocessor-face)
+ "Directives and roles"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-comment-face 'font-lock-comment-face
+ "Comments"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-emphasis1-face
+ ;; XEmacs compatibility
+ (if (facep 'italic)
+ ''italic
+ 'italic)
+ "Simple emphasis"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-emphasis2-face
+ ;; XEmacs compatibility
+ (if (facep 'bold)
+ ''bold
+ 'bold)
+ "Double emphasis"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-literal-face 'font-lock-string-face
+ "Literal text"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-reference-face 'font-lock-variable-name-face
+ "References to a definition"
+ :group 'rst-faces
+ :type '(face))
+
+;; Faces for displaying items on several levels; these definitions define
+;; different shades of grey where the lightest one is used for level 1
+(defconst rst-level-face-max 6
+ "Maximum depth of level faces defined")
+(defconst rst-level-face-base-color "grey"
+ "The base color to be used for creating level faces")
+(defconst rst-level-face-base-light 85
+ "The lightness factor for the base color")
+(defconst rst-level-face-format-light "%2d"
+ "The format for the lightness factor for the base color")
+(defconst rst-level-face-step-light -7
+ "The step width to use for next color")
+
+;; Define the faces
+(let ((i 1))
+ (while (<= i rst-level-face-max)
+ (let ((sym (intern (format "rst-level-%d-face" i)))
+ (doc (format "Face for showing section title text at level %d" i))
+ (col (format (concat "%s" rst-level-face-format-light)
+ rst-level-face-base-color
+ (+ (* (1- i) rst-level-face-step-light)
+ rst-level-face-base-light))))
+ (make-empty-face sym)
+ (set-face-doc-string sym doc)
+ (set-face-background sym col)
+ (set sym sym)
+ (setq i (1+ i)))))
+
+(defcustom rst-adornment-faces-alist
+ '((1 . rst-level-1-face)
+ (2 . rst-level-2-face)
+ (3 . rst-level-3-face)
+ (4 . rst-level-4-face)
+ (5 . rst-level-5-face)
+ (6 . rst-level-6-face)
+ (t . font-lock-keyword-face)
+ (nil . font-lock-keyword-face))
+ "Provides faces for the various adornment types. Key is a number (for the
+section title text of that level), t (for transitions) or nil (for section
+title adornment)."
+ :group 'rst-faces
+ :type '(alist :key-type (choice (integer :tag "Section level")
+ (boolean :tag "transitions (on) / section title adornment (off)"))
+ :value-type (face)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; FIXME: Code from `restructuredtext.el' should be integrated
+
+(defvar rst-mode-syntax-table nil
+ "Syntax table used while in rst mode.")
+
+(unless rst-mode-syntax-table
+ (setq rst-mode-syntax-table (make-syntax-table text-mode-syntax-table))
+ (modify-syntax-entry ?$ "." rst-mode-syntax-table)
+ (modify-syntax-entry ?% "." rst-mode-syntax-table)
+ (modify-syntax-entry ?& "." rst-mode-syntax-table)
+ (modify-syntax-entry ?' "." rst-mode-syntax-table)
+ (modify-syntax-entry ?* "." rst-mode-syntax-table)
+ (modify-syntax-entry ?+ "." rst-mode-syntax-table)
+ (modify-syntax-entry ?. "_" rst-mode-syntax-table)
+ (modify-syntax-entry ?/ "." rst-mode-syntax-table)
+ (modify-syntax-entry ?< "." rst-mode-syntax-table)
+ (modify-syntax-entry ?= "." rst-mode-syntax-table)
+ (modify-syntax-entry ?> "." rst-mode-syntax-table)
+ (modify-syntax-entry ?\\ "\\" rst-mode-syntax-table)
+ (modify-syntax-entry ?| "." rst-mode-syntax-table)
+ (modify-syntax-entry ?_ "." rst-mode-syntax-table)
+ )
+
+(defvar rst-mode-abbrev-table nil
+ "Abbrev table used while in rst mode.")
+(define-abbrev-table 'rst-mode-abbrev-table ())
+
+;; FIXME: Movement keys to skip forward / backward over or mark an indented
+;; block could be defined; keys to markup section titles based on
+;; `rst-adornment-level-alist' would be useful
+(defvar rst-mode-map nil
+ "Keymap for rst mode. This inherits from Text mode.")
+
+(unless rst-mode-map
+ (setq rst-mode-map (copy-keymap text-mode-map)))
+
+(defun rst-mode ()
+ "Major mode for editing reStructuredText documents.
+
+You may customize `rst-mode-lazy' to switch font-locking of blocks.
+
+\\{rst-mode-map}
+Turning on `rst-mode' calls the normal hooks `text-mode-hook' and
+`rst-mode-hook'."
+ (interactive)
+ (kill-all-local-variables)
+
+ ;; Maps and tables
+ (use-local-map rst-mode-map)
+ (setq local-abbrev-table rst-mode-abbrev-table)
+ (set-syntax-table rst-mode-syntax-table)
+
+ ;; For editing text
+ ;;
+ ;; FIXME: It would be better if this matches more exactly the start of a reST
+ ;; paragraph; however, this not always possible with a simple regex because
+ ;; paragraphs are determined by indentation of the following line
+ (set (make-local-variable 'paragraph-start)
+ (concat page-delimiter "\\|[ \t]*$"))
+ (if (eq ?^ (aref paragraph-start 0))
+ (setq paragraph-start (substring paragraph-start 1)))
+ (set (make-local-variable 'paragraph-separate) paragraph-start)
+ (set (make-local-variable 'indent-line-function) 'indent-relative-maybe)
+ (set (make-local-variable 'adaptive-fill-mode) t)
+ (set (make-local-variable 'comment-start) ".. ")
+
+ ;; Special variables
+ (make-local-variable 'rst-adornment-level-alist)
+
+ ;; Font lock
+ (set (make-local-variable 'font-lock-defaults)
+ '(rst-font-lock-keywords-function
+ t nil nil nil
+ (font-lock-multiline . t)
+ (font-lock-mark-block-function . mark-paragraph)))
+ (when (boundp 'font-lock-support-mode)
+ ;; rst-mode has its own mind about font-lock-support-mode
+ (make-local-variable 'font-lock-support-mode)
+ (cond
+ ((and (not rst-mode-lazy) (not font-lock-support-mode)))
+ ;; No support mode set and none required - leave it alone
+ ((or (not font-lock-support-mode) ;; No support mode set (but required)
+ (symbolp font-lock-support-mode)) ;; or a fixed mode for all
+ (setq font-lock-support-mode
+ (list (cons 'rst-mode (and rst-mode-lazy 'lazy-lock-mode))
+ (cons t font-lock-support-mode))))
+ ((and (listp font-lock-support-mode)
+ (not (assoc 'rst-mode font-lock-support-mode)))
+ ;; A list of modes missing rst-mode
+ (setq font-lock-support-mode
+ (append '((cons 'rst-mode (and rst-mode-lazy 'lazy-lock-mode)))
+ font-lock-support-mode)))))
+
+ ;; Names and hooks
+ (setq mode-name "reST")
+ (setq major-mode 'rst-mode)
+ (run-hooks 'text-mode-hook)
+ (run-hooks 'rst-mode-hook))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Font lock
+
+(defun rst-font-lock-keywords-function ()
+ "Returns keywords to highlight in rst mode according to current settings."
+ ;; The reST-links in the comments below all relate to sections in
+ ;; http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html
+ (let* ( ;; This gets big - so let's define some abbreviations
+ ;; horizontal white space
+ (re-hws "[\t ]")
+ ;; beginning of line with possible indentation
+ (re-bol (concat "^" re-hws "*"))
+ ;; Separates block lead-ins from their content
+ (re-blksep1 (concat "\\(" re-hws "+\\|$\\)"))
+ ;; explicit markup tag
+ (re-emt "\\.\\.")
+ ;; explicit markup start
+ (re-ems (concat re-emt re-hws "+"))
+ ;; inline markup prefix
+ (re-imp1 (concat "\\(^\\|" re-hws "\\|[-'\"([{</:]\\)"))
+ ;; inline markup suffix
+ (re-ims1 (concat "\\(" re-hws "\\|[]-'\")}>/:.,;!?\\]\\|$\\)"))
+ ;; symbol character
+ (re-sym1 "\\(\\sw\\|\\s_\\)")
+ ;; inline markup content begin
+ (re-imbeg2 "\\(\\S \\|\\S \\([^")
+
+ ;; There seems to be a bug leading to error "Stack overflow in regexp
+ ;; matcher" when "|" or "\\*" are the characters searched for
+ (re-imendbeg
+ (if (< emacs-major-version 21)
+ "]"
+ "\\]\\|\\\\."))
+ ;; inline markup content end
+ (re-imend (concat re-imendbeg "\\)*[^\t \\\\]\\)"))
+ ;; inline markup content without asterisk
+ (re-ima2 (concat re-imbeg2 "*" re-imend))
+ ;; inline markup content without backquote
+ (re-imb2 (concat re-imbeg2 "`" re-imend))
+ ;; inline markup content without vertical bar
+ (re-imv2 (concat re-imbeg2 "|" re-imend))
+ ;; Supported URI schemes
+ (re-uris1 "\\(acap\\|cid\\|data\\|dav\\|fax\\|file\\|ftp\\|gopher\\|http\\|https\\|imap\\|ldap\\|mailto\\|mid\\|modem\\|news\\|nfs\\|nntp\\|pop\\|prospero\\|rtsp\\|service\\|sip\\|tel\\|telnet\\|tip\\|urn\\|vemmi\\|wais\\)")
+ ;; Line starting with adornment and optional whitespace; complete
+ ;; adornment is in (match-string 1); there must be at least 3
+ ;; characters because otherwise explicit markup start would be
+ ;; recognized
+ (re-ado2 (concat "^\\(\\(["
+ (if (or
+ (< emacs-major-version 21)
+ (save-match-data
+ (string-match "XEmacs\\|Lucid" emacs-version)))
+ "^a-zA-Z0-9 \t\x00-\x1F"
+ "^[:word:][:space:][:cntrl:]")
+ "]\\)\\2\\2+\\)" re-hws "*$"))
+ )
+ (list
+ ;; FIXME: Block markup is not recognized in blocks after explicit markup
+ ;; start
+
+ ;; Simple `Body Elements`_
+ ;; `Bullet Lists`_
+ (list
+ (concat re-bol "\\([-*+]" re-blksep1 "\\)")
+ 1 rst-block-face)
+ ;; `Enumerated Lists`_
+ (list
+ (concat re-bol "\\((?\\([0-9]+\\|[A-Za-z]\\|[IVXLCMivxlcm]+\\)[.)]" re-blksep1 "\\)")
+ 1 rst-block-face)
+ ;; `Definition Lists`_ FIXME: missing
+ ;; `Field Lists`_
+ (list
+ (concat re-bol "\\(:[^:]+:\\)" re-blksep1)
+ 1 rst-external-face)
+ ;; `Option Lists`_
+ (list
+ (concat re-bol "\\(\\(\\(\\([-+/]\\|--\\)\\sw\\(-\\|\\sw\\)*\\([ =]\\S +\\)?\\)\\(,[\t ]\\)?\\)+\\)\\($\\|[\t ]\\{2\\}\\)")
+ 1 rst-block-face)
+
+ ;; `Tables`_ FIXME: missing
+
+ ;; All the `Explicit Markup Blocks`_
+ ;; `Footnotes`_ / `Citations`_
+ (list
+ (concat re-bol "\\(" re-ems "\\[[^[]+\\]\\)" re-blksep1)
+ 1 rst-definition-face)
+ ;; `Directives`_ / `Substitution Definitions`_
+ (list
+ (concat re-bol "\\(" re-ems "\\)\\(\\(|[^|]+|[\t ]+\\)?\\)\\(" re-sym1 "+::\\)" re-blksep1)
+ (list 1 rst-directive-face)
+ (list 2 rst-definition-face)
+ (list 4 rst-directive-face))
+ ;; `Hyperlink Targets`_
+ (list
+ (concat re-bol "\\(" re-ems "_\\([^:\\`]\\|\\\\.\\|`[^`]+`\\)+:\\)" re-blksep1)
+ 1 rst-definition-face)
+ (list
+ (concat re-bol "\\(__\\)" re-blksep1)
+ 1 rst-definition-face)
+
+ ;; All `Inline Markup`_
+ ;; FIXME: Condition 5 preventing fontification of e.g. "*" not implemented
+ ;; `Strong Emphasis`_
+ (list
+ (concat re-imp1 "\\(\\*\\*" re-ima2 "\\*\\*\\)" re-ims1)
+ 2 rst-emphasis2-face)
+ ;; `Emphasis`_
+ (list
+ (concat re-imp1 "\\(\\*" re-ima2 "\\*\\)" re-ims1)
+ 2 rst-emphasis1-face)
+ ;; `Inline Literals`_
+ (list
+ (concat re-imp1 "\\(``" re-imb2 "``\\)" re-ims1)
+ 2 rst-literal-face)
+ ;; `Inline Internal Targets`_
+ (list
+ (concat re-imp1 "\\(_`" re-imb2 "`\\)" re-ims1)
+ 2 rst-definition-face)
+ ;; `Hyperlink References`_
+ ;; FIXME: `Embedded URIs`_ not considered
+ (list
+ (concat re-imp1 "\\(\\(`" re-imb2 "`\\|\\sw+\\)__?\\)" re-ims1)
+ 2 rst-reference-face)
+ ;; `Interpreted Text`_
+ (list
+ (concat re-imp1 "\\(\\(:" re-sym1 "+:\\)?\\)\\(`" re-imb2 "`\\)\\(\\(:" re-sym1 "+:\\)?\\)" re-ims1)
+ (list 2 rst-directive-face)
+ (list 5 rst-external-face)
+ (list 8 rst-directive-face))
+ ;; `Footnote References`_ / `Citation References`_
+ (list
+ (concat re-imp1 "\\(\\[[^]]+\\]_\\)" re-ims1)
+ 2 rst-reference-face)
+ ;; `Substitution References`_
+ (list
+ (concat re-imp1 "\\(|" re-imv2 "|\\)" re-ims1)
+ 2 rst-reference-face)
+ ;; `Standalone Hyperlinks`_
+ (list
+ ;; FIXME: This takes it easy by using a whitespace as delimiter
+ (concat re-imp1 "\\(" re-uris1 ":\\S +\\)" re-ims1)
+ 2 rst-definition-face)
+ (list
+ (concat re-imp1 "\\(" re-sym1 "+@" re-sym1 "+\\)" re-ims1)
+ 2 rst-definition-face)
+
+ ;; Do all block fontification as late as possible so 'append works
+
+ ;; Sections_ / Transitions_
+ (append
+ (list
+ re-ado2)
+ (if (not rst-mode-lazy)
+ (list 1 rst-block-face)
+ (list
+ (list 'rst-font-lock-handle-adornment
+ '(progn
+ (setq rst-font-lock-adornment-point (match-end 1))
+ (point-max))
+ nil
+ (list 1 '(cdr (assoc nil rst-adornment-faces-alist))
+ 'append t)
+ (list 2 '(cdr (assoc rst-font-lock-level rst-adornment-faces-alist))
+ 'append t)
+ (list 3 '(cdr (assoc nil rst-adornment-faces-alist))
+ 'append t)))))
+
+ ;; `Comments`_
+ (append
+ (list
+ (concat re-bol "\\(" re-ems "\\)\[^[|_]\\([^:]\\|:\\([^:]\\|$\\)\\)*$")
+ (list 1 rst-comment-face))
+ (if rst-mode-lazy
+ (list
+ (list 'rst-font-lock-find-unindented-line
+ '(progn
+ (setq rst-font-lock-indentation-point (match-end 1))
+ (point-max))
+ nil
+ (list 0 rst-comment-face 'append)))))
+ (append
+ (list
+ (concat re-bol "\\(" re-emt "\\)\\(\\s *\\)$")
+ (list 1 rst-comment-face)
+ (list 2 rst-comment-face))
+ (if rst-mode-lazy
+ (list
+ (list 'rst-font-lock-find-unindented-line
+ '(progn
+ (setq rst-font-lock-indentation-point 'next)
+ (point-max))
+ nil
+ (list 0 rst-comment-face 'append)))))
+
+ ;; `Literal Blocks`_
+ (append
+ (list
+ (concat re-bol "\\(\\([^.\n]\\|\\.[^.\n]\\).*\\)?\\(::\\)$")
+ (list 3 rst-block-face))
+ (if rst-mode-lazy
+ (list
+ (list 'rst-font-lock-find-unindented-line
+ '(progn
+ (setq rst-font-lock-indentation-point t)
+ (point-max))
+ nil
+ (list 0 rst-literal-face 'append)))))
+ )))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Indented blocks
+
+(defun rst-forward-indented-block (&optional column limit)
+ "Move forward across one indented block.
+Find the next non-empty line which is not indented at least to COLUMN (defaults
+to the column of the point). Moves point to first character of this line or the
+first empty line immediately before it and returns that position. If there is
+no such line before LIMIT (defaults to the end of the buffer) returns nil and
+point is not moved."
+ (interactive)
+ (let ((clm (or column (current-column)))
+ (start (point))
+ fnd beg cand)
+ (if (not limit)
+ (setq limit (point-max)))
+ (save-match-data
+ (while (and (not fnd) (< (point) limit))
+ (forward-line 1)
+ (when (< (point) limit)
+ (setq beg (point))
+ (if (looking-at "\\s *$")
+ (setq cand (or cand beg)) ; An empty line is a candidate
+ (move-to-column clm)
+ ;; FIXME: No indentation [(zerop clm)] must be handled in some
+ ;; useful way - though it is not clear what this should mean at all
+ (if (string-match
+ "^\\s *$" (buffer-substring-no-properties beg (point)))
+ (setq cand nil) ; An indented line resets a candidate
+ (setq fnd (or cand beg)))))))
+ (goto-char (or fnd start))
+ fnd))
+
+;; Stores the point where the current indentation ends if a number. If `next'
+;; indicates `rst-font-lock-find-unindented-line' shall take the indentation
+;; from the next line if this is not empty. If non-nil indicates
+;; `rst-font-lock-find-unindented-line' shall take the indentation from the
+;; next non-empty line. Also used as a trigger for
+;; `rst-font-lock-find-unindented-line'.
+(defvar rst-font-lock-indentation-point nil)
+
+(defun rst-font-lock-find-unindented-line (limit)
+ (let* ((ind-pnt rst-font-lock-indentation-point)
+ (beg-pnt ind-pnt))
+ ;; May run only once - enforce this
+ (setq rst-font-lock-indentation-point nil)
+ (when (and ind-pnt (not (numberp ind-pnt)))
+ ;; Find indentation point in next line if any
+ (setq ind-pnt
+ (save-excursion
+ (save-match-data
+ (if (eq ind-pnt 'next)
+ (when (and (zerop (forward-line 1)) (< (point) limit))
+ (setq beg-pnt (point))
+ (when (not (looking-at "\\s *$"))
+ (looking-at "\\s *")
+ (match-end 0)))
+ (while (and (zerop (forward-line 1)) (< (point) limit)
+ (looking-at "\\s *$")))
+ (when (< (point) limit)
+ (setq beg-pnt (point))
+ (looking-at "\\s *")
+ (match-end 0)))))))
+ (when ind-pnt
+ (goto-char ind-pnt)
+ ;; Always succeeds because the limit set by PRE-MATCH-FORM is the
+ ;; ultimate point to find
+ (goto-char (or (rst-forward-indented-block nil limit) limit))
+ (set-match-data (list beg-pnt (point)))
+ t)))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Adornments
+
+;; Stores the point where the current adornment ends. Also used as a trigger
+;; for `rst-font-lock-handle-adornment'.
+(defvar rst-font-lock-adornment-point nil)
+
+;; Here `rst-font-lock-handle-adornment' stores the section level of the
+;; current adornment or t for a transition.
+(defvar rst-font-lock-level nil)
+
+;; FIXME: It would be good if this could be used to markup section titles of
+;; given level with a special key; it would be even better to be able to
+;; customize this so it can be used for a generally available personal style
+;;
+;; FIXME: There should be some way to reset and reload this variable - probably
+;; a special key
+;;
+;; FIXME: Some support for `outline-mode' would be nice which should be based
+;; on this information
+(defvar rst-adornment-level-alist nil
+ "Associates adornments with section levels.
+The key is a two character string. The first character is the adornment
+character. The second character distinguishes underline section titles (`u')
+from overline/underline section titles (`o'). The value is the section level.
+
+This is made buffer local on start and adornments found during font lock are
+entered.")
+
+;; Returns section level for adornment key KEY. Adds new section level if KEY
+;; is not found and ADD. If KEY is not a string it is simply returned.
+(defun rst-adornment-level (key &optional add)
+ (let ((fnd (assoc key rst-adornment-level-alist))
+ (new 1))
+ (cond
+ ((not (stringp key))
+ key)
+ (fnd
+ (cdr fnd))
+ (add
+ (while (rassoc new rst-adornment-level-alist)
+ (setq new (1+ new)))
+ (setq rst-adornment-level-alist
+ (append rst-adornment-level-alist (list (cons key new))))
+ new))))
+
+;; Classifies adornment for section titles and transitions. ADORNMENT is the
+;; complete adornment string as found in the buffer. END is the point after the
+;; last character of ADORNMENT. For overline section adornment LIMIT limits the
+;; search for the matching underline. Returns a list. The first entry is t for
+;; a transition, or a key string for `rst-adornment-level' for a section title.
+;; The following eight values forming four match groups as can be used for
+;; `set-match-data'. First match group contains the maximum points of the whole
+;; construct. Second and last match group matched pure section title adornment
+;; while third match group matched the section title text or the transition.
+;; Each group but the first may or may not exist.
+(defun rst-classify-adornment (adornment end limit)
+ (save-excursion
+ (save-match-data
+ (goto-char end)
+ (let ((ado-ch (aref adornment 0))
+ (ado-re (regexp-quote adornment))
+ (end-pnt (point))
+ (beg-pnt (progn
+ (forward-line 0)
+ (point)))
+ (nxt-emp
+ (save-excursion
+ (or (not (zerop (forward-line 1)))
+ (looking-at "\\s *$"))))
+ (prv-emp
+ (save-excursion
+ (or (not (zerop (forward-line -1)))
+ (looking-at "\\s *$"))))
+ key beg-ovr end-ovr beg-txt end-txt beg-und end-und)
+ (cond
+ ((and nxt-emp prv-emp)
+ ;; A transition
+ (setq key t)
+ (setq beg-txt beg-pnt)
+ (setq end-txt end-pnt))
+ (prv-emp
+ ;; An overline
+ (setq key (concat (list ado-ch) "o"))
+ (setq beg-ovr beg-pnt)
+ (setq end-ovr end-pnt)
+ (forward-line 1)
+ (setq beg-txt (point))
+ (while (and (< (point) limit) (not end-txt))
+ (if (looking-at "\\s *$")
+ ;; No underline found
+ (setq end-txt (1- (point)))
+ (when (looking-at (concat "\\(" ado-re "\\)\\s *$"))
+ (setq end-und (match-end 1))
+ (setq beg-und (point))
+ (setq end-txt (1- beg-und))))
+ (forward-line 1)))
+ (t
+ ;; An underline
+ (setq key (concat (list ado-ch) "u"))
+ (setq beg-und beg-pnt)
+ (setq end-und end-pnt)
+ (setq end-txt (1- beg-und))
+ (setq beg-txt (progn
+ (if (re-search-backward "^\\s *$" 1 'move)
+ (forward-line 1))
+ (point)))))
+ (list key
+ (or beg-ovr beg-txt beg-und)
+ (or end-und end-txt end-und)
+ beg-ovr end-ovr beg-txt end-txt beg-und end-und)))))
+
+;; Handles adornments for font-locking section titles and transitions. Returns
+;; three match groups. First and last match group matched pure overline /
+;; underline adornment while second group matched section title text. Each
+;; group may not exist.
+(defun rst-font-lock-handle-adornment (limit)
+ (let ((ado-pnt rst-font-lock-adornment-point))
+ ;; May run only once - enforce this
+ (setq rst-font-lock-adornment-point nil)
+ (if ado-pnt
+ (let* ((ado (rst-classify-adornment (match-string-no-properties 1)
+ ado-pnt limit))
+ (key (car ado))
+ (mtc (cdr ado)))
+ (setq rst-font-lock-level (rst-adornment-level key t))
+ (goto-char (nth 1 mtc))
+ (set-match-data mtc)
+ t))))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;; rst-mode.el ends here
diff --git a/emacs/rst.el b/emacs/rst.el
new file mode 100644
index 0000000..6caa778
--- /dev/null
+++ b/emacs/rst.el
@@ -0,0 +1,3431 @@
+;;; rst.el --- Mode for viewing and editing reStructuredText-documents.
+
+;; Copyright 2003-2008 by Martin Blais, Stefan Merten, and David Goodger.
+
+;; Authors: Martin Blais <blais at furius.ca>,
+;; Stefan Merten <smerten at oekonux.de>,
+;; David Goodger <goodger at python.org>
+
+;; This program is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License version 2,
+;; as published by the Free Software Foundation.
+;;
+;; This program is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License version 2
+;; along with this program and available at
+;; http://docutils.sf.net/licenses/gpl.txt and at
+;; http://www.gnu.org/licenses/gpl.txt.
+
+;;; Commentary:
+
+;; This package provides major mode rst-mode, which supports documents marked up
+;; using the reStructuredText format. Support includes font locking as well as
+;; some convenience functions for editing. It does this by defining a Emacs
+;; major mode: rst-mode (ReST). This mode is derived from text-mode (and
+;; inherits much of it). This package also contains:
+;;
+;; - Functions to automatically adjust and cycle the section underline
+;; decorations;
+;; - A mode that displays the table of contents and allows you to jump anywhere
+;; from it;
+;; - Functions to insert and automatically update a TOC in your source
+;; document;
+;; - Font-lock highlighting of notable reStructuredText structures;
+;; - Some other convenience functions.
+;;
+;; See the accompanying document in the docutils documentation about
+;; the contents of this package and how to use it.
+;;
+;; For more information about reStructuredText, see
+;; http://docutils.sourceforge.net/rst.html
+;;
+;; For full details on how to use the contents of this file, see
+;; http://docutils.sourceforge.net/docs/user/emacs.html
+;;
+;;
+;; There are a number of convenient keybindings provided by rst-mode. The main
+;; one is
+;;
+;; C-c C-a (also C-=): rst-adjust
+;;
+;; Updates or rotates the section title around point or promotes/demotes the
+;; decorations within the region (see full details below). Note that C-= is a
+;; good binding, since it allows you to specify a negative arg easily with C--
+;; C-= (easy to type), as well as ordinary prefix arg with C-u C-=.
+;;
+;; For more on bindings, see rst-mode-map below. There are also many variables
+;; that can be customized, look for defcustom and defvar in this file.
+;;
+;; If you use the table-of-contents feature, you may want to add a hook to
+;; update the TOC automatically everytime you adjust a section title::
+;;
+;; (add-hook 'rst-adjust-hook 'rst-toc-update)
+;;
+;; Syntax highlighting: font-lock is enabled by default. If you want to turn off
+;; syntax highlighting to rst-mode, you can use the following::
+;;
+;; (setq font-lock-global-modes '(not rst-mode ...))
+;;
+
+
+;; CUSTOMIZATION
+;;
+;; rst
+;; ---
+;; This group contains some general customizable features.
+;;
+;; The group is contained in the wp group.
+;;
+;; rst-faces
+;; ---------
+;; This group contains all necessary for customizing fonts. The default
+;; settings use standard font-lock-*-face's so if you set these to your
+;; liking they are probably good in rst-mode also.
+;;
+;; The group is contained in the faces group as well as in the rst group.
+;;
+;; rst-faces-defaults
+;; ------------------
+;; This group contains all necessary for customizing the default fonts used for
+;; section title faces.
+;;
+;; The general idea for section title faces is to have a non-default background
+;; but do not change the background. The section level is shown by the
+;; lightness of the background color. If you like this general idea of
+;; generating faces for section titles but do not like the details this group
+;; is the point where you can customize the details. If you do not like the
+;; general idea, however, you should customize the faces used in
+;; rst-adornment-faces-alist.
+;;
+;; Note: If you are using a dark background please make sure the variable
+;; frame-background-mode is set to the symbol dark. This triggers
+;; some default values which are probably right for you.
+;;
+;; The group is contained in the rst-faces group.
+;;
+;; All customizable features have a comment explaining their meaning. Refer to
+;; the customization of your Emacs (try ``M-x customize``).
+
+
+;;; DOWNLOAD
+
+;; The latest version of this file lies in the docutils source code repository:
+;; http://svn.berlios.de/svnroot/repos/docutils/trunk/docutils/tools/editors/emacs/rst.el
+
+
+;;; INSTALLATION
+
+;; Add the following lines to your `.emacs' file:
+;;
+;; (require 'rst)
+;;
+;; If you are using `.txt' as a standard extension for reST files as
+;; http://docutils.sourceforge.net/FAQ.html#what-s-the-standard-filename-extension-for-a-restructuredtext-file
+;; suggests you may use one of the `Local Variables in Files' mechanism Emacs
+;; provides to set the major mode automatically. For instance you may use::
+;;
+;; .. -*- mode: rst -*-
+;;
+;; in the very first line of your file. The following code is useful if you want
+;; to automatically enter rst-mode from any file with compatible extensions:
+;;
+;; (setq auto-mode-alist
+;; (append '(("\\.txt$" . rst-mode)
+;; ("\\.rst$" . rst-mode)
+;; ("\\.rest$" . rst-mode)) auto-mode-alist))
+;;
+
+;;; BUGS
+
+;; - rst-enumeration-region: Select a single paragraph, with the top at one
+;; blank line before the beginning, and it will fail.
+;; - The active region goes away when we shift it left or right, and this
+;; prevents us from refilling it automatically when shifting many times.
+;; - The suggested decorations when adjusting should not have to cycle
+;; below one below the last section decoration level preceding the
+;; cursor. We need to fix that.
+
+;;; TODO LIST
+
+;; rst-toc-insert features
+;; ------------------------
+;; - rst-toc-insert: We should parse the contents:: options to figure out how
+;; deep to render the inserted TOC.
+;; - On load, detect any existing TOCs and set the properties for links.
+;; - TOC insertion should have an option to add empty lines.
+;; - TOC insertion should deal with multiple lines.
+;; - There is a bug on redo after undo of adjust when rst-adjust-hook uses the
+;; automatic toc update. The cursor ends up in the TOC and this is
+;; annoying. Gotta fix that.
+;; - numbering: automatically detect if we have a section-numbering directive in
+;; the corresponding section, to render the toc.
+;;
+;; bulleted and enumerated list items
+;; ----------------------------------
+;; - We need to provide way to rebullet bulleted lists, and that would include
+;; automatic enumeration as well.
+;;
+;; Other
+;; -----
+;; - It would be nice to differentiate between text files using
+;; reStructuredText_ and other general text files. If we had a
+;; function to automatically guess whether a .txt file is following the
+;; reStructuredText_ conventions, we could trigger rst-mode without
+;; having to hard-code this in every text file, nor forcing the user to
+;; add a local mode variable at the top of the file.
+;; We could perform this guessing by searching for a valid decoration
+;; at the top of the document or searching for reStructuredText_
+;; directives further on.
+;;
+;; - We should support imenu in our major mode, with the menu filled with the
+;; section titles (this should be really easy).
+;;
+;; - We should rename "adornment" to "decoration" or vice-versa in this
+;; document (Stefan's code ("adornment") vs Martin ("decoration")), maybe some
+;; functions even overlap.
+;;
+;; - We need to automatically recenter on rst-forward-section movement commands.
+
+
+;;; HISTORY
+;;
+
+;;; CODE
+
+
+(defgroup rst nil "Support for reStructuredText documents"
+ :group 'wp
+ :version "21.1"
+ :link '(url-link "http://docutils.sourceforge.net/rst.html"))
+
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Define some generic support functions.
+
+(require 'cl) ;; We need this for destructuring-bind below.
+
+;; Generic Filter function.
+(unless (fboundp 'filter)
+ (defun filter (pred list)
+ "Returns a list of all the elements fulfilling the pred requirement (that
+is for which (pred elem) is true)"
+ (if list
+ (let ((head (car list))
+ (tail (filter pred (cdr list))))
+ (if (funcall pred head)
+ (cons head tail)
+ tail)))))
+
+
+;; From emacs-22
+(unless (fboundp 'line-number-at-pos)
+ (defun line-number-at-pos (&optional pos)
+ "Return (narrowed) buffer line number at position POS.
+ If POS is nil, use current buffer location."
+ (let ((opoint (or pos (point))) start)
+ (save-excursion
+ (goto-char (point-min))
+ (setq start (point))
+ (goto-char opoint)
+ (forward-line 0)
+ (1+ (count-lines start (point)))))) )
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Mode definition.
+
+(defconst rst-use-unicode
+ (string-equal "\u0020" " ")
+ "Non-nil if we can use unicode characters.")
+
+;; Key bindings.
+(defvar rst-mode-map
+ (let ((map (make-sparse-keymap)))
+
+ ;;
+ ;; Section Decorations.
+ ;;
+ ;; The adjustment function that decorates or rotates a section title.
+ (define-key map [(control c) (control a)] 'rst-adjust)
+ (define-key map [(control c) (control ?=)] 'rst-adjust)
+ (define-key map [(control ?=)] 'rst-adjust) ;; (Does not work on the Mac OSX.)
+ ;; Display the hierarchy of decorations implied by the current document contents.
+ (define-key map [(control c) (control h)] 'rst-display-decorations-hierarchy)
+ ;; Homogeneize the decorations in the document.
+ (define-key map [(control c) (control s)] 'rst-straighten-decorations)
+;; (define-key map [(control c) (control s)] 'rst-straighten-deco-spacing)
+
+ ;;
+ ;; Section Movement and Selection.
+ ;;
+ ;; Mark the subsection where the cursor is.
+ (define-key map [(control c) (control m)] 'rst-mark-section)
+ ;; Move forward/backward between section titles.
+ (define-key map [(control c) (control n)] 'rst-forward-section)
+ (define-key map [(control c) (control p)] 'rst-backward-section)
+
+ ;;
+ ;; Operating on Blocks of Text.
+ ;;
+ ;; Makes paragraphs in region as a bullet list.
+ (define-key map [(control c) (control b)] 'rst-bullet-list-region)
+ ;; Makes paragraphs in region as a enumeration.
+ (define-key map [(control c) (control e)] 'rst-enumerate-region)
+ ;; Converts bullets to an enumeration.
+ (define-key map [(control c) (control v)] 'rst-convert-bullets-to-enumeration)
+ ;; Makes region a line-block.
+ (define-key map [(control c) (control d)] 'rst-line-block-region)
+ ;; Make sure that all the bullets in the region are consistent.
+ (define-key map [(control c) (control w)] 'rst-straighten-bullets-region)
+ ;; Shift region left or right (taking into account of enumerations/bullets, etc.).
+ (define-key map [(control c) (control l)] 'rst-shift-region-left)
+ (define-key map [(control c) (control r)] 'rst-shift-region-right)
+ ;; Comment/uncomment the active region.
+ (define-key map [(control c) (control c)] 'comment-region)
+
+ ;;
+ ;; Table-of-Contents Features.
+ ;;
+ ;; Enter a TOC buffer to view and move to a specific section.
+ (define-key map [(control c) (control t)] 'rst-toc)
+ ;; Insert a TOC here.
+ (define-key map [(control c) (control i)] 'rst-toc-insert)
+ ;; Update the document's TOC (without changing the cursor position).
+ (define-key map [(control c) (control u)] 'rst-toc-update)
+ ;; Got to the section under the cursor (cursor must be in TOC).
+ (define-key map [(control c) (control f)] 'rst-goto-section)
+
+ ;;
+ ;; Converting Documents from Emacs.
+ ;;
+ ;; Run one of two pre-configured toolset commands on the document.
+ (define-key map [(control c) (?1)] 'rst-compile)
+ (define-key map [(control c) (?2)] 'rst-compile-alt-toolset)
+ ;; Convert the active region to pseudo-xml using the docutils tools.
+ (define-key map [(control c) (?3)] 'rst-compile-pseudo-region)
+ ;; Convert the current document to PDF and launch a viewer on the results.
+ (define-key map [(control c) (?4)] 'rst-compile-pdf-preview)
+ ;; Convert the current document to S5 slides and view in a web browser.
+ (define-key map [(control c) (?5)] 'rst-compile-slides-preview)
+
+ map)
+ "Keymap for ReStructuredText mode commands. This inherits from Text mode.")
+
+
+;; Abbrevs.
+(defvar rst-mode-abbrev-table nil
+ "Abbrev table used while in rst mode.")
+(define-abbrev-table 'rst-mode-abbrev-table
+ '(
+ ("contents" ".. contents::\n..\n " nil 0)
+ ("con" ".. contents::\n..\n " nil 0)
+ ("cont" "[...]" nil 0)
+ ("skip" "\n\n[...]\n\n " nil 0)
+ ("seq" "\n\n[...]\n\n " nil 0)
+ ;; FIXME: Add footnotes, links, and more.
+ ))
+
+
+;; Syntax table.
+(defvar rst-mode-syntax-table
+ (let ((st (copy-syntax-table text-mode-syntax-table)))
+
+ (modify-syntax-entry ?$ "." st)
+ (modify-syntax-entry ?% "." st)
+ (modify-syntax-entry ?& "." st)
+ (modify-syntax-entry ?' "." st)
+ (modify-syntax-entry ?* "." st)
+ (modify-syntax-entry ?+ "." st)
+ (modify-syntax-entry ?. "_" st)
+ (modify-syntax-entry ?/ "." st)
+ (modify-syntax-entry ?< "." st)
+ (modify-syntax-entry ?= "." st)
+ (modify-syntax-entry ?> "." st)
+ (modify-syntax-entry ?\\ "\\" st)
+ (modify-syntax-entry ?| "." st)
+ (modify-syntax-entry ?_ "." st)
+ (when rst-use-unicode
+ ;; Use strings because unicode literals are not understood before Emacs
+ ;; 22
+ (modify-syntax-entry (aref "\u00ab" 0) "." st)
+ (modify-syntax-entry (aref "\u00bb" 0) "." st)
+ (modify-syntax-entry (aref "\u2018" 0) "." st)
+ (modify-syntax-entry (aref "\u2019" 0) "." st)
+ (modify-syntax-entry (aref "\u201c" 0) "." st)
+ (modify-syntax-entry (aref "\u201d" 0) "." st))
+
+ st)
+ "Syntax table used while in `rst-mode'.")
+
+
+(defcustom rst-mode-hook nil
+ "Hook run when Rst Mode is turned on. The hook for Text Mode is run before
+ this one."
+ :group 'rst
+ :type '(hook))
+
+
+;;;###autoload
+(define-derived-mode rst-mode text-mode "ReST"
+ :abbrev-table rst-mode-abbrev-table
+ :syntax-table rst-mode-syntax-table
+ :group 'rst
+ "Major mode for editing reStructuredText documents.
+
+There are a number of convenient keybindings provided by
+rst-mode. The main one is \[rst-adjust\], it updates or rotates
+the section title around point or promotes/demotes the
+decorations within the region (see full details below). Use
+negative prefix arg to rotate in the other direction.
+\\{rst-mode-map}
+
+Turning on `rst-mode' calls the normal hooks `text-mode-hook' and
+`rst-mode-hook'. This mode also supports font-lock highlighting."
+
+ (set (make-local-variable 'paragraph-separate) paragraph-start)
+ (set (make-local-variable 'indent-line-function)
+ (if (<= emacs-major-version 21)
+ 'indent-relative-maybe
+ 'indent-relative))
+ (set (make-local-variable 'paragraph-start)
+ "\f\\|>*[ \t]*$\\|>*[ \t]*[-+*] \\|>*[ \t]*[0-9#]+\\. ")
+ (set (make-local-variable 'adaptive-fill-mode) t)
+
+ ;; FIXME: No need to reset this.
+ ;; (set (make-local-variable 'indent-line-function) 'indent-relative)
+
+ ;; The details of the following comment setup is important because it affects
+ ;; auto-fill, and it is pretty common in running text to have an ellipsis
+ ;; ("...") which trips because of the rest comment syntax (".. ").
+ (set (make-local-variable 'comment-start) ".. ")
+ (set (make-local-variable 'comment-start-skip) "^\\.\\. ")
+ (set (make-local-variable 'comment-multi-line) nil)
+
+ ;; Special variables
+ (make-local-variable 'rst-adornment-level-alist)
+
+ ;; Font lock
+ (set (make-local-variable 'font-lock-defaults)
+ '(rst-font-lock-keywords
+ t nil nil nil
+ (font-lock-multiline . t)
+ (font-lock-mark-block-function . mark-paragraph)))
+ (when (boundp 'font-lock-support-mode)
+ ;; rst-mode does not need font-lock-support-mode and works not well with
+ ;; jit-lock-mode because reST is not made for machines
+ (set (make-local-variable 'font-lock-support-mode) nil)))
+
+;;;###autoload
+(define-minor-mode rst-minor-mode
+ "ReST Minor Mode.
+Toggle ReST minor mode.
+With no argument, this command toggles the mode.
+Non-null prefix argument turns on the mode.
+Null prefix argument turns off the mode.
+
+When ReST minor mode is enabled, the ReST mode
+keybindings are installed on top of the major
+mode bindings. Use this for modes derived from
+text-mode, like mail-mode.."
+ ;; The initial value.
+ nil
+ ;; The indicator for the mode line.
+ " ReST"
+ ;; The minor mode bindings.
+ rst-mode-map
+ :group 'rst)
+
+;; FIXME: can I somehow install these too?
+;; :abbrev-table rst-mode-abbrev-table
+;; :syntax-table rst-mode-syntax-table
+
+
+
+
+
+;; Bulleted item lists.
+(defcustom rst-bullets
+ '(?- ?* ?+)
+ "List of all possible bullet characters for bulleted lists."
+ :group 'rst)
+
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Section Decoration Adjusment
+;; ============================
+;;
+;; The following functions implement a smart automatic title sectioning feature.
+;; The idea is that with the cursor sitting on a section title, we try to get as
+;; much information from context and try to do the best thing automatically.
+;; This function can be invoked many times and/or with prefix argument to rotate
+;; between the various sectioning decorations.
+;;
+;; Definitions: the two forms of sectioning define semantically separate section
+;; levels. A sectioning DECORATION consists in:
+;;
+;; - a CHARACTER
+;;
+;; - a STYLE which can be either of 'simple' or 'over-and-under'.
+;;
+;; - an INDENT (meaningful for the over-and-under style only) which determines
+;; how many characters and over-and-under style is hanging outside of the
+;; title at the beginning and ending.
+;;
+;; Important note: an existing decoration must be formed by at least two
+;; characters to be recognized.
+;;
+;; Here are two examples of decorations (| represents the window border, column
+;; 0):
+;;
+;; |
+;; 1. char: '-' e |Some Title
+;; style: simple |----------
+;; |
+;; 2. char: '=' |==============
+;; style: over-and-under | Some Title
+;; indent: 2 |==============
+;; |
+;;
+;; Some notes:
+;;
+;; - The underlining character that is used depends on context. The file is
+;; scanned to find other sections and an appropriate character is selected.
+;; If the function is invoked on a section that is complete, the character is
+;; rotated among the existing section decorations.
+;;
+;; Note that when rotating the characters, if we come to the end of the
+;; hierarchy of decorations, the variable rst-preferred-decorations is
+;; consulted to propose a new underline decoration, and if continued, we cycle
+;; the decorations all over again. Set this variable to nil if you want to
+;; limit the underlining character propositions to the existing decorations in
+;; the file.
+;;
+;; - A prefix argument can be used to alternate the style.
+;;
+;; - An underline/overline that is not extended to the column at which it should
+;; be hanging is dubbed INCOMPLETE. For example::
+;;
+;; |Some Title
+;; |-------
+;;
+;; Examples of default invocation:
+;;
+;; |Some Title ---> |Some Title
+;; | |----------
+;;
+;; |Some Title ---> |Some Title
+;; |----- |----------
+;;
+;; | |------------
+;; | Some Title ---> | Some Title
+;; | |------------
+;;
+;; In over-and-under style, when alternating the style, a variable is
+;; available to select how much default indent to use (it can be zero). Note
+;; that if the current section decoration already has an indent, we don't
+;; adjust it to the default, we rather use the current indent that is already
+;; there for adjustment (unless we cycle, in which case we use the indent
+;; that has been found previously).
+
+(defgroup rst-adjust nil
+ "Settings for adjustment and cycling of section title
+decorations."
+ :group 'rst
+ :version "21.1")
+
+(defcustom rst-preferred-decorations '( (?= over-and-under 1)
+ (?= simple 0)
+ (?- simple 0)
+ (?~ simple 0)
+ (?+ simple 0)
+ (?` simple 0)
+ (?# simple 0)
+ (?@ simple 0) )
+ "Preferred ordering of section title decorations.
+
+This sequence is consulted to offer a new decoration suggestion
+when we rotate the underlines at the end of the existing
+hierarchy of characters, or when there is no existing section
+title in the file."
+ :group 'rst-adjust)
+
+
+(defcustom rst-default-indent 1
+ "Number of characters to indent the section title.
+
+THis is used for when toggling decoration styles, when switching
+from a simple decoration style to a over-and-under decoration
+style."
+ :group 'rst-adjust)
+
+
+(defvar rst-section-text-regexp "^[ \t]*\\S-*\\w\\S-*"
+ "Regular expression for valid section title text.")
+
+
+(defun rst-line-homogeneous-p (&optional accept-special)
+ "Return true if the line is homogeneous.
+
+Predicate that returns the unique char if the current line is
+composed only of a single repeated non-whitespace character. This
+returns the char even if there is whitespace at the beginning of
+the line.
+
+If ACCEPT-SPECIAL is specified we do not ignore special sequences
+which normally we would ignore when doing a search on many lines.
+For example, normally we have cases to ignore commonly occuring
+patterns, such as :: or ...; with the flag do not ignore them."
+ (save-excursion
+ (back-to-indentation)
+ (unless (looking-at "\n")
+ (let ((c (thing-at-point 'char)))
+ (if (and (looking-at (format "[%s]+[ \t]*$" c))
+ (or accept-special
+ (and
+ ;; Common patterns.
+ (not (looking-at "::[ \t]*$"))
+ (not (looking-at "\\.\\.\\.[ \t]*$"))
+ ;; Discard one char line
+ (not (looking-at ".[ \t]*$"))
+ )))
+ (string-to-char c))
+ ))
+ ))
+
+(defun rst-line-homogeneous-nodent-p (&optional accept-special)
+ "Return true if the line is homogeneous with no indent.
+See `rst-line-homogeneous-p' about ACCEPT-SPECIAL."
+ (save-excursion
+ (beginning-of-line)
+ (if (looking-at "^[ \t]+")
+ nil
+ (rst-line-homogeneous-p accept-special)
+ )))
+
+
+(defun rst-compare-decorations (deco1 deco2)
+ "Compare decorations.
+Returns true if both DECO1 and DECO2 decorations are equal,
+according to restructured text semantics (only the character and
+the style are compared, the indentation does not matter."
+ (and (eq (car deco1) (car deco2))
+ (eq (cadr deco1) (cadr deco2))))
+
+
+(defun rst-get-decoration-match (hier deco)
+ "Return the index (level) in hierarchy HIER of decoration DECO.
+This basically just searches for the item using the appropriate
+comparison and returns the index. We return nil if the item is
+not found."
+ (let ((cur hier))
+ (while (and cur (not (rst-compare-decorations (car cur) deco)))
+ (setq cur (cdr cur)))
+ cur))
+
+
+(defun rst-suggest-new-decoration (alldecos &optional prev)
+ "Suggest a new, different decoration from all that have been seen.
+
+ALLDECOS is the set of all decorations, including the line
+numbers. PREV is the optional previous decoration, in order to
+suggest a better match."
+
+ ;; For all the preferred decorations...
+ (let* (
+ ;; If 'prev' is given, reorder the list to start searching after the
+ ;; match.
+ (fplist
+ (cdr (rst-get-decoration-match rst-preferred-decorations prev)))
+
+ ;; List of candidates to search.
+ (curpotential (append fplist rst-preferred-decorations)))
+ (while
+ ;; For all the decorations...
+ (let ((cur alldecos)
+ found)
+ (while (and cur (not found))
+ (if (rst-compare-decorations (car cur) (car curpotential))
+ ;; Found it!
+ (setq found (car curpotential))
+ (setq cur (cdr cur))))
+ found)
+
+ (setq curpotential (cdr curpotential)))
+
+ (copy-list (car curpotential)) ))
+
+(defun rst-delete-entire-line ()
+ "Delete the entire current line without using the `kill-ring'."
+ (delete-region (line-beginning-position) (min (+ 1 (line-end-position))
+ (point-max))))
+
+(defun rst-update-section (char style &optional indent)
+ "Unconditionally update the style of a section decoration.
+
+Do this using the given character CHAR, with STYLE 'simple or
+'over-and-under, and with indent INDENT. If the STYLE is
+'simple, whitespace before the title is removed (indent is always
+assume to be 0).
+
+If there are existing overline and/or underline from the
+existing decoration, they are removed before adding the
+requested decoration."
+
+ (interactive)
+ (let (marker
+ len)
+
+ (end-of-line)
+ (setq marker (point-marker))
+
+ ;; Fixup whitespace at the beginning and end of the line
+ (if (or (null indent) (eq style 'simple))
+ (setq indent 0))
+ (beginning-of-line)
+ (delete-horizontal-space)
+ (insert (make-string indent ? ))
+
+ (end-of-line)
+ (delete-horizontal-space)
+
+ ;; Set the current column, we're at the end of the title line
+ (setq len (+ (current-column) indent))
+
+ ;; Remove previous line if it consists only of a single repeated character
+ (save-excursion
+ (forward-line -1)
+ (and (rst-line-homogeneous-p 1)
+ ;; Avoid removing the underline of a title right above us.
+ (save-excursion (forward-line -1)
+ (not (looking-at rst-section-text-regexp)))
+ (rst-delete-entire-line)))
+
+ ;; Remove following line if it consists only of a single repeated
+ ;; character
+ (save-excursion
+ (forward-line +1)
+ (and (rst-line-homogeneous-p 1)
+ (rst-delete-entire-line))
+ ;; Add a newline if we're at the end of the buffer, for the subsequence
+ ;; inserting of the underline
+ (if (= (point) (buffer-end 1))
+ (newline 1)))
+
+ ;; Insert overline
+ (if (eq style 'over-and-under)
+ (save-excursion
+ (beginning-of-line)
+ (open-line 1)
+ (insert (make-string len char))))
+
+ ;; Insert underline
+ (forward-line +1)
+ (open-line 1)
+ (insert (make-string len char))
+
+ (forward-line +1)
+ (goto-char marker)
+ ))
+
+
+(defun rst-normalize-cursor-position ()
+ "Normalize the cursor position.
+If the cursor is on a decoration line or an empty line , place it
+on the section title line (at the end). Returns the line offset
+by which the cursor was moved. This works both over or under a
+line."
+ (if (save-excursion (beginning-of-line)
+ (or (rst-line-homogeneous-p 1)
+ (looking-at "^[ \t]*$")))
+ (progn
+ (beginning-of-line)
+ (cond
+ ((save-excursion (forward-line -1)
+ (beginning-of-line)
+ (and (looking-at rst-section-text-regexp)
+ (not (rst-line-homogeneous-p 1))))
+ (progn (forward-line -1) -1))
+ ((save-excursion (forward-line +1)
+ (beginning-of-line)
+ (and (looking-at rst-section-text-regexp)
+ (not (rst-line-homogeneous-p 1))))
+ (progn (forward-line +1) +1))
+ (t 0)))
+ 0 ))
+
+
+(defun rst-find-all-decorations ()
+ "Find all the decorations in the file.
+Return a list of (line, decoration) pairs. Each decoration
+consists in a (char, style, indent) triple.
+
+This function does not detect the hierarchy of decorations, it
+just finds all of them in a file. You can then invoke another
+function to remove redundancies and inconsistencies."
+
+ (let (positions
+ (curline 1))
+ ;; Iterate over all the section titles/decorations in the file.
+ (save-excursion
+ (goto-char (point-min))
+ (while (< (point) (buffer-end 1))
+ (if (rst-line-homogeneous-nodent-p)
+ (progn
+ (setq curline (+ curline (rst-normalize-cursor-position)))
+
+ ;; Here we have found a potential site for a decoration,
+ ;; characterize it.
+ (let ((deco (rst-get-decoration)))
+ (if (cadr deco) ;; Style is existing.
+ ;; Found a real decoration site.
+ (progn
+ (push (cons curline deco) positions)
+ ;; Push beyond the underline.
+ (forward-line 1)
+ (setq curline (+ curline 1))
+ )))
+ ))
+ (forward-line 1)
+ (setq curline (+ curline 1))
+ ))
+ (reverse positions)))
+
+
+(defun rst-infer-hierarchy (decorations)
+ "Build a hierarchy of decorations using the list of given DECORATIONS.
+
+This function expects a list of (char, style, indent) decoration
+specifications, in order that they appear in a file, and will
+infer a hierarchy of section levels by removing decorations that
+have already been seen in a forward traversal of the decorations,
+comparing just the character and style.
+
+Similarly returns a list of (char, style, indent), where each
+list element should be unique."
+
+ (let ((hierarchy-alist (list)))
+ (dolist (x decorations)
+ (let ((char (car x))
+ (style (cadr x)))
+ (unless (assoc (cons char style) hierarchy-alist)
+ (push (cons (cons char style) x) hierarchy-alist))
+ ))
+
+ (mapcar 'cdr (nreverse hierarchy-alist))
+ ))
+
+
+(defun rst-get-hierarchy (&optional alldecos ignore)
+ "Return the hierarchy of section titles in the file.
+
+Return a list of decorations that represents the hierarchy of
+section titles in the file. Reuse the list of decorations
+already computed in ALLDECOS if present. If the line number in
+IGNORE is specified, the decoration found on that line (if there
+is one) is not taken into account when building the hierarchy."
+ (let ((all (or alldecos (rst-find-all-decorations))))
+ (setq all (assq-delete-all ignore all))
+ (rst-infer-hierarchy (mapcar 'cdr all))))
+
+
+(defun rst-get-decoration (&optional point)
+ "Get the decoration at POINT.
+
+Looks around point and finds the characteristics of the
+decoration that is found there. We assume that the cursor is
+already placed on the title line (and not on the overline or
+underline).
+
+This function returns a (char, style, indent) triple. If the
+characters of overline and underline are different, we return
+the underline character. The indent is always calculated. A
+decoration can be said to exist if the style is not nil.
+
+A point can be specified to go to the given location before
+extracting the decoration."
+
+ (let (char style indent)
+ (save-excursion
+ (if point (goto-char point))
+ (beginning-of-line)
+ (if (looking-at rst-section-text-regexp)
+ (let* ((over (save-excursion
+ (forward-line -1)
+ (rst-line-homogeneous-nodent-p)))
+
+ (under (save-excursion
+ (forward-line +1)
+ (rst-line-homogeneous-nodent-p)))
+ )
+
+ ;; Check that the line above the overline is not part of a title
+ ;; above it.
+ (if (and over
+ (save-excursion
+ (and (equal (forward-line -2) 0)
+ (looking-at rst-section-text-regexp))))
+ (setq over nil))
+
+ (cond
+ ;; No decoration found, leave all return values nil.
+ ((and (eq over nil) (eq under nil)))
+
+ ;; Overline only, leave all return values nil.
+ ;;
+ ;; Note: we don't return the overline character, but it could
+ ;; perhaps in some cases be used to do something.
+ ((and over (eq under nil)))
+
+ ;; Underline only.
+ ((and under (eq over nil))
+ (setq char under
+ style 'simple))
+
+ ;; Both overline and underline.
+ (t
+ (setq char under
+ style 'over-and-under))
+ )
+ )
+ )
+ ;; Find indentation.
+ (setq indent (save-excursion (back-to-indentation) (current-column)))
+ )
+ ;; Return values.
+ (list char style indent)))
+
+
+(defun rst-get-decorations-around (&optional alldecos)
+ "Return the decorations around point.
+
+Given the list of all decorations ALLDECOS (with positions), find
+the decorations before and after the given point. A list of the
+previous and next decorations is returned."
+ (let* ((all (or alldecos (rst-find-all-decorations)))
+ (curline (line-number-at-pos))
+ prev next
+ (cur all))
+
+ ;; Search for the decorations around the current line.
+ (while (and cur (< (caar cur) curline))
+ (setq prev cur
+ cur (cdr cur)))
+ ;; 'cur' is the following decoration.
+
+ (if (and cur (caar cur))
+ (setq next (if (= curline (caar cur)) (cdr cur) cur)))
+
+ (mapcar 'cdar (list prev next))
+ ))
+
+
+(defun rst-decoration-complete-p (deco)
+ "Return true if the decoration DECO around POINT is complete."
+ ;; Note: we assume that the detection of the overline as being the underline
+ ;; of a preceding title has already been detected, and has been eliminated
+ ;; from the decoration that is given to us.
+
+ ;; There is some sectioning already present, so check if the current
+ ;; sectioning is complete and correct.
+ (let* ((char (car deco))
+ (style (cadr deco))
+ (indent (caddr deco))
+ (endcol (save-excursion (end-of-line) (current-column)))
+ )
+ (if char
+ (let ((exps (concat "^"
+ (regexp-quote (make-string (+ endcol indent) char))
+ "$")))
+ (and
+ (save-excursion (forward-line +1)
+ (beginning-of-line)
+ (looking-at exps))
+ (or (not (eq style 'over-and-under))
+ (save-excursion (forward-line -1)
+ (beginning-of-line)
+ (looking-at exps))))
+ ))
+ ))
+
+
+(defun rst-get-next-decoration
+ (curdeco hier &optional suggestion reverse-direction)
+ "Get the next decoration for CURDECO, in given hierarchy HIER.
+If suggesting, suggest for new decoration SUGGESTION.
+REVERSE-DIRECTION is used to reverse the cycling order."
+
+ (let* (
+ (char (car curdeco))
+ (style (cadr curdeco))
+
+ ;; Build a new list of decorations for the rotation.
+ (rotdecos
+ (append hier
+ ;; Suggest a new decoration.
+ (list suggestion
+ ;; If nothing to suggest, use first decoration.
+ (car hier)))) )
+ (or
+ ;; Search for next decoration.
+ (cadr
+ (let ((cur (if reverse-direction rotdecos
+ (reverse rotdecos))))
+ (while (and cur
+ (not (and (eq char (caar cur))
+ (eq style (cadar cur)))))
+ (setq cur (cdr cur)))
+ cur))
+
+ ;; If not found, take the first of all decorations.
+ suggestion
+ )))
+
+
+(defun rst-adjust ()
+ "Auto-adjust the decoration around point.
+
+Adjust/rotate the section decoration for the section title
+around point or promote/demote the decorations inside the region,
+depending on if the region is active. This function is meant to
+be invoked possibly multiple times, and can vary its behaviour
+with a positive prefix argument (toggle style), or with a
+negative prefix argument (alternate behaviour).
+
+This function is the main focus of this module and is a bit of a
+swiss knife. It is meant as the single most essential function
+to be bound to invoke to adjust the decorations of a section
+title in restructuredtext. It tries to deal with all the
+possible cases gracefully and to do `the right thing' in all
+cases.
+
+See the documentations of `rst-adjust-decoration' and
+`rst-promote-region' for full details.
+
+Prefix Arguments
+================
+
+The method can take either (but not both) of
+
+a. a (non-negative) prefix argument, which means to toggle the
+ decoration style. Invoke with a prefix arg for example;
+
+b. a negative numerical argument, which generally inverts the
+ direction of search in the file or hierarchy. Invoke with C--
+ prefix for example."
+ (interactive)
+
+ (let* (;; Save our original position on the current line.
+ (origpt (set-marker (make-marker) (point)))
+
+ ;; Parse the positive and negative prefix arguments.
+ (reverse-direction
+ (and current-prefix-arg
+ (< (prefix-numeric-value current-prefix-arg) 0)))
+ (toggle-style
+ (and current-prefix-arg (not reverse-direction))))
+
+ (if (rst-portable-mark-active-p)
+ ;; Adjust decorations within region.
+ (rst-promote-region current-prefix-arg)
+ ;; Adjust decoration around point.
+ (rst-adjust-decoration toggle-style reverse-direction))
+
+ ;; Run the hooks to run after adjusting.
+ (run-hooks 'rst-adjust-hook)
+
+ ;; Make sure to reset the cursor position properly after we're done.
+ (goto-char origpt)
+
+ ))
+
+(defvar rst-adjust-hook nil
+ "Hooks to be run after running `rst-adjust'.")
+
+(defvar rst-new-decoration-down nil
+ "If true, a new decoration being added will be initialized to
+ be one level down from the previous decoration. If nil, a new
+ decoration will be equal to the level of the previous
+ decoration.")
+
+(defun rst-adjust-decoration (&optional toggle-style reverse-direction)
+"Adjust/rotate the section decoration for the section title around point.
+
+This function is meant to be invoked possibly multiple times, and
+can vary its behaviour with a true TOGGLE-STYLE argument, or with
+a REVERSE-DIRECTION argument.
+
+General Behaviour
+=================
+
+The next action it takes depends on context around the point, and
+it is meant to be invoked possibly more than once to rotate among
+the various possibilities. Basically, this function deals with:
+
+- adding a decoration if the title does not have one;
+
+- adjusting the length of the underline characters to fit a
+ modified title;
+
+- rotating the decoration in the set of already existing
+ sectioning decorations used in the file;
+
+- switching between simple and over-and-under styles.
+
+You should normally not have to read all the following, just
+invoke the method and it will do the most obvious thing that you
+would expect.
+
+
+Decoration Definitions
+======================
+
+The decorations consist in
+
+1. a CHARACTER
+
+2. a STYLE which can be either of 'simple' or 'over-and-under'.
+
+3. an INDENT (meaningful for the over-and-under style only)
+ which determines how many characters and over-and-under
+ style is hanging outside of the title at the beginning and
+ ending.
+
+See source code for mode details.
+
+
+Detailed Behaviour Description
+==============================
+
+Here are the gory details of the algorithm (it seems quite
+complicated, but really, it does the most obvious thing in all
+the particular cases):
+
+Before applying the decoration change, the cursor is placed on
+the closest line that could contain a section title.
+
+Case 1: No Decoration
+---------------------
+
+If the current line has no decoration around it,
+
+- search backwards for the last previous decoration, and apply
+ the decoration one level lower to the current line. If there
+ is no defined level below this previous decoration, we suggest
+ the most appropriate of the `rst-preferred-decorations'.
+
+ If REVERSE-DIRECTION is true, we simply use the previous
+ decoration found directly.
+
+- if there is no decoration found in the given direction, we use
+ the first of `rst-preferred-decorations'.
+
+The prefix argument forces a toggle of the prescribed decoration
+style.
+
+Case 2: Incomplete Decoration
+-----------------------------
+
+If the current line does have an existing decoration, but the
+decoration is incomplete, that is, the underline/overline does
+not extend to exactly the end of the title line (it is either too
+short or too long), we simply extend the length of the
+underlines/overlines to fit exactly the section title.
+
+If the prefix argument is given, we toggle the style of the
+decoration as well.
+
+REVERSE-DIRECTION has no effect in this case.
+
+Case 3: Complete Existing Decoration
+------------------------------------
+
+If the decoration is complete (i.e. the underline (overline)
+length is already adjusted to the end of the title line), we
+search/parse the file to establish the hierarchy of all the
+decorations (making sure not to include the decoration around
+point), and we rotate the current title's decoration from within
+that list (by default, going *down* the hierarchy that is present
+in the file, i.e. to a lower section level). This is meant to be
+used potentially multiple times, until the desired decoration is
+found around the title.
+
+If we hit the boundary of the hierarchy, exactly one choice from
+the list of preferred decorations is suggested/chosen, the first
+of those decoration that has not been seen in the file yet (and
+not including the decoration around point), and the next
+invocation rolls over to the other end of the hierarchy (i.e. it
+cycles). This allows you to avoid having to set which character
+to use by always using the
+
+If REVERSE-DIRECTION is true, the effect is to change the
+direction of rotation in the hierarchy of decorations, thus
+instead going *up* the hierarchy.
+
+However, if there is a non-negative prefix argument, we do not
+rotate the decoration, but instead simply toggle the style of the
+current decoration (this should be the most common way to toggle
+the style of an existing complete decoration).
+
+
+Point Location
+==============
+
+The invocation of this function can be carried out anywhere
+within the section title line, on an existing underline or
+overline, as well as on an empty line following a section title.
+This is meant to be as convenient as possible.
+
+
+Indented Sections
+=================
+
+Indented section titles such as ::
+
+ My Title
+ --------
+
+are illegal in restructuredtext and thus not recognized by the
+parser. This code will thus not work in a way that would support
+indented sections (it would be ambiguous anyway).
+
+
+Joint Sections
+==============
+
+Section titles that are right next to each other may not be
+treated well. More work might be needed to support those, and
+special conditions on the completeness of existing decorations
+might be required to make it non-ambiguous.
+
+For now we assume that the decorations are disjoint, that is,
+there is at least a single line between the titles/decoration
+lines.
+
+
+Suggested Binding
+=================
+
+We suggest that you bind this function on C-=. It is close to
+C-- so a negative argument can be easily specified with a flick
+of the right hand fingers and the binding is unused in `text-mode'."
+ (interactive)
+
+ ;; If we were invoked directly, parse the prefix arguments into the
+ ;; arguments of the function.
+ (if current-prefix-arg
+ (setq reverse-direction
+ (and current-prefix-arg
+ (< (prefix-numeric-value current-prefix-arg) 0))
+
+ toggle-style
+ (and current-prefix-arg (not reverse-direction))))
+
+ (let* (;; Check if we're on an underline around a section title, and move the
+ ;; cursor to the title if this is the case.
+ (moved (rst-normalize-cursor-position))
+
+ ;; Find the decoration and completeness around point.
+ (curdeco (rst-get-decoration))
+ (char (car curdeco))
+ (style (cadr curdeco))
+ (indent (caddr curdeco))
+
+ ;; New values to be computed.
+ char-new style-new indent-new
+ )
+
+ ;; We've moved the cursor... if we're not looking at some text, we have
+ ;; nothing to do.
+ (if (save-excursion (beginning-of-line)
+ (looking-at rst-section-text-regexp))
+ (progn
+ (cond
+ ;;-------------------------------------------------------------------
+ ;; Case 1: No Decoration
+ ((and (eq char nil) (eq style nil))
+
+ (let* ((alldecos (rst-find-all-decorations))
+
+ (around (rst-get-decorations-around alldecos))
+ (prev (car around))
+ cur
+
+ (hier (rst-get-hierarchy alldecos))
+ )
+
+ ;; Advance one level down.
+ (setq cur
+ (if prev
+ (if (not reverse-direction)
+ (or (funcall (if rst-new-decoration-down 'cadr 'car)
+ (rst-get-decoration-match hier prev))
+ (rst-suggest-new-decoration hier prev))
+ prev)
+ (copy-list (car rst-preferred-decorations))
+ ))
+
+ ;; Invert the style if requested.
+ (if toggle-style
+ (setcar (cdr cur) (if (eq (cadr cur) 'simple)
+ 'over-and-under 'simple)) )
+
+ (setq char-new (car cur)
+ style-new (cadr cur)
+ indent-new (caddr cur))
+ ))
+
+ ;;-------------------------------------------------------------------
+ ;; Case 2: Incomplete Decoration
+ ((not (rst-decoration-complete-p curdeco))
+
+ ;; Invert the style if requested.
+ (if toggle-style
+ (setq style (if (eq style 'simple) 'over-and-under 'simple)))
+
+ (setq char-new char
+ style-new style
+ indent-new indent))
+
+ ;;-------------------------------------------------------------------
+ ;; Case 3: Complete Existing Decoration
+ (t
+ (if toggle-style
+
+ ;; Simply switch the style of the current decoration.
+ (setq char-new char
+ style-new (if (eq style 'simple) 'over-and-under 'simple)
+ indent-new rst-default-indent)
+
+ ;; Else, we rotate, ignoring the decoration around the current
+ ;; line...
+ (let* ((alldecos (rst-find-all-decorations))
+
+ (hier (rst-get-hierarchy alldecos (line-number-at-pos)))
+
+ ;; Suggestion, in case we need to come up with something
+ ;; new
+ (suggestion (rst-suggest-new-decoration
+ hier
+ (car (rst-get-decorations-around alldecos))))
+
+ (nextdeco (rst-get-next-decoration
+ curdeco hier suggestion reverse-direction))
+
+ )
+
+ ;; Indent, if present, always overrides the prescribed indent.
+ (setq char-new (car nextdeco)
+ style-new (cadr nextdeco)
+ indent-new (caddr nextdeco))
+
+ )))
+ )
+
+ ;; Override indent with present indent!
+ (setq indent-new (if (> indent 0) indent indent-new))
+
+ (if (and char-new style-new)
+ (rst-update-section char-new style-new indent-new))
+ ))
+
+
+ ;; Correct the position of the cursor to more accurately reflect where it
+ ;; was located when the function was invoked.
+ (unless (= moved 0)
+ (forward-line (- moved))
+ (end-of-line))
+
+ ))
+
+;; Maintain an alias for compatibility.
+(defalias 'rst-adjust-section-title 'rst-adjust)
+
+
+(defun rst-promote-region (&optional demote)
+ "Promote the section titles within the region.
+
+With argument DEMOTE or a prefix argument, demote the
+section titles instead. The algorithm used at the boundaries of
+the hierarchy is similar to that used by `rst-adjust-decoration'."
+ (interactive)
+
+ (let* ((demote (or current-prefix-arg demote))
+ (alldecos (rst-find-all-decorations))
+ (cur alldecos)
+
+ (hier (rst-get-hierarchy alldecos))
+ (suggestion (rst-suggest-new-decoration hier))
+
+ (region-begin-line (line-number-at-pos (region-beginning)))
+ (region-end-line (line-number-at-pos (region-end)))
+
+ marker-list
+ )
+
+ ;; Skip the markers that come before the region beginning
+ (while (and cur (< (caar cur) region-begin-line))
+ (setq cur (cdr cur)))
+
+ ;; Create a list of markers for all the decorations which are found within
+ ;; the region.
+ (save-excursion
+ (let (m line)
+ (while (and cur (< (setq line (caar cur)) region-end-line))
+ (setq m (make-marker))
+ (goto-line line)
+ (push (list (set-marker m (point)) (cdar cur)) marker-list)
+ (setq cur (cdr cur)) ))
+
+ ;; Apply modifications.
+ (let (nextdeco)
+ (dolist (p marker-list)
+ ;; Go to the decoration to promote.
+ (goto-char (car p))
+
+ ;; Rotate the next decoration.
+ (setq nextdeco (rst-get-next-decoration
+ (cadr p) hier suggestion demote))
+
+ ;; Update the decoration.
+ (apply 'rst-update-section nextdeco)
+
+ ;; Clear marker to avoid slowing down the editing after we're done.
+ (set-marker (car p) nil)
+ ))
+ (setq deactivate-mark nil)
+ )))
+
+
+
+(defun rst-display-decorations-hierarchy (&optional decorations)
+ "Display the current file's section title decorations hierarchy.
+This function expects a list of (char, style, indent) triples in
+DECORATIONS."
+ (interactive)
+
+ (if (not decorations)
+ (setq decorations (rst-get-hierarchy)))
+ (with-output-to-temp-buffer "*rest section hierarchy*"
+ (let ((level 1))
+ (with-current-buffer standard-output
+ (dolist (x decorations)
+ (insert (format "\nSection Level %d" level))
+ (apply 'rst-update-section x)
+ (goto-char (point-max))
+ (insert "\n")
+ (incf level)
+ ))
+ )))
+
+(defun rst-straighten-decorations ()
+ "Redo all the decorations in the current buffer.
+This is done using our preferred set of decorations. This can be
+used, for example, when using somebody else's copy of a document,
+in order to adapt it to our preferred style."
+ (interactive)
+ (save-excursion
+ (let* ((alldecos (rst-find-all-decorations))
+ (hier (rst-get-hierarchy alldecos))
+
+ ;; Get a list of pairs of (level . marker)
+ (levels-and-markers (mapcar
+ (lambda (deco)
+ (cons (position (cdr deco) hier :test 'equal)
+ (let ((m (make-marker)))
+ (goto-line (car deco))
+ (set-marker m (point))
+ m)))
+ alldecos))
+ )
+ (dolist (lm levels-and-markers)
+ ;; Go to the appropriate position
+ (goto-char (cdr lm))
+
+ ;; Apply the new styule
+ (apply 'rst-update-section (nth (car lm) rst-preferred-decorations))
+
+ ;; Reset the market to avoid slowing down editing until it gets GC'ed
+ (set-marker (cdr lm) nil)
+ )
+ )))
+
+
+
+
+(defun rst-straighten-deco-spacing ()
+ "Adjust the spacing before and after decorations in the entire document.
+The spacing will be set to two blank lines before the first two
+section levels, and one blank line before any of the other
+section levels."
+;; FIXME: we need to take care of subtitle at some point.
+ (interactive)
+ (save-excursion
+ (let* ((alldecos (rst-find-all-decorations)))
+
+ ;; Work the list from the end, so that we don't have to use markers to
+ ;; adjust for the changes in the document.
+ (dolist (deco (nreverse alldecos))
+ ;; Go to the appropriate position.
+ (goto-line (car deco))
+ (insert "@\n")
+;; FIXME: todo, we
+ )
+ )))
+
+
+(defun rst-find-pfx-in-region (beg end pfx-re)
+ "Find all the positions of prefixes in region between BEG and END.
+This is used to find bullets and enumerated list items. PFX-RE
+is a regular expression for matching the lines with items."
+ (let (pfx)
+ (save-excursion
+ (goto-char beg)
+ (while (< (point) end)
+ (back-to-indentation)
+ (when (and
+ (looking-at pfx-re)
+ (let ((pfx-col (current-column)))
+ (save-excursion
+ (forward-line -1)
+ (back-to-indentation)
+ (or (looking-at "^[ \t]*$")
+ (> (current-column) pfx-col)
+ (and (= (current-column) pfx-col)
+ (looking-at pfx-re))))))
+ (setq pfx (cons (cons (point) (current-column))
+ pfx)))
+ (forward-line 1)) )
+ (nreverse pfx)))
+
+(defvar rst-re-bullets
+ (format "\\([%s][ \t]\\)[^ \t]" (regexp-quote (concat rst-bullets)))
+ "Regexp for finding bullets.")
+
+(defvar rst-re-enumerations
+ "\\(\\(#\\|[0-9]+\\)\\.[ \t]\\)[^ \t]"
+ "Regexp for finding bullets.")
+
+(defvar rst-re-items
+ (format "\\(%s\\|%s\\)[^ \t]"
+ (format "[%s][ \t]" (regexp-quote (concat rst-bullets)))
+ "\\(#\\|[0-9]+\\)\\.[ \t]")
+ "Regexp for finding bullets.")
+
+(defvar rst-preferred-bullets
+ '(?- ?* ?+)
+ "List of favourite bullets to set for straightening bullets.")
+
+(defun rst-straighten-bullets-region (beg end)
+ "Make all the bulleted list items in the region consistent.
+The region is specified between BEG and END. You can use this
+after you have merged multiple bulleted lists to make them use
+the same/correct/consistent bullet characters.
+
+See variable `rst-preferred-bullets' for the list of bullets to
+adjust. If bullets are found on levels beyond the
+`rst-preferred-bullets' list, they are not modified."
+ (interactive "r")
+
+ (let ((bullets (rst-find-pfx-in-region beg end
+ rst-re-bullets))
+ (levtable (make-hash-table :size 4)))
+
+ ;; Create a map of levels to list of positions.
+ (dolist (x bullets)
+ (let ((key (cdr x)))
+ (puthash key
+ (append (gethash key levtable (list))
+ (list (car x)))
+ levtable)))
+
+ ;; Sort this map and create a new map of prefix char and list of positions.
+ (let (poslist)
+ (maphash (lambda (x y) (setq poslist (cons (cons x y) poslist))) levtable)
+
+ (mapcar* (lambda (x char)
+ ;; Apply the characters.
+ (dolist (pos (cdr x))
+ (goto-char pos)
+ (delete-char 1)
+ (insert (char-to-string char))))
+
+ ;; Sorted list of indent . positions
+ (sort poslist (lambda (x y) (<= (car x) (car y))))
+
+ ;; List of preferred bullets.
+ rst-preferred-bullets)
+
+ )))
+
+(defun rst-rstrip (str)
+ "Strips the whitespace at the end of string STR."
+ (string-match "[ \t\n]*\\'" str)
+ (substring str 0 (match-beginning 0)))
+
+(defun rst-get-stripped-line ()
+ "Return the line at cursor, stripped from whitespace."
+ (re-search-forward "\\S-.*\\S-" (line-end-position))
+ (buffer-substring-no-properties (match-beginning 0)
+ (match-end 0)) )
+
+(defun rst-section-tree (alldecos)
+ "Get the hierarchical tree of section titles.
+
+Returns a hierarchical tree of the sections titles in the
+document, for decorations ALLDECOS. This can be used to generate
+a table of contents for the document. The top node will always
+be a nil node, with the top level titles as children (there may
+potentially be more than one).
+
+Each section title consists in a cons of the stripped title
+string and a marker to the section in the original text document.
+
+If there are missing section levels, the section titles are
+inserted automatically, and the title string is set to nil, and
+the marker set to the first non-nil child of itself.
+Conceptually, the nil nodes--i.e. those which have no title--are
+to be considered as being the same line as their first non-nil
+child. This has advantages later in processing the graph."
+
+ (let* ((hier (rst-get-hierarchy alldecos))
+ (levels (make-hash-table :test 'equal :size 10))
+ lines)
+
+ (let ((lev 0))
+ (dolist (deco hier)
+ ;; Compare just the character and indent in the hash table.
+ (puthash (cons (car deco) (cadr deco)) lev levels)
+ (incf lev)))
+
+ ;; Create a list of lines that contains (text, level, marker) for each
+ ;; decoration.
+ (save-excursion
+ (setq lines
+ (mapcar (lambda (deco)
+ (goto-line (car deco))
+ (list (gethash (cons (cadr deco) (caddr deco)) levels)
+ (rst-get-stripped-line)
+ (let ((m (make-marker)))
+ (beginning-of-line 1)
+ (set-marker m (point)))
+ ))
+ alldecos)))
+
+ (let ((lcontnr (cons nil lines)))
+ (rst-section-tree-rec lcontnr -1))))
+
+
+(defun rst-section-tree-rec (decos lev)
+ "Recursive guts of the section tree construction.
+DECOS is a cons cell whose cdr is the remaining list of
+decorations, and we change it as we consume them. LEV is the
+current level of that node. This function returns a pair of the
+subtree that was built. This treats the decos list
+destructively."
+
+ (let ((ndeco (cadr decos))
+ node
+ children)
+
+ ;; If the next decoration matches our level
+ (when (and ndeco (= (car ndeco) lev))
+ ;; Pop the next decoration and create the current node with it
+ (setcdr decos (cddr decos))
+ (setq node (cdr ndeco)) )
+ ;; Else we let the node title/marker be unset.
+
+ ;; Build the child nodes
+ (while (and (cdr decos) (> (caadr decos) lev))
+ (setq children
+ (cons (rst-section-tree-rec decos (1+ lev))
+ children)))
+ (setq children (reverse children))
+
+ ;; If node is still unset, we use the marker of the first child.
+ (when (eq node nil)
+ (setq node (cons nil (cdaar children))))
+
+ ;; Return this node with its children.
+ (cons node children)
+ ))
+
+
+(defun rst-section-tree-point (node &optional point)
+ "Find tree node at point.
+Given a computed and valid section tree in NODE and a point
+POINT (default being the current point in the current buffer),
+find and return the node within the sectree where the cursor
+lives.
+
+Return values: a pair of (parent path, container subtree). The
+parent path is simply a list of the nodes above the container
+subtree node that we're returning."
+
+ (let (path outtree)
+
+ (let* ((curpoint (or point (point))))
+
+ ;; Check if we are before the current node.
+ (if (and (cadar node) (>= curpoint (cadar node)))
+
+ ;; Iterate all the children, looking for one that might contain the
+ ;; current section.
+ (let ((curnode (cdr node))
+ last)
+
+ (while (and curnode (>= curpoint (cadaar curnode)))
+ (setq last curnode
+ curnode (cdr curnode)))
+
+ (if last
+ (let ((sub (rst-section-tree-point (car last) curpoint)))
+ (setq path (car sub)
+ outtree (cdr sub)))
+ (setq outtree node))
+
+ )))
+ (cons (cons (car node) path) outtree)
+ ))
+
+
+(defun rst-toc-insert (&optional pfxarg)
+ "Insert a simple text rendering of the table of contents.
+By default the top level is ignored if there is only one, because
+we assume that the document will have a single title.
+
+If a numeric prefix argument PFXARG is given, insert the TOC up
+to the specified level.
+
+The TOC is inserted indented at the current column."
+
+ (interactive "P")
+
+ (let* (;; Check maximum level override
+ (rst-toc-insert-max-level
+ (if (and (integerp pfxarg) (> (prefix-numeric-value pfxarg) 0))
+ (prefix-numeric-value pfxarg) rst-toc-insert-max-level))
+
+ ;; Get the section tree for the current cursor point.
+ (sectree-pair
+ (rst-section-tree-point
+ (rst-section-tree (rst-find-all-decorations))))
+
+ ;; Figure out initial indent.
+ (initial-indent (make-string (current-column) ? ))
+ (init-point (point)))
+
+ (when (cddr sectree-pair)
+ (rst-toc-insert-node (cdr sectree-pair) 0 initial-indent "")
+
+ ;; Fixup for the first line.
+ (delete-region init-point (+ init-point (length initial-indent)))
+
+ ;; Delete the last newline added.
+ (delete-backward-char 1)
+ )))
+
+
+(defgroup rst-toc nil
+ "Settings for reStructuredText table of contents."
+ :group 'rst
+ :version "21.1")
+
+(defcustom rst-toc-indent 2
+ "Indentation for table-of-contents display.
+Also used for formatting insertion, when numbering is disabled."
+ :group 'rst-toc)
+
+(defcustom rst-toc-insert-style 'fixed
+ "Insertion style for table-of-contents.
+Set this to one of the following values to determine numbering and
+indentation style:
+- plain: no numbering (fixed indentation)
+- fixed: numbering, but fixed indentation
+- aligned: numbering, titles aligned under each other
+- listed: numbering, with dashes like list items (EXPERIMENTAL)"
+ :group 'rst-toc)
+
+(defcustom rst-toc-insert-number-separator " "
+ "Separator that goes between the TOC number and the title."
+ :group 'rst-toc)
+
+;; This is used to avoid having to change the user's mode.
+(defvar rst-toc-insert-click-keymap
+ (let ((map (make-sparse-keymap)))
+ (define-key map [mouse-1] 'rst-toc-mode-mouse-goto)
+ map)
+ "(Internal) What happens when you click on propertized text in the TOC.")
+
+(defcustom rst-toc-insert-max-level nil
+ "If non-nil, maximum depth of the inserted TOC."
+ :group 'rst-toc)
+
+(defun rst-toc-insert-node (node level indent pfx)
+ "Insert tree node NODE in table-of-contents.
+Recursive function that does printing of the inserted toc. LEVEL
+is the depth level of the sections in the tree. INDENT bis the
+indentation string. PFX is the prefix numbering, that includes
+the alignment necessary for all the children of level to
+align."
+
+ ;; Note: we do child numbering from the parent, so we start number the
+ ;; children one level before we print them.
+ (let ((do-print (> level 0))
+ (count 1))
+ (when do-print
+ (insert indent)
+ (let ((b (point)))
+ (unless (equal rst-toc-insert-style 'plain)
+ (insert pfx rst-toc-insert-number-separator))
+ (insert (or (caar node) "[missing node]"))
+ ;; Add properties to the text, even though in normal text mode it
+ ;; won't be doing anything for now. Not sure that I want to change
+ ;; mode stuff. At least the highlighting gives the idea that this
+ ;; is generated automatically.
+ (put-text-property b (point) 'mouse-face 'highlight)
+ (put-text-property b (point) 'rst-toc-target (cadar node))
+ (put-text-property b (point) 'keymap rst-toc-insert-click-keymap)
+
+ )
+ (insert "\n")
+
+ ;; Prepare indent for children.
+ (setq indent
+ (cond
+ ((eq rst-toc-insert-style 'plain)
+ (concat indent (make-string rst-toc-indent ? )))
+
+ ((eq rst-toc-insert-style 'fixed)
+ (concat indent (make-string rst-toc-indent ? )))
+
+ ((eq rst-toc-insert-style 'aligned)
+ (concat indent (make-string (+ (length pfx) 2) ? )))
+
+ ((eq rst-toc-insert-style 'listed)
+ (concat (substring indent 0 -3)
+ (concat (make-string (+ (length pfx) 2) ? ) " - ")))
+ ))
+ )
+
+ (if (or (eq rst-toc-insert-max-level nil)
+ (< level rst-toc-insert-max-level))
+ (let ((do-child-numbering (>= level 0))
+ fmt)
+ (if do-child-numbering
+ (progn
+ ;; Add a separating dot if there is already a prefix
+ (if (> (length pfx) 0)
+ (setq pfx (concat (rst-rstrip pfx) ".")))
+
+ ;; Calculate the amount of space that the prefix will require
+ ;; for the numbers.
+ (if (cdr node)
+ (setq fmt (format "%%-%dd"
+ (1+ (floor (log10 (length
+ (cdr node))))))))
+ ))
+
+ (dolist (child (cdr node))
+ (rst-toc-insert-node child
+ (1+ level)
+ indent
+ (if do-child-numbering
+ (concat pfx (format fmt count)) pfx))
+ (incf count)))
+
+ )))
+
+
+(defun rst-toc-insert-find-delete-contents ()
+ "Find and deletes an existing comment after the first contents directive.
+Delete that region. Return t if found and the cursor is left after the comment."
+ (goto-char (point-min))
+ ;; We look for the following and the following only (in other words, if your
+ ;; syntax differs, this won't work. If you would like a more flexible thing,
+ ;; contact the author, I just can't imagine that this requirement is
+ ;; unreasonable for now).
+ ;;
+ ;; .. contents:: [...anything here...]
+ ;; ..
+ ;; XXXXXXXX
+ ;; XXXXXXXX
+ ;; [more lines]
+ ;;
+ (let ((beg
+ (re-search-forward "^\\.\\. contents[ \t]*::\\(.*\\)\n\\.\\."
+ nil t))
+ last-real)
+ (when beg
+ ;; Look for the first line that starts at the first column.
+ (forward-line 1)
+ (beginning-of-line)
+ (while (and
+ (< (point) (point-max))
+ (or (and (looking-at "[ \t]+[^ \t]") (setq last-real (point)) t)
+ (looking-at "[ \t]*$")))
+ (forward-line 1)
+ )
+ (if last-real
+ (progn
+ (goto-char last-real)
+ (end-of-line)
+ (delete-region beg (point)))
+ (goto-char beg))
+ t
+ )))
+
+(defun rst-toc-update ()
+ "Automatically find the contents section of a document and update.
+Updates the inserted TOC if present. You can use this in your
+file-write hook to always make it up-to-date automatically."
+ (interactive)
+ (let ((p (point)))
+ (save-excursion
+ (when (rst-toc-insert-find-delete-contents)
+ (insert "\n ")
+ (rst-toc-insert)
+ ))
+ ;; Somehow save-excursion does not really work well.
+ (goto-char p))
+ ;; Note: always return nil, because this may be used as a hook.
+ )
+
+;; Note: we cannot bind the TOC update on file write because it messes with
+;; undo. If we disable undo, since it adds and removes characters, the
+;; positions in the undo list are not making sense anymore. Dunno what to do
+;; with this, it would be nice to update when saving.
+;;
+;; (add-hook 'write-contents-hooks 'rst-toc-update-fun)
+;; (defun rst-toc-update-fun ()
+;; ;; Disable undo for the write file hook.
+;; (let ((buffer-undo-list t)) (rst-toc-update) ))
+
+(defalias 'rst-toc-insert-update 'rst-toc-update) ;; backwards compat.
+
+;;------------------------------------------------------------------------------
+
+(defun rst-toc-node (node level)
+ "Recursive function that does insert NODE at LEVEL in the table-of-contents."
+
+ (if (> level 0)
+ (let ((b (point)))
+ ;; Insert line text.
+ (insert (make-string (* rst-toc-indent (1- level)) ? ))
+ (insert (or (caar node) "[missing node]"))
+
+ ;; Highlight lines.
+ (put-text-property b (point) 'mouse-face 'highlight)
+
+ ;; Add link on lines.
+ (put-text-property b (point) 'rst-toc-target (cadar node))
+
+ (insert "\n")
+ ))
+
+ (dolist (child (cdr node))
+ (rst-toc-node child (1+ level))))
+
+(defun rst-toc-count-lines (node target-node)
+ "Count the number of lines from NODE to the TARGET-NODE node.
+This recursive function returns a cons of the number of
+additional lines that have been counted for its node and children
+and 't if the node has been found."
+
+ (let ((count 1)
+ found)
+ (if (eq node target-node)
+ (setq found t)
+ (let ((child (cdr node)))
+ (while (and child (not found))
+ (let ((cl (rst-toc-count-lines (car child) target-node)))
+ (setq count (+ count (car cl))
+ found (cdr cl)
+ child (cdr child))))))
+ (cons count found)))
+
+
+(defun rst-toc ()
+ "Display a table-of-contents.
+Finds all the section titles and their decorations in the
+file, and displays a hierarchically-organized list of the
+titles, which is essentially a table-of-contents of the
+document.
+
+The Emacs buffer can be navigated, and selecting a section
+brings the cursor in that section."
+ (interactive)
+ (let* ((curbuf (current-buffer))
+
+ ;; Get the section tree
+ (alldecos (rst-find-all-decorations))
+ (sectree (rst-section-tree alldecos))
+
+ (our-node (cdr (rst-section-tree-point sectree)))
+ line
+
+ ;; Create a temporary buffer.
+ (buf (get-buffer-create rst-toc-buffer-name))
+ )
+
+ (with-current-buffer buf
+ (let ((inhibit-read-only t))
+ (rst-toc-mode)
+ (delete-region (point-min) (point-max))
+ (insert (format "Table of Contents: %s\n" (or (caar sectree) "")))
+ (put-text-property (point-min) (point)
+ 'face (list '(background-color . "gray")))
+ (rst-toc-node sectree 0)
+
+ ;; Count the lines to our found node.
+ (let ((linefound (rst-toc-count-lines sectree our-node)))
+ (setq line (if (cdr linefound) (car linefound) 0)))
+ ))
+ (display-buffer buf)
+ (pop-to-buffer buf)
+
+ ;; Save the buffer to return to.
+ (set (make-local-variable 'rst-toc-return-buffer) curbuf)
+
+ ;; Move the cursor near the right section in the TOC.
+ (goto-line line)
+ ))
+
+
+(defun rst-toc-mode-find-section ()
+ "Get the section from text property at point."
+ (let ((pos (get-text-property (point) 'rst-toc-target)))
+ (unless pos
+ (error "No section on this line"))
+ (unless (buffer-live-p (marker-buffer pos))
+ (error "Buffer for this section was killed"))
+ pos))
+
+(defvar rst-toc-buffer-name "*Table of Contents*"
+ "Name of the Table of Contents buffer.")
+
+(defun rst-goto-section (&optional kill)
+ "Go to the section the current line describes."
+ (interactive)
+ (let ((pos (rst-toc-mode-find-section)))
+ (when kill
+ (kill-buffer (get-buffer rst-toc-buffer-name)))
+ (pop-to-buffer (marker-buffer pos))
+ (goto-char pos)
+ ;; FIXME: make the recentering conditional on scroll.
+ (recenter 5)))
+
+(defun rst-toc-mode-goto-section ()
+ "Go to the section the current line describes and kill the toc buffer."
+ (interactive)
+ (rst-goto-section t))
+
+(defun rst-toc-mode-mouse-goto (event)
+ "In `rst-toc' mode, go to the occurrence whose line you click on.
+EVENT is the input event."
+ (interactive "e")
+ (let (pos)
+ (save-excursion
+ (set-buffer (window-buffer (posn-window (event-end event))))
+ (save-excursion
+ (goto-char (posn-point (event-end event)))
+ (setq pos (rst-toc-mode-find-section))))
+ (pop-to-buffer (marker-buffer pos))
+ (goto-char pos)
+ (recenter 5)))
+
+(defun rst-toc-mode-mouse-goto-kill (event)
+ (interactive "e")
+ (call-interactively 'rst-toc-mode-mouse-goto event)
+ (kill-buffer (get-buffer rst-toc-buffer-name)))
+
+(defvar rst-toc-return-buffer nil
+ "Buffer local variable that is used to return to the original
+ buffer from the TOC.")
+
+(defun rst-toc-quit-window ()
+ (interactive)
+ (quit-window)
+ (pop-to-buffer rst-toc-return-buffer))
+
+(defvar rst-toc-mode-map
+ (let ((map (make-sparse-keymap)))
+ (define-key map [mouse-1] 'rst-toc-mode-mouse-goto-kill)
+ (define-key map [mouse-2] 'rst-toc-mode-mouse-goto)
+ (define-key map "\C-m" 'rst-toc-mode-goto-section)
+ (define-key map "f" 'rst-toc-mode-goto-section)
+ (define-key map "q" 'rst-toc-quit-window)
+ (define-key map "z" 'kill-this-buffer)
+ map)
+ "Keymap for `rst-toc-mode'.")
+
+(put 'rst-toc-mode 'mode-class 'special)
+
+(defun rst-toc-mode ()
+ "Major mode for output from \\[rst-toc], the table-of-contents for the document."
+ (interactive)
+ (kill-all-local-variables)
+ (use-local-map rst-toc-mode-map)
+ (setq major-mode 'rst-toc-mode)
+ (setq mode-name "ReST-TOC")
+ (setq buffer-read-only t)
+ )
+
+;; Note: use occur-mode (replace.el) as a good example to complete missing
+;; features.
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; Section movement commands.
+;;
+
+(defun rst-forward-section (&optional offset)
+ "Skip to the next restructured text section title.
+ OFFSET specifies how many titles to skip. Use a negative OFFSET to move
+ backwards in the file (default is to use 1)."
+ (interactive)
+ (let* (;; Default value for offset.
+ (offset (or offset 1))
+
+ ;; Get all the decorations in the file, with their line numbers.
+ (alldecos (rst-find-all-decorations))
+
+ ;; Get the current line.
+ (curline (line-number-at-pos))
+
+ (cur alldecos)
+ (idx 0)
+ )
+
+ ;; Find the index of the "next" decoration w.r.t. to the current line.
+ (while (and cur (< (caar cur) curline))
+ (setq cur (cdr cur))
+ (incf idx))
+ ;; 'cur' is the decoration on or following the current line.
+
+ (if (and (> offset 0) cur (= (caar cur) curline))
+ (incf idx))
+
+ ;; Find the final index.
+ (setq idx (+ idx (if (> offset 0) (- offset 1) offset)))
+ (setq cur (nth idx alldecos))
+
+ ;; If the index is positive, goto the line, otherwise go to the buffer
+ ;; boundaries.
+ (if (and cur (>= idx 0))
+ (goto-line (car cur))
+ (if (> offset 0) (goto-char (point-max)) (goto-char (point-min))))
+ ))
+
+(defun rst-backward-section ()
+ "Like rst-forward-section, except move back one title.
+With a prefix argument, move backward by a page."
+ (interactive)
+ (rst-forward-section -1))
+
+(defun rst-mark-section (&optional arg allow-extend)
+ "Select the section that point is currently in."
+ ;; Cloned from mark-paragraph.
+ (interactive "p\np")
+ (unless arg (setq arg 1))
+ (when (zerop arg)
+ (error "Cannot mark zero sections"))
+ (cond ((and allow-extend
+ (or (and (eq last-command this-command) (mark t))
+ (rst-portable-mark-active-p)))
+ (set-mark
+ (save-excursion
+ (goto-char (mark))
+ (rst-forward-section arg)
+ (point))))
+ (t
+ (rst-forward-section arg)
+ (push-mark nil t t)
+ (rst-forward-section (- arg)))))
+
+
+
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Functions to work on item lists (e.g. indent/dedent, enumerate), which are
+;; always 2 or 3 characters apart horizontally with rest.
+
+;; (FIXME: there is currently a bug that makes the region go away when we do that.)
+(defvar rst-shift-fill-region nil
+ "Set to true if you want to automatically re-fill the region that is being
+shifted.")
+
+(defun rst-find-leftmost-column (beg end)
+ "Finds the leftmost column in the region."
+ (let ((mincol 1000))
+ (save-excursion
+ (goto-char beg)
+ (while (< (point) end)
+ (back-to-indentation)
+ (unless (looking-at "[ \t]*$")
+ (setq mincol (min mincol (current-column))))
+ (forward-line 1)
+ ))
+ mincol))
+
+
+;; What we really need to do is compute all the possible alignment possibilities
+;; and then select one.
+;;
+;; .. line-block::
+;;
+;; a) sdjsds
+;;
+;; - sdjsd jsjds
+;;
+;; sdsdsjdsj
+;;
+;; 11. sjdss jddjs
+;;
+;; * * * * * * *
+;;
+;; Move backwards, accumulate the beginning positions, and also the second
+;; positions, in case the line matches the bullet pattern, and then sort.
+
+(defun rst-compute-bullet-tabs (&optional pt)
+ "Search backwards from point (or point PT if specified) to
+build the list of possible horizontal alignment points that
+includes the beginning and contents of a restructuredtext
+bulleted or enumerated list item. Return a sorted list
+of (column-number . line) pairs."
+ (save-excursion
+ (when pt (goto-char pt))
+
+ ;; We work our way backwards and towards the left.
+ (let ((leftcol 100000) ;; Current column.
+ (tablist nil) ;; List of tab positions.
+ )
+
+ ;; Start by skipping the current line.
+ (beginning-of-line 0)
+
+ ;; Search backwards for each line.
+ (while (and (> (point) (point-min))
+ (> leftcol 0))
+
+ ;; Skip empty lines.
+ (unless (looking-at "^[ \t]*$")
+ ;; Inspect the current non-empty line
+ (back-to-indentation)
+
+ ;; Skip lines that are beyond the current column (we want to move
+ ;; towards the left).
+ (let ((col (current-column)))
+ (when (< col leftcol)
+
+ ;; Add the beginning of the line as a tabbing point.
+ (unless (memq col (mapcar 'car tablist))
+ (setq tablist (cons (cons col (point)) tablist)))
+
+ ;; Look at the line to figure out if it is a bulleted or enumerate
+ ;; list item.
+ (when (looking-at
+ (concat
+ "\\(?:"
+ "\\(\\(?:[0-9a-zA-Z#]\\{1,3\\}[.):-]\\|[*+-]\\)[ \t]+\\)[^ \t\n]"
+ "\\|"
+ (format "\\(%s%s+[ \t]+\\)[^ \t\n]"
+ (regexp-quote (thing-at-point 'char))
+ (regexp-quote (thing-at-point 'char)))
+ "\\)"
+ ))
+ ;; Add the column of the contained item.
+ (let* ((matchlen (length (or (match-string 1) (match-string 2))))
+ (newcol (+ col matchlen)))
+ (unless (or (>= newcol leftcol)
+ (memq (+ col matchlen) (mapcar 'car tablist)))
+ (setq tablist (cons
+ (cons (+ col matchlen) (+ (point) matchlen))
+ tablist))))
+ )
+
+ (setq leftcol col)
+ )))
+
+ ;; Move backwards one line.
+ (beginning-of-line 0))
+
+ (sort tablist (lambda (x y) (<= (car x) (car y))))
+ )))
+
+(defun rst-debug-print-tabs (tablist)
+ "A routine that inserts a line and places special characters at
+the tab points in the given tablist."
+ (beginning-of-line)
+ (insert (concat "\n" (make-string 1000 ? ) "\n"))
+ (beginning-of-line 0)
+ (dolist (col tablist)
+ (beginning-of-line)
+ (forward-char (car col))
+ (delete-char 1)
+ (insert "@")
+ ))
+
+(defun rst-debug-mark-found (tablist)
+ "A routine that inserts a line and places special characters at
+the tab points in the given tablist."
+ (dolist (col tablist)
+ (when (cdr col)
+ (goto-char (cdr col))
+ (insert "@"))))
+
+
+(defvar rst-shift-basic-offset 2
+ "Basic horizontal shift distance when there is no preceding alignment tabs.")
+
+(defun rst-shift-region-guts (find-next-fun offset-fun)
+ "(See rst-shift-region-right for a description.)"
+ (let* ((mbeg (set-marker (make-marker) (region-beginning)))
+ (mend (set-marker (make-marker) (region-end)))
+ (tabs (rst-compute-bullet-tabs mbeg))
+ (leftmostcol (rst-find-leftmost-column (region-beginning) (region-end)))
+ )
+ ;; Add basic offset tabs at the end of the list. This is a better
+ ;; implementation technique than hysteresis and a basic offset because it
+ ;; insures that movement in both directions is consistently using the same
+ ;; column positions. This makes it more predictable.
+ (setq tabs
+ (append tabs
+ (mapcar (lambda (x) (cons x nil))
+ (let ((maxcol 120)
+ (max-lisp-eval-depth 2000))
+ (flet ((addnum (x)
+ (if (> x maxcol)
+ nil
+ (cons x (addnum
+ (+ x rst-shift-basic-offset))))))
+ (addnum (or (caar (last tabs)) 0))))
+ )))
+
+ ;; (For debugging.)
+ ;;; (save-excursion (goto-char mbeg) (forward-char -1) (rst-debug-print-tabs tabs))))
+ ;;; (print tabs)
+ ;;; (save-excursion (rst-debug-mark-found tabs))
+
+ ;; Apply the indent.
+ (indent-rigidly
+ mbeg mend
+
+ ;; Find the next tab after the leftmost columnt.
+ (let ((tab (funcall find-next-fun tabs leftmostcol)))
+
+ (if tab
+ (progn
+ (when (cdar tab)
+ (message "Aligned on '%s'"
+ (save-excursion
+ (goto-char (cdar tab))
+ (buffer-substring-no-properties
+ (line-beginning-position)
+ (line-end-position))))
+ )
+ (- (caar tab) leftmostcol)) ;; Num chars.
+
+ ;; Otherwise use the basic offset
+ (funcall offset-fun rst-shift-basic-offset)
+ )))
+
+ ;; Optionally reindent.
+ (when rst-shift-fill-region
+ (fill-region mbeg mend))
+ ))
+
+(defun rst-shift-region-right (pfxarg)
+ "Indent region ridigly, by a few characters to the right. This
+function first computes all possible alignment columns by
+inspecting the lines preceding the region for bulleted or
+enumerated list items. If the leftmost column is beyond the
+preceding lines, the region is moved to the right by
+rst-shift-basic-offset. With a prefix argument, do not
+automatically fill the region."
+ (interactive "P")
+ (let ((rst-shift-fill-region
+ (if (not pfxarg) rst-shift-fill-region)))
+ (rst-shift-region-guts (lambda (tabs leftmostcol)
+ (let ((cur tabs))
+ (while (and cur (<= (caar cur) leftmostcol))
+ (setq cur (cdr cur)))
+ cur))
+ 'identity
+ )))
+
+(defun rst-shift-region-left (pfxarg)
+ "Like rst-shift-region-right, except we move to the left.
+Also, if invoked with a negative prefix arg, the entire
+indentation is removed, up to the leftmost character in the
+region, and automatic filling is disabled."
+ (interactive "P")
+ (let ((mbeg (set-marker (make-marker) (region-beginning)))
+ (mend (set-marker (make-marker) (region-end)))
+ (leftmostcol (rst-find-leftmost-column
+ (region-beginning) (region-end)))
+ (rst-shift-fill-region
+ (if (not pfxarg) rst-shift-fill-region)))
+
+ (when (> leftmostcol 0)
+ (if (and pfxarg (< (prefix-numeric-value pfxarg) 0))
+ (progn
+ (indent-rigidly (region-beginning) (region-end) (- leftmostcol))
+ (when rst-shift-fill-region
+ (fill-region mbeg mend))
+ )
+ (rst-shift-region-guts (lambda (tabs leftmostcol)
+ (let ((cur (reverse tabs)))
+ (while (and cur (>= (caar cur) leftmostcol))
+ (setq cur (cdr cur)))
+ cur))
+ '-
+ ))
+ )))
+
+
+;;------------------------------------------------------------------------------
+
+;; FIXME: these next functions should become part of a larger effort to redo the
+;; bullets in bulletted lists. The enumerate would just be one of the possible
+;; outputs.
+;;
+;; FIXME: TODO we need to do the enumeration removal as well.
+
+(defun rst-enumerate-region (beg end)
+ "Add enumeration to all the leftmost paragraphs in the given region.
+The region is specified between BEG and END. With prefix argument,
+do all lines instead of just paragraphs."
+ (interactive "r")
+ (let ((count 0)
+ (last-insert-len nil))
+ (rst-iterate-leftmost-paragraphs
+ beg end (not current-prefix-arg)
+ (let ((ins-string (format "%d. " (incf count))))
+ (setq last-insert-len (length ins-string))
+ (insert ins-string))
+ (insert (make-string last-insert-len ?\ ))
+ )))
+
+(defun rst-bullet-list-region (beg end)
+ "Add bullets to all the leftmost paragraphs in the given region.
+The region is specified between BEG and END. With prefix argument,
+do all lines instead of just paragraphs."
+ (interactive "r")
+ (rst-iterate-leftmost-paragraphs
+ beg end (not current-prefix-arg)
+ (insert "- ")
+ (insert " ")
+ ))
+
+(defmacro rst-iterate-leftmost-paragraphs
+ (beg end first-only body-consequent body-alternative)
+ "FIXME This definition is old and deprecated / we need to move
+to the newer version below:
+
+Call FUN at the beginning of each line, with an argument that
+specifies whether we are at the first line of a paragraph that
+starts at the leftmost column of the given region BEG and END.
+Set FIRST-ONLY to true if you want to callback on the first line
+of each paragraph only."
+ `(save-excursion
+ (let ((leftcol (rst-find-leftmost-column ,beg ,end))
+ (endm (set-marker (make-marker) ,end))
+ ,(when first-only '(in-par nil))
+ )
+
+ (do* (;; Iterate lines
+ (l (progn (goto-char ,beg) (back-to-indentation))
+ (progn (forward-line 1) (back-to-indentation)))
+
+ (previous nil valid)
+
+ (curcol (current-column)
+ (current-column))
+
+ (valid (and (= curcol leftcol)
+ (not (looking-at "[ \t]*$")))
+ (and (= curcol leftcol)
+ (not (looking-at "[ \t]*$"))))
+ )
+ ((>= (point-marker) endm))
+
+ (if (if ,first-only
+ (and valid (not previous))
+ valid)
+ ,body-consequent
+ ,body-alternative)
+
+ ))))
+
+
+(defmacro rst-iterate-leftmost-paragraphs-2 (spec &rest body)
+ "Evaluate BODY for each line in region defined by BEG END.
+LEFTMOST is set to true if the line is one of the leftmost of the
+entire paragraph. PARABEGIN is set to true if the line is the
+first of a paragraph."
+ (destructuring-bind
+ (beg end parabegin leftmost isleftmost isempty) spec
+
+ `(save-excursion
+ (let ((,leftmost (rst-find-leftmost-column ,beg ,end))
+ (endm (set-marker (make-marker) ,end))
+ (in-par nil)
+ )
+
+ (do* (;; Iterate lines
+ (l (progn (goto-char ,beg) (back-to-indentation))
+ (progn (forward-line 1) (back-to-indentation)))
+
+ (empty-line-previous nil ,isempty)
+
+ (,isempty (looking-at "[ \t]*$")
+ (looking-at "[ \t]*$"))
+
+ (,parabegin (not ,isempty)
+ (and empty-line-previous
+ (not ,isempty)))
+
+ (,isleftmost (and (not ,isempty)
+ (= (current-column) ,leftmost))
+ (and (not ,isempty)
+ (= (current-column) ,leftmost)))
+ )
+ ((>= (point-marker) endm))
+
+ (progn , at body)
+
+ )))))
+
+
+;; FIXME: there are some problems left with the following function
+;; implementation:
+;;
+;; * It does not deal with a varying number of digits appropriately
+;; * It does not deal with multiple levels independently, and it should.
+;;
+;; I suppose it does 90% of the job for now.
+
+(defun rst-convert-bullets-to-enumeration (beg end)
+ "Convert all the bulleted items and enumerated items in the
+ region to enumerated lists, renumbering as necessary."
+ (interactive "r")
+ (let* (;; Find items and convert the positions to markers.
+ (items (mapcar
+ (lambda (x)
+ (cons (let ((m (make-marker)))
+ (set-marker m (car x))
+ m)
+ (cdr x)))
+ (rst-find-pfx-in-region beg end rst-re-items)))
+ (count 1)
+ )
+ (save-excursion
+ (dolist (x items)
+ (goto-char (car x))
+ (looking-at rst-re-items)
+ (replace-match (format "%d. " count) nil nil nil 1)
+ (incf count)
+ ))
+ ))
+
+
+
+;;------------------------------------------------------------------------------
+
+(defun rst-line-block-region (rbeg rend &optional pfxarg)
+ "Toggle line block prefixes for a region. With prefix argument
+set the empty lines too."
+ (interactive "r\nP")
+ (let ((comment-start "| ")
+ (comment-end "")
+ (comment-start-skip "| ")
+ (comment-style 'indent)
+ (force current-prefix-arg))
+ (rst-iterate-leftmost-paragraphs-2
+ (rbeg rend parbegin leftmost isleft isempty)
+ (if force
+ (progn
+ (move-to-column leftmost t)
+ (delete-region (point) (+ (point) (- (current-indentation) leftmost)))
+ (insert "| "))
+ (when (not isempty)
+ (move-to-column leftmost)
+ (delete-region (point) (+ (point) (- (current-indentation) leftmost)))
+ (insert "| ")))
+ )))
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(require 'font-lock)
+
+(defgroup rst-faces nil "Faces used in Rst Mode"
+ :group 'rst
+ :group 'faces
+ :version "21.1")
+
+(defcustom rst-block-face 'font-lock-keyword-face
+ "All syntax marking up a special block"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-external-face 'font-lock-type-face
+ "Field names and interpreted text"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-definition-face 'font-lock-function-name-face
+ "All other defining constructs"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-directive-face
+ ;; XEmacs compatibility
+ (if (boundp 'font-lock-builtin-face)
+ 'font-lock-builtin-face
+ 'font-lock-preprocessor-face)
+ "Directives and roles"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-comment-face 'font-lock-comment-face
+ "Comments"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-emphasis1-face
+ ;; XEmacs compatibility
+ (if (facep 'italic)
+ ''italic
+ 'italic)
+ "Simple emphasis"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-emphasis2-face
+ ;; XEmacs compatibility
+ (if (facep 'bold)
+ ''bold
+ 'bold)
+ "Double emphasis"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-literal-face 'font-lock-string-face
+ "Literal text"
+ :group 'rst-faces
+ :type '(face))
+
+(defcustom rst-reference-face 'font-lock-variable-name-face
+ "References to a definition"
+ :group 'rst-faces
+ :type '(face))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(defgroup rst-faces-defaults nil
+ "Values used to generate default faces for section titles on all levels.
+Tweak these if you are content with how section title faces are built in
+general but you do not like the details."
+ :group 'rst-faces
+ :version "21.1")
+
+(defun rst-define-level-faces ()
+ "Define the faces for the section title text faces from the values."
+ ;; All variables used here must be checked in `rst-set-level-default'
+ (let ((i 1))
+ (while (<= i rst-level-face-max)
+ (let ((sym (intern (format "rst-level-%d-face" i)))
+ (doc (format "Face for showing section title text at level %d" i))
+ (col (format (concat "%s" rst-level-face-format-light)
+ rst-level-face-base-color
+ (+ (* (1- i) rst-level-face-step-light)
+ rst-level-face-base-light))))
+ (make-empty-face sym)
+ (set-face-doc-string sym doc)
+ (set-face-background sym col)
+ (set sym sym)
+ (setq i (1+ i))))))
+
+(defun rst-set-level-default (sym val)
+ "Set a customized value affecting section title text face and recompute the
+faces."
+ (custom-set-default sym val)
+ ;; Also defines the faces initially when all values are available
+ (and (boundp 'rst-level-face-max)
+ (boundp 'rst-level-face-format-light)
+ (boundp 'rst-level-face-base-color)
+ (boundp 'rst-level-face-step-light)
+ (boundp 'rst-level-face-base-light)
+ (rst-define-level-faces)))
+
+;; Faces for displaying items on several levels; these definitions define
+;; different shades of grey where the lightest one (i.e. least contrasting) is
+;; used for level 1
+(defcustom rst-level-face-max 6
+ "Maximum depth of levels for which section title faces are defined."
+ :group 'rst-faces-defaults
+ :type '(integer)
+ :set 'rst-set-level-default)
+(defcustom rst-level-face-base-color "grey"
+ "The base name of the color to be used for creating background colors in
+ection title faces for all levels."
+ :group 'rst-faces-defaults
+ :type '(string)
+ :set 'rst-set-level-default)
+(defcustom rst-level-face-base-light
+ (if (eq frame-background-mode 'dark)
+ 15
+ 85)
+ "The lightness factor for the base color. This value is used for level 1. The
+default depends on whether the value of `frame-background-mode' is `dark' or
+not."
+ :group 'rst-faces-defaults
+ :type '(integer)
+ :set 'rst-set-level-default)
+(defcustom rst-level-face-format-light "%2d"
+ "The format for the lightness factor appended to the base name of the color.
+This value is expanded by `format' with an integer."
+ :group 'rst-faces-defaults
+ :type '(string)
+ :set 'rst-set-level-default)
+(defcustom rst-level-face-step-light
+ (if (eq frame-background-mode 'dark)
+ 7
+ -7)
+ "The step width to use for the next color. The formula
+
+ `rst-level-face-base-light'
+ + (`rst-level-face-max' - 1) * `rst-level-face-step-light'
+
+must result in a color level which appended to `rst-level-face-base-color'
+using `rst-level-face-format-light' results in a valid color such as `grey50'.
+This color is used as background for section title text on level
+`rst-level-face-max'."
+ :group 'rst-faces-defaults
+ :type '(integer)
+ :set 'rst-set-level-default)
+
+(defcustom rst-adornment-faces-alist
+ (let ((alist '((t . font-lock-keyword-face)
+ (nil . font-lock-keyword-face)))
+ (i 1))
+ (while (<= i rst-level-face-max)
+ (nconc alist (list (cons i (intern (format "rst-level-%d-face" i)))))
+ (setq i (1+ i)))
+ alist)
+ "Provides faces for the various adornment types. Key is a number (for the
+section title text of that level), t (for transitions) or nil (for section
+title adornment). If you generally do not like how section title text faces are
+set up tweak here. If the general idea is ok for you but you do not like the
+details check the Rst Faces Defaults group."
+ :group 'rst-faces
+ :type '(alist
+ :key-type
+ (choice
+ (integer
+ :tag
+ "Section level (may not be bigger than `rst-level-face-max')")
+ (boolean :tag "transitions (on) / section title adornment (off)"))
+ :value-type (face))
+ :set-after '(rst-level-face-max))
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Font lock
+
+(defconst rst-use-char-classes
+ (string-match "[[:alpha:]]" "b")
+ "Non-nil if we can use the character classes in our regexps.")
+
+(defvar rst-font-lock-keywords
+ ;; The reST-links in the comments below all relate to sections in
+ ;; http://docutils.sourceforge.net/docs/ref/rst/restructuredtext.html
+ (let* ( ;; This gets big - so let's define some abbreviations; the trailing
+ ;; numbers in the names give the number of regex groups contained
+ ;; horizontal white space
+ (re-hws "[\t ]")
+ ;; beginning of line with possible indentation
+ (re-bol (concat "^" re-hws "*"))
+ ;; Separates block lead-ins from their content
+ (re-blksep1 (concat "\\(" re-hws "+\\|$\\)"))
+ ;; explicit markup tag
+ (re-emt "\\.\\.")
+ ;; explicit markup start
+ (re-ems (concat re-emt re-hws "+"))
+ ;; inline markup prefix
+ (re-imp1 (concat "\\(^\\|" re-hws "\\|[-'\"([{<"
+ (if rst-use-unicode
+ "\u2018\u201c\u00ab\u2019"
+ "")
+ "/:]\\)"))
+ ;; inline markup suffix
+ (re-ims1 (concat "\\(" re-hws "\\|[]-'\")}>"
+ (if rst-use-unicode
+ "\u2019\u201d\u00bb"
+ "")
+ "/:.,;!?\\]\\|$\\)"))
+ ;; symbol character
+ (re-sym1 "\\(\\sw\\|\\s_\\)")
+ ;; inline markup content begin
+ (re-imbeg2 "\\(\\S \\|\\S \\([^")
+
+ ;; There seems to be a bug leading to error "Stack overflow in regexp
+ ;; matcher" when "|" or "\\*" are the characters searched for
+ (re-imendbeg
+ (if (< emacs-major-version 21)
+ "]"
+ "\\]\\|\\\\."))
+ ;; inline markup content end
+ (re-imend (concat re-imendbeg "\\)*[^\t \\\\]\\)"))
+ ;; inline markup content without asterisk
+ (re-ima2 (concat re-imbeg2 "*" re-imend))
+ ;; inline markup content without backquote
+ (re-imb2 (concat re-imbeg2 "`" re-imend))
+ ;; inline markup content without vertical bar
+ (re-imv2 (concat re-imbeg2 "|" re-imend))
+ ;; Supported URI schemes
+ (re-uris1 "\\(acap\\|cid\\|data\\|dav\\|fax\\|file\\|ftp\\|gopher\\|http\\|https\\|imap\\|ldap\\|mailto\\|mid\\|modem\\|news\\|nfs\\|nntp\\|pop\\|prospero\\|rtsp\\|service\\|sip\\|tel\\|telnet\\|tip\\|urn\\|vemmi\\|wais\\)")
+ ;; Line starting with adornment and optional whitespace; complete
+ ;; adornment is in (match-string 1); there must be at least 3
+ ;; characters because otherwise explicit markup start would be
+ ;; recognized
+ (re-ado2 (concat "^\\(\\(["
+ (if rst-use-char-classes
+ "^[:word:][:space:][:cntrl:]"
+ "^\\w \t\x00-\x1F")
+ "]\\)\\2\\2+\\)" re-hws "*$"))
+ )
+ (list
+ ;; FIXME: Block markup is not recognized in blocks after explicit markup
+ ;; start
+
+ ;; Simple `Body Elements`_
+ ;; `Bullet Lists`_
+ (list
+ (concat re-bol "\\([-*+]" re-blksep1 "\\)")
+ 1 rst-block-face)
+ ;; `Enumerated Lists`_
+ (list
+ (concat re-bol "\\((?\\(#\\|[0-9]+\\|[A-Za-z]\\|[IVXLCMivxlcm]+\\)[.)]"
+ re-blksep1 "\\)")
+ 1 rst-block-face)
+ ;; `Definition Lists`_ FIXME: missing
+ ;; `Field Lists`_
+ (list
+ (concat re-bol "\\(:[^:\n]+:\\)" re-blksep1)
+ 1 rst-external-face)
+ ;; `Option Lists`_
+ (list
+ (concat re-bol "\\(\\(\\(\\([-+/]\\|--\\)\\sw\\(-\\|\\sw\\)*"
+ "\\([ =]\\S +\\)?\\)\\(,[\t ]\\)?\\)+\\)\\($\\|[\t ]\\{2\\}\\)")
+ 1 rst-block-face)
+
+ ;; `Tables`_ FIXME: missing
+
+ ;; All the `Explicit Markup Blocks`_
+ ;; `Footnotes`_ / `Citations`_
+ (list
+ (concat re-bol "\\(" re-ems "\\[[^[\n]+\\]\\)" re-blksep1)
+ 1 rst-definition-face)
+ ;; `Directives`_ / `Substitution Definitions`_
+ (list
+ (concat re-bol "\\(" re-ems "\\)\\(\\(|[^|\n]+|[\t ]+\\)?\\)\\("
+ re-sym1 "+::\\)" re-blksep1)
+ (list 1 rst-directive-face)
+ (list 2 rst-definition-face)
+ (list 4 rst-directive-face))
+ ;; `Hyperlink Targets`_
+ (list
+ (concat re-bol "\\(" re-ems "_\\([^:\\`\n]\\|\\\\.\\|`[^`\n]+`\\)+:\\)"
+ re-blksep1)
+ 1 rst-definition-face)
+ (list
+ (concat re-bol "\\(__\\)" re-blksep1)
+ 1 rst-definition-face)
+
+ ;; All `Inline Markup`_
+ ;; FIXME: Condition 5 preventing fontification of e.g. "*" not implemented
+ ;; `Strong Emphasis`_
+ (list
+ (concat re-imp1 "\\(\\*\\*" re-ima2 "\\*\\*\\)" re-ims1)
+ 2 rst-emphasis2-face)
+ ;; `Emphasis`_
+ (list
+ (concat re-imp1 "\\(\\*" re-ima2 "\\*\\)" re-ims1)
+ 2 rst-emphasis1-face)
+ ;; `Inline Literals`_
+ (list
+ (concat re-imp1 "\\(``" re-imb2 "``\\)" re-ims1)
+ 2 rst-literal-face)
+ ;; `Inline Internal Targets`_
+ (list
+ (concat re-imp1 "\\(_`" re-imb2 "`\\)" re-ims1)
+ 2 rst-definition-face)
+ ;; `Hyperlink References`_
+ ;; FIXME: `Embedded URIs`_ not considered
+ (list
+ (concat re-imp1 "\\(\\(`" re-imb2 "`\\|\\(\\sw\\(\\sw\\|-\\)+\\sw\\)\\)__?\\)" re-ims1)
+ 2 rst-reference-face)
+ ;; `Interpreted Text`_
+ (list
+ (concat re-imp1 "\\(\\(:" re-sym1 "+:\\)?\\)\\(`" re-imb2 "`\\)\\(\\(:"
+ re-sym1 "+:\\)?\\)" re-ims1)
+ (list 2 rst-directive-face)
+ (list 5 rst-external-face)
+ (list 8 rst-directive-face))
+ ;; `Footnote References`_ / `Citation References`_
+ (list
+ (concat re-imp1 "\\(\\[[^]]+\\]_\\)" re-ims1)
+ 2 rst-reference-face)
+ ;; `Substitution References`_
+ (list
+ (concat re-imp1 "\\(|" re-imv2 "|\\)" re-ims1)
+ 2 rst-reference-face)
+ ;; `Standalone Hyperlinks`_
+ (list
+ ;; FIXME: This takes it easy by using a whitespace as delimiter
+ (concat re-imp1 "\\(" re-uris1 ":\\S +\\)" re-ims1)
+ 2 rst-definition-face)
+ (list
+ (concat re-imp1 "\\(" re-sym1 "+@" re-sym1 "+\\)" re-ims1)
+ 2 rst-definition-face)
+
+ ;; Do all block fontification as late as possible so 'append works
+
+ ;; Sections_ / Transitions_
+ (list
+ re-ado2
+ (list 'rst-font-lock-handle-adornment-match
+ '(rst-font-lock-handle-adornment-limit
+ (match-string-no-properties 1) (match-end 1))
+ nil
+ (list 1 '(cdr (assoc nil rst-adornment-faces-alist))
+ 'append t)
+ (list 2 '(cdr (assoc rst-font-lock-adornment-level
+ rst-adornment-faces-alist))
+ 'append t)
+ (list 3 '(cdr (assoc nil rst-adornment-faces-alist))
+ 'append t)))
+
+ ;; `Comments`_
+ (list
+ (concat re-bol "\\(" re-ems "\\)\[^[|_]\\([^:\n]\\|:\\([^:\n]\\|$\\)\\)*$")
+ (list 1 rst-comment-face)
+ (list 'rst-font-lock-find-unindented-line-match
+ '(rst-font-lock-find-unindented-line-limit (match-end 1))
+ nil
+ (list 0 rst-comment-face 'append)))
+ (list
+ (concat re-bol "\\(" re-emt "\\)\\(\\s *\\)$")
+ (list 1 rst-comment-face)
+ (list 2 rst-comment-face)
+ (list 'rst-font-lock-find-unindented-line-match
+ '(rst-font-lock-find-unindented-line-limit 'next)
+ nil
+ (list 0 rst-comment-face 'append)))
+
+ ;; `Literal Blocks`_
+ (list
+ (concat re-bol "\\(\\([^.\n]\\|\\.[^.\n]\\).*\\)?\\(::\\)$")
+ (list 3 rst-block-face)
+ (list 'rst-font-lock-find-unindented-line-match
+ '(rst-font-lock-find-unindented-line-limit t)
+ nil
+ (list 0 rst-literal-face 'append)))
+
+ ;; `Doctest Blocks`_
+ (list
+ (concat re-bol "\\(>>>\\|\\.\\.\\.\\)\\(.+\\)")
+ (list 1 rst-block-face)
+ (list 2 rst-literal-face))
+ ))
+ "Returns keywords to highlight in rst mode according to current settings.")
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Indented blocks
+
+(defun rst-forward-indented-block (&optional column limit)
+ "Move forward across one indented block.
+Find the next non-empty line which is not indented at least to COLUMN (defaults
+to the column of the point). Moves point to first character of this line or the
+first empty line immediately before it and returns that position. If there is
+no such line before LIMIT (defaults to the end of the buffer) returns nil and
+point is not moved."
+ (interactive)
+ (let ((clm (or column (current-column)))
+ (start (point))
+ fnd beg cand)
+ (if (not limit)
+ (setq limit (point-max)))
+ (save-match-data
+ (while (and (not fnd) (< (point) limit))
+ (forward-line 1)
+ (when (< (point) limit)
+ (setq beg (point))
+ (if (looking-at "\\s *$")
+ (setq cand (or cand beg)) ; An empty line is a candidate
+ (move-to-column clm)
+ ;; FIXME: No indentation [(zerop clm)] must be handled in some
+ ;; useful way - though it is not clear what this should mean at all
+ (if (string-match
+ "^\\s *$" (buffer-substring-no-properties beg (point)))
+ (setq cand nil) ; An indented line resets a candidate
+ (setq fnd (or cand beg)))))))
+ (goto-char (or fnd start))
+ fnd))
+
+;; Beginning of the match if `rst-font-lock-find-unindented-line-end'.
+(defvar rst-font-lock-find-unindented-line-begin nil)
+
+;; End of the match as determined by
+;; `rst-font-lock-find-unindented-line-limit'. Also used as a trigger for
+;; `rst-font-lock-find-unindented-line-match'.
+(defvar rst-font-lock-find-unindented-line-end nil)
+
+;; Finds the next unindented line relative to indenation at IND-PNT and returns
+;; this point, the end of the buffer or nil if nothing found. If IND-PNT is
+;; `next' takes the indentation from the next line if this is not empty. If
+;; IND-PNT is non-nil but not a number takes the indentation from the next
+;; non-empty line.
+(defun rst-font-lock-find-unindented-line-limit (ind-pnt)
+ (setq rst-font-lock-find-unindented-line-begin ind-pnt)
+ (setq rst-font-lock-find-unindented-line-end
+ (save-excursion
+ (when (not (numberp ind-pnt))
+ ;; Find indentation point in next line if any
+ (setq ind-pnt
+ ;; FIXME: Should be refactored to two different functions
+ ;; giving their result to this function, may be
+ ;; integrated in caller
+ (save-match-data
+ (if (eq ind-pnt 'next)
+ (when (and (zerop (forward-line 1))
+ (< (point) (point-max)))
+ ;; Not at EOF
+ (setq rst-font-lock-find-unindented-line-begin (point))
+ (when (not (looking-at "\\s *$"))
+ ;; Use end of indentation if non-empty line
+ (looking-at "\\s *")
+ (match-end 0)))
+ ;; Skip until non-empty line or EOF
+ (while (and (zerop (forward-line 1))
+ (< (point) (point-max))
+ (looking-at "\\s *$")))
+ (when (< (point) (point-max))
+ ;; Not at EOF
+ (setq rst-font-lock-find-unindented-line-begin (point))
+ (looking-at "\\s *")
+ (match-end 0))))))
+ (when ind-pnt
+ (goto-char ind-pnt)
+ (or (rst-forward-indented-block nil (point-max))
+ (point-max))))))
+
+;; Sets the match found by `rst-font-lock-find-unindented-line-limit' the first
+;; time called or nil.
+(defun rst-font-lock-find-unindented-line-match (limit)
+ (when rst-font-lock-find-unindented-line-end
+ (set-match-data
+ (list rst-font-lock-find-unindented-line-begin
+ rst-font-lock-find-unindented-line-end))
+ ;; Make sure this is called only once
+ (setq rst-font-lock-find-unindented-line-end nil)
+ t))
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Adornments
+
+;; Here `rst-font-lock-handle-adornment-match' stores the section level of the
+;; current adornment or t for a transition.
+(defvar rst-font-lock-adornment-level nil)
+
+;; FIXME: It would be good if this could be used to markup section titles of
+;; given level with a special key; it would be even better to be able to
+;; customize this so it can be used for a generally available personal style
+;;
+;; FIXME: There should be some way to reset and reload this variable - probably
+;; a special key
+;;
+;; FIXME: Some support for `outline-mode' would be nice which should be based
+;; on this information
+(defvar rst-adornment-level-alist nil
+ "Associates adornments with section levels.
+The key is a two character string. The first character is the adornment
+character. The second character distinguishes underline section titles (`u')
+from overline/underline section titles (`o'). The value is the section level.
+
+This is made buffer local on start and adornments found during font lock are
+entered.")
+
+;; Returns section level for adornment key KEY. Adds new section level if KEY
+;; is not found and ADD. If KEY is not a string it is simply returned.
+(defun rst-adornment-level (key &optional add)
+ (let ((fnd (assoc key rst-adornment-level-alist))
+ (new 1))
+ (cond
+ ((not (stringp key))
+ key)
+ (fnd
+ (cdr fnd))
+ (add
+ (while (rassoc new rst-adornment-level-alist)
+ (setq new (1+ new)))
+ (setq rst-adornment-level-alist
+ (append rst-adornment-level-alist (list (cons key new))))
+ new))))
+
+;; Classifies adornment for section titles and transitions. ADORNMENT is the
+;; complete adornment string as found in the buffer. END is the point after the
+;; last character of ADORNMENT. For overline section adornment LIMIT limits the
+;; search for the matching underline. Returns a list. The first entry is t for
+;; a transition, or a key string for `rst-adornment-level' for a section title.
+;; The following eight values forming four match groups as can be used for
+;; `set-match-data'. First match group contains the maximum points of the whole
+;; construct. Second and last match group matched pure section title adornment
+;; while third match group matched the section title text or the transition.
+;; Each group but the first may or may not exist.
+(defun rst-classify-adornment (adornment end limit)
+ (save-excursion
+ (save-match-data
+ (goto-char end)
+ (let ((ado-ch (aref adornment 0))
+ (ado-re (regexp-quote adornment))
+ (end-pnt (point))
+ (beg-pnt (progn
+ (forward-line 0)
+ (point)))
+ (nxt-emp
+ (save-excursion
+ (or (not (zerop (forward-line 1)))
+ (looking-at "\\s *$"))))
+ (prv-emp
+ (save-excursion
+ (or (not (zerop (forward-line -1)))
+ (looking-at "\\s *$"))))
+ key beg-ovr end-ovr beg-txt end-txt beg-und end-und)
+ (cond
+ ((and nxt-emp prv-emp)
+ ;; A transition
+ (setq key t)
+ (setq beg-txt beg-pnt)
+ (setq end-txt end-pnt))
+ (prv-emp
+ ;; An overline
+ (setq key (concat (list ado-ch) "o"))
+ (setq beg-ovr beg-pnt)
+ (setq end-ovr end-pnt)
+ (forward-line 1)
+ (setq beg-txt (point))
+ (while (and (<= (point) limit) (not end-txt))
+ (if (or (= (point) limit) (looking-at "\\s *$"))
+ ;; No underline found
+ (setq end-txt (1- (point)))
+ (when (looking-at (concat "\\(" ado-re "\\)\\s *$"))
+ (setq end-und (match-end 1))
+ (setq beg-und (point))
+ (setq end-txt (1- beg-und))))
+ (forward-line 1)))
+ (t
+ ;; An underline
+ (setq key (concat (list ado-ch) "u"))
+ (setq beg-und beg-pnt)
+ (setq end-und end-pnt)
+ (setq end-txt (1- beg-und))
+ (setq beg-txt (progn
+ (goto-char end-txt)
+ (forward-line 0)
+ (point)))
+ (when (and (zerop (forward-line -1))
+ (looking-at (concat "\\(" ado-re "\\)\\s *$")))
+ ;; There is a matching overline
+ (setq key (concat (list ado-ch) "o"))
+ (setq beg-ovr (point))
+ (setq end-ovr (match-end 1)))))
+ (list key
+ (or beg-ovr beg-txt beg-und)
+ (or end-und end-txt end-und)
+ beg-ovr end-ovr beg-txt end-txt beg-und end-und)))))
+
+;; Stores the result of `rst-classify-adornment'. Also used as a trigger
+;; for `rst-font-lock-handle-adornment-match'.
+(defvar rst-font-lock-adornment-data nil)
+
+;; Determines limit for adornments for font-locking section titles and
+;; transitions. In fact it determines all things necessary and puts the result
+;; to `rst-font-lock-adornment-data'. ADO is the complete adornment matched.
+;; ADO-END is the point where ADO ends. Returns the point where the whole
+;; adorned construct ends.
+(defun rst-font-lock-handle-adornment-limit (ado ado-end)
+ (let ((ado-data (rst-classify-adornment ado ado-end (point-max))))
+ (setq rst-font-lock-adornment-level (rst-adornment-level (car ado-data) t))
+ (setq rst-font-lock-adornment-data (cdr ado-data))
+ (goto-char (nth 1 ado-data))
+ (nth 2 ado-data)))
+
+;; Sets the match found by `rst-font-lock-handle-adornment-limit' the first
+;; time called or nil.
+(defun rst-font-lock-handle-adornment-match (limit)
+ (let ((ado-data rst-font-lock-adornment-data))
+ ;; May run only once - enforce this
+ (setq rst-font-lock-adornment-data nil)
+ (when ado-data
+ (goto-char (nth 1 ado-data))
+ (set-match-data ado-data)
+ t)))
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; Support for conversion from within Emacs
+
+(defgroup rst-compile nil
+ "Settings for support of conversion of reStructuredText
+document with \\[rst-compile]."
+ :group 'rst
+ :version "21.1")
+
+(defvar rst-compile-toolsets
+ '((html . ("rst2html.py" ".html" nil))
+ (latex . ("rst2latex.py" ".tex" nil))
+ (newlatex . ("rst2newlatex.py" ".tex" nil))
+ (pseudoxml . ("rst2pseudoxml.py" ".xml" nil))
+ (xml . ("rst2xml.py" ".xml" nil)))
+ "An association list of the toolset to a list of the (command to use,
+extension of produced filename, options to the tool (nil or a
+string)) to be used for converting the document.")
+
+;; Note for Python programmers not familiar with association lists: you can set
+;; values in an alists like this, e.g. :
+;; (setcdr (assq 'html rst-compile-toolsets)
+;; '("rst2html.py" ".htm" "--stylesheet=/docutils.css"))
+
+
+(defvar rst-compile-primary-toolset 'html
+ "The default toolset for rst-compile.")
+
+(defvar rst-compile-secondary-toolset 'latex
+ "The default toolset for rst-compile with a prefix argument.")
+
+(defun rst-compile-find-conf ()
+ "Look for the configuration file in the parents of the current path."
+ (interactive)
+ (let ((file-name "docutils.conf")
+ (buffer-file (buffer-file-name)))
+ ;; Move up in the dir hierarchy till we find a change log file.
+ (let* ((dir (file-name-directory buffer-file))
+ (prevdir nil))
+ (while (and (or (not (string= dir prevdir))
+ (setq dir nil)
+ nil)
+ (not (file-exists-p (concat dir file-name))))
+ ;; Move up to the parent dir and try again.
+ (setq prevdir dir)
+ (setq dir (expand-file-name (file-name-directory
+ (directory-file-name
+ (file-name-directory dir)))))
+ )
+ (or (and dir (concat dir file-name)) nil)
+ )))
+
+
+(require 'compile)
+
+(defun rst-compile (&optional pfxarg)
+ "Compile command to convert reST document into some output file.
+Attempts to find configuration file, if it can, overrides the
+options. There are two commands to choose from, with a prefix
+argument, select the alternative toolset."
+ (interactive "P")
+ ;; Note: maybe we want to check if there is a Makefile too and not do anything
+ ;; if that is the case. I dunno.
+ (let* ((toolset (cdr (assq (if pfxarg
+ rst-compile-secondary-toolset
+ rst-compile-primary-toolset)
+ rst-compile-toolsets)))
+ (command (car toolset))
+ (extension (cadr toolset))
+ (options (caddr toolset))
+ (conffile (rst-compile-find-conf))
+ (bufname (file-name-nondirectory buffer-file-name))
+ (outname (file-name-sans-extension bufname)))
+
+ ;; Set compile-command before invocation of compile.
+ (set (make-local-variable 'compile-command)
+ (mapconcat 'identity
+ (list command
+ (or options "")
+ (if conffile
+ (concat "--config=\"" conffile "\"")
+ "")
+ bufname
+ (concat outname extension))
+ " "))
+
+ ;; Invoke the compile command.
+ (if (or compilation-read-command current-prefix-arg)
+ (call-interactively 'compile)
+ (compile compile-command))
+ ))
+
+(defun rst-compile-alt-toolset ()
+ "Compile command with the alternative toolset."
+ (interactive)
+ (rst-compile 't))
+
+(defun rst-compile-pseudo-region ()
+ "Show the pseudo-XML rendering of the current active region, or
+of the entire buffer, if the region is not selected."
+ (interactive)
+ (with-output-to-temp-buffer "*pseudoxml*"
+ (shell-command-on-region
+ (if mark-active (region-beginning) (point-min))
+ (if mark-active (region-end) (point-max))
+ "rst2pseudoxml.py"
+ standard-output)))
+
+(defvar rst-pdf-program "xpdf"
+ "Program used to preview PDF files.")
+
+(defun rst-compile-pdf-preview ()
+ "Convert the document to a PDF file and launch a preview program."
+ (interactive)
+ (let* ((tmp-filename "/tmp/out.pdf")
+ (command (format "rst2pdf.py %s %s && %s %s"
+ buffer-file-name tmp-filename
+ rst-pdf-program tmp-filename)))
+ (start-process-shell-command "rst-pdf-preview" nil command)
+ ;; Note: you could also use (compile command) to view the compilation
+ ;; output.
+ ))
+
+(defvar rst-slides-program "firefox"
+ "Program used to preview S5 slides.")
+
+(defun rst-compile-slides-preview ()
+ "Convert the document to an S5 slide presentation and launch a preview program."
+ (interactive)
+ (let* ((tmp-filename "/tmp/slides.html")
+ (command (format "rst2s5.py %s %s && %s %s"
+ buffer-file-name tmp-filename
+ rst-slides-program tmp-filename)))
+ (start-process-shell-command "rst-slides-preview" nil command)
+ ;; Note: you could also use (compile command) to view the compilation
+ ;; output.
+ ))
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; Generic text functions that are more convenient than the defaults.
+;;
+
+(defun rst-replace-lines (fromchar tochar)
+ "Replace flush-left lines, consisting of multiple FROMCHAR characters,
+with equal-length lines of TOCHAR."
+ (interactive "\
+cSearch for flush-left lines of char:
+cand replace with char: ")
+ (save-excursion
+ (let* ((fromstr (string fromchar))
+ (searchre (concat "^" (regexp-quote fromstr) "+ *$"))
+ (found 0))
+ (condition-case err
+ (while t
+ (search-forward-regexp searchre)
+ (setq found (1+ found))
+ (search-backward fromstr) ;; point will be *before* last char
+ (setq p (1+ (point)))
+ (beginning-of-line)
+ (setq l (- p (point)))
+ (rst-delete-entire-line)
+ (insert-char tochar l))
+ (search-failed
+ (message (format "%d lines replaced." found)))))))
+
+(defun rst-join-paragraph ()
+ "Join lines in current paragraph into one line, removing end-of-lines."
+ (interactive)
+ (let ((fill-column 65000)) ; some big number
+ (call-interactively 'fill-paragraph)))
+
+(defun rst-force-fill-paragraph ()
+ "Fill paragraph at point, first joining the paragraph's lines into one.
+This is useful for filling list item paragraphs."
+ (interactive)
+ (rst-join-paragraph)
+ (fill-paragraph nil))
+
+
+;; Generic character repeater function.
+;; For sections, better to use the specialized function above, but this can
+;; be useful for creating separators.
+(defun rst-repeat-last-character (&optional tofill)
+ "Fills the current line up to the length of the preceding line (if not
+empty), using the last character on the current line. If the preceding line is
+empty, we use the fill-column.
+
+If a prefix argument is provided, use the next line rather than the preceding
+line.
+
+If the current line is longer than the desired length, shave the characters off
+the current line to fit the desired length.
+
+As an added convenience, if the command is repeated immediately, the alternative
+column is used (fill-column vs. end of previous/next line)."
+ (interactive)
+ (let* ((curcol (current-column))
+ (curline (+ (count-lines (point-min) (point))
+ (if (eq curcol 0) 1 0)))
+ (lbp (line-beginning-position 0))
+ (prevcol (if (and (= curline 1) (not current-prefix-arg))
+ fill-column
+ (save-excursion
+ (forward-line (if current-prefix-arg 1 -1))
+ (end-of-line)
+ (skip-chars-backward " \t" lbp)
+ (let ((cc (current-column)))
+ (if (= cc 0) fill-column cc)))))
+ (rightmost-column
+ (cond (tofill fill-column)
+ ((equal last-command 'rst-repeat-last-character)
+ (if (= curcol fill-column) prevcol fill-column))
+ (t (save-excursion
+ (if (= prevcol 0) fill-column prevcol)))
+ )) )
+ (end-of-line)
+ (if (> (current-column) rightmost-column)
+ ;; shave characters off the end
+ (delete-region (- (point)
+ (- (current-column) rightmost-column))
+ (point))
+ ;; fill with last characters
+ (insert-char (preceding-char)
+ (- rightmost-column (current-column))))
+ ))
+
+
+(defun rst-portable-mark-active-p ()
+ "A portable function that returns non-nil if the mark is active."
+ (cond
+ ((fboundp 'region-active-p) (region-active-p))
+ ((boundp 'transient-mark-mode) transient-mark-mode mark-active)))
+
+
+
+(provide 'rst)
+;;; rst.el ends here
diff --git a/examples/grammars/Makefile b/examples/grammars/Makefile
new file mode 100644
index 0000000..b19e127
--- /dev/null
+++ b/examples/grammars/Makefile
@@ -0,0 +1,38 @@
+# NLTK: Documentation Makefile
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+DATADIR = ../../../nltk_data
+PUBLISH = $(DATADIR)/packages/grammars
+
+PACKAGE_DIRS = book_grammars sample_grammars #basque_grammars spanish_grammars
+PACKAGES := $(addsuffix .zip, $(PACKAGE_DIRS))
+
+ZIP = zip
+
+define remove
+ $(if $(wildcard $1), rm $1,)
+endef
+
+all: publish
+
+ci:
+ git ci -m "updated grammar files"
+
+zip: clean $(PACKAGES)
+
+
+clean:
+ $(call remove, *.zip)
+
+%.zip: %
+ $(ZIP) -r $< $<
+# git add *zip
+
+publish: zip
+ cp $(PACKAGES) $(PUBLISH)
+ $(MAKE) -C $(DATADIR) grammars
+ $(MAKE) -C $(DATADIR) pkg_index
diff --git a/examples/grammars/basque_grammars/basque1.cfg b/examples/grammars/basque_grammars/basque1.cfg
new file mode 100755
index 0000000..39d74cb
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque1.cfg
@@ -0,0 +1,11 @@
+ P -> IS AS
+ AS -> IS ADI
+ AS -> ADI
+ IS -> IM erl_atz
+ IM -> ize_arr
+ IM -> ize_izb
+ ADI -> adt
+ erl_atz -> "k" | "a"
+ ize_arr -> "ardo" | "egunkari" | "baloi"
+ ize_izb -> "Pintxo" | "Kepa"
+ adt -> "dakar" | "darama"
diff --git a/examples/grammars/basque_grammars/basque1.fcfg b/examples/grammars/basque_grammars/basque1.fcfg
new file mode 100755
index 0000000..61faea5
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque1.fcfg
@@ -0,0 +1,22 @@
+% start AS
+# ############################
+# Grammar Rules
+# ############################
+# AS expansion rules
+AS[ergnum=?n1, absnum=?n2] -> IS[kas=erg, num=?n1] AS[ergnum=?n1, absnum=?n2]
+AS[ergnum=?n1, absnum=?n2] -> AS[ergnum=?n1, absnum=?n2] IS[kas=erg, num=?n1]
+AS[ergnum=?n1, absnum=?n2] -> IS[kas=abs, num=?n2] AS[ergnum=?n1, absnum=?n2]
+AS[ergnum=?n1, absnum=?n2] -> AS[ergnum=?n1, absnum=?n2] IS[kas=abs, num=?n2]
+IS[kas=?k, num=?n] -> ize[azp=arr] knmdek[kas=?k, num=?n]
+AS[ergnum=?n1, absnum=?n2] -> adt[ergnum=?n1, absnum=?n2]
+# ############################
+# Lexicon
+# ############################
+adt[ergnum=hu, absnum=hu] -> 'dakar' | 'darama'
+adt[ergnum=hk, absnum=hu] -> 'dakarte' | 'daramate'
+knmdek[kas=erg, num=hu] -> 'ak'
+knmdek[kas=erg, num=hk] -> 'ek'
+knmdek[kas=abs, num=hk] -> 'ak'
+knmdek[kas=abs, num=hu] -> 'a'
+ize[azp=arr] -> 'zakur' | 'gizon'
+
diff --git a/examples/grammars/basque_grammars/basque1.pcfg b/examples/grammars/basque_grammars/basque1.pcfg
new file mode 100755
index 0000000..162aa85
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque1.pcfg
@@ -0,0 +1,21 @@
+ as -> mendekoa as [0.15]
+ as -> adlg mendekoa as [0.31]
+ as -> adlg adlg mendekoa as [0.08]
+ as -> adi adl [0.46]
+ mendekoa -> adlg mendekoa [0.37]
+ mendekoa -> adlg adlg mendekoa [0.09]
+ mendekoa -> 'joatea' [0.18]
+ mendekoa -> 'joateko' [0.27]
+ mendekoa -> 'sartzera' [0.09]
+ adi -> 'esan' [0.5]
+ adi -> 'debekatzen' [0.33]
+ adi -> 'eraman' [0.17]
+ adl -> 'zuen' [0.17]
+ adl -> 'zioten' [0.83]
+ adlg -> 'bozgorailuarekin' [0.28]
+ adlg -> 'euskal_presoekin' [0.18]
+ adlg -> 'epaitegian' [0.09]
+ adlg -> 'mendira' [0.18]
+ adlg -> 'ejertzitoan' [0.09]
+ adlg -> 'derrigorrean' [0.09]
+ adlg -> 'lagunekin' [0.09]
diff --git a/examples/grammars/basque_grammars/basque1.regexp b/examples/grammars/basque_grammars/basque1.regexp
new file mode 100755
index 0000000..2dcb91e
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque1.regexp
@@ -0,0 +1 @@
+NP: {<IZE.*>+<ADJ.*>*<DET.*>*} """ # adjetibo edo determinatzaileei loturiko izenak nahiz izen segidak topatzen ditu
diff --git a/examples/grammars/basque_grammars/basque2.cfg b/examples/grammars/basque_grammars/basque2.cfg
new file mode 100755
index 0000000..929c70a
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque2.cfg
@@ -0,0 +1,10 @@
+ S -> is as
+ is -> ize adj | ior
+ ize -> 'gaizkile' | 'epaile' | 'bizilagun'
+ adj -> 'gaiztoek' | 'gaiztoak' | 'kanpotarrak' | 'kanpotarrek' | 'berriak' | 'berriek'
+ ior -> 'haiek' | 'hark'
+ as -> mendekoa as | adlg mendekoa as | adlg adlg mendekoa as | adi adl
+ mendekoa -> adlg mendekoa | adlg adlg mendekoa | 'joatea' | 'joateko' | 'sartzera'
+ adi -> 'esan' | 'debekatzen' | 'eraman'
+ adl -> 'zuen' |'zioten'
+ adlg -> 'bozgorailuarekin' | 'euskal_presoekin' | 'epaitegian' | 'mendira' | 'ejertzitoan' | 'derrigorrean' | 'lagunekin'
diff --git a/examples/grammars/basque_grammars/basque2.fcfg b/examples/grammars/basque_grammars/basque2.fcfg
new file mode 100755
index 0000000..5865a84
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque2.fcfg
@@ -0,0 +1,26 @@
+% start S
+# ############################
+# Grammar Rules
+# ############################
+S -> IS[kas=erg] AS/IS
+# IS erregelak
+IS[kas=?k, num=?n] -> ize[azp=arr] knmdek[kas=?k, num=?n]
+IS[kas=?k, num=?n] -> ize[azp=ber] knmdek[kas=?k, num=?n]
+IS[kas=?k, num=?n]/IS ->
+# AS erregelak
+AS[ergnum=?n1, absnum=?n2]/?x -> IS[kas=abs, num=?n1]/?x AS[ergnum=?n1, absnum=?n2]
+AS[ergnum=?n1, absnum=?n2] -> adi adl[ergnum=?n1, absnum=?n2]
+# ############################
+# Lexicon
+# ############################
+knmdek[kas=erg, num=hu] -> 'ak'
+knmdek[kas=erg, num=hk] -> 'ek'
+knmdek[kas=abs, num=hk] -> 'ak'
+knmdek[kas=abs, num=hu] -> 'a'
+ize[azp=arr] -> 'bizilagun' | 'aita' | 'gizon' | 'emakume'
+ize[azp=ber] -> 'Kepa' | 'Ainara'
+adi -> 'ekarri' | 'eraman' | 'puskatu' | 'lapurtu'
+adl[ergnum=hu, absnum=hu] -> 'du' | 'zuen'
+adl[ergnum=hk, absnum=hu] -> 'dute' | 'zuten'
+adl[ergnum=hu, absnum=hk] -> 'ditu' | 'zituen'
+adl[ergnum=hk, absnum=hk] -> 'dituzte' | 'zituzten'
diff --git a/examples/grammars/basque_grammars/basque2.pcfg b/examples/grammars/basque_grammars/basque2.pcfg
new file mode 100755
index 0000000..b022778
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque2.pcfg
@@ -0,0 +1,4 @@
+ IS -> IZE_ARR [0.5] | IZE_ARR ADJ [0.3] | IS LOT IS [0.2]
+ IZE_ARR -> 'gizon' [0.1] | 'emakume' [0.2] | 'ume' [0.3] | IZE_ARR LOT IZE_ARR [0.4]
+ ADJ -> 'zaharrak' [0.4] | 'gazteak' [0.6]
+ LOT -> 'eta' [0.9] | 'edo' [0.1]
diff --git a/examples/grammars/basque_grammars/basque2.regexp b/examples/grammars/basque_grammars/basque2.regexp
new file mode 100755
index 0000000..4f18460
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque2.regexp
@@ -0,0 +1,2 @@
+ NP: {<IZE.*><ADJ.*>*<DET.*>*} # adjetibo edo determinatzaileei loturiko izenak topatzen ditu
+ NP: {<IZE.*>+} # izen segidak topatzen ditu
diff --git a/examples/grammars/basque_grammars/basque3.cfg b/examples/grammars/basque_grammars/basque3.cfg
new file mode 100755
index 0000000..5e56868
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque3.cfg
@@ -0,0 +1,4 @@
+ IS -> IZE_ARR | IZE_ARR ADJ | IS LOT IS
+ IZE_ARR -> 'gizon' | 'emakume' | 'ume' | IZE_ARR LOT IZE_ARR
+ ADJ -> 'zaharrak' | 'gazteak'
+ LOT -> 'eta' | 'edo'
diff --git a/examples/grammars/basque_grammars/basque3.fcfg b/examples/grammars/basque_grammars/basque3.fcfg
new file mode 100755
index 0000000..8d69c9f
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque3.fcfg
@@ -0,0 +1,36 @@
+% start S
+# ############################
+# Grammar Rules
+# ############################
+
+## NORK-NOR Kasuak
+
+S -> IS[kas=erg] AS/IS
+# IS erregelak
+IS[kas=?k, num=?n] -> ize[azp=arr] knmdek[kas=?k, num=?n]
+IS[kas=?k, num=?n] -> ize[azp=ber] knmdek[kas=?k, num=?n]
+
+IS[kas=?k, num=?n]/IS ->
+
+# AS erregelak
+AS[ergnum=?n1, absnum=?n2]/?x -> IS[kas=abs, num=?n1]/?x AS[ergnum=?n1, absnum=?n2]
+AS[ergnum=?n1, absnum=?n2] -> adi adl[ergnum=?n1, absnum=?n2]
+# ############################
+# Lexicon
+# ############################
+
+knmdek[kas=erg, num=hu] -> 'ak'
+knmdek[kas=erg, num=hk] -> 'ek'
+
+knmdek[kas=abs, num=hk] -> 'ak'
+knmdek[kas=abs, num=hu] -> 'a'
+
+ize[azp=arr] -> 'bizilagun' | 'aita' | 'gizon' | 'emakume'
+ize[azp=ber] -> 'Kepa' | 'Ainara'
+
+adi -> 'ekarri' | 'eraman' | 'puskatu' | 'lapurtu'
+
+adl[ergnum=hu, absnum=hu] -> 'du' | 'zuen'
+adl[ergnum=hk, absnum=hu] -> 'dute' | 'zuten'
+adl[ergnum=hu, absnum=hk] -> 'ditu' | 'zituen'
+adl[ergnum=hk, absnum=hk] -> 'dituzte' | 'zituzten'
diff --git a/examples/grammars/basque_grammars/basque3.regexp b/examples/grammars/basque_grammars/basque3.regexp
new file mode 100755
index 0000000..3ea283a
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque3.regexp
@@ -0,0 +1,3 @@
+IS:
+ {<.*>+} # Edozer Onartzen Duen Chunkerra
+ }<ADI.*|ADL.*|ADT.*|PUNT.*|POST.*|LOT.*|ADB.*>+{ # Chink Bezala Barneratu Aditzak (ADI.*, ADT.* eta ADL.*), Adberbioak (ADB.*), Preposizioak (POST.*), Loturak (LOT.*) Eta Puntuazio Ikurrak (PUNT.*)
diff --git a/examples/grammars/basque_grammars/basque4.regexp b/examples/grammars/basque_grammars/basque4.regexp
new file mode 100755
index 0000000..6b5f8a8
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque4.regexp
@@ -0,0 +1,3 @@
+ IS: {(<ADJ.*>*<DET.*>*<IZE.*>+<ADJ.*>*<DET.*>*)*} #noun phrase chunks
+ AS: {(<ADI.*|ADL.*|ADT.*>*)+} # verb phrase chunks
+ PS: {<POST.*>+} # prepositional phrase chunks
diff --git a/examples/grammars/basque_grammars/basque5.regexp b/examples/grammars/basque_grammars/basque5.regexp
new file mode 100755
index 0000000..996abcd
--- /dev/null
+++ b/examples/grammars/basque_grammars/basque5.regexp
@@ -0,0 +1,5 @@
+ IS: {(<ADJ.*>*<DET.*>*<IZE.*>*<ADJ.*>*<DET.*>*)*} #noun phrase chunks
+ AS: {(<ADI.*|ADL.*|ADT.*>+)+<POST.*>*} # verb phrase chunks
+ PS: {<POST.*>+} # prepositional phrase chunks
+ S: {<IS><AS>}
+ {<AS><IS>} # Chunk NP, VP
diff --git a/examples/grammars/book_grammars/background.fol b/examples/grammars/book_grammars/background.fol
new file mode 100644
index 0000000..f9d32f0
--- /dev/null
+++ b/examples/grammars/book_grammars/background.fol
@@ -0,0 +1,22 @@
+## Natural Language Toolkit: background1.fol
+##
+## Illustration of simple knowledge base for use with inference tools.
+## To accompany sem4.fcfg
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+all x. (boxerdog(x) -> dog(x))
+all x. (boxer(x) -> person(x))
+
+all x. (-(dog(x) & person(x)))
+
+all x. (married(x) <-> exists y. marry(x,y))
+all x. (bark(x) -> dog(x))
+
+all x. all y. (marry(x,y) -> (person(x) & person(y)))
+
+(-(Vincent = Mia))
+(-(Vincent = Fido))
+(-(Mia = Fido))
diff --git a/examples/grammars/book_grammars/discourse.fcfg b/examples/grammars/book_grammars/discourse.fcfg
new file mode 100644
index 0000000..903ff90
--- /dev/null
+++ b/examples/grammars/book_grammars/discourse.fcfg
@@ -0,0 +1,125 @@
+## Natural Language Toolkit: discourse.fcfg
+##
+## Grammar to illustrate simple 2-3 sentence discourse processing.
+##
+## Developed as an extension of sem3.fcfg
+## Main additions:
+## - a few more lexical entries (including 'no' and 'the')
+## - 'is', 'does' and auxiliary negation
+## - Predicate categories, including predicate nominals and adjectives
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+############################
+# Grammar Rules
+#############################
+
+S[SEM = <app(?subj,?vp)>] -> NP[NUM=?n,SEM=?subj] VP[NUM=?n,SEM=?vp]
+
+NP[NUM=?n,SEM=<app(?det,?nom)> ] -> Det[NUM=?n,SEM=?det] Nom[NUM=?n,SEM=?nom]
+NP[LOC=?l,NUM=?n,SEM=?np] -> PropN[LOC=?l,NUM=?n,SEM=?np]
+
+NP[-LOC,NUM=sg,SEM=<\Q. (- exists x. (person(x) & Q(x)))>] -> 'nobody' | 'Nobody'
+NP[-LOC,NUM=sg,SEM=<\Q. exists x. (person(x) & Q(x))>] -> 'somebody' | 'Somebody'
+
+## Copular predicates
+Pred[SEM=?prd] -> PredN[SEM=?prd] | PP[+LOC,+PRED,SEM=?prd] | Adj[SEM=?prd]
+
+## Predicative NPs
+## Doesn't bLOCk 'is every dog', but determiner SEMantics is ignored
+PredN[NUM=?n, SEM=?nom] -> Det[NUM=?n] Nom[NUM=?n, SEM=?nom]
+
+Nom[NUM=?n,SEM=?nom] -> N[NUM=?n,SEM=?nom]
+Nom[NUM=?n,SEM=<app(?pp,?nom)>] -> N[NUM=?n,SEM=?nom] PP[SEM=?pp]
+
+## Transitive verbs
+VP[NUM=?n,SEM=<app(?v,?obj)>] -> TV[NUM=?n,SEM=?v] NP[SEM=?obj]
+
+## Copular VPs
+VP[NUM=?n,SEM=<app(?v,?prd)>] -> AuxP[+COP,NUM=?n,SEM=?v] Pred[SEM=?prd]
+
+## Do auxiliaries
+VP[+neg,NUM=?n,SEM=<app(?v,?vp)>] -> AuxP[-COP,NUM=?n,SEM=?v] VP[NUM=pl,SEM=?vp]
+
+AuxP[COP=?c,NUM=?n,SEM=<app(?neg,?aux)>] -> Aux[COP=?c,NUM=?n,SEM=?aux] Neg[SEM=?neg]
+AuxP[COP=?c,NUM=?n,SEM=?aux] -> Aux[COP=?c,NUM=?n,SEM=?aux]
+
+## Intransitive verbs
+VP[NUM=?n,SEM=?v] -> IV[NUM=?n,SEM=?v]
+
+## VP-level PPs
+VP[NUM=?n,SEM=<app(?pp,?vp)>] -> VP[NUM=?n,SEM=?vp] PP[-PRED,SEM=?pp]
+
+PP[LOC=?l,PRED=?prd,SEM=<app(?p,?np)>] -> P[LOC=?l,PRED=?prd,SEM=?p] NP[LOC=?l,SEM=?np]
+
+#############################
+# Lexical Rules
+#############################
+
+PropN[-LOC,NUM=sg,SEM=<\P.P(John)>] -> 'John'
+PropN[-LOC,NUM=sg,SEM=<\P.P(Mary)>] -> 'Mary'
+PropN[-LOC,NUM=sg,SEM=<\P.P(Suzie)>] -> 'Suzie'
+PropN[-LOC,NUM=sg,SEM=<\P.P(Vincent)>] -> 'Vincent'
+PropN[-LOC,NUM=sg,SEM=<\P.P(Mia)>] -> 'Mia'
+PropN[-LOC,NUM=sg,SEM=<\P.P(Marsellus)>] -> 'Marsellus'
+PropN[-LOC,NUM=sg,SEM=<\P.P(Fido)>] -> 'Fido'
+PropN[+LOC, NUM=sg,SEM=<\P.P(Noosa)>] -> 'Noosa'
+
+NP[-LOC, NUM=sg, SEM=<\P.\x.P(x)>] -> 'who' | 'Who'
+
+Det[NUM=sg,SEM=<\P Q.all x.(P(x) -> Q(x))>] -> 'every' | 'Every'
+Det[NUM=pl,SEM=<\P Q.all x.(P(x) -> Q(x))>] -> 'all' | 'All'
+Det[SEM=<\P Q.exists x.(P(x) & Q(x))>] -> 'some' | 'Some'
+Det[NUM=sg,SEM=<\P Q.exists x.(P(x) & Q(x))>] -> 'a' | 'A'
+Det[NUM=sg,SEM=<\P Q.(- exists x.(P(x) & Q(x)))>] -> 'no' | 'No'
+Det[NUM=sg,SEM=<\P Q.exists x.((P(x) & Q(x)) & all y.(P(y) -> (x = y)))>] -> 'the' | 'The'
+
+N[NUM=sg,SEM=<\x.boy(x)>] -> 'boy'
+N[NUM=pl,SEM=<\x.boy(x)>] -> 'boys'
+N[NUM=sg,SEM=<\x.girl(x)>] -> 'girl'
+N[NUM=pl,SEM=<\x.girl(x)>] -> 'girls'
+N[NUM=sg,SEM=<\x.dog(x)>] -> 'dog'
+N[NUM=pl,SEM=<\x.dog(x)>] -> 'dogs'
+N[NUM=sg,SEM=<\x.student(x)>] -> 'student'
+N[NUM=pl,SEM=<\x.student(x)>] -> 'students'
+N[NUM=sg,SEM=<\x.person(x)>] -> 'person'
+N[NUM=pl,SEM=<\x.person(x)>] -> 'persons'
+N[NUM=sg,SEM=<\x.boxerdog(x)>] -> 'boxer'
+N[NUM=pl,SEM=<\x.boxerdog(x)>] -> 'boxers'
+N[NUM=sg,SEM=<\x.boxer(x)>] -> 'boxer'
+N[NUM=pl,SEM=<\x.boxer(x)>] -> 'boxers'
+N[NUM=sg,SEM=<\x.garden(x)>] -> 'garden'
+N[NUM=sg,SEM=<\x.kitchen(x)>] -> 'kitchen'
+
+Adj[SEM=<\x.happy(x)>] -> 'happy'
+Adj[SEM=<\x.drunk(x)>] -> 'drunk'
+Adj[SEM=<\x.married(x)>] -> 'married'
+
+TV[NUM=sg,SEM=<\X y.X(\x.chase(y,x))>,tns=pres] -> 'chases'
+TV[NUM=pl,SEM=<\X y.X(\x.chase(y,x))>,tns=pres] -> 'chase'
+TV[NUM=sg,SEM=<\X y.X(\x.marry(y,x))>,tns=pres] -> 'marries'
+TV[NUM=pl,SEM=<\X y.X(\x.marry(y,x))>,tns=pres] -> 'marry'
+TV[NUM=sg,SEM=<\X y.X(\x.know(y,x))>,tns=pres] -> 'knows'
+TV[NUM=pl,SEM=<\X y.X(\x.know(y,x))>,tns=pres] -> 'know'
+TV[NUM=sg,SEM=<\X y.X(\x.see(y,x))>,tns=pres] -> 'sees'
+TV[NUM=pl,SEM=<\X y.X(\x.see(y,x))>,tns=pres] -> 'see'
+IV[NUM=sg,SEM=<\x.bark(x)>,tns=pres] -> 'barks'
+IV[NUM=pl,SEM=<\x.bark(x)>,tns=pres] -> 'bark'
+IV[NUM=sg,SEM=<\x.walk(x)>,tns=pres] -> 'walks'
+IV[NUM=pl,SEM=<\x.walk(x)>,tns=pres] -> 'walk'
+IV[NUM=pl,SEM=<\x.dance(x)>,tns=pres] -> 'dance'
+IV[NUM=sg,SEM=<\x.dance(x)>,tns=pres] -> 'dances'
+
+Aux[+COP,NUM=sg,SEM=<\P x.P(x)>,tns=pres] -> 'is'
+Aux[+COP,NUM=pl,SEM=<\P x.P(x)>,tns=pres] -> 'are'
+Aux[-COP,NUM=sg,SEM=<\P x.P(x)>,tns=pres] -> 'does'
+Aux[-COP,NUM=pl,SEM=<\P x.P(x)>,tns=pres] -> 'do'
+
+P[+LOC,-PRED,SEM=<\X P x.X(\y.(P(x) & in(x,y)))>] -> 'in'
+P[+LOC,+PRED,SEM=<\X x.X(\y.in(x,y))>] -> 'in'
+P[-LOC,SEM=<\X P x.X(\y.(P(x) & with(x,y)))>] -> 'with'
+
+Neg[SEM=<\T P.T(\x.(- P(x)))>] -> 'not'
diff --git a/examples/grammars/book_grammars/drt.fcfg b/examples/grammars/book_grammars/drt.fcfg
new file mode 100644
index 0000000..1bda654
--- /dev/null
+++ b/examples/grammars/book_grammars/drt.fcfg
@@ -0,0 +1,82 @@
+## Natural Language Toolkit: drt.fcfg
+##
+## Author: Dan Garrette <dhgarrette at gmail.com>
+## URL: <http://nltk.orgt>
+## For license information, see LICENSE.TXT
+
+% start S
+############################
+# Grammar Rules
+#############################
+
+S[SEM = <app(?subj,?vp)>] -> NP[NUM=?n,SEM=?subj] VP[NUM=?n,SEM=?vp]
+
+NP[NUM=?n,SEM=<app(?det,?nom)> ] -> Det[NUM=?n,SEM=?det] Nom[NUM=?n,SEM=?nom]
+NP[LOC=?l,NUM=?n,SEM=?np] -> PropN[LOC=?l,NUM=?n,SEM=?np]
+
+Nom[NUM=?n,SEM=?nom] -> N[NUM=?n,SEM=?nom]
+Nom[NUM=?n,SEM=<app(?pp,?nom)>] -> N[NUM=?n,SEM=?nom] PP[SEM=?pp]
+
+VP[NUM=?n,SEM=?v] -> IV[NUM=?n,SEM=?v]
+VP[NUM=?n,SEM=<app(?v,?obj)>] -> TV[NUM=?n,SEM=?v] NP[SEM=?obj]
+
+#############################
+# Lexical Rules
+#############################
+
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[Angus(x)])+P(x))>] -> 'Angus'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[Irene(x)])+P(x))>] -> 'Irene'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[John(x)])+P(x))>] -> 'John'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[Mary(x)])+P(x))>] -> 'Mary'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[Suzie(x)])+P(x))>] -> 'Suzie'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[Vincent(x)])+P(x))>] -> 'Vincent'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[Mia(x)])+P(x))>] -> 'Mia'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[Marsellus(x)])+P(x))>] -> 'Marsellus'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[Fido(x)])+P(x))>] -> 'Fido'
+PropN[+LOC,NUM=sg,SEM=<\P.(DRS([x],[Noosa(x)])+P(x))>] -> 'Noosa'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[PRO(x)])+P(x))>] -> 'he'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[PRO(x)])+P(x))>] -> 'she'
+PropN[-LOC,NUM=sg,SEM=<\P.(DRS([x],[PRO(x)])+P(x))>] -> 'it'
+
+Det[NUM=sg,SEM=<\P Q.DRS([],[((DRS([x],[])+P(x)) implies Q(x))])>] -> 'every' | 'Every'
+Det[NUM=pl,SEM=<\P Q.DRS([],[((DRS([x],[])+P(x)) implies Q(x))])>] -> 'all' | 'All'
+Det[SEM=<\P Q.((DRS([x],[])+P(x))+Q(x))>] -> 'some' | 'Some'
+Det[NUM=sg,SEM=<\P Q.((DRS([x],[])+P(x))+Q(x))>] -> 'a' | 'A'
+Det[NUM=sg,SEM=<\P Q.(not ((DRS([x],[])+P(x))+Q(x)))>] -> 'no' | 'No'
+
+N[NUM=sg,SEM=<\x.DRS([],[boy(x)])>] -> 'boy'
+N[NUM=pl,SEM=<\x.DRS([],[boy(x)])>] -> 'boys'
+N[NUM=sg,SEM=<\x.DRS([],[girl(x)])>] -> 'girl'
+N[NUM=pl,SEM=<\x.DRS([],[girl(x)])>] -> 'girls'
+N[NUM=sg,SEM=<\x.DRS([],[dog(x)])>] -> 'dog'
+N[NUM=pl,SEM=<\x.DRS([],[dog(x)])>] -> 'dogs'
+N[NUM=sg,SEM=<\x.DRS([],[student(x)])>] -> 'student'
+N[NUM=pl,SEM=<\x.DRS([],[student(x)])>] -> 'students'
+N[NUM=sg,SEM=<\x.DRS([],[person(x)])>] -> 'person'
+N[NUM=pl,SEM=<\x.DRS([],[person(x)])>] -> 'persons'
+N[NUM=sg,SEM=<\x.DRS([],[boxerdog(x)])>] -> 'boxer'
+N[NUM=pl,SEM=<\x.DRS([],[boxerdog(x)])>] -> 'boxers'
+N[NUM=sg,SEM=<\x.DRS([],[boxer(x)])>] -> 'boxer'
+N[NUM=pl,SEM=<\x.DRS([],[boxer(x)])>] -> 'boxers'
+N[NUM=sg,SEM=<\x.DRS([],[garden(x)])>] -> 'garden'
+N[NUM=sg,SEM=<\x.DRS([],[kitchen(x)])>] -> 'kitchen'
+
+IV[NUM=sg,SEM=<\x.DRS([],[bark(x)])>,tns=pres] -> 'barks'
+IV[NUM=pl,SEM=<\x.DRS([],[bark(x)])>,tns=pres] -> 'bark'
+IV[NUM=sg,SEM=<\x.DRS([],[walk(x)])>,tns=pres] -> 'walks'
+IV[NUM=pl,SEM=<\x.DRS([],[walk(x)])>,tns=pres] -> 'walk'
+IV[NUM=pl,SEM=<\x.DRS([],[dance(x)])>,tns=pres] -> 'dance'
+IV[NUM=sg,SEM=<\x.DRS([],[dance(x)])>,tns=pres] -> 'dances'
+
+TV[NUM=sg,SEM=<\X x.X(\y.DRS([],[own(x,y)]))>,tns=pres] -> 'owns'
+TV[NUM=pl,SEM=<\X x.X(\y.DRS([],[own(x,y)]))>,tns=pres] -> 'own'
+TV[NUM=sg,SEM=<\X x.X(\y.DRS([],[bite(x,y)]))>,tns=pres] -> 'bites'
+TV[NUM=pl,SEM=<\X x.X(\y.DRS([],[bite(x,y)]))>,tns=pres] -> 'bite'
+TV[NUM=sg,SEM=<\X x.X(\y.DRS([],[chase(x,y)]))>,tns=pres] -> 'chases'
+TV[NUM=pl,SEM=<\X x.X(\y.DRS([],[chase(x,y)]))>,tns=pres] -> 'chase'
+TV[NUM=sg,SEM=<\X x.X(\y.DRS([],[marry(x,y)]))>,tns=pres] -> 'marries'
+TV[NUM=pl,SEM=<\X x.X(\y.DRS([],[marry(x,y)]))>,tns=pres] -> 'marry'
+TV[NUM=sg,SEM=<\X x.X(\y.DRS([],[know(x,y)]))>,tns=pres] -> 'knows'
+TV[NUM=pl,SEM=<\X x.X(\y.DRS([],[know(x,y)]))>,tns=pres] -> 'know'
+TV[NUM=sg,SEM=<\X x.X(\y.DRS([],[see(x,y)]))>,tns=pres] -> 'sees'
+TV[NUM=pl,SEM=<\X x.X(\y.DRS([],[see(x,y)]))>,tns=pres] -> 'see'
diff --git a/examples/grammars/book_grammars/feat0.fcfg b/examples/grammars/book_grammars/feat0.fcfg
new file mode 100644
index 0000000..2fc8c01
--- /dev/null
+++ b/examples/grammars/book_grammars/feat0.fcfg
@@ -0,0 +1,49 @@
+## Natural Language Toolkit: feat0.fcfg
+##
+## First example of a feature-based grammar for English, illustrating
+## value-sharing of NUM and TENSE features.
+## Used in Feature-Based Grammars chapter.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+# ###################
+# Grammar Productions
+# ###################
+
+# S expansion productions
+S -> NP[NUM=?n] VP[NUM=?n]
+
+# NP expansion productions
+NP[NUM=?n] -> N[NUM=?n]
+NP[NUM=?n] -> PropN[NUM=?n]
+NP[NUM=?n] -> Det[NUM=?n] N[NUM=?n]
+NP[NUM=pl] -> N[NUM=pl]
+
+# VP expansion productions
+VP[TENSE=?t, NUM=?n] -> IV[TENSE=?t, NUM=?n]
+VP[TENSE=?t, NUM=?n] -> TV[TENSE=?t, NUM=?n] NP
+
+# ###################
+# Lexical Productions
+# ###################
+
+Det[NUM=sg] -> 'this' | 'every'
+Det[NUM=pl] -> 'these' | 'all'
+Det -> 'the' | 'some' | 'several'
+
+PropN[NUM=sg]-> 'Kim' | 'Jody'
+
+N[NUM=sg] -> 'dog' | 'girl' | 'car' | 'child'
+N[NUM=pl] -> 'dogs' | 'girls' | 'cars' | 'children'
+
+IV[TENSE=pres, NUM=sg] -> 'disappears' | 'walks'
+TV[TENSE=pres, NUM=sg] -> 'sees' | 'likes'
+
+IV[TENSE=pres, NUM=pl] -> 'disappear' | 'walk'
+TV[TENSE=pres, NUM=pl] -> 'see' | 'like'
+
+IV[TENSE=past] -> 'disappeared' | 'walked'
+TV[TENSE=past] -> 'saw' | 'liked'
diff --git a/examples/grammars/book_grammars/feat1.fcfg b/examples/grammars/book_grammars/feat1.fcfg
new file mode 100644
index 0000000..b1e2784
--- /dev/null
+++ b/examples/grammars/book_grammars/feat1.fcfg
@@ -0,0 +1,55 @@
+## Natural Language Toolkit: feat1.fcfg
+##
+## Second example of a feature-based grammar, illustrating
+## SUBCAT and slash features. Also introduces SBar and embedded
+## clauses.
+## Used in Feature-Based Grammars chapter.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+# ###################
+# Grammar Productions
+# ###################
+
+S[-INV] -> NP VP
+S[-INV]/?x -> NP VP/?x
+
+S[-INV] -> NP S/NP
+S[-INV] -> Adv[+NEG] S[+INV]
+
+S[+INV] -> V[+AUX] NP VP
+S[+INV]/?x -> V[+AUX] NP VP/?x
+
+SBar -> Comp S[-INV]
+SBar/?x -> Comp S[-INV]/?x
+
+VP -> V[SUBCAT=intrans, -AUX]
+
+VP -> V[SUBCAT=trans, -AUX] NP
+VP/?x -> V[SUBCAT=trans, -AUX] NP/?x
+
+VP -> V[SUBCAT=clause, -AUX] SBar
+VP/?x -> V[SUBCAT=clause, -AUX] SBar/?x
+
+VP -> V[+AUX] VP
+VP/?x -> V[+AUX] VP/?x
+
+# ###################
+# Lexical Productions
+# ###################
+V[SUBCAT=intrans, -AUX] -> 'walk' | 'sing'
+V[SUBCAT=trans, -AUX] -> 'see' | 'like'
+V[SUBCAT=clause, -AUX] -> 'say' | 'claim'
+V[+AUX] -> 'do' | 'can'
+
+NP[-WH] -> 'you' | 'cats'
+NP[+WH] -> 'who'
+
+Adv[+NEG] -> 'rarely' | 'never'
+
+NP/NP ->
+
+Comp -> 'that'
diff --git a/examples/grammars/book_grammars/german.fcfg b/examples/grammars/book_grammars/german.fcfg
new file mode 100644
index 0000000..0ef79df
--- /dev/null
+++ b/examples/grammars/book_grammars/german.fcfg
@@ -0,0 +1,86 @@
+## Natural Language Toolkit: german.fcfg
+##
+## Example of a feature-based grammar for German, illustrating
+## CASE and AGR features (PER, GND, NUM) working as a bundle.
+## Used in Feature-Based Grammars chapter.
+##
+## Author: Michaela Atterer <atterer at ims.uni-stuttgart.de>
+## Ewan Klein <ewan at inf.ed.ac.uk>
+##
+## Plural transitive verbs productions by Jordan Boyd-Graber (ezubaric at users.sourceforge.net)
+
+% start S
+#####################
+# Grammar Productions
+#####################
+S -> NP[CASE=nom, AGR=?a] VP[AGR=?a]
+
+NP[CASE=?c, AGR=?a] -> PRO[CASE=?c, AGR=?a]
+NP[CASE=?c, AGR=?a] -> Det[CASE=?c, AGR=?a] N[CASE=?c, AGR=?a]
+
+VP[AGR=?a] -> IV[AGR=?a]
+VP[AGR=?a] -> TV[OBJCASE=?c, AGR=?a] NP[CASE=?c]
+
+#####################
+# Lexical Productions
+#####################
+# Singular determiners
+
+# masc
+Det[CASE=nom, AGR=[GND=masc,PER=3,NUM=sg]] -> 'der'
+Det[CASE=dat, AGR=[GND=masc,PER=3,NUM=sg]] -> 'dem'
+Det[CASE=acc, AGR=[GND=masc,PER=3,NUM=sg]] -> 'den'
+
+# fem
+Det[CASE=nom, AGR=[GND=fem,PER=3,NUM=sg]] -> 'die'
+Det[CASE=dat, AGR=[GND=fem,PER=3,NUM=sg]] -> 'der'
+Det[CASE=acc, AGR=[GND=fem,PER=3,NUM=sg]] -> 'die'
+
+# Plural determiners
+Det[CASE=nom, AGR=[PER=3,NUM=pl]] -> 'die'
+Det[CASE=dat, AGR=[PER=3,NUM=pl]] -> 'den'
+Det[CASE=acc, AGR=[PER=3,NUM=pl]] -> 'die'
+
+# Nouns
+N[AGR=[GND=masc,PER=3,NUM=sg]] -> 'Hund'
+N[CASE=nom, AGR=[GND=masc,PER=3,NUM=pl]] -> 'Hunde'
+N[CASE=dat, AGR=[GND=masc,PER=3,NUM=pl]] -> 'Hunden'
+N[CASE=acc, AGR=[GND=masc,PER=3,NUM=pl]] -> 'Hunde'
+
+N[AGR=[GND=fem,PER=3,NUM=sg]] -> 'Katze'
+N[AGR=[GND=fem,PER=3,NUM=pl]] -> 'Katzen'
+
+# Pronouns
+PRO[CASE=nom, AGR=[PER=1,NUM=sg]] -> 'ich'
+PRO[CASE=acc, AGR=[PER=1,NUM=sg]] -> 'mich'
+PRO[CASE=dat, AGR=[PER=1,NUM=sg]] -> 'mir'
+PRO[CASE=nom, AGR=[PER=2,NUM=sg]] -> 'du'
+PRO[CASE=nom, AGR=[PER=3,NUM=sg]] -> 'er' | 'sie' | 'es'
+PRO[CASE=nom, AGR=[PER=1,NUM=pl]] -> 'wir'
+PRO[CASE=acc, AGR=[PER=1,NUM=pl]] -> 'uns'
+PRO[CASE=dat, AGR=[PER=1,NUM=pl]] -> 'uns'
+PRO[CASE=nom, AGR=[PER=2,NUM=pl]] -> 'ihr'
+PRO[CASE=nom, AGR=[PER=3,NUM=pl]] -> 'sie'
+
+# Verbs
+IV[AGR=[NUM=sg,PER=1]] -> 'komme'
+IV[AGR=[NUM=sg,PER=2]] -> 'kommst'
+IV[AGR=[NUM=sg,PER=3]] -> 'kommt'
+IV[AGR=[NUM=pl, PER=1]] -> 'kommen'
+IV[AGR=[NUM=pl, PER=2]] -> 'kommt'
+IV[AGR=[NUM=pl, PER=3]] -> 'kommen'
+
+TV[OBJCASE=acc, AGR=[NUM=sg,PER=1]] -> 'sehe' | 'mag'
+TV[OBJCASE=acc, AGR=[NUM=sg,PER=2]] -> 'siehst' | 'magst'
+TV[OBJCASE=acc, AGR=[NUM=sg,PER=3]] -> 'sieht' | 'mag'
+TV[OBJCASE=dat, AGR=[NUM=sg,PER=1]] -> 'folge' | 'helfe'
+TV[OBJCASE=dat, AGR=[NUM=sg,PER=2]] -> 'folgst' | 'hilfst'
+TV[OBJCASE=dat, AGR=[NUM=sg,PER=3]] -> 'folgt' | 'hilft'
+TV[OBJCASE=acc, AGR=[NUM=pl,PER=1]] -> 'sehen' | 'moegen'
+TV[OBJCASE=acc, AGR=[NUM=pl,PER=2]] -> 'sieht' | 'moegt'
+TV[OBJCASE=acc, AGR=[NUM=pl,PER=3]] -> 'sehen' | 'moegen'
+TV[OBJCASE=dat, AGR=[NUM=pl,PER=1]] -> 'folgen' | 'helfen'
+TV[OBJCASE=dat, AGR=[NUM=pl,PER=2]] -> 'folgt' | 'helft'
+TV[OBJCASE=dat, AGR=[NUM=pl,PER=3]] -> 'folgen' | 'helfen'
+
+
diff --git a/examples/grammars/book_grammars/simple-sem.fcfg b/examples/grammars/book_grammars/simple-sem.fcfg
new file mode 100644
index 0000000..473fed2
--- /dev/null
+++ b/examples/grammars/book_grammars/simple-sem.fcfg
@@ -0,0 +1,65 @@
+## Natural Language Toolkit: sem3.fcfg
+##
+## Alternative simple grammar with transitive verbs and
+## quantifiers for the book.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+
+% start S
+############################
+# Grammar Rules
+#############################
+
+S[SEM = <?subj(?vp)>] -> NP[NUM=?n,SEM=?subj] VP[NUM=?n,SEM=?vp]
+
+NP[NUM=?n,SEM=<?det(?nom)> ] -> Det[NUM=?n,SEM=?det] Nom[NUM=?n,SEM=?nom]
+NP[LOC=?l,NUM=?n,SEM=?np] -> PropN[LOC=?l,NUM=?n,SEM=?np]
+
+Nom[NUM=?n,SEM=?nom] -> N[NUM=?n,SEM=?nom]
+
+VP[NUM=?n,SEM=?v] -> IV[NUM=?n,SEM=?v]
+VP[NUM=?n,SEM=<?v(?obj)>] -> TV[NUM=?n,SEM=?v] NP[SEM=?obj]
+VP[NUM=?n,SEM=<?v(?obj,?pp)>] -> DTV[NUM=?n,SEM=?v] NP[SEM=?obj] PP[+TO,SEM=?pp]
+
+PP[+TO, SEM=?np] -> P[+TO] NP[SEM=?np]
+
+#############################
+# Lexical Rules
+#############################
+
+PropN[-LOC,NUM=sg,SEM=<\P.P(angus)>] -> 'Angus'
+PropN[-LOC,NUM=sg,SEM=<\P.P(cyril)>] -> 'Cyril'
+PropN[-LOC,NUM=sg,SEM=<\P.P(irene)>] -> 'Irene'
+
+Det[NUM=sg,SEM=<\P Q.all x.(P(x) -> Q(x))>] -> 'every'
+Det[NUM=pl,SEM=<\P Q.all x.(P(x) -> Q(x))>] -> 'all'
+Det[SEM=<\P Q.exists x.(P(x) & Q(x))>] -> 'some'
+Det[NUM=sg,SEM=<\P Q.exists x.(P(x) & Q(x))>] -> 'a'
+Det[NUM=sg,SEM=<\P Q.exists x.(P(x) & Q(x))>] -> 'an'
+
+N[NUM=sg,SEM=<\x.man(x)>] -> 'man'
+N[NUM=sg,SEM=<\x.girl(x)>] -> 'girl'
+N[NUM=sg,SEM=<\x.boy(x)>] -> 'boy'
+N[NUM=sg,SEM=<\x.bone(x)>] -> 'bone'
+N[NUM=sg,SEM=<\x.ankle(x)>] -> 'ankle'
+N[NUM=sg,SEM=<\x.dog(x)>] -> 'dog'
+N[NUM=pl,SEM=<\x.dog(x)>] -> 'dogs'
+
+IV[NUM=sg,SEM=<\x.bark(x)>,TNS=pres] -> 'barks'
+IV[NUM=pl,SEM=<\x.bark(x)>,TNS=pres] -> 'bark'
+IV[NUM=sg,SEM=<\x.walk(x)>,TNS=pres] -> 'walks'
+IV[NUM=pl,SEM=<\x.walk(x)>,TNS=pres] -> 'walk'
+TV[NUM=sg,SEM=<\X x.X(\y.chase(x,y))>,TNS=pres] -> 'chases'
+TV[NUM=pl,SEM=<\X x.X(\y.chase(x,y))>,TNS=pres] -> 'chase'
+TV[NUM=sg,SEM=<\X x.X(\y.see(x,y))>,TNS=pres] -> 'sees'
+TV[NUM=pl,SEM=<\X x.X(\y.see(x,y))>,TNS=pres] -> 'see'
+TV[NUM=sg,SEM=<\X x.X(\y.bite(x,y))>,TNS=pres] -> 'bites'
+TV[NUM=pl,SEM=<\X x.X(\y.bite(x,y))>,TNS=pres] -> 'bite'
+DTV[NUM=sg,SEM=<\Y X x.X(\z.Y(\y.give(x,y,z)))>,TNS=pres] -> 'gives'
+DTV[NUM=pl,SEM=<\Y X x.X(\z.Y(\y.give(x,y,z)))>,TNS=pres] -> 'give'
+
+P[+to] -> 'to'
+
diff --git a/examples/grammars/book_grammars/sql0.fcfg b/examples/grammars/book_grammars/sql0.fcfg
new file mode 100644
index 0000000..888e8a6
--- /dev/null
+++ b/examples/grammars/book_grammars/sql0.fcfg
@@ -0,0 +1,32 @@
+## Natural Language Toolkit: sql.fcfg
+##
+## Deliberately naive string-based grammar for
+## deriving SQL queries from English
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[SEM=(?np + WHERE + ?vp)] -> NP[SEM=?np] VP[SEM=?vp]
+
+VP[SEM=(?v + ?pp)] -> IV[SEM=?v] PP[SEM=?pp]
+VP[SEM=(?v + ?ap)] -> IV[SEM=?v] AP[SEM=?ap]
+NP[SEM=(?det + ?n)] -> Det[SEM=?det] N[SEM=?n]
+PP[SEM=(?p + ?np)] -> P[SEM=?p] NP[SEM=?np]
+AP[SEM=?pp] -> A[SEM=?a] PP[SEM=?pp]
+
+NP[SEM='Country="greece"'] -> 'Greece'
+NP[SEM='Country="china"'] -> 'China'
+
+Det[SEM='SELECT'] -> 'Which' | 'What'
+
+N[SEM='City FROM city_table'] -> 'cities'
+
+IV[SEM=''] -> 'are'
+A[SEM=''] -> 'located'
+P[SEM=''] -> 'in'
+
+
+
diff --git a/examples/grammars/book_grammars/sql1.fcfg b/examples/grammars/book_grammars/sql1.fcfg
new file mode 100644
index 0000000..8638e13
--- /dev/null
+++ b/examples/grammars/book_grammars/sql1.fcfg
@@ -0,0 +1,44 @@
+## Natural Language Toolkit: sql.fcfg
+##
+## Deliberately naive string-based grammar for
+## deriving SQL queries from English
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[SEM=(?np + WHERE + ?vp)] -> NP[SEM=?np] VP[SEM=?vp]
+
+VP[SEM=(?v + ?pp)] -> IV[SEM=?v] PP[SEM=?pp]
+VP[SEM=(?v + ?ap)] -> IV[SEM=?v] AP[SEM=?ap]
+VP[SEM=(?v + ?np)] -> TV[SEM=?v] NP[SEM=?np]
+VP[SEM=(?vp1 + ?c + ?vp2)] -> VP[SEM=?vp1] Conj[SEM=?c] VP[SEM=?vp2]
+
+NP[SEM=(?det + ?n)] -> Det[SEM=?det] N[SEM=?n]
+NP[SEM=(?n + ?pp)] -> N[SEM=?n] PP[SEM=?pp]
+NP[SEM=?n] -> N[SEM=?n] | CardN[SEM=?n]
+
+## NB Numbers in the Chat-80 database represent thousands.
+CardN[SEM='1000'] -> '1,000,000'
+
+PP[SEM=(?p + ?np)] -> P[SEM=?p] NP[SEM=?np]
+AP[SEM=?pp] -> A[SEM=?a] PP[SEM=?pp]
+
+NP[SEM='Country="greece"'] -> 'Greece'
+NP[SEM='Country="china"'] -> 'China'
+
+Det[SEM='SELECT'] -> 'Which' | 'What'
+Conj[SEM='AND'] -> 'and'
+
+N[SEM='City FROM city_table'] -> 'cities'
+N[SEM='Population'] -> 'populations'
+
+IV[SEM=''] -> 'are'
+TV[SEM=''] -> 'have'
+A -> 'located'
+P[SEM=''] -> 'in'
+P[SEM='>'] -> 'above'
+
+
diff --git a/examples/grammars/book_grammars/storage.fcfg b/examples/grammars/book_grammars/storage.fcfg
new file mode 100644
index 0000000..fb0c17a
--- /dev/null
+++ b/examples/grammars/book_grammars/storage.fcfg
@@ -0,0 +1,54 @@
+## Natural Language Toolkit: storage.fcfg
+##
+## Feature-based grammar that implements Cooper storage by dividing the
+## semantics for each phrase into two pieces: the core semantics
+## ('SEM','CORE') and a sequence of binding operators ('SEM','STORE').
+## Each binding operator is encoded as a logic term <bo(quant, var)>,
+## where <quant> is a quantifier expression and the individual variable
+## <@var> specifies the 'address' of the quantifier in the core
+## semantics. and <predicate> is a predicate describing that variable.
+
+## In order for this grammar to generate the correct results, all
+## variables of the form <@var> must be instantiated (i.e., replaced
+## by unique new variables) whenever they are used. This can be
+## accomplished by using the InstantiateVarsChart class when parsing.
+##
+## Author: Edward Loper <edloper at gmail.com>,
+## Ewan Klein <ewan at inf.ed.ac.uk>
+## Robin Cooper <robin.cooper at ling.gu.se>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+%start S
+
+S[SEM=[CORE=<?vp(?subj)>, STORE=(?b1+?b2)]] -> NP[SEM=[CORE=?subj, STORE=?b1]] VP[SEM=[CORE=?vp, STORE=?b2]]
+
+VP[SEM=?s] -> IV[SEM=?s]
+VP[SEM=[CORE=<?v(?obj)>, STORE=(?b1+?b2)]] -> TV[SEM=[CORE=?v, STORE=?b1]] NP[SEM=[CORE=?obj, STORE=?b2]]
+VP[SEM=[CORE=<?v(?pp)(?obj)>, STORE=(?b1+?b2+?b3)]] -> DTV[SEM=[CORE=?v, STORE=?b1]] NP[SEM=[CORE=?obj, STORE=?b2]] PP[+TO, SEM=[CORE=?pp, STORE=?b3]]
+
+NP[SEM=[CORE=<@x>, STORE=((<bo(?det(?n), at x)>)+?b1+?b2)]] -> Det[SEM=[CORE=?det, STORE=?b1]] N[SEM=[CORE=?n, STORE=?b2]]
+
+PP[+TO, SEM=[CORE=?np, STORE=?b1]] -> P NP[SEM=[CORE=?np, STORE=?b1]]
+
+# Lexical items:
+Det[SEM=[CORE=<\Q P.exists x.(Q(x) & P(x))>, STORE=(/)]] -> 'a'
+Det[SEM=[CORE=<\Q P.all x.(Q(x) implies P(x))>, STORE=(/)]] -> 'every'
+
+N[SEM=[CORE=<dog>, STORE=(/)]] -> 'dog'
+N[SEM=[CORE=<bone>, STORE=(/)]] -> 'bone'
+N[SEM=[CORE=<girl>, STORE=(/)]] -> 'girl'
+N[SEM=[CORE=<man>, STORE=(/)]] -> 'man'
+
+IV[SEM=[CORE=<\x.smile(x)>, STORE=(/)]] -> 'smiles'
+IV[SEM=[CORE=<\x.walk(x)>, STORE=(/)]] -> 'walks'
+
+TV[SEM=[CORE=<\y x.feed(x,y)>, STORE=(/)]] -> 'feeds'
+TV[SEM=[CORE=<\y x.chase(x,y)>, STORE=(/)]] -> 'chases'
+
+DTV[SEM=[CORE=<\z y x.give(x,y,z)>, STORE=(/)]] -> 'gives'
+
+NP[SEM=[CORE=<@x>, STORE=(<bo(\P.P(angus), at x)>)]] -> 'Angus'
+NP[SEM=[CORE=<@x>, STORE=(<bo(\P.P(cyril), at x)>)]] -> 'Cyril'
+
+P[+TO] -> 'to'
diff --git a/examples/grammars/sample_grammars/background0.fol b/examples/grammars/sample_grammars/background0.fol
new file mode 100644
index 0000000..d6fe8ef
--- /dev/null
+++ b/examples/grammars/sample_grammars/background0.fol
@@ -0,0 +1,19 @@
+## Natural Language Toolkit: background0.fol
+##
+## Illustration of simple knowledge base for use with inference tools.
+## To accompany sem4.fcfg
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+all x. (boxerdog(x) -> dog(x))
+all x. (boxer(x) -> person(x))
+
+all x. (-(dog(x) & person(x)))
+
+some x. boxer(x)
+some x. boxerdog(x)
+
+
+
diff --git a/examples/grammars/sample_grammars/bindop.fcfg b/examples/grammars/sample_grammars/bindop.fcfg
new file mode 100644
index 0000000..c1243b1
--- /dev/null
+++ b/examples/grammars/sample_grammars/bindop.fcfg
@@ -0,0 +1,42 @@
+## Natural Language Toolkit: sem0.fcfg
+##
+## Feature-based grammar that divides the semantics for each element
+## into two pieces: the core semantics, with path ('SEM','CORE'), and a set of
+## binding operators, with path ('SEM','BO'). Each binding operator is encoded
+## as a lambda-calculus expression <bo(expr, @var)>, specifying
+## that <@var> is an individual variable that should be instantiated,
+## and <expr> is an expression that can bind that variable.
+##
+## In order for this grammar to generate the correct results, all
+## variables of the form <@var> must be instantiated (i.e., replaced
+## by unique new variables) whenever they are used. This can be
+## accomplished by using the InstantiateVarsChart class when parsing.
+##
+## Author: Edward Loper <edloper at gmail.com>,
+## Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+%start S
+## Grammar summary:
+## S -> NP VP
+## VP -> TV NP | IV
+## NP -> Det N | proper nouns...
+## TV -> transitive verbs...
+## IV -> intransitive verbs...
+## Det -> determiners...
+
+S[SEM=[CORE=<?vp(?subj)>, BO={?b1+?b2}]] -> NP[SEM=[CORE=?subj, BO=?b1]] VP[SEM=[CORE=?vp, BO=?b2]]
+
+VP[SEM=[CORE=<?v(?obj)>, BO={?b1+?b2}]] -> TV[SEM=[CORE=?v, BO=?b1]] NP[SEM=[CORE=?obj, BO=?b2]]
+
+VP[SEM=?s] -> IV[SEM=?s]
+
+NP[SEM=[CORE=<@x>, BO={{<bo(?det(?n), @x)>}+?b1+?b2}]] -> Det[SEM=[CORE=?det, BO=?b1]] N[SEM=[CORE=?n, BO=?b2]]
+
+# Lexical items:
+Det[SEM=[CORE=<\Q P.exists x.(Q(x) & P(x))>, BO={/}]] -> 'a'
+N[SEM=[CORE=<dog>, BO={/}]] -> 'dog' | 'cat' | 'mouse'
+IV[SEM=[CORE=<\x.bark(x)>, BO={/}]] -> 'barks' | 'eats' | 'walks'
+TV[SEM=[CORE=<\x y.feed(y,x)>, BO={/}]] -> 'feeds' | 'walks'
+NP[SEM=[CORE=<@x>, BO={<bo(\P. P(John), @x)>}]] -> 'john' | 'alex'
diff --git a/examples/grammars/sample_grammars/chat80.fcfg b/examples/grammars/sample_grammars/chat80.fcfg
new file mode 100644
index 0000000..e023295
--- /dev/null
+++ b/examples/grammars/sample_grammars/chat80.fcfg
@@ -0,0 +1,95 @@
+## Natural Language Toolkit: chat80.fcfg
+##
+##
+## Grammar used to illustrate querying the Chat-80 database.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+# ###########################
+# Grammar Rules
+# ############################
+
+S[SEM=<app(?subj,?vp)>] -> NP[-PRED,NUM=?n,SEM=?subj] VP[NUM=?n,SEM=?vp]
+
+Rel[NUM=?n,SEM=<app(?comp,?vp)>] -> Comp[SEM=?comp] VP[NUM=?n,SEM=?vp]
+
+NP[-PRED, NUM=pl,SEM=<(\P Q. exists x. (Q(x) and P(x)) ?nom)>] -> Nom[NUM=pl,SEM=?nom]
+NP[WH=?wh,-PRED,NUM=?n,SEM=<app(?det,?nom)>] -> Det[WH=?wh, NUM=?n,SEM=?det] Nom[NUM=?n,SEM=?nom]
+
+
+NP[+PRED,NUM=sg,SEM=?nom] -> Det[NUM=sg,SEM=?det] Nom[NUM=sg,SEM=?nom]
+NP[+PRED,NUM=pl,SEM=?nom] -> Nom[NUM=pl,SEM=?nom]
+
+NP[LOC=?l,NUM=?n,SEM=?np] -> PropN[LOC=?l,NUM=?n,SEM=?np]
+
+Nom[NUM=?n,SEM=?nom] -> N[NUM=?n,SEM=?nom]
+Nom[NUM=sg,SEM=<app(?pp,?nom)>] -> N[subcat=11,NUM=sg,SEM=?nom] PP[pform=of,SEM=?pp]
+Nom[NUM=?n,SEM=<app(?mod,?nom)>] -> Nom[NUM=?n,SEM=?nom] Rel[NUM=?n,SEM=?mod]
+Nom[NUM=?n,SEM=<app(?adj,?nom)>] -> A[SEM=?adj] Nom[NUM=?n,SEM=?nom]
+
+##VP[NUM=?n,SEM=?v] -> V[SUBCAT=1,NUM=?n,SEM=?v]
+VP[NUM=?n,SEM=<app(?v,?obj)>] -> V[SUBCAT=2, NUM=?n,SEM=?v] NP[-PRED,SEM=?obj]
+VP[NUM=?n,SEM=<app(?v,?PRED)>] -> V[SUBCAT=3, NUM=?n,SEM=?v] NP[+PRED,SEM=?PRED]
+
+PP[PFORM=?pf,SEM=<app(?p,?np)>] -> P[PFORM=?pf, LOC=?l,SEM=?p] NP[LOC=?l,SEM=?np]
+
+
+# ############################
+# Lexical Rules
+# ############################
+
+% include chat_pnames.cfg
+
+Comp[SEM=<\P Q x.(P(x) and Q(x))>] -> 'that'
+
+NP[+WH, NUM=sg, SEM=<\P.\x.P(x)>] -> 'what'
+
+Det[-WH,NUM=sg,SEM=<\P Q. all x. (P(x) -> Q(x))>] -> 'every'
+Det[-WH,NUM=pl,SEM=<\P Q. all x. (P(x) -> Q(x))>] -> 'all'
+Det[-WH,SEM=<\P Q. exists x. (P(x) & Q(x))>] -> 'some'
+Det[-WH,NUM=sg,SEM=<\P Q. exists x. (P(x) & Q(x))>] -> 'a'
+Det[-WH,NUM=sg,SEM=<\P Q. exists x. (P(x) & Q(x))>] -> 'the'
+Det[+WH,SEM=<\P Q x. (Q(x) & P(x))>] -> 'which'
+
+N[SUBCAT=10,NUM=sg,SEM=<\x.city(x)>] -> 'city'
+N[SUBCAT=10,NUM=pl,SEM=<\x.city(x)>] -> 'cities'
+N[SUBCAT=10,NUM=sg,SEM=<\x.continent(x)>] -> 'continent'
+N[SUBCAT=10,NUM=pl,SEM=<\x.continent(x)>] -> 'continents'
+N[SUBCAT=10,NUM=sg,SEM=<\x.country(x)>] -> 'country'
+N[SUBCAT=10,NUM=pl,SEM=<\x.country(x)>] -> 'countries'
+N[SUBCAT=10,NUM=sg,SEM=<\x.sea(x)>] -> 'sea'
+N[SUBCAT=10,NUM=pl,SEM=<\x.sea(x)>] -> 'seas'
+N[SUBCAT=10,NUM=sg,SEM=<\x.ocean(x)>] -> 'ocean'
+N[SUBCAT=10,NUM=pl,SEM=<\x.ocean(x)>] -> 'oceans'
+
+PL[SEM=<\P Q. exists x. (P(x) & Q(x))>] -> ' '
+
+N[SUBCAT=11,NUM=sg,SEM=<\x y.area_of(x,y))>] -> 'area'
+N[SUBCAT=11,NUM=sg,SEM=<\x y.capital_of(x,y))>] -> 'capital'
+N[SUBCAT=11,NUM=sg,SEM=<\x y.currency_of(x,y))>] -> 'currency'
+N[SUBCAT=11,NUM=sg,SEM=<\x y.region_of(x,y))>] -> 'region'
+N[SUBCAT=11,NUM=sg,SEM=<\x y.longitude_of(x,y))>] -> 'longitude'
+N[SUBCAT=11,NUM=sg,SEM=<\x y.latitude_of(x,y))>] -> 'latitude'
+N[SUBCAT=11,NUM=sg,SEM=<\x y.population_of(x,y))>] -> 'population'
+
+
+## V[SUBCAT=3,NUM=sg,SEM=<\X y.(X \x.(x = y))>,tns=pres] -> 'is'
+## V[SUBCAT=3,NUM=pl,SEM=<\P.P))>,tns=pres] -> 'are'
+V[SUBCAT=3,NUM=sg,SEM=<\P.P>,tns=pres] -> 'is'
+V[SUBCAT=3,NUM=pl,SEM=<\P.P>,tns=pres] -> 'are'
+V[SUBCAT=2,NUM=sg,SEM=<\X y.(X \x.border(y,x))>,tns=pres] -> 'borders'
+V[SUBCAT=2,NUM=pl,SEM=<\X y.(X \x.border(y,x))>,tns=pres] -> 'border'
+V[SUBCAT=2,NUM=sg,SEM=<\X y.(X \x.contain(y,x))>,tns=pres] -> 'contains'
+V[SUBCAT=2,NUM=pl,SEM=<\X y.(X \x.contain(y,x))>,tns=pres] -> 'contain'
+
+A[SEM=<\P x.(contain(asia,x) & P(x))>] -> 'Asian'
+
+P[PFORM=of,SEM=<\X.X>] -> 'of'
+P[+LOC,SEM=<\X P x.(X \y.(P(x) & in(x,y)))>] -> 'in'
+P[-LOC,SEM=<\X P x.(X \y.(P(x) & with(x,y)))>] -> 'with'
+
+
+
diff --git a/examples/grammars/sample_grammars/chat_pnames.fcfg b/examples/grammars/sample_grammars/chat_pnames.fcfg
new file mode 100644
index 0000000..c32c768
--- /dev/null
+++ b/examples/grammars/sample_grammars/chat_pnames.fcfg
@@ -0,0 +1,545 @@
+
+##################################################################
+# Lexical rules automatically generated by running 'chat80.py -x'.
+##################################################################
+
+PropN[num=sg, sem=<\P.P(abidjan)>] -> 'Abidjan'
+PropN[num=sg, sem=<\P.P(abu_dhabi)>] -> 'Abu_Dhabi'
+PropN[num=sg, sem=<\P.P(accra)>] -> 'Accra'
+PropN[num=sg, sem=<\P.P(addis_ababa)>] -> 'Addis_Ababa'
+PropN[num=sg, sem=<\P.P(aden)>] -> 'Aden'
+PropN[num=sg, sem=<\P.P(afghani)>] -> 'Afghani'
+PropN[num=sg, sem=<\P.P(afghanistan)>] -> 'Afghanistan'
+PropN[num=sg, sem=<\P.P(africa)>] -> 'Africa'
+PropN[num=sg, sem=<\P.P(albania)>] -> 'Albania'
+PropN[num=sg, sem=<\P.P(algeria)>] -> 'Algeria'
+PropN[num=sg, sem=<\P.P(algiers)>] -> 'Algiers'
+PropN[num=sg, sem=<\P.P(amazon)>] -> 'Amazon'
+PropN[num=sg, sem=<\P.P(america)>] -> 'America'
+PropN[num=sg, sem=<\P.P(amman)>] -> 'Amman'
+PropN[num=sg, sem=<\P.P(amsterdam)>] -> 'Amsterdam'
+PropN[num=sg, sem=<\P.P(amu_darya)>] -> 'Amu_Darya'
+PropN[num=sg, sem=<\P.P(amur)>] -> 'Amur'
+PropN[num=sg, sem=<\P.P(andorra)>] -> 'Andorra'
+PropN[num=sg, sem=<\P.P(andorra_la_villa)>] -> 'Andorra_La_Villa'
+PropN[num=sg, sem=<\P.P(angola)>] -> 'Angola'
+PropN[num=sg, sem=<\P.P(ankara)>] -> 'Ankara'
+PropN[num=sg, sem=<\P.P(antarctic_circle)>] -> 'Antarctic_Circle'
+PropN[num=sg, sem=<\P.P(antarctica)>] -> 'Antarctica'
+PropN[num=sg, sem=<\P.P(apia)>] -> 'Apia'
+PropN[num=sg, sem=<\P.P(arctic_circle)>] -> 'Arctic_Circle'
+PropN[num=sg, sem=<\P.P(arctic_ocean)>] -> 'Arctic_Ocean'
+PropN[num=sg, sem=<\P.P(argentina)>] -> 'Argentina'
+PropN[num=sg, sem=<\P.P(ariary)>] -> 'Ariary'
+PropN[num=sg, sem=<\P.P(asia)>] -> 'Asia'
+PropN[num=sg, sem=<\P.P(asuncion)>] -> 'Asuncion'
+PropN[num=sg, sem=<\P.P(athens)>] -> 'Athens'
+PropN[num=sg, sem=<\P.P(atlantic)>] -> 'Atlantic'
+PropN[num=sg, sem=<\P.P(australasia)>] -> 'Australasia'
+PropN[num=sg, sem=<\P.P(australia)>] -> 'Australia'
+PropN[num=sg, sem=<\P.P(australian_dollar)>] -> 'Australian_Dollar'
+PropN[num=sg, sem=<\P.P(austria)>] -> 'Austria'
+PropN[num=sg, sem=<\P.P(baghdad)>] -> 'Baghdad'
+PropN[num=sg, sem=<\P.P(bahamas)>] -> 'Bahamas'
+PropN[num=sg, sem=<\P.P(bahamian_dollar)>] -> 'Bahamian_Dollar'
+PropN[num=sg, sem=<\P.P(bahrain)>] -> 'Bahrain'
+PropN[num=sg, sem=<\P.P(baht)>] -> 'Baht'
+PropN[num=sg, sem=<\P.P(balboa)>] -> 'Balboa'
+PropN[num=sg, sem=<\P.P(baltic)>] -> 'Baltic'
+PropN[num=sg, sem=<\P.P(bamako)>] -> 'Bamako'
+PropN[num=sg, sem=<\P.P(bangkok)>] -> 'Bangkok'
+PropN[num=sg, sem=<\P.P(bangladesh)>] -> 'Bangladesh'
+PropN[num=sg, sem=<\P.P(bangui)>] -> 'Bangui'
+PropN[num=sg, sem=<\P.P(banjul)>] -> 'Banjul'
+PropN[num=sg, sem=<\P.P(barbados)>] -> 'Barbados'
+PropN[num=sg, sem=<\P.P(barcelona)>] -> 'Barcelona'
+PropN[num=sg, sem=<\P.P(beirut)>] -> 'Beirut'
+PropN[num=sg, sem=<\P.P(belgium)>] -> 'Belgium'
+PropN[num=sg, sem=<\P.P(belgrade)>] -> 'Belgrade'
+PropN[num=sg, sem=<\P.P(belize)>] -> 'Belize'
+PropN[num=sg, sem=<\P.P(belize_town)>] -> 'Belize_Town'
+PropN[num=sg, sem=<\P.P(berlin)>] -> 'Berlin'
+PropN[num=sg, sem=<\P.P(bern)>] -> 'Bern'
+PropN[num=sg, sem=<\P.P(bhutan)>] -> 'Bhutan'
+PropN[num=sg, sem=<\P.P(birmingham)>] -> 'Birmingham'
+PropN[num=sg, sem=<\P.P(bissau)>] -> 'Bissau'
+PropN[num=sg, sem=<\P.P(black_sea)>] -> 'Black_Sea'
+PropN[num=sg, sem=<\P.P(bogota)>] -> 'Bogota'
+PropN[num=sg, sem=<\P.P(bolivar)>] -> 'Bolivar'
+PropN[num=sg, sem=<\P.P(bolivia)>] -> 'Bolivia'
+PropN[num=sg, sem=<\P.P(bombay)>] -> 'Bombay'
+PropN[num=sg, sem=<\P.P(bonn)>] -> 'Bonn'
+PropN[num=sg, sem=<\P.P(botswana)>] -> 'Botswana'
+PropN[num=sg, sem=<\P.P(brahmaputra)>] -> 'Brahmaputra'
+PropN[num=sg, sem=<\P.P(brasilia)>] -> 'Brasilia'
+PropN[num=sg, sem=<\P.P(brazil)>] -> 'Brazil'
+PropN[num=sg, sem=<\P.P(brazzaville)>] -> 'Brazzaville'
+PropN[num=sg, sem=<\P.P(bridgetown)>] -> 'Bridgetown'
+PropN[num=sg, sem=<\P.P(brussels)>] -> 'Brussels'
+PropN[num=sg, sem=<\P.P(bucharest)>] -> 'Bucharest'
+PropN[num=sg, sem=<\P.P(budapest)>] -> 'Budapest'
+PropN[num=sg, sem=<\P.P(buenos_aires)>] -> 'Buenos_Aires'
+PropN[num=sg, sem=<\P.P(bujumbura)>] -> 'Bujumbura'
+PropN[num=sg, sem=<\P.P(bulgaria)>] -> 'Bulgaria'
+PropN[num=sg, sem=<\P.P(burma)>] -> 'Burma'
+PropN[num=sg, sem=<\P.P(burundi)>] -> 'Burundi'
+PropN[num=sg, sem=<\P.P(cairo)>] -> 'Cairo'
+PropN[num=sg, sem=<\P.P(calcutta)>] -> 'Calcutta'
+PropN[num=sg, sem=<\P.P(cambodia)>] -> 'Cambodia'
+PropN[num=sg, sem=<\P.P(cameroon)>] -> 'Cameroon'
+PropN[num=sg, sem=<\P.P(canada)>] -> 'Canada'
+PropN[num=sg, sem=<\P.P(canadian_dollar)>] -> 'Canadian_Dollar'
+PropN[num=sg, sem=<\P.P(canberra)>] -> 'Canberra'
+PropN[num=sg, sem=<\P.P(canton)>] -> 'Canton'
+PropN[num=sg, sem=<\P.P(caracas)>] -> 'Caracas'
+PropN[num=sg, sem=<\P.P(caribbean)>] -> 'Caribbean'
+PropN[num=sg, sem=<\P.P(caspian)>] -> 'Caspian'
+PropN[num=sg, sem=<\P.P(cayenne)>] -> 'Cayenne'
+PropN[num=sg, sem=<\P.P(cedi)>] -> 'Cedi'
+PropN[num=sg, sem=<\P.P(central_africa)>] -> 'Central_Africa'
+PropN[num=sg, sem=<\P.P(central_african_republic)>] -> 'Central_African_Republic'
+PropN[num=sg, sem=<\P.P(central_america)>] -> 'Central_America'
+PropN[num=sg, sem=<\P.P(cfa_franc)>] -> 'Cfa_Franc'
+PropN[num=sg, sem=<\P.P(chad)>] -> 'Chad'
+PropN[num=sg, sem=<\P.P(chicago)>] -> 'Chicago'
+PropN[num=sg, sem=<\P.P(chile)>] -> 'Chile'
+PropN[num=sg, sem=<\P.P(china)>] -> 'China'
+PropN[num=sg, sem=<\P.P(chungking)>] -> 'Chungking'
+PropN[num=sg, sem=<\P.P(colombia)>] -> 'Colombia'
+PropN[num=sg, sem=<\P.P(colombo)>] -> 'Colombo'
+PropN[num=sg, sem=<\P.P(colon)>] -> 'Colon'
+PropN[num=sg, sem=<\P.P(colorado)>] -> 'Colorado'
+PropN[num=sg, sem=<\P.P(conakry)>] -> 'Conakry'
+PropN[num=sg, sem=<\P.P(congo)>] -> 'Congo'
+PropN[num=sg, sem=<\P.P(congo_river)>] -> 'Congo_River'
+PropN[num=sg, sem=<\P.P(copenhagen)>] -> 'Copenhagen'
+PropN[num=sg, sem=<\P.P(cordoba)>] -> 'Cordoba'
+PropN[num=sg, sem=<\P.P(costa_rica)>] -> 'Costa_Rica'
+PropN[num=sg, sem=<\P.P(cruzeiro)>] -> 'Cruzeiro'
+PropN[num=sg, sem=<\P.P(cuba)>] -> 'Cuba'
+PropN[num=sg, sem=<\P.P(cubango)>] -> 'Cubango'
+PropN[num=sg, sem=<\P.P(cyprus)>] -> 'Cyprus'
+PropN[num=sg, sem=<\P.P(czechoslovakia)>] -> 'Czechoslovakia'
+PropN[num=sg, sem=<\P.P(dacca)>] -> 'Dacca'
+PropN[num=sg, sem=<\P.P(dahomey)>] -> 'Dahomey'
+PropN[num=sg, sem=<\P.P(dairen)>] -> 'Dairen'
+PropN[num=sg, sem=<\P.P(dakar)>] -> 'Dakar'
+PropN[num=sg, sem=<\P.P(dalasi)>] -> 'Dalasi'
+PropN[num=sg, sem=<\P.P(damascus)>] -> 'Damascus'
+PropN[num=sg, sem=<\P.P(danube)>] -> 'Danube'
+PropN[num=sg, sem=<\P.P(dar_es_salaam)>] -> 'Dar_Es_Salaam'
+PropN[num=sg, sem=<\P.P(ddr_mark)>] -> 'Ddr_Mark'
+PropN[num=sg, sem=<\P.P(delhi)>] -> 'Delhi'
+PropN[num=sg, sem=<\P.P(denmark)>] -> 'Denmark'
+PropN[num=sg, sem=<\P.P(detroit)>] -> 'Detroit'
+PropN[num=sg, sem=<\P.P(deutsche_mark)>] -> 'Deutsche_Mark'
+PropN[num=sg, sem=<\P.P(dinar)>] -> 'Dinar'
+PropN[num=sg, sem=<\P.P(dirham)>] -> 'Dirham'
+PropN[num=sg, sem=<\P.P(djibouti)>] -> 'Djibouti'
+PropN[num=sg, sem=<\P.P(doha)>] -> 'Doha'
+PropN[num=sg, sem=<\P.P(dollar)>] -> 'Dollar'
+PropN[num=sg, sem=<\P.P(dominican_republic)>] -> 'Dominican_Republic'
+PropN[num=sg, sem=<\P.P(don)>] -> 'Don'
+PropN[num=sg, sem=<\P.P(dong)>] -> 'Dong'
+PropN[num=sg, sem=<\P.P(drachma)>] -> 'Drachma'
+PropN[num=sg, sem=<\P.P(dublin)>] -> 'Dublin'
+PropN[num=sg, sem=<\P.P(east_africa)>] -> 'East_Africa'
+PropN[num=sg, sem=<\P.P(east_berlin)>] -> 'East_Berlin'
+PropN[num=sg, sem=<\P.P(east_caribbean_dollar)>] -> 'East_Caribbean_Dollar'
+PropN[num=sg, sem=<\P.P(east_carribean_dollar)>] -> 'East_Carribean_Dollar'
+PropN[num=sg, sem=<\P.P(east_germany)>] -> 'East_Germany'
+PropN[num=sg, sem=<\P.P(eastern_europe)>] -> 'Eastern_Europe'
+PropN[num=sg, sem=<\P.P(ecuador)>] -> 'Ecuador'
+PropN[num=sg, sem=<\P.P(egypt)>] -> 'Egypt'
+PropN[num=sg, sem=<\P.P(egyptian_pound)>] -> 'Egyptian_Pound'
+PropN[num=sg, sem=<\P.P(eire)>] -> 'Eire'
+PropN[num=sg, sem=<\P.P(el_salvador)>] -> 'El_Salvador'
+PropN[num=sg, sem=<\P.P(elbe)>] -> 'Elbe'
+PropN[num=sg, sem=<\P.P(equator)>] -> 'Equator'
+PropN[num=sg, sem=<\P.P(equatorial_guinea)>] -> 'Equatorial_Guinea'
+PropN[num=sg, sem=<\P.P(escudo)>] -> 'Escudo'
+PropN[num=sg, sem=<\P.P(ethiopean_dollar)>] -> 'Ethiopean_Dollar'
+PropN[num=sg, sem=<\P.P(ethiopia)>] -> 'Ethiopia'
+PropN[num=sg, sem=<\P.P(euphrates)>] -> 'Euphrates'
+PropN[num=sg, sem=<\P.P(europe)>] -> 'Europe'
+PropN[num=sg, sem=<\P.P(far_east)>] -> 'Far_East'
+PropN[num=sg, sem=<\P.P(fiji)>] -> 'Fiji'
+PropN[num=sg, sem=<\P.P(fiji_dollar)>] -> 'Fiji_Dollar'
+PropN[num=sg, sem=<\P.P(finland)>] -> 'Finland'
+PropN[num=sg, sem=<\P.P(forint)>] -> 'Forint'
+PropN[num=sg, sem=<\P.P(franc)>] -> 'Franc'
+PropN[num=sg, sem=<\P.P(franc_peseta)>] -> 'Franc_Peseta'
+PropN[num=sg, sem=<\P.P(france)>] -> 'France'
+PropN[num=sg, sem=<\P.P(freetown)>] -> 'Freetown'
+PropN[num=sg, sem=<\P.P(french_franc)>] -> 'French_Franc'
+PropN[num=sg, sem=<\P.P(french_guiana)>] -> 'French_Guiana'
+PropN[num=sg, sem=<\P.P(gabon)>] -> 'Gabon'
+PropN[num=sg, sem=<\P.P(gaborone)>] -> 'Gaborone'
+PropN[num=sg, sem=<\P.P(gambia)>] -> 'Gambia'
+PropN[num=sg, sem=<\P.P(ganges)>] -> 'Ganges'
+PropN[num=sg, sem=<\P.P(georgetown)>] -> 'Georgetown'
+PropN[num=sg, sem=<\P.P(ghana)>] -> 'Ghana'
+PropN[num=sg, sem=<\P.P(glasgow)>] -> 'Glasgow'
+PropN[num=sg, sem=<\P.P(gourde)>] -> 'Gourde'
+PropN[num=sg, sem=<\P.P(greece)>] -> 'Greece'
+PropN[num=sg, sem=<\P.P(greenland)>] -> 'Greenland'
+PropN[num=sg, sem=<\P.P(grenada)>] -> 'Grenada'
+PropN[num=sg, sem=<\P.P(guarani)>] -> 'Guarani'
+PropN[num=sg, sem=<\P.P(guatamala_city)>] -> 'Guatamala_City'
+PropN[num=sg, sem=<\P.P(guatemala)>] -> 'Guatemala'
+PropN[num=sg, sem=<\P.P(guilder)>] -> 'Guilder'
+PropN[num=sg, sem=<\P.P(guinea)>] -> 'Guinea'
+PropN[num=sg, sem=<\P.P(guinea_bissau)>] -> 'Guinea_Bissau'
+PropN[num=sg, sem=<\P.P(guyana)>] -> 'Guyana'
+PropN[num=sg, sem=<\P.P(guyana_dollar)>] -> 'Guyana_Dollar'
+PropN[num=sg, sem=<\P.P(haiti)>] -> 'Haiti'
+PropN[num=sg, sem=<\P.P(hamburg)>] -> 'Hamburg'
+PropN[num=sg, sem=<\P.P(hanoi)>] -> 'Hanoi'
+PropN[num=sg, sem=<\P.P(harbin)>] -> 'Harbin'
+PropN[num=sg, sem=<\P.P(havana)>] -> 'Havana'
+PropN[num=sg, sem=<\P.P(helsinki)>] -> 'Helsinki'
+PropN[num=sg, sem=<\P.P(honduras)>] -> 'Honduras'
+PropN[num=sg, sem=<\P.P(hongkong)>] -> 'Hongkong'
+PropN[num=sg, sem=<\P.P(hongkong_city)>] -> 'Hongkong_City'
+PropN[num=sg, sem=<\P.P(hungary)>] -> 'Hungary'
+PropN[num=sg, sem=<\P.P(hwang_ho)>] -> 'Hwang_Ho'
+PropN[num=sg, sem=<\P.P(hyderabad)>] -> 'Hyderabad'
+PropN[num=sg, sem=<\P.P(iceland)>] -> 'Iceland'
+PropN[num=sg, sem=<\P.P(india)>] -> 'India'
+PropN[num=sg, sem=<\P.P(indian_ocean)>] -> 'Indian_Ocean'
+PropN[num=sg, sem=<\P.P(indian_rupee)>] -> 'Indian_Rupee'
+PropN[num=sg, sem=<\P.P(indian_subcontinent)>] -> 'Indian_Subcontinent'
+PropN[num=sg, sem=<\P.P(indonesia)>] -> 'Indonesia'
+PropN[num=sg, sem=<\P.P(indus)>] -> 'Indus'
+PropN[num=sg, sem=<\P.P(iran)>] -> 'Iran'
+PropN[num=sg, sem=<\P.P(iraq)>] -> 'Iraq'
+PropN[num=sg, sem=<\P.P(irish_pound)>] -> 'Irish_Pound'
+PropN[num=sg, sem=<\P.P(irrawaddy)>] -> 'Irrawaddy'
+PropN[num=sg, sem=<\P.P(islamad)>] -> 'Islamad'
+PropN[num=sg, sem=<\P.P(israel)>] -> 'Israel'
+PropN[num=sg, sem=<\P.P(israeli_pound)>] -> 'Israeli_Pound'
+PropN[num=sg, sem=<\P.P(istanbul)>] -> 'Istanbul'
+PropN[num=sg, sem=<\P.P(italian_lira)>] -> 'Italian_Lira'
+PropN[num=sg, sem=<\P.P(italy)>] -> 'Italy'
+PropN[num=sg, sem=<\P.P(ivory_coast)>] -> 'Ivory_Coast'
+PropN[num=sg, sem=<\P.P(jakarta)>] -> 'Jakarta'
+PropN[num=sg, sem=<\P.P(jamaica)>] -> 'Jamaica'
+PropN[num=sg, sem=<\P.P(jamaican_dollar)>] -> 'Jamaican_Dollar'
+PropN[num=sg, sem=<\P.P(japan)>] -> 'Japan'
+PropN[num=sg, sem=<\P.P(jerusalem)>] -> 'Jerusalem'
+PropN[num=sg, sem=<\P.P(johannesburg)>] -> 'Johannesburg'
+PropN[num=sg, sem=<\P.P(jordan)>] -> 'Jordan'
+PropN[num=sg, sem=<\P.P(kabul)>] -> 'Kabul'
+PropN[num=sg, sem=<\P.P(kampala)>] -> 'Kampala'
+PropN[num=sg, sem=<\P.P(karachi)>] -> 'Karachi'
+PropN[num=sg, sem=<\P.P(katmandu)>] -> 'Katmandu'
+PropN[num=sg, sem=<\P.P(kenya)>] -> 'Kenya'
+PropN[num=sg, sem=<\P.P(kenya_shilling)>] -> 'Kenya_Shilling'
+PropN[num=sg, sem=<\P.P(khartoum)>] -> 'Khartoum'
+PropN[num=sg, sem=<\P.P(kiev)>] -> 'Kiev'
+PropN[num=sg, sem=<\P.P(kigali)>] -> 'Kigali'
+PropN[num=sg, sem=<\P.P(kingston)>] -> 'Kingston'
+PropN[num=sg, sem=<\P.P(kinshasa)>] -> 'Kinshasa'
+PropN[num=sg, sem=<\P.P(kip)>] -> 'Kip'
+PropN[num=sg, sem=<\P.P(kobe)>] -> 'Kobe'
+PropN[num=sg, sem=<\P.P(koruna)>] -> 'Koruna'
+PropN[num=sg, sem=<\P.P(kowloon)>] -> 'Kowloon'
+PropN[num=sg, sem=<\P.P(krona)>] -> 'Krona'
+PropN[num=sg, sem=<\P.P(krone)>] -> 'Krone'
+PropN[num=sg, sem=<\P.P(kuala_lumpa)>] -> 'Kuala_Lumpa'
+PropN[num=sg, sem=<\P.P(kuwait)>] -> 'Kuwait'
+PropN[num=sg, sem=<\P.P(kuwait_city)>] -> 'Kuwait_City'
+PropN[num=sg, sem=<\P.P(kuwaiti_dinar)>] -> 'Kuwaiti_Dinar'
+PropN[num=sg, sem=<\P.P(kwacha)>] -> 'Kwacha'
+PropN[num=sg, sem=<\P.P(kyat)>] -> 'Kyat'
+PropN[num=sg, sem=<\P.P(kyoto)>] -> 'Kyoto'
+PropN[num=sg, sem=<\P.P(lagos)>] -> 'Lagos'
+PropN[num=sg, sem=<\P.P(laos)>] -> 'Laos'
+PropN[num=sg, sem=<\P.P(lebanese_pound)>] -> 'Lebanese_Pound'
+PropN[num=sg, sem=<\P.P(lebanon)>] -> 'Lebanon'
+PropN[num=sg, sem=<\P.P(lek)>] -> 'Lek'
+PropN[num=sg, sem=<\P.P(lempira)>] -> 'Lempira'
+PropN[num=sg, sem=<\P.P(lena)>] -> 'Lena'
+PropN[num=sg, sem=<\P.P(leningrad)>] -> 'Leningrad'
+PropN[num=sg, sem=<\P.P(leone)>] -> 'Leone'
+PropN[num=sg, sem=<\P.P(lesotho)>] -> 'Lesotho'
+PropN[num=sg, sem=<\P.P(leu)>] -> 'Leu'
+PropN[num=sg, sem=<\P.P(lev)>] -> 'Lev'
+PropN[num=sg, sem=<\P.P(liberia)>] -> 'Liberia'
+PropN[num=sg, sem=<\P.P(libreville)>] -> 'Libreville'
+PropN[num=sg, sem=<\P.P(libya)>] -> 'Libya'
+PropN[num=sg, sem=<\P.P(libyan_dinar)>] -> 'Libyan_Dinar'
+PropN[num=sg, sem=<\P.P(liechtenstein)>] -> 'Liechtenstein'
+PropN[num=sg, sem=<\P.P(lilageru)>] -> 'Lilageru'
+PropN[num=sg, sem=<\P.P(lima)>] -> 'Lima'
+PropN[num=sg, sem=<\P.P(limpopo)>] -> 'Limpopo'
+PropN[num=sg, sem=<\P.P(lira)>] -> 'Lira'
+PropN[num=sg, sem=<\P.P(lisbon)>] -> 'Lisbon'
+PropN[num=sg, sem=<\P.P(lome)>] -> 'Lome'
+PropN[num=sg, sem=<\P.P(london)>] -> 'London'
+PropN[num=sg, sem=<\P.P(los_angeles)>] -> 'Los_Angeles'
+PropN[num=sg, sem=<\P.P(luanda)>] -> 'Luanda'
+PropN[num=sg, sem=<\P.P(lusaka)>] -> 'Lusaka'
+PropN[num=sg, sem=<\P.P(luxembourg)>] -> 'Luxembourg'
+PropN[num=sg, sem=<\P.P(luxembourg_franc)>] -> 'Luxembourg_Franc'
+PropN[num=sg, sem=<\P.P(mackenzie)>] -> 'Mackenzie'
+PropN[num=sg, sem=<\P.P(madras)>] -> 'Madras'
+PropN[num=sg, sem=<\P.P(madrid)>] -> 'Madrid'
+PropN[num=sg, sem=<\P.P(malagasy)>] -> 'Malagasy'
+PropN[num=sg, sem=<\P.P(malawi)>] -> 'Malawi'
+PropN[num=sg, sem=<\P.P(malaysia)>] -> 'Malaysia'
+PropN[num=sg, sem=<\P.P(malaysian_dollar)>] -> 'Malaysian_Dollar'
+PropN[num=sg, sem=<\P.P(maldives)>] -> 'Maldives'
+PropN[num=sg, sem=<\P.P(male)>] -> 'Male'
+PropN[num=sg, sem=<\P.P(mali)>] -> 'Mali'
+PropN[num=sg, sem=<\P.P(mali_franc)>] -> 'Mali_Franc'
+PropN[num=sg, sem=<\P.P(malta)>] -> 'Malta'
+PropN[num=sg, sem=<\P.P(managua)>] -> 'Managua'
+PropN[num=sg, sem=<\P.P(manama)>] -> 'Manama'
+PropN[num=sg, sem=<\P.P(manila)>] -> 'Manila'
+PropN[num=sg, sem=<\P.P(maputo)>] -> 'Maputo'
+PropN[num=sg, sem=<\P.P(markka)>] -> 'Markka'
+PropN[num=sg, sem=<\P.P(masero)>] -> 'Masero'
+PropN[num=sg, sem=<\P.P(mauritania)>] -> 'Mauritania'
+PropN[num=sg, sem=<\P.P(mauritius)>] -> 'Mauritius'
+PropN[num=sg, sem=<\P.P(mbabane)>] -> 'Mbabane'
+PropN[num=sg, sem=<\P.P(mediterranean)>] -> 'the_Mediterranean'
+PropN[num=sg, sem=<\P.P(mekong)>] -> 'Mekong'
+PropN[num=sg, sem=<\P.P(melbourne)>] -> 'Melbourne'
+PropN[num=sg, sem=<\P.P(mexico)>] -> 'Mexico'
+PropN[num=sg, sem=<\P.P(mexico_city)>] -> 'Mexico_City'
+PropN[num=sg, sem=<\P.P(middle_east)>] -> 'Middle_East'
+PropN[num=sg, sem=<\P.P(milan)>] -> 'Milan'
+PropN[num=sg, sem=<\P.P(mississippi)>] -> 'Mississippi'
+PropN[num=sg, sem=<\P.P(mogadishu)>] -> 'Mogadishu'
+PropN[num=sg, sem=<\P.P(monaco)>] -> 'Monaco'
+PropN[num=sg, sem=<\P.P(mongolia)>] -> 'Mongolia'
+PropN[num=sg, sem=<\P.P(monrovia)>] -> 'Monrovia'
+PropN[num=sg, sem=<\P.P(montevideo)>] -> 'Montevideo'
+PropN[num=sg, sem=<\P.P(montreal)>] -> 'Montreal'
+PropN[num=sg, sem=<\P.P(morocco)>] -> 'Morocco'
+PropN[num=sg, sem=<\P.P(moscow)>] -> 'Moscow'
+PropN[num=sg, sem=<\P.P(mozambique)>] -> 'Mozambique'
+PropN[num=sg, sem=<\P.P(mukden)>] -> 'Mukden'
+PropN[num=sg, sem=<\P.P(murray)>] -> 'Murray'
+PropN[num=sg, sem=<\P.P(muscat)>] -> 'Muscat'
+PropN[num=sg, sem=<\P.P(n_djamena)>] -> 'N_Djamena'
+PropN[num=sg, sem=<\P.P(nagoya)>] -> 'Nagoya'
+PropN[num=sg, sem=<\P.P(naira)>] -> 'Naira'
+PropN[num=sg, sem=<\P.P(nairobi)>] -> 'Nairobi'
+PropN[num=sg, sem=<\P.P(nanking)>] -> 'Nanking'
+PropN[num=sg, sem=<\P.P(naples)>] -> 'Naples'
+PropN[num=sg, sem=<\P.P(nassau)>] -> 'Nassau'
+PropN[num=sg, sem=<\P.P(nepal)>] -> 'Nepal'
+PropN[num=sg, sem=<\P.P(nepalese_rupee)>] -> 'Nepalese_Rupee'
+PropN[num=sg, sem=<\P.P(netherlands)>] -> 'Netherlands'
+PropN[num=sg, sem=<\P.P(new_delhi)>] -> 'New_Delhi'
+PropN[num=sg, sem=<\P.P(new_york)>] -> 'New_York'
+PropN[num=sg, sem=<\P.P(new_zealand)>] -> 'New_Zealand'
+PropN[num=sg, sem=<\P.P(new_zealand_dollar)>] -> 'New_Zealand_Dollar'
+PropN[num=sg, sem=<\P.P(niamey)>] -> 'Niamey'
+PropN[num=sg, sem=<\P.P(nicaragua)>] -> 'Nicaragua'
+PropN[num=sg, sem=<\P.P(nicosia)>] -> 'Nicosia'
+PropN[num=sg, sem=<\P.P(niger)>] -> 'Niger'
+PropN[num=sg, sem=<\P.P(niger_river)>] -> 'Niger_River'
+PropN[num=sg, sem=<\P.P(nigeria)>] -> 'Nigeria'
+PropN[num=sg, sem=<\P.P(nile)>] -> 'Nile'
+PropN[num=sg, sem=<\P.P(north_africa)>] -> 'North_Africa'
+PropN[num=sg, sem=<\P.P(north_america)>] -> 'North_America'
+PropN[num=sg, sem=<\P.P(north_korea)>] -> 'North_Korea'
+PropN[num=sg, sem=<\P.P(northern_asia)>] -> 'Northern_Asia'
+PropN[num=sg, sem=<\P.P(norway)>] -> 'Norway'
+PropN[num=sg, sem=<\P.P(nouakchott)>] -> 'Nouakchott'
+PropN[num=sg, sem=<\P.P(nukualofa)>] -> 'Nukualofa'
+PropN[num=sg, sem=<\P.P(ob)>] -> 'Ob'
+PropN[num=sg, sem=<\P.P(oder)>] -> 'Oder'
+PropN[num=sg, sem=<\P.P(oman)>] -> 'Oman'
+PropN[num=sg, sem=<\P.P(orange)>] -> 'Orange'
+PropN[num=sg, sem=<\P.P(orinoco)>] -> 'Orinoco'
+PropN[num=sg, sem=<\P.P(osaka)>] -> 'Osaka'
+PropN[num=sg, sem=<\P.P(oslo)>] -> 'Oslo'
+PropN[num=sg, sem=<\P.P(ottawa)>] -> 'Ottawa'
+PropN[num=sg, sem=<\P.P(ouagadougou)>] -> 'Ouagadougou'
+PropN[num=sg, sem=<\P.P(ouguiya)>] -> 'Ouguiya'
+PropN[num=sg, sem=<\P.P(pa_anga)>] -> 'Pa_Anga'
+PropN[num=sg, sem=<\P.P(pacific)>] -> 'Pacific'
+PropN[num=sg, sem=<\P.P(pakistan)>] -> 'Pakistan'
+PropN[num=sg, sem=<\P.P(panama)>] -> 'Panama'
+PropN[num=sg, sem=<\P.P(papua_new_guinea)>] -> 'Papua_New_Guinea'
+PropN[num=sg, sem=<\P.P(paraguay)>] -> 'Paraguay'
+PropN[num=sg, sem=<\P.P(paramaribo)>] -> 'Paramaribo'
+PropN[num=sg, sem=<\P.P(parana)>] -> 'Parana'
+PropN[num=sg, sem=<\P.P(paris)>] -> 'Paris'
+PropN[num=sg, sem=<\P.P(pataca)>] -> 'Pataca'
+PropN[num=sg, sem=<\P.P(peking)>] -> 'Peking'
+PropN[num=sg, sem=<\P.P(persian_gulf)>] -> 'Persian_Gulf'
+PropN[num=sg, sem=<\P.P(peru)>] -> 'Peru'
+PropN[num=sg, sem=<\P.P(peseta)>] -> 'Peseta'
+PropN[num=sg, sem=<\P.P(peso)>] -> 'Peso'
+PropN[num=sg, sem=<\P.P(peveta)>] -> 'Peveta'
+PropN[num=sg, sem=<\P.P(philadelphia)>] -> 'Philadelphia'
+PropN[num=sg, sem=<\P.P(philippines)>] -> 'Philippines'
+PropN[num=sg, sem=<\P.P(phnom_penh)>] -> 'Phnom_Penh'
+PropN[num=sg, sem=<\P.P(piso)>] -> 'Piso'
+PropN[num=sg, sem=<\P.P(poland)>] -> 'Poland'
+PropN[num=sg, sem=<\P.P(port_au_prince)>] -> 'Port_Au_Prince'
+PropN[num=sg, sem=<\P.P(port_harcourt)>] -> 'Port_Harcourt'
+PropN[num=sg, sem=<\P.P(port_louis)>] -> 'Port_Louis'
+PropN[num=sg, sem=<\P.P(port_of_spain)>] -> 'Port_Of_Spain'
+PropN[num=sg, sem=<\P.P(porto_novo)>] -> 'Porto_Novo'
+PropN[num=sg, sem=<\P.P(portugal)>] -> 'Portugal'
+PropN[num=sg, sem=<\P.P(pound)>] -> 'Pound'
+PropN[num=sg, sem=<\P.P(prague)>] -> 'Prague'
+PropN[num=sg, sem=<\P.P(pretoria)>] -> 'Pretoria'
+PropN[num=sg, sem=<\P.P(pusan)>] -> 'Pusan'
+PropN[num=sg, sem=<\P.P(pvongvang)>] -> 'Pvongvang'
+PropN[num=sg, sem=<\P.P(qatar)>] -> 'Qatar'
+PropN[num=sg, sem=<\P.P(quetzal)>] -> 'Quetzal'
+PropN[num=sg, sem=<\P.P(quezon_city)>] -> 'Quezon_City'
+PropN[num=sg, sem=<\P.P(quito)>] -> 'Quito'
+PropN[num=sg, sem=<\P.P(rabat)>] -> 'Rabat'
+PropN[num=sg, sem=<\P.P(rand)>] -> 'Rand'
+PropN[num=sg, sem=<\P.P(rangoon)>] -> 'Rangoon'
+PropN[num=sg, sem=<\P.P(red_sea)>] -> 'Red_Sea'
+PropN[num=sg, sem=<\P.P(reykjavik)>] -> 'Reykjavik'
+PropN[num=sg, sem=<\P.P(rhine)>] -> 'Rhine'
+PropN[num=sg, sem=<\P.P(rhodesian_dollar)>] -> 'Rhodesian_Dollar'
+PropN[num=sg, sem=<\P.P(rhone)>] -> 'Rhone'
+PropN[num=sg, sem=<\P.P(rial)>] -> 'Rial'
+PropN[num=sg, sem=<\P.P(riel)>] -> 'Riel'
+PropN[num=sg, sem=<\P.P(rio_de_janeiro)>] -> 'Rio_De_Janeiro'
+PropN[num=sg, sem=<\P.P(rio_grande)>] -> 'Rio_Grande'
+PropN[num=sg, sem=<\P.P(riyadh)>] -> 'Riyadh'
+PropN[num=sg, sem=<\P.P(riyal)>] -> 'Riyal'
+PropN[num=sg, sem=<\P.P(riyal_omani)>] -> 'Riyal_Omani'
+PropN[num=sg, sem=<\P.P(romania)>] -> 'Romania'
+PropN[num=sg, sem=<\P.P(rome)>] -> 'Rome'
+PropN[num=sg, sem=<\P.P(ruble)>] -> 'Ruble'
+PropN[num=sg, sem=<\P.P(rupee)>] -> 'Rupee'
+PropN[num=sg, sem=<\P.P(rupiah)>] -> 'Rupiah'
+PropN[num=sg, sem=<\P.P(rwanda)>] -> 'Rwanda'
+PropN[num=sg, sem=<\P.P(rwanda_franc)>] -> 'Rwanda_Franc'
+PropN[num=sg, sem=<\P.P(saigon)>] -> 'Saigon'
+PropN[num=sg, sem=<\P.P(salisbury)>] -> 'Salisbury'
+PropN[num=sg, sem=<\P.P(salween)>] -> 'Salween'
+PropN[num=sg, sem=<\P.P(san_jose)>] -> 'San_Jose'
+PropN[num=sg, sem=<\P.P(san_marino)>] -> 'San_Marino'
+PropN[num=sg, sem=<\P.P(san_salvador)>] -> 'San_Salvador'
+PropN[num=sg, sem=<\P.P(sana)>] -> 'Sana'
+PropN[num=sg, sem=<\P.P(santa_domingo)>] -> 'Santa_Domingo'
+PropN[num=sg, sem=<\P.P(santa_isabel)>] -> 'Santa_Isabel'
+PropN[num=sg, sem=<\P.P(santiago)>] -> 'Santiago'
+PropN[num=sg, sem=<\P.P(sao_paulo)>] -> 'Sao_Paulo'
+PropN[num=sg, sem=<\P.P(saudi_arabia)>] -> 'Saudi_Arabia'
+PropN[num=sg, sem=<\P.P(scandinavia)>] -> 'Scandinavia'
+PropN[num=sg, sem=<\P.P(schilling)>] -> 'Schilling'
+PropN[num=sg, sem=<\P.P(senegal)>] -> 'Senegal'
+PropN[num=sg, sem=<\P.P(senegal_river)>] -> 'Senegal_River'
+PropN[num=sg, sem=<\P.P(seoul)>] -> 'Seoul'
+PropN[num=sg, sem=<\P.P(seychelles)>] -> 'Seychelles'
+PropN[num=sg, sem=<\P.P(shanghai)>] -> 'Shanghai'
+PropN[num=sg, sem=<\P.P(sian)>] -> 'Sian'
+PropN[num=sg, sem=<\P.P(sierra_leone)>] -> 'Sierra_Leone'
+PropN[num=sg, sem=<\P.P(singapore)>] -> 'Singapore'
+PropN[num=sg, sem=<\P.P(singapore_city)>] -> 'Singapore_City'
+PropN[num=sg, sem=<\P.P(singapore_dollar)>] -> 'Singapore_Dollar'
+PropN[num=sg, sem=<\P.P(sofia)>] -> 'Sofia'
+PropN[num=sg, sem=<\P.P(sol)>] -> 'Sol'
+PropN[num=sg, sem=<\P.P(somali_shilling)>] -> 'Somali_Shilling'
+PropN[num=sg, sem=<\P.P(somalia)>] -> 'Somalia'
+PropN[num=sg, sem=<\P.P(south_africa)>] -> 'South_Africa'
+PropN[num=sg, sem=<\P.P(south_african_rand)>] -> 'South_African_Rand'
+PropN[num=sg, sem=<\P.P(south_america)>] -> 'South_America'
+PropN[num=sg, sem=<\P.P(south_korea)>] -> 'South_Korea'
+PropN[num=sg, sem=<\P.P(south_yemen)>] -> 'South_Yemen'
+PropN[num=sg, sem=<\P.P(southeast_east)>] -> 'Southeast_East'
+PropN[num=sg, sem=<\P.P(southern_africa)>] -> 'Southern_Africa'
+PropN[num=sg, sem=<\P.P(southern_europe)>] -> 'Southern_Europe'
+PropN[num=sg, sem=<\P.P(southern_ocean)>] -> 'Southern_Ocean'
+PropN[num=sg, sem=<\P.P(soviet_union)>] -> 'Soviet_Union'
+PropN[num=sg, sem=<\P.P(spain)>] -> 'Spain'
+PropN[num=sg, sem=<\P.P(sri_lanka)>] -> 'Sri_Lanka'
+PropN[num=sg, sem=<\P.P(st_georges)>] -> 'St_Georges'
+PropN[num=sg, sem=<\P.P(stockholm)>] -> 'Stockholm'
+PropN[num=sg, sem=<\P.P(sucre)>] -> 'Sucre'
+PropN[num=sg, sem=<\P.P(sudan)>] -> 'Sudan'
+PropN[num=sg, sem=<\P.P(surinam)>] -> 'Surinam'
+PropN[num=sg, sem=<\P.P(suva)>] -> 'Suva'
+PropN[num=sg, sem=<\P.P(swaziland)>] -> 'Swaziland'
+PropN[num=sg, sem=<\P.P(sweden)>] -> 'Sweden'
+PropN[num=sg, sem=<\P.P(swiss_franc)>] -> 'Swiss_Franc'
+PropN[num=sg, sem=<\P.P(switzerland)>] -> 'Switzerland'
+PropN[num=sg, sem=<\P.P(sydney)>] -> 'Sydney'
+PropN[num=sg, sem=<\P.P(syli)>] -> 'Syli'
+PropN[num=sg, sem=<\P.P(syria)>] -> 'Syria'
+PropN[num=sg, sem=<\P.P(syrian_pound)>] -> 'Syrian_Pound'
+PropN[num=sg, sem=<\P.P(tagus)>] -> 'Tagus'
+PropN[num=sg, sem=<\P.P(taipei)>] -> 'Taipei'
+PropN[num=sg, sem=<\P.P(taiwan)>] -> 'Taiwan'
+PropN[num=sg, sem=<\P.P(taiwan_dollar)>] -> 'Taiwan_Dollar'
+PropN[num=sg, sem=<\P.P(taka)>] -> 'Taka'
+PropN[num=sg, sem=<\P.P(tala)>] -> 'Tala'
+PropN[num=sg, sem=<\P.P(tananarive)>] -> 'Tananarive'
+PropN[num=sg, sem=<\P.P(tanzania)>] -> 'Tanzania'
+PropN[num=sg, sem=<\P.P(tanzanian_shilling)>] -> 'Tanzanian_Shilling'
+PropN[num=sg, sem=<\P.P(tegucigalpa)>] -> 'Tegucigalpa'
+PropN[num=sg, sem=<\P.P(tehran)>] -> 'Tehran'
+PropN[num=sg, sem=<\P.P(thailand)>] -> 'Thailand'
+PropN[num=sg, sem=<\P.P(thimphu)>] -> 'Thimphu'
+PropN[num=sg, sem=<\P.P(tientsin)>] -> 'Tientsin'
+PropN[num=sg, sem=<\P.P(tighrik)>] -> 'Tighrik'
+PropN[num=sg, sem=<\P.P(tirana)>] -> 'Tirana'
+PropN[num=sg, sem=<\P.P(togo)>] -> 'Togo'
+PropN[num=sg, sem=<\P.P(tokyo)>] -> 'Tokyo'
+PropN[num=sg, sem=<\P.P(tonga)>] -> 'Tonga'
+PropN[num=sg, sem=<\P.P(toronto)>] -> 'Toronto'
+PropN[num=sg, sem=<\P.P(trinidad_and_tobago)>] -> 'Trinidad_And_Tobago'
+PropN[num=sg, sem=<\P.P(trinidad_and_tobago_dollar)>] -> 'Trinidad_And_Tobago_Dollar'
+PropN[num=sg, sem=<\P.P(tripoli)>] -> 'Tripoli'
+PropN[num=sg, sem=<\P.P(tropic_of_cancer)>] -> 'Tropic_Of_Cancer'
+PropN[num=sg, sem=<\P.P(tropic_of_capricorn)>] -> 'Tropic_Of_Capricorn'
+PropN[num=sg, sem=<\P.P(tunis)>] -> 'Tunis'
+PropN[num=sg, sem=<\P.P(tunisia)>] -> 'Tunisia'
+PropN[num=sg, sem=<\P.P(turkey)>] -> 'Turkey'
+PropN[num=sg, sem=<\P.P(uganda)>] -> 'Uganda'
+PropN[num=sg, sem=<\P.P(uganda_shilling)>] -> 'Uganda_Shilling'
+PropN[num=sg, sem=<\P.P(ulan_bator)>] -> 'Ulan_Bator'
+PropN[num=sg, sem=<\P.P(united_arab_emirates)>] -> 'United_Arab_Emirates'
+PropN[num=sg, sem=<\P.P(united_kingdom)>] -> 'United_Kingdom'
+PropN[num=sg, sem=<\P.P(united_states)>] -> 'United_States'
+PropN[num=sg, sem=<\P.P(upper_volta)>] -> 'Upper_Volta'
+PropN[num=sg, sem=<\P.P(uruguay)>] -> 'Uruguay'
+PropN[num=sg, sem=<\P.P(us_dollar)>] -> 'Us_Dollar'
+PropN[num=sg, sem=<\P.P(vaduz)>] -> 'Vaduz'
+PropN[num=sg, sem=<\P.P(valetta)>] -> 'Valetta'
+PropN[num=sg, sem=<\P.P(venezuela)>] -> 'Venezuela'
+PropN[num=sg, sem=<\P.P(victoria)>] -> 'Victoria'
+PropN[num=sg, sem=<\P.P(vienna)>] -> 'Vienna'
+PropN[num=sg, sem=<\P.P(vientiane)>] -> 'Vientiane'
+PropN[num=sg, sem=<\P.P(vietnam)>] -> 'Vietnam'
+PropN[num=sg, sem=<\P.P(vistula)>] -> 'Vistula'
+PropN[num=sg, sem=<\P.P(volga)>] -> 'Volga'
+PropN[num=sg, sem=<\P.P(volta)>] -> 'Volta'
+PropN[num=sg, sem=<\P.P(warsaw)>] -> 'Warsaw'
+PropN[num=sg, sem=<\P.P(washington)>] -> 'Washington'
+PropN[num=sg, sem=<\P.P(wellington)>] -> 'Wellington'
+PropN[num=sg, sem=<\P.P(west_africa)>] -> 'West_Africa'
+PropN[num=sg, sem=<\P.P(west_germany)>] -> 'West_Germany'
+PropN[num=sg, sem=<\P.P(western_europe)>] -> 'Western_Europe'
+PropN[num=sg, sem=<\P.P(western_samoa)>] -> 'Western_Samoa'
+PropN[num=sg, sem=<\P.P(won)>] -> 'Won'
+PropN[num=sg, sem=<\P.P(yangtze)>] -> 'Yangtze'
+PropN[num=sg, sem=<\P.P(yaounde)>] -> 'Yaounde'
+PropN[num=sg, sem=<\P.P(yemen)>] -> 'Yemen'
+PropN[num=sg, sem=<\P.P(yen)>] -> 'Yen'
+PropN[num=sg, sem=<\P.P(yenisei)>] -> 'Yenisei'
+PropN[num=sg, sem=<\P.P(yokohama)>] -> 'Yokohama'
+PropN[num=sg, sem=<\P.P(yuan)>] -> 'Yuan'
+PropN[num=sg, sem=<\P.P(yugoslavia)>] -> 'Yugoslavia'
+PropN[num=sg, sem=<\P.P(yukon)>] -> 'Yukon'
+PropN[num=sg, sem=<\P.P(zaire)>] -> 'Zaire'
+PropN[num=sg, sem=<\P.P(zambesi)>] -> 'Zambesi'
+PropN[num=sg, sem=<\P.P(zambia)>] -> 'Zambia'
+PropN[num=sg, sem=<\P.P(zimbabwe)>] -> 'Zimbabwe'
+PropN[num=sg, sem=<\P.P(zloty)>] -> 'Zloty'
+PropN[num=sg, sem=<\P.P(zomba)>] -> 'Zomba'
diff --git a/examples/grammars/sample_grammars/dep_test2.dep b/examples/grammars/sample_grammars/dep_test2.dep
new file mode 100644
index 0000000..4bcd74e
--- /dev/null
+++ b/examples/grammars/sample_grammars/dep_test2.dep
@@ -0,0 +1,4 @@
+1 John _ NNP _ _ 2 SUBJ _ _
+2 sees _ VB _ _ 0 ROOT _ _
+3 a _ DT _ _ 4 SPEC _ _
+4 dog _ NN _ _ 2 OBJ _ _
diff --git a/examples/grammars/sample_grammars/drt_glue.semtype b/examples/grammars/sample_grammars/drt_glue.semtype
new file mode 100644
index 0000000..dba4ead
--- /dev/null
+++ b/examples/grammars/sample_grammars/drt_glue.semtype
@@ -0,0 +1,61 @@
+########################################################################
+# DRT-Glue Semantics Formulas Using DRT and Event Representation
+#
+# Entries are made up of three parts, separated by colons (":")
+#
+# 1) The semtype name.
+# - May appear multiple times with different relationship sets (3)
+# - May "extend" other semtypes: "type(parent)"
+#
+# 2) The glue formulas.
+# - A comma-separated list of tuples representing glue formulas
+# - If the entry is an extension, then the listed formulas will be added to
+# the list from the super type
+#
+# 3) The relationship set (OPTIONAL)
+# - If not specified, then assume the entry covers ALL relationship sets
+# - If the entry is an extension, then the relationship set dictates which
+# particular entry should be extended. If no relationship set is
+# specified, then every entry of the parent type is extended.
+#
+########################################################################
+
+#Quantifiers
+def_art : (\P Q.([x],[((([y],[])+Q(y)) <-> (x = y)), P(x)]), ((v -o r) -o ((f -o Var) -o Var)))
+ex_quant : (\P Q.(([x],[])+P(x)+Q(x)), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+univ_quant : (\P Q.([],[((([x],[])+P(x)) -> Q(x))]), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+no_quant : (\P Q.(-(([x],[])+P(x)+Q(x))), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+
+#Nouns
+NN : (\Q.(([x],[<word>(x)])+Q(x)), ((f -o Var) -o Var)) : [] # treat a noun missing its spec as implicitly existentially quantified
+
+NN : (\x.([],[<word>(x)]), (v -o r)) : [spec]
+NN : (\P Q.(([x],[]) + P(x) + Q(x)), ((v -o r) -o ((f -o var) -o var))), (\x.([],[<word>(x)]), (v -o r)) : [] # treat a noun missing its spec as implicitly existentially quantified
+NNP : (\P Q.(([x],[]) + P(x) + Q(x)), ((v -o r) -o ((f -o var) -o var))), (\x.([],[<word>(x)]), (v -o r))
+NNS(NN)
+PRP : (\P Q.(([x],[]) + P(x) + Q(x)), ((v -o r) -o ((f -o var) -o var))), (\x.([],[PRO(x)]), (v -o r))
+
+#Verbs
+VB : (\x.([],[<word>(x)]), (subj -o f)) : [subj] #iv
+VB : (\x y.([],[<word>(x,y)]), (subj -o (obj -o f))) : [subj, obj] #tv
+VB : (\y.exists x.([],[<word>(x,y)]), (obj -o f)) : [obj] #incomplete tv
+VB : (\x y z.([],[<word>(x,y,z)]), (subj -o (obj -o (theme -o f)))) : [subj, obj, theme] #dtv
+VB : (\y z.exists x.([],[<word>(x,y,z)]), obj -o (theme -o f)) : [obj, theme] #incomplete dtv
+VB : (\x z.exists y.([],[<word>(x,y,z)]), subj -o (theme -o f)) : [subj, theme] #incomplete dtv
+VB : (\z.exists x y.([],[<word>(x,y,z)]), theme -o f) : [theme] #incomplete dtv
+VB : (\x y.([],[<word>(x,y)]), (subj -o (comp -o f))) : [subj, comp] #tv_comp
+VB : (\x P.([],[<word>(x,P)]), (subj -o ((xcomp.subj -o xcomp) -o f))) : [subj, xcomp] #equi
+VB : (\x y P.([],[<word>(x,y,P)]), (subj -o (obj -o ((xcomp.subj -o xcomp) -o f)))) : [subj, obj, xcomp] # object equi
+VB : (\P.([],[<word>(P)]), (xcomp -o f)) : [xcomp] #raising
+VBD(VB) : (\P.PAST(P), (f -o f))
+VBZ(VB)
+
+#Modifiers
+nmod : (\x.([],[<word>(x)]), f), (\P Q x.(P(x)+Q(x)), (f -o ((super.v -o super.r) -o (super.v -o super.r))))
+JJ(nmod)
+vmod : (\x.([],[<word>(x)]), f), (\P Q x.P(Q(x)), (f -o (super -o super)))
+RB(vmod)
+tense : (\P.([],[<word>(P)]), (super.f -o super.f))
+
+#Conjunctions
+cc_clause : (\P Q.(P + Q), (a -o (b -o f)))
diff --git a/examples/grammars/sample_grammars/drt_glue_event.semtype b/examples/grammars/sample_grammars/drt_glue_event.semtype
new file mode 100644
index 0000000..bddb43a
--- /dev/null
+++ b/examples/grammars/sample_grammars/drt_glue_event.semtype
@@ -0,0 +1,62 @@
+########################################################################
+# DRT-Glue Semantics Formulas Using DRT and Event Representation
+#
+# Entries are made up of three parts, separated by colons (":")
+#
+# 1) The semtype name.
+# - May appear multiple times with different relationship sets (3)
+# - May "extend" other semtypes: "type(parent)"
+#
+# 2) The glue formulas.
+# - A comma-separated list of tuples representing glue formulas
+# - If the entry is an extension, then the listed formulas will be added to
+# the list from the super type
+#
+# 3) The relationship set (OPTIONAL)
+# - If not specified, then assume the entry covers ALL relationship sets
+# - If the entry is an extension, then the relationship set dictates which
+# particular entry should be extended. If no relationship set is
+# specified, then every entry of the parent type is extended.
+#
+########################################################################
+
+#Quantifiers
+def_art : (\P Q.([x],[((([y],[])+Q(y)) <-> (x = y)), P(x)]), ((v -o r) -o ((f -o Var) -o Var)))
+ex_quant : (\P Q.(([x],[])+P(x)+Q(x)), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+univ_quant : (\P Q.([],[((([x],[])+P(x)) -> Q(x))]), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+no_quant : (\P Q.(-(([x],[])+P(x)+Q(x))), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+
+#Nouns
+NN : (\x.([],[<word>(x)]), (v -o r)) : [spec]
+NN : (\P Q e.(([x],[]) + P(x) + Q(x,e)), ((v -o r) -o ((f -o var) -o var))), (\x.([],[<word>(x)]), (v -o r)) : [] # treat a noun missing its spec as implicitly existentially quantified
+NNP : (\P Q e.(([x],[]) + P(x) + Q(x,e)), ((v -o r) -o ((f -o var) -o var))), (\x.([],[<word>(x)]), (v -o r))
+NNS(NN)
+PRP : (\P Q e.(([x],[]) + P(x) + Q(x,e)), ((v -o r) -o ((f -o var) -o var))), (\x.([],[PRO(x)]), (v -o r))
+
+#Verbs
+VB : (\x e.([],[<word>(e),subj(e,x)]), (subj -o f)) [subj] #iv
+VB : (\x y e.([],[<word>(e), subj(e,x), obj(e,y)]), (subj -o (obj -o f))) : [subj, obj] #tv
+VB : (\x y z e.([],[<word>(e), subj(e,x), obj(e,y), theme(e,z)]), (subj -o (obj -o (theme -o f)))) : [subj, obj, theme] #dtv
+VB : (\y z e.([x],[<word>(e), subj(e,x), obj(e,y), theme(e,z)]), obj -o (theme -o f)) : [obj, theme] #incomplete dtv
+VB : (\x z e.([y],[<word>(e), subj(e,x), obj(e,y), theme(e,z)]), subj -o (theme -o f)) : [subj, theme] #incomplete dtv
+VB : (\z e.([x,y],[<word>(e), subj(e,x), obj(e,y), theme(e,z)]), theme -o f) : [theme] #incomplete dtv
+VB : (\x y e.(([],[<word>(e), subj(e,x), comp(e,y)])+P(e)), (subj -o (comp -o f))) : [subj, comp] #tv_comp
+VB : (\x P e.([],[<word>(e), subj(e,x), xcomp(e,P)]), (subj -o ((xcomp.subj -o xcomp) -o f))) : [subj, xcomp] #equi
+VB : (\x y P e.([],[<word>(e), subj(e,x), obj(e,y), (xcomp e P)]), (subj -o (obj -o ((xcomp.subj -o xcomp) -o f)))) : [subj, obj, xcomp] # object equi
+VB : (\P e.([],[<word>(e), xcomp(e,P)]), (xcomp -o f)) : [xcomp] #raising
+VBD(VB) : (\P.PAST(P), (f -o f))
+VBZ(VB)
+
+#Modifiers
+nmod : (\x.([],[<word>(x)]), f), (\P Q x.(P(x)+Q(x)), (f -o ((super.v -o super.r) -o (super.v -o super.r)))) : []
+JJ(nmod) : []
+vmod : (\x.([],[<word>(x)]), f), (\P Q x.P(Q(x)), (f -o (super -o super))) : []
+RB(vmod) : []
+tense(vmod) : []
+
+#Prepositions
+IN : (\P Q e1.P(\x e2.(([],[<word>(e2,x)]) + Q(e2)),e1), ((subj -o subj.var) -o subj.var) -o (super -o super)) : [subj]
+IN(vmod) : []
+
+#Conjunctions
+cc_clause : (\P Q.(P + Q), (a -o (b -o f)))
diff --git a/examples/grammars/sample_grammars/event.fcfg b/examples/grammars/sample_grammars/event.fcfg
new file mode 100644
index 0000000..96179b4
--- /dev/null
+++ b/examples/grammars/sample_grammars/event.fcfg
@@ -0,0 +1,72 @@
+## Natural Language Toolkit: event.fcfg
+##
+## Illustrating Davidson-style event semantics
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+############################
+# Grammar Rules
+#############################
+
+S[sem = <exists e.?subj(e,?vp)>] -> NP[num=?n,sem=?subj] VP[num=?n,sem=?vp]
+
+NP[num=?n,sem=<?det(?nom)> ] -> Det[num=?n,sem=?det] Nom[num=?n,sem=?nom]
+NP[loc=?l,num=?n,sem=?np] -> PropN[loc=?l,num=?n,sem=?np]
+
+Nom[num=?n,sem=?nom] -> N[num=?n,sem=?nom]
+Nom[num=?n,sem=<?pp(?nom)>] -> N[num=?n,sem=?nom] PP[sem=?pp]
+
+VP[num=?n,sem=?v] -> IV[num=?n,sem=?v]
+VP[num=?n,sem=<?v(?obj)>] -> TV[num=?n,sem=?v] NP[sem=?obj]
+VP[num=?n,sem=<?v(?obj,?pp)>] -> DTV[num=?n,sem=?v] NP[sem=?obj] PP[+to, sem=?pp]
+
+
+VP[num=?n,sem=<?pp(?vp)>] -> VP[num=?n,sem=?vp] PP[sem=?pp]
+VP[num=?n,sem=<?adv(?vp)>] -> VP[num=?n,sem=?vp] Adv[sem=?adv]
+
+PP[sem=<?p(?np)>] -> P[loc=?l,sem=?p] NP[loc=?l,sem=?np]
+
+#############################
+# Lexical Rules
+#############################
+
+PropN[-loc,num=sg,sem=<\e R.R(e,angus)>] -> 'Angus'
+PropN[-loc,num=sg,sem=<\e R.R(e,pat)>] -> 'Pat'
+PropN[-loc,num=sg,sem=<\e R.R(e,irene)>] -> 'Irene'
+PropN[-loc,num=sg,sem=<\e R.R(e,cyril)>] -> 'Cyril'
+PropN[+loc, num=sg,sem=<\e R.R(e,stockbridge)>] -> 'Stockbridge'
+
+NP[-loc, num=sg, sem=<\P.\x.P(x)>] -> 'who'
+
+Det[num=sg,sem=<\P R e.all x.(P(x) -> R(e,x))>] -> 'every'
+Det[num=pl,sem=<\P R e.all x.(P(x) -> R(e,x))>] -> 'all'
+Det[sem=<\P R e.exists x.(P(x) & R(e,x))>] -> 'some'
+Det[num=sg,sem=<\P R e.exists x.(P(x) & R(e,x))>] -> 'a'
+
+N[num=sg,sem=<boy>] -> 'boy'
+N[num=pl,sem=<boy>] -> 'boys'
+N[num=sg,sem=<girl>] -> 'girl'
+N[num=pl,sem=<girl>] -> 'girls'
+N[num=sg,sem=<bone>] -> 'bone'
+N[num=sg,sem=<dog>] -> 'dog'
+
+IV[num=sg,sem=<\e x.(bark(e) & agent(e,x))>,tns=pres] -> 'barks'
+IV[num=pl,sem=<\e x.(bark(e) & agent(e,x))>,tns=pres] -> 'bark'
+IV[num=sg,sem=<\e x.(walk(e) & agent(e,x))>,tns=pres] -> 'walks'
+IV[num=pl,sem=<\e x.( walk(e) & agent(e,x))>,tns=pres] -> 'walk'
+TV[num=sg,sem=<\X y.X(\e x.(chase(e) & agent(e,y) & patient(e,x)))>,tns=pres] -> 'chases'
+TV[num=pl,sem=<\X y.X(\e x.(chase(e) & agent(e,y) & patient(e,x)))>,tns=pres] -> 'chase'
+TV[num=sg,sem=<\X y.X(\e x.(see(e) & agent(e,y) & patient(e,x)))>,tns=pres] -> 'sees'
+TV[num=pl,sem=<\X y.X(\e x.(see(e) & agent(e,y) & patient(e,x)))>,tns=pres] -> 'see'
+DTV[num=sg,sem=<\Y X x.X(\z.Y(\e y.(give(e) & agent(e,x) & theme(e,y) & recip(e,z))))>,tns=pres] -> 'gives'
+DTV[num=pl,sem=<\Y X x.X(\z.Y(\e y.(give(e) & agent(e,x) & theme(e,y) & recip(e,z))))>,tns=pres] -> 'give'
+
+P[+loc,sem=<\X P e.X(\y.(P(e) & in(e,y)))>] -> 'in'
+P[-loc,sem=<\X P e.X(\y.(P(e) & with(e,y)))>] -> 'with'
+P[+to,sem=<\X.X>] -> 'to'
+
+Adv[sem=<\R e x.(slow(e) & R(e,x))>] -> 'slowly'
+Adv[sem=<\R e x.(thoughtful(e) & R(e,x))>] -> 'thoughtfully'
diff --git a/examples/grammars/sample_grammars/glue.semtype b/examples/grammars/sample_grammars/glue.semtype
new file mode 100644
index 0000000..910ee19
--- /dev/null
+++ b/examples/grammars/sample_grammars/glue.semtype
@@ -0,0 +1,59 @@
+########################################################################
+# Glue Semantics Formulas Using Event Representation
+#
+# Entries are made up of three parts, separated by colons (":")
+#
+# 1) The semtype name.
+# - May appear multiple times with different relationship sets (3)
+# - May "extend" other semtypes: "type(parent)"
+#
+# 2) The glue formulas.
+# - A comma-separated list of tuples representing glue formulas
+# - If the entry is an extension, then the listed formulas will be added to
+# the list from the super type
+#
+# 3) The relationship set (OPTIONAL)
+# - If not specified, then assume the entry covers ALL relationship sets
+# - If the entry is an extension, then the relationship set dictates which
+# particular entry should be extended. If no relationship set is
+# specified, then every entry of the parent type is extended.
+#
+########################################################################
+
+#Quantifiers
+def_art : (\P Q.exists x.(P(x) & all y.(Q(y) <-> (x = y))), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+ex_quant : (\P Q.exists x.(P(x) & Q(x)), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+univ_quant : (\P Q.all x.(P(x) -> Q(x)), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+no_quant : (\P Q.-exists x.(P(x) & Q(x)), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+
+#Nouns
+NN : (\x.<word>(x), (v -o r)) : [spec]
+NN : (\P Q.exists x.(P(x) & Q(x)), ((v -o r) -o ((f -o var) -o var))), (\x.<word>(x), (v -o r)) : [] # treat a noun missing its spec as implicitly existentially quantified
+NNP : (\P Q.exists x.(P(x) & Q(x)), ((v -o r) -o ((f -o var) -o var))), (\x.<word>(x), (v -o r))
+NNS(NN)
+PRP : (\P Q.exists x.(P(x) & Q(x)), ((v -o r) -o ((f -o var) -o var))), (\x.PRO(x), (v -o r))
+
+#Verbs
+VB : (\x.<word>(x), (subj -o f)) : [subj] #iv
+VB : (\x y.<word>(x,y), (subj -o (obj -o f))) : [subj, obj] #tv
+VB : (\y.exists x.<word>(x,y), (obj -o f)) : [obj] #incomplete tv
+VB : (\x y z.<word>(x,y,z), (subj -o (obj -o (theme -o f)))) : [subj, obj, theme] #dtv
+VB : (\y z.exists x.<word>(x,y,z), obj -o (theme -o f)) : [obj, theme] #incomplete dtv
+VB : (\x z.exists y.<word>(x,y,z), subj -o (theme -o f)) : [subj, theme] #incomplete dtv
+VB : (\z.exists x y.<word>(x,y,z), theme -o f) : [theme] #incomplete dtv
+VB : (\x y.<word>(x,y), (subj -o (comp -o f))) : [subj, comp] #tv_comp
+VB : (\x P.<word>(x,P), (subj -o ((xcomp.subj -o xcomp) -o f))) : [subj, xcomp] #equi
+VB : (\x y P.<word>(x,y,P), (subj -o (obj -o ((xcomp.subj -o xcomp) -o f)))) : [subj, obj, xcomp] # object equi
+VB : (\P.<word>(P), (xcomp -o f)) : [xcomp] #raising
+VBD(VB) : (\P.PAST(P), (f -o f))
+VBZ(VB)
+
+#Modifiers
+nmod : (\Q P x.(P(x) & Q(x)), (f -o ((super.v -o super.r) -o (super.v -o super.r)))), (\x.<word>(x), f)
+JJ(nmod)
+vmod : (\P.<word>(P), (super.f -o super.f))
+RB(vmod)
+tense : (\P.<word>(P), (super.f -o super.f))
+
+#Conjunctions
+cc_clause : (\P Q.(P & Q), (a -o (b -o f)))
diff --git a/examples/grammars/sample_grammars/glue_event.semtype b/examples/grammars/sample_grammars/glue_event.semtype
new file mode 100644
index 0000000..8120a8e
--- /dev/null
+++ b/examples/grammars/sample_grammars/glue_event.semtype
@@ -0,0 +1,66 @@
+########################################################################
+# Glue Semantics Formulas Using Event Representation
+#
+# Entries are made up of three parts, separated by colons (":")
+#
+# 1) The semtype name.
+# - May appear multiple times with different relationship sets (3)
+# - May "extend" other semtypes: "type(parent)"
+#
+# 2) The glue formulas.
+# - A comma-separated list of tuples representing glue formulas
+# - If the entry is an extension, then the listed formulas will be added to
+# the list from the super type
+#
+# 3) The relationship set (OPTIONAL)
+# - If not specified, then assume the entry covers ALL relationship sets
+# - If the entry is an extension, then the relationship set dictates which
+# particular entry should be extended. If no relationship set is
+# specified, then every entry of the parent type is extended.
+#
+########################################################################
+
+#Quantifiers
+def_art : (\P Q.exists x.(P(x) & all y.(Q(y) <-> (x = y))), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+ex_quant : (\P Q.exists x.(P(x) & Q(x)), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+univ_quant : (\P Q.all x.(P(x) -> Q(x)), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+no_quant : (\P Q.-exists x.(P(x) & Q(x)), ((super.v -o super.r) -o ((super.f -o super.var) -o super.var)))
+
+#Nouns
+NN : (\x.<word>(x), (v -o r)) : [spec]
+NN : (\P Q e.exists x.(P(x) & Q(x,e)), ((v -o r) -o ((f -o var) -o var))), (\x.<word>(x), (v -o r)) : [] # treat a noun missing its spec as implicitly existentially quantified
+NNP : (\P Q e.exists x.(P(x) & Q(x,e)), ((v -o r) -o ((f -o var) -o var))), (\x.<word>(x), (v -o r))
+NNS(NN)
+PRP : (\P Q e.exists x.(P(x) & Q(x,e)), ((v -o r) -o ((f -o var) -o var))), (\x.PRO(x), (v -o r))
+
+#Verbs
+VB : (\x e.(<word>(e) & subj(e,x)), (subj -o f)) : [subj] #iv
+VB : (\x y e.(<word>(e) & subj(e,x) & obj(e,y)), (subj -o (obj -o f))) : [subj, obj] #tv
+VB : (\y e.exists x.(<word>(e) & subj(e,x) & obj(e,y)), (obj -o f)) : [obj] #incomplete tv
+VB : (\x y z e.(<word>(e) & subj(e,x) & obj(e,y) & theme(e,z)), (subj -o (obj -o (theme -o f)))) : [subj, obj, theme] #dtv
+VB : (\y z e.exists x.(<word>(e) & subj(e,x) & obj(e,y) & theme(e,z)), obj -o (theme -o f)) : [obj, theme] #incomplete dtv
+VB : (\x z e.exists y.(<word>(e) & subj(e,x) & obj(e,y) & theme(e,z)), subj -o (theme -o f)) : [subj, theme] #incomplete dtv
+VB : (\z e.exists x y.(<word>(e) & subj(e,x) & obj(e,y) & theme(e,z)), theme -o f) : [theme] #incomplete dtv
+VB : (\x y e.(<word>(e) & subj(e,x) & comp(e,y) & P(e)), (subj -o (comp -o f))) : [subj, comp] #tv_comp
+VB : (\x P e.(<word>(e) & subj(e,x) & xcomp(e,P)), (subj -o ((xcomp.subj -o xcomp) -o f))) : [subj, xcomp] #equi
+VB : (\x y P e.(<word>(e) & subj(e,x) & obj(e,y) & (xcomp e P)), (subj -o (obj -o ((xcomp.subj -o xcomp) -o f)))) : [subj, obj, xcomp] # object equi
+VB : (\P e.(<word>(e) & xcomp(e,P)), (xcomp -o f)) : [xcomp] #raising
+VBD(VB) : (\P.PAST(P), (f -o f))
+VBZ(VB)
+
+#Auxillary Verbs
+MD : (\P Q e1.P(\x e2.(<word>(e2,x) & Q(e2)),e1), ((subj -o subj.var) -o subj.var) -o (main -o main)) : [subj]
+
+#Modifiers
+nmod : (\Q P x.(P(x) & Q(x)), (f -o ((super.v -o super.r) -o (super.v -o super.r)))), (\x.<word>(x), f) : []
+JJ(nmod) : []
+vmod : (\P.<word>(P), (super.f -o super.f)) : []
+RB(vmod) : []
+tense : (\P.<word>(P), (super.f -o super.f)) : []
+
+#Prepositions
+IN : (\P Q e1.P(\x e2.(<word>(e2,x) & Q(e2)),e1), ((subj -o subj.var) -o subj.var) -o (super -o super)) : [subj]
+IN(vmod) : []
+
+#Conjunctions
+cc_clause : (\P Q.(P & Q), (a -o (b -o f)))
diff --git a/examples/grammars/sample_grammars/glue_train.conll b/examples/grammars/sample_grammars/glue_train.conll
new file mode 100644
index 0000000..e97e148
--- /dev/null
+++ b/examples/grammars/sample_grammars/glue_train.conll
@@ -0,0 +1,27 @@
+1 John _ NNP _ _ 2 SUBJ _ _
+2 runs _ VB _ _ 0 ROOT _ _
+
+1 a _ DT _ _ 2 SPEC _ _
+2 man _ NN _ _ 3 SUBJ _ _
+3 runs _ VB _ _ 0 ROOT _ _
+
+1 John _ NNP _ _ 2 SUBJ _ _
+2 sees _ VB _ _ 0 ROOT _ _
+3 Mary _ NNP _ _ 2 OBJ _ _
+
+1 every _ DT _ _ 2 SPEC _ _
+2 girl _ NN _ _ 3 SUBJ _ _
+3 chases _ VB _ _ 0 ROOT _ _
+4 an _ DT _ _ 5 SPEC _ _
+5 animal _ NN _ _ 3 OBJ _ _
+
+1 Bill _ NNP _ _ 2 SUBJ _ _
+2 sees _ VB _ _ 0 ROOT _ _
+3 a _ DT _ _ 4 SPEC _ _
+4 dog _ NN _ _ 2 OBJ _ _
+
+1 every _ DT _ _ 2 SPEC _ _
+2 girl _ NN _ _ 3 SUBJ _ _
+3 chases _ VB _ _ 0 ROOT _ _
+4 John _ NNP _ _ 3 OBJ _ _
+
diff --git a/examples/grammars/sample_grammars/gluesemantics.fcfg b/examples/grammars/sample_grammars/gluesemantics.fcfg
new file mode 100644
index 0000000..4dff781
--- /dev/null
+++ b/examples/grammars/sample_grammars/gluesemantics.fcfg
@@ -0,0 +1,131 @@
+% start S
+
+#############################
+# Grammar Rules
+#############################
+
+# S expansion rules
+S -> NP[num=?n, case=nom] VP[num=?n]
+S -> S CC[sem=cc_clause] S
+
+# NP expansion rules
+NP[num=?n, gender=?g] -> Det[num=?n] N[num=?n, gender=?g]
+NP[num=?n, gender=?g] -> PropN[num=?n, gender=?g]
+NP[num=?n, case=?c, gender=?g] -> Pro[num=?n, case=?c, gender=?g]
+NP[num=pl, gender=?g] -> N[num=pl]
+NP[num=?n, gender=?g] -> NP[num=?n, gender=?g] PP
+NP[num=pl] -> NP CC[sem=cc_np] NP
+
+# N's can have Adjectives in front
+N[num=?n] -> JJ[type=attributive] N[num=?n]
+
+# JJs can have ADVs in front
+JJ -> ADV JJ
+
+# VP expansion rules
+VP[tense=?t, num=?n] -> IV[tense=?t, num=?n]
+VP[tense=?t, num=?n] -> TV[tense=?t, num=?n] NP[case=acc]
+VP[tense=?t, num=?n] -> TVComp[tense=?t, num=?n] S
+VP[tense=?t, num=?n] -> DTV[tense=?t, num=?n] NP[case=acc] NP[case=acc]
+VP[tense=?t, num=?n] -> EquiV[tense=?t, num=?n] TO VP[tense=inf]
+VP[tense=?t, num=?n] -> ObjEquiV[tense=?t, num=?n] NP[case=acc] TO VP[tense=inf]
+VP[tense=?t, num=?n] -> RaisingV[tense=?t, num=?n] TO VP[tense=inf]
+VP[tense=?t, num=?n] -> ADV VP[tense=?t, num=?n]
+VP[tense=?t, num=?n] -> VP[tense=?t, num=?n] PP
+VP[tense=?t, num=?n] -> VP[tense=?t, num=?n] CC[sem=cc_vp] VP[tense=?t, num=?n]
+
+# PP expansion
+PP -> IN NP
+
+# Det types
+Det[num=sg] -> DT
+Det[num=pl] -> DTS
+Det -> AT
+Det[num=?n] -> DTI[num=?n]
+Det[num=?n] -> ABN[num=?n]
+
+
+#############################
+# Lexical Rules
+#############################
+
+DT -> 'this' | 'each'
+DTS -> 'these'
+AT[num=sg, sem=ex_quant] -> 'a' | 'an'
+AT[sem=art_def] -> 'the'
+DTI[num=sg, sem=univ_quant] -> 'every'
+DTI[sem=ex_quant] -> 'some'
+ABN[num=sg] -> 'half'
+ABN[num=pl, sem=univ_quant] -> 'all'
+
+PropN[num=sg, gender=m, sem=pn] -> 'Kim' | 'Jody' | 'Mary' | 'Sue'
+PropN[num=sg, gender=m, sem=pn] -> 'David' | 'John' | 'Tom'
+PropN[num=pl, sem=pn] -> 'JM'
+
+N[num=sg, sem=n] -> 'boy' | 'car' | 'cat' | 'child' | 'criminal' | 'dog' | 'gift' | 'girl' | 'man' | 'mouse' | 'person' | 'pizza' | 'racketeer' | 'sandwich' | 'senator' | 'student' | 'telescope' | 'thing' | 'unicorn' | 'woman'
+N[num=pl, sem=n] -> 'boys' | 'cars' | 'cats' | 'children' | 'criminals' | 'dogs' | 'gifts' | 'girls' | 'men' | 'mice' | 'people' | 'pizzas' | 'racketeers' | 'sandwiches' | 'senators' | 'students' | 'telescopes' | 'things' | 'unicorns' | 'women'
+
+IV[tense=pres, num=sg, sem=iv] -> 'approaches' | 'comes' | 'disappears' | 'goes' | 'leaves' | 'vanishes' | 'walks' | 'yawns'
+IV[tense=pres, num=pl, sem=iv] -> 'approach' | 'come' | 'disappear' | 'go' | 'leave' | 'vanish' | 'walk' | 'yawn'
+IV[tense=past, num=?n, sem=iv] -> 'approached' | 'came' | 'disappeared' | 'went' | 'left' | 'vanished' | 'walked' | 'yawned'
+IV[tense=inf, num=na, sem=iv] -> 'approach' | 'come' | 'disappear' | 'go' | 'leave' | 'vanish' | 'walk' | 'yawn'
+
+TV[tense=pres, num=sg, sem=tv] -> 'chases' | 'eats' | 'finds' | 'likes' | 'sees' | 'orders'
+TV[tense=pres, num=pl, sem=tv] -> 'chase' | 'eat' | 'find' | 'like' | 'see' | 'order'
+TV[tense=past, num=?n, sem=tv] -> 'chased' | 'ate' | 'found' | 'liked' | 'saw' | 'ordered'
+TV[tense=inf, num=na, sem=tv] -> 'chase' | 'eat' | 'find' | 'like' | 'see' | 'order'
+
+DTV[tense=pres, num=sg, sem=dtv] -> 'gives'
+DTV[tense=pres, num=pl, sem=dtv] -> 'give'
+DTV[tense=past, num=?n, sem=dtv] -> 'gave'
+DTV[tense=inf, num=na, sem=dtv] -> 'give'
+
+TVComp[tense=pres, num=sg, sem=tv_comp] -> 'believes'
+TVComp[tense=pres, num=pl, sem=tv_comp] -> 'believe'
+TVComp[tense=past, num=?n, sem=tv_comp] -> 'believed'
+TVComp[tense=inf, num=na, sem=tv_comp] -> 'believe'
+
+EquiV[tense=pres, num=sg, sem=equi] -> 'tries'
+EquiV[tense=pres, num=pl, sem=equi] -> 'try'
+EquiV[tense=past, num=?n, sem=equi] -> 'tried'
+EquiV[tense=inf, num=na, sem=equi] -> 'try'
+
+ObjEquiV[tense=pres, num=sg, sem=obj_equi] -> 'persuades'
+ObjEquiV[tense=pres, num=pl, sem=obj_equi] -> 'persuade'
+ObjEquiV[tense=past, num=?n, sem=obj_equi] -> 'persuaded'
+ObjEquiV[tense=inf, num=na, sem=obj_equi] -> 'persuade'
+
+RaisingV[tense=pres, num=sg, sem=raising] -> 'seems'
+RaisingV[tense=pres, num=pl, sem=raising] -> 'seem'
+RaisingV[tense=past, num=?n, sem=raising] -> 'seemed'
+RaisingV[tense=inf, num=na, sem=raising] -> 'seem'
+
+#infinitive marker
+TO -> 'to'
+
+JJ[type=attributive, sem=adj_attributive_intersective] -> 'gray' | 'swedish'
+JJ[type=attributive, sem=adj_attributive_nonintersective] -> 'alleged'
+JJ[type=attributive, sem=adj_attributive_relative_intersective] -> 'big' | 'fat'
+JJ[type=attributive, sem=adj_attributive_relative_nonintersective] -> 'confessed' | 'former'
+JJ[type=predicative, sem=adj_predicative] -> 'gray' | 'swedish'
+
+ADV[sem=adv] -> 'apparently' | 'possibly' | 'very'
+ADV[sem=adv_ModifyingRelativeAdj] -> 'very'
+
+CC[sem=cc_clause] -> 'and'
+CC[sem=cc_np] -> 'and'
+CC[sem=cc_vp] -> 'and'
+
+IN -> 'at' | 'by' | 'from' | 'on' | 'with'
+
+Pro[num=sg, gender=m, -reflex, case=nom, sem=pro] -> 'he'
+Pro[num=sg, gender=m, -reflex, case=acc, sem=pro] -> 'him'
+Pro[num=sg, gender=m, +reflex, case=acc, sem=pro] -> 'himself'
+Pro[num=sg, gender=f, -reflex, sem=pro] -> 'her'
+Pro[num=sg, gender=f, +reflex, case=acc, sem=pro] -> 'herself'
+Pro[num=sg, gender=n, -reflex, sem=pro] -> 'it'
+Pro[num=sg, gender=n, +reflex, case=acc, sem=pro] -> 'itself'
+Pro[num=pl, -reflex, case=nom, sem=pro] -> 'they'
+Pro[num=pl, -reflex, case=acc, sem=pro] -> 'them'
+Pro[num=pl, +reflex, case=acc, sem=pro] -> 'themselves'
+Pro[num=pl, +reflex, case=acc, sem=recip] -> 'eachother'
diff --git a/examples/grammars/sample_grammars/hole.fcfg b/examples/grammars/sample_grammars/hole.fcfg
new file mode 100644
index 0000000..b8e58a3
--- /dev/null
+++ b/examples/grammars/sample_grammars/hole.fcfg
@@ -0,0 +1,23 @@
+## Natural Language Toolkit: hole.fcfg
+##
+## Minimal feature-based grammar with lambda semantics for use by the hole.py
+## module for Hole Semantics (see Blackburn and Bos).
+##
+## Author: Dan Garrette <DHGarrette at gmail.com>
+## Robin Cooper <robin.cooper at ling.gu.se>
+## URL: <http://nltk.org>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[SEM=<?subj(?vp)>] -> NP[SEM=?subj] VP[SEM=?vp]
+VP[SEM=?v] -> IV[SEM=?v]
+VP[NUM=?n,SEM=<?v(?obj)>] -> TV[NUM=?n,SEM=?v] NP[SEM=?obj]
+NP[SEM=<?det(?n)>] -> Det[SEM=?det] N[SEM=?n]
+
+Det[SEM=<\P Q h l.exists h1 l1 l2 l3 x.(ALL(l2,x,l3) & IMP(l3,l1,h1) & LEQ(l,h1) & LEQ(l2,h) & P(x)(h)(l1) & Q(x)(h)(l) & HOLE(h) & HOLE(h1) & LABEL(l) & LABEL(l1) & LABEL(l2) & LABEL(l3))>] -> 'every'
+Det[SEM=<\P Q h l.exists h1 l1 l2 l3 x.(EXISTS(l2,x,l3) & AND(l3,l1,h1) & LEQ(l,h1) & LEQ(l2,h) & P(x)(h)(l1) & Q(x)(h)(l) & HOLE(h) & HOLE(h1) & LABEL(l) & LABEL(l1) & LABEL(l2) & LABEL(l3))>] -> 'a'
+N[SEM=<\x h l.(PRED(l,girl,x) & LEQ(l,h) & HOLE(h) & LABEL(l))>] -> 'girl'
+N[SEM=<\x h l.(PRED(l,dog,x) & LEQ(l,h) & HOLE(h) & LABEL(l))>] -> 'dog'
+IV[SEM=<\x h l.(PRED(l,bark,x) & LEQ(l,h) & HOLE(h) & LABEL(l))>] -> 'barks'
+TV[SEM=<\P x.P(\y h l.(PRED(l,chase,x,y) & LEQ(l,h) & HOLE(h) & LABEL(l)))>] -> 'chases'
diff --git a/examples/grammars/sample_grammars/np.fcfg b/examples/grammars/sample_grammars/np.fcfg
new file mode 100644
index 0000000..ddd2a39
--- /dev/null
+++ b/examples/grammars/sample_grammars/np.fcfg
@@ -0,0 +1,12 @@
+% start NP
+NP[AGR=?a] -> Det[AGR=?a] N[AGR=?a]
+Det[AGR=[NUM='sg', PER=3]] -> 'this' | 'that'
+Det[AGR=[NUM='pl', PER=3]] -> 'these' | 'those'
+Det[AGR=[NUM='pl', PER=1]] -> 'we'
+Det[AGR=[PER=2]] -> 'you'
+N[AGR=[NUM='sg', GND='m']] -> 'boy'
+N[AGR=[NUM='pl', GND='m']] -> 'boys'
+N[AGR=[NUM='sg', GND='f']] -> 'girl'
+N[AGR=[NUM='pl', GND='f']] -> 'girls'
+N[AGR=[NUM='sg']] -> 'student'
+N[AGR=[NUM='pl']] -> 'students'
diff --git a/examples/grammars/sample_grammars/sem0.fcfg b/examples/grammars/sample_grammars/sem0.fcfg
new file mode 100644
index 0000000..9bc4ef4
--- /dev/null
+++ b/examples/grammars/sample_grammars/sem0.fcfg
@@ -0,0 +1,14 @@
+## Natural Language Toolkit: sem0.fcfg
+##
+## Minimal feature-based grammar with lambda semantics.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[SEM=<?vp(?subj)>] -> NP[SEM=?subj] VP[SEM=?vp]
+VP[SEM=?v] -> V[SEM=?v]
+NP[SEM=<cyril>] -> 'Cyril'
+V[SEM=<\x.bark(x)>] -> 'barks'
diff --git a/examples/grammars/sample_grammars/sem1.fcfg b/examples/grammars/sample_grammars/sem1.fcfg
new file mode 100644
index 0000000..401092b
--- /dev/null
+++ b/examples/grammars/sample_grammars/sem1.fcfg
@@ -0,0 +1,19 @@
+## Natural Language Toolkit: sem1.fcfg
+##
+## Minimal feature-based grammar to illustrate the interpretation of
+## determiner phrases.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[SEM = <?subj(?vp)>] -> NP[SEM=?subj] VP[SEM=?vp]
+VP[SEM=?v] -> IV[SEM=?v]
+NP[SEM=<?det(?n)>] -> Det[SEM=?det] N[SEM=?n]
+
+Det[SEM=<\Q P.exists x.(Q(x) & P(x))>] -> 'a'
+Det[SEM=<\Q P.all x.(Q(x) -> P(x))>] -> 'every'
+N[SEM=<\x.dog(x)>] -> 'dog'
+IV[SEM=<\x.bark(x)>] -> 'barks'
diff --git a/examples/grammars/sample_grammars/sem2.fcfg b/examples/grammars/sample_grammars/sem2.fcfg
new file mode 100644
index 0000000..1bd4c98
--- /dev/null
+++ b/examples/grammars/sample_grammars/sem2.fcfg
@@ -0,0 +1,68 @@
+## Natural Language Toolkit: sem2.fcfg
+##
+## Longer feature-based grammar with more quantifers, and illustrating
+## transitive verbs and prepositional phrases (PPs). The
+## interpretation of PPs is a bit weird and could do with further
+## work.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+############################
+# Grammar Rules
+#############################
+
+S[SEM = <?subj(?vp)>] -> NP[NUM=?n,SEM=?subj] VP[NUM=?n,SEM=?vp]
+
+NP[NUM=?n,SEM=<?det(?nom)> ] -> Det[NUM=?n,SEM=?det] Nom[NUM=?n,SEM=?nom]
+NP[LOC=?l,NUM=?n,SEM=?np] -> PropN[LOC=?l,NUM=?n,SEM=?np]
+
+Nom[NUM=?n,SEM=?nom] -> N[NUM=?n,SEM=?nom]
+Nom[NUM=?n,SEM=<?pp(?nom)>] -> N[NUM=?n,SEM=?nom] PP[SEM=?pp]
+
+VP[NUM=?n,SEM=<?v(?obj)>] -> TV[NUM=?n,SEM=?v] NP[SEM=?obj]
+VP[NUM=?n,SEM=?v] -> IV[NUM=?n,SEM=?v]
+
+VP[NUM=?n,SEM=<?pp(?vp)>] -> VP[NUM=?n,SEM=?vp] PP[SEM=?pp]
+
+PP[SEM=<?p(?np)>] -> P[LOC=?l,SEM=?p] NP[LOC=?l,SEM=?np]
+
+#############################
+# Lexical Rules
+#############################
+
+PropN[-LOC,NUM=sg,SEM=<\P.P(john)>] -> 'John'
+PropN[-LOC,NUM=sg,SEM=<\P.P(mary)>] -> 'Mary'
+PropN[-LOC,NUM=sg,SEM=<\P.P(suzie)>] -> 'Suzie'
+PropN[-LOC,NUM=sg,SEM=<\P.P(fido)>] -> 'Fido'
+PropN[+LOC, NUM=sg,SEM=<\P.P(noosa)>] -> 'Noosa'
+
+NP[-LOC, NUM=sg, SEM=<\P.\x.P(x)>] -> 'who'
+
+Det[NUM=sg,SEM=<\P Q.all x.(P(x) -> Q(x))>] -> 'every'
+Det[NUM=pl,SEM=<\P Q.all x.(P(x) -> Q(x))>] -> 'all'
+Det[SEM=<\P Q.exists x.(P(x) & Q(x))>] -> 'some'
+Det[NUM=sg,SEM=<\P Q.exists x.(P(x) & Q(x))>] -> 'a'
+
+N[NUM=sg,SEM=<\x.boy(x)>] -> 'boy'
+N[NUM=pl,SEM=<\x.boy(x)>] -> 'boys'
+N[NUM=sg,SEM=<\x.girl(x)>] -> 'girl'
+N[NUM=pl,SEM=<\x.girl(x)>] -> 'girls'
+N[NUM=sg,SEM=<\x.dog(x)>] -> 'dog'
+N[NUM=pl,SEM=<\x.dog(x)>] -> 'dogs'
+
+TV[NUM=sg,SEM=<\X y.X(\x.chase(y,x))>,TNS=pres] -> 'chases'
+TV[NUM=pl,SEM=<\X y.X(\x.chase(y,x))>,TNS=pres] -> 'chase'
+TV[NUM=sg,SEM=<\X y.X(\x.see(y,x))>,TNS=pres] -> 'sees'
+TV[NUM=pl,SEM=<\X y.X(\x.see(y,x))>,TNS=pres] -> 'see'
+TV[NUM=sg,SEM=<\X y.X(\x.chase(y,x))>,TNS=pres] -> 'chases'
+TV[NUM=pl,SEM=<\X y.X(\x.chase(y,x))>,TNS=pres] -> 'chase'
+IV[NUM=sg,SEM=<\x.bark(x)>,TNS=pres] -> 'barks'
+IV[NUM=pl,SEM=<\x.bark(x)>,TNS=pres] -> 'bark'
+IV[NUM=sg,SEM=<\x.walk(x)>,TNS=pres] -> 'walks'
+IV[NUM=pl,SEM=<\x.walk(x)>,TNS=pres] -> 'walk'
+
+P[+LOC,SEM=<\X P x.X(\y.(P(x) & in(x,y)))>] -> 'in'
+P[-LOC,SEM=<\X P x.X(\y.(P(x) & with(x,y)))>] -> 'with'
diff --git a/examples/grammars/sample_grammars/sql.fcfg b/examples/grammars/sample_grammars/sql.fcfg
new file mode 100644
index 0000000..abdd830
--- /dev/null
+++ b/examples/grammars/sample_grammars/sql.fcfg
@@ -0,0 +1,27 @@
+## Natural Language Toolkit: sql.fcfg
+##
+## Deliberately naive string-based grammar for
+## deriving SQL queries from English
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[sem=(?np + ?vp)] -> NP[sem=?np] VP[sem=?vp]
+
+VP[sem=(?v + ?pp)] -> IV[sem=?v] PP[sem=?pp]
+VP[sem=(?v + ?np)] -> TV[sem=?v] NP[sem=?np]
+
+NP[sem=(?det + ?n)] -> Det[sem=?det] N[sem=?n]
+NP[sem='Country="japan"'] -> 'Japan'
+NP[sem='Country="united_states"'] -> 'USA'
+
+Det[sem='SELECT'] -> 'Which'
+N[sem='City FROM city_table'] -> 'cities'
+
+IV[sem='WHERE'] -> 'are'
+PP[sem=?np] -> P[sem=?p] NP[sem=?np]
+P -> 'in'
+
diff --git a/examples/grammars/sample_grammars/toy.cfg b/examples/grammars/sample_grammars/toy.cfg
new file mode 100644
index 0000000..0977292
--- /dev/null
+++ b/examples/grammars/sample_grammars/toy.cfg
@@ -0,0 +1,9 @@
+S -> NP VP
+PP -> P NP
+NP -> Det N | NP PP
+VP -> V NP | VP PP
+Det -> 'a' | 'the'
+N -> 'dog' | 'cat'
+V -> 'chased' | 'sat'
+P -> 'on' | 'in'
+
diff --git a/examples/grammars/sample_grammars/valuation1.val b/examples/grammars/sample_grammars/valuation1.val
new file mode 100644
index 0000000..a66dae5
--- /dev/null
+++ b/examples/grammars/sample_grammars/valuation1.val
@@ -0,0 +1,15 @@
+john => b1
+mary => g1
+suzie => g2
+fido => d1
+tess => d2
+noosa => n
+girl => {g1, g2}
+boy => {b1, b2}
+dog => {d1, d2}
+bark => {d1, d2}
+walk => {b1, g2, d1}
+chase => {(b1, g1), (b2, g1), (g1, d1), (g2, d2)}
+see => {(b1, g1), (b2, d2), (g1, b1),(d2, b1), (g2, n)}
+in => {(b1, n), (b2, n), (d2, n)}
+with => {(b1, g1), (g1, b1), (d1, b1), (b1, d1)}
diff --git a/examples/grammars/spanish_grammars/spanish1.cfg b/examples/grammars/spanish_grammars/spanish1.cfg
new file mode 100755
index 0000000..581c73a
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish1.cfg
@@ -0,0 +1,10 @@
+ S -> SN SV
+ SV -> v SN
+ SV -> v
+ SN -> det GN
+ GN -> nom_com
+ GN -> nom_prop
+ det -> "el" | "la" | "los" | "las" | "un" | "una" | "unos" | "unas"
+ nom_com -> "vecino" | "ladrones" | "mujeres" | "bosques" | "noche" | "flauta" | "ventana"
+ nom_prop -> "Jose" | "Lucas" | "Pedro" | "Marta"
+ v -> "toca" | "moja" | "adoran" | "robaron" | "escondieron" | "rompió"
diff --git a/examples/grammars/spanish_grammars/spanish1.fcfg b/examples/grammars/spanish_grammars/spanish1.fcfg
new file mode 100755
index 0000000..6601c39
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish1.fcfg
@@ -0,0 +1,32 @@
+% start S
+# ############################
+# Grammar Rules
+# ############################
+S -> SN[num=?n,gen=?g] SV[num=?n,tiempo=?t]
+SN[num=?n,gen=?g,+PROP] -> NP[num=?n]
+SN[num=?n,gen=?g,-PROP] -> DET[num=?n,gen=?g] NC[num=?n,gen=?g]
+SN[num=plural,gen=?g,-PROP] -> DET[num=plural,gen=?g] NC[num=plural,gen=?g]
+SV[tiempo=?t,num=?n] -> VI[tiempo=?t,num=?n]
+SV[tiempo=?t,num=?n] -> VT[tiempo=?t,num=?n] SN[-PROP]
+SV[tiempo=?t,num=?n] -> VT[tiempo=?t,num=?n] PREP SN
+# ############################
+# Lexical Rules
+# ############################
+DET[num=singular,gen=masculino] -> 'un' | 'el'
+DET[num=singular,gen=femenino] -> 'una' | 'la'
+DET[num=plural,gen=masculino] -> 'unos' | 'los'
+DET[num=plural,gen=femenino] -> 'unas' | 'las'
+PREP -> 'a'
+NP[num=singular] -> 'Miguel' | 'Sara' | 'Pedro'
+NC[num=singular,gen=masculino] -> 'perro' | 'gato' | 'vecino' | 'profesor'
+NC[num=singular,gen=femenino] -> 'perra' | 'gata' | 'vecina' | 'profesora'
+NC[num=plural,gen=masculino] -> 'perros' | 'gatos' | 'vecinos' | 'profesores'
+NC[num=plural,gen=femenino] -> 'perras' | 'gatas' | 'vecinas' | 'profesoras'
+VI[tiempo=pasado,num=singular] -> 'desaparecio' | 'anduvo' | 'murio'
+VI[tiempo=presente,num=singular] -> 'desaparece' | 'anda' | 'muere'
+VI[tiempo=pasado,num=plural] -> 'desaparecion' | 'anduvieron' | 'murieron'
+VI[tiempo=presente,num=plural] -> 'desaparecen' | 'andan' | 'mueren'
+VT[tiempo=pasado,num=singular] -> 'vio' | 'adoró' | 'gritó' | 'odio'
+VT[tiempo=presente,num=singular] -> 've' | 'adora' | 'grita' | 'odia'
+VT[tiempo=pasado,num=plural] -> 'vieron' | 'adoraron' | 'gritaron' | 'odiaron'
+VT[tiempo=presente,num=plural] -> 'ven' | 'adoran' | 'gritan' | 'odian'
diff --git a/examples/grammars/spanish_grammars/spanish1.pcfg b/examples/grammars/spanish_grammars/spanish1.pcfg
new file mode 100755
index 0000000..490f6ab
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish1.pcfg
@@ -0,0 +1,9 @@
+ S -> SN SV [1.0]
+ SV -> VTrans SN [0.4]
+ SV -> VIntrans [0.3]
+ SV -> VSupl SN SN [0.3]
+ VTrans -> "bebió" [1.0]
+ VIntrans -> "murió" [1.0]
+ VSupl -> "regaló" [1.0]
+ SN -> "flores" [0.6]
+ SN -> "agua" [0.4]
diff --git a/examples/grammars/spanish_grammars/spanish1.regexp b/examples/grammars/spanish_grammars/spanish1.regexp
new file mode 100755
index 0000000..cb08d09
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish1.regexp
@@ -0,0 +1,2 @@
+
+NP: {<d.*>*<n.*>+<a.*>*} # busca determinantes y adjetivos que acompañen a nombres
diff --git a/examples/grammars/spanish_grammars/spanish2.cfg b/examples/grammars/spanish_grammars/spanish2.cfg
new file mode 100755
index 0000000..e67359b
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish2.cfg
@@ -0,0 +1,8 @@
+ S -> SN SV
+ SP -> P SN
+ SN -> Det N | SN SP
+ SV -> V SN | SV SP
+ Det -> "el" | "la" | "un" | "una" | "los" | "las"
+ N -> "tren" | "telescopio" | "noticia" | "mesa" | "hombre" | "casa" | "amiga"
+ V -> "vio" | "leí" | "encontró"
+ P -> "en" | "sobre" | "con" | "de" | "a"
diff --git a/examples/grammars/spanish_grammars/spanish2.fcfg b/examples/grammars/spanish_grammars/spanish2.fcfg
new file mode 100755
index 0000000..b3b3404
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish2.fcfg
@@ -0,0 +1,17 @@
+% start S
+# ############################
+# Grammar Rules
+# ############################
+S -> SN S/SN
+S/?x -> SV/?x
+S/?x -> V[+aux] COMP SV/?x
+SN/SN ->
+SV/?x -> V[-aux] SN/?x
+# ############################
+# Lexical Rules
+# ############################
+V[-aux] -> 'adoras' | 'odias'
+V[+aux] -> 'dices'
+
+SN -> 'quien' | 'que'
+COMP -> 'que'
diff --git a/examples/grammars/spanish_grammars/spanish2.pcfg b/examples/grammars/spanish_grammars/spanish2.pcfg
new file mode 100755
index 0000000..93b9f63
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish2.pcfg
@@ -0,0 +1,4 @@
+ SN -> N [0.5]| N Adj [0.3]| SN Conj SN [0.2]
+ N -> 'hombres' [0.1]| 'mujeres' [0.2]| 'niños' [0.3]| N Conj N [0.4]
+ Adj -> 'mayores' [0.3]| 'jovenes' [0.7]
+ Conj -> 'y' [0.6]| 'o' [0.3] | 'e' [0.1]
diff --git a/examples/grammars/spanish_grammars/spanish2.regexp b/examples/grammars/spanish_grammars/spanish2.regexp
new file mode 100755
index 0000000..01275cb
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish2.regexp
@@ -0,0 +1,4 @@
+
+ NP: {<d.*>*<n.*><a.*>*} # Busca det + nombre + adjetivo
+ NP: {<d.*>*<n.*>+} # Busca seguidas de nombres
+
diff --git a/examples/grammars/spanish_grammars/spanish3.cfg b/examples/grammars/spanish_grammars/spanish3.cfg
new file mode 100755
index 0000000..81665bb
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish3.cfg
@@ -0,0 +1,4 @@
+ SN -> N | N Adj | SN Conj SN
+ N -> 'hombres' | 'mujeres' | 'niños' | N Conj N
+ Adj -> 'mayores' | 'jovenes'
+ Conj -> 'y' | 'o' | 'e'
diff --git a/examples/grammars/spanish_grammars/spanish3.regexp b/examples/grammars/spanish_grammars/spanish3.regexp
new file mode 100755
index 0000000..b5a5d11
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish3.regexp
@@ -0,0 +1,4 @@
+
+ SN:
+ {<.*>+} # Crea Un Chunk Con Cualquier Cosa
+ }<v.*|sp.*|F.*>+{ # Considerar Como Chink Apariciones De Verbos (v.*), Preposiciones (sp.*) y Signos De Puntuación (F.*)
diff --git a/examples/grammars/spanish_grammars/spanish4.regexp b/examples/grammars/spanish_grammars/spanish4.regexp
new file mode 100755
index 0000000..aee335d
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish4.regexp
@@ -0,0 +1,5 @@
+
+ SN: {<d.*>?<n.*>+<a.*>*} # noun phrase chunks
+ SV: {<p.*>?<v.*>} # verb phrase chunks
+ SP: {<sp.*>} # prepositional phrase chunks
+
diff --git a/examples/grammars/spanish_grammars/spanish5.regexp b/examples/grammars/spanish_grammars/spanish5.regexp
new file mode 100755
index 0000000..7df465b
--- /dev/null
+++ b/examples/grammars/spanish_grammars/spanish5.regexp
@@ -0,0 +1,6 @@
+
+ SN: {<d.*>?<n.*>+<a.*>*} # noun phrase chunks
+ SV: {<p.*>?<v.*>+<SN|sp.*|S>*} # verb phrase chunks
+ SP: {<sp.*>} # prepositional phrase chunks
+ S: {<SN><SV>} # Chunk NP, VP
+
diff --git a/examples/school/README b/examples/school/README
new file mode 100644
index 0000000..21cfc9f
--- /dev/null
+++ b/examples/school/README
@@ -0,0 +1,3 @@
+The files in this directory were created for teaching computational
+linguistics in secondary school English classes. For instructions
+and lesson plans, please see http://nltk.org/index.php/Electronic_Grammar
diff --git a/examples/school/categories.py b/examples/school/categories.py
new file mode 100644
index 0000000..72352ae
--- /dev/null
+++ b/examples/school/categories.py
@@ -0,0 +1,222 @@
+from __future__ import print_function
+
+from words import *
+from nltk.wordnet import *
+from operator import itemgetter
+import nltk
+import re
+from string import join
+
+def build_word_associations():
+ cfd = nltk.ConditionalFreqDist()
+
+ # get a list of all English stop words
+ stopwords_list = nltk.corpus.stopwords.words('english')
+
+ # count words that occur within a window of size 5 ahead of other words
+ for sentence in nltk.corpus.brown.tagged_sents():
+ sentence = [(token.lower(), tag) for (token, tag) in sentence if token.lower() not in stopwords_list]
+ for (index, (token, tag)) in enumerate(sentence):
+ if token not in stopwords_list:
+ window = sentence[index+1:index+5]
+ for (window_token, window_tag) in window:
+ if window_token not in stopwords_list and window_tag[0] is tag[0]:
+ cfd[token].inc(window_token)
+ return cfd
+
+def associate():
+ while True:
+ word = raw_input("Enter a word: ")
+ for i in range(100):
+ next = cfd[word].max()
+ if next:
+ print("->", next,)
+ word = next
+ else:
+ break
+ print()
+
+def build_word_contexts(words):
+ contexts_to_words = {}
+ words = [w.lower() for w in words]
+ for i in range(1,len(words)-1):
+ context = words[i-1]+"_"+words[i+1]
+ if context not in contexts_to_words:
+ contexts_to_words[context] = []
+ contexts_to_words[context].append(words[i])
+ # inverted structure, tracking frequency
+ words_to_contexts = {}
+ for context in contexts_to_words:
+ for word in contexts_to_words[context]:
+ if word not in words_to_contexts:
+ words_to_contexts[word] = []
+ words_to_contexts[word].append(context)
+ return words_to_contexts, contexts_to_words
+
+def search_contexts(words):
+ words_to_contexts, contexts_to_words = build_word_contexts(words)
+ while True:
+ hits = []
+ word = raw_input("word> ")
+ if word not in words_to_contexts:
+ print("Word not found")
+ continue
+ contexts = words_to_contexts[word]
+ for w in words_to_contexts: # all words
+ for context in words_to_contexts[w]:
+ if context in contexts:
+ hits.append(w)
+ hit_freqs = count_words(hits).items()
+ sorted_hits = sorted(hit_freqs, key=itemgetter(1), reverse=True)
+ words = [word for (word, count) in sorted_hits[1:] if count > 1]
+ print(join(words))
+
+def lookup(word):
+ for category in [N, V, ADJ, ADV]:
+ if word in category:
+ for synset in category[word]:
+ print(category[word], ":", synset.gloss)
+
+############################################
+# Simple Tagger
+############################################
+
+# map brown pos tags
+# http://khnt.hit.uib.no/icame/manuals/brown/INDEX.HTM
+
+def map1(tag):
+ tag = re.sub(r'fw-', '', tag) # foreign words
+ tag = re.sub(r'-[th]l', '', tag) # headlines, titles
+ tag = re.sub(r'-nc', '', tag) # cited
+ tag = re.sub(r'ber?', 'vb', tag) # verb "to be"
+ tag = re.sub(r'hv', 'vb', tag) # verb "to have"
+ tag = re.sub(r'do', 'vb', tag) # verb "to do"
+ tag = re.sub(r'nc', 'nn', tag) # cited word
+ tag = re.sub(r'z', '', tag) # third-person singular
+ return tag
+
+def map2(tag):
+ tag = re.sub(r'\bj[^-+]*', 'J', tag) # adjectives
+ tag = re.sub(r'\bp[^-+]*', 'P', tag) # pronouns
+ tag = re.sub(r'\bm[^-+]*', 'M', tag) # modals
+ tag = re.sub(r'\bq[^-+]*', 'Q', tag) # qualifiers
+ tag = re.sub(r'\babl', 'Q', tag) # qualifiers
+ tag = re.sub(r'\bab[nx]', 'D', tag) # determiners
+ tag = re.sub(r'\bap', 'D', tag) # determiners
+ tag = re.sub(r'\bd[^-+]*', 'D', tag) # determiners
+ tag = re.sub(r'\bat', 'D', tag) # determiners
+ tag = re.sub(r'\bw[^-+]*', 'W', tag) # wh words
+ tag = re.sub(r'\br[^-+]*', 'R', tag) # adverbs
+ tag = re.sub(r'\bto', 'T', tag) # "to"
+ tag = re.sub(r'\bc[cs]', 'C', tag) # conjunctions
+ tag = re.sub(r's', '', tag) # plurals
+ tag = re.sub(r'\bin', 'I', tag) # prepositions
+ tag = re.sub(r'\buh', 'U', tag) # interjections (uh)
+ tag = re.sub(r'\bex', 'E', tag) # existential "there"
+ tag = re.sub(r'\bvbn', 'VN', tag) # past participle
+ tag = re.sub(r'\bvbd', 'VD', tag) # past tense
+ tag = re.sub(r'\bvbg', 'VG', tag) # gerund
+ tag = re.sub(r'\bvb', 'V', tag) # verb
+ tag = re.sub(r'\bnn', 'N', tag) # noun
+ tag = re.sub(r'\bnp', 'NP', tag) # proper noun
+ tag = re.sub(r'\bnr', 'NR', tag) # adverbial noun
+ tag = re.sub(r'\bex', 'E', tag) # existential "there"
+ tag = re.sub(r'\bod', 'OD', tag) # ordinal
+ tag = re.sub(r'\bcd', 'CD', tag) # cardinal
+ tag = re.sub(r'-t', '', tag) # misc
+ tag = re.sub(r'[a-z\*]', '', tag) # misc
+ return tag
+
+def map(tag):
+ return map2(map1(tag.lower()))
+
+# print(sorted(set(map2(map1(tag)) for s in brown.tagged() for w,tag in s)))
+
+def load_brown_corpus(sections):
+ global map
+ corpus = nltk.corpus.brown.tagged_sents(tuple(sections))
+ return [[(w.lower(), map(t)) for w, t in sent] for sent in corpus]
+
+def train_tagger(corpus):
+ t0 = nltk.tag.Default('N')
+ t1 = nltk.tag.Unigram(cutoff=0, backoff=t0)
+ t2 = nltk.tag.Bigram(cutoff=0, backoff=t1)
+ t3 = nltk.tag.Trigram(cutoff=1, backoff=t2)
+
+ t1.train(corpus, verbose=True)
+ t2.train(corpus, verbose=True)
+ t3.train(corpus, verbose=True)
+ return t3
+
+def tag(corpus):
+ print("Training tagger...")
+ tagger = train_tagger(corpus)
+ while True:
+ text = raw_input("sentence> ")
+ words = text.split()
+ print(join(word+"/"+tag for word, tag in tagger.tag(words)))
+
+WORD_OR_TAG = '[^/ ]+'
+BOUNDARY = r'\b'
+
+def process(pattern):
+ new = []
+ for term in pattern.split():
+ if re.match('[A-Z]+$', term):
+ new.append(BOUNDARY + WORD_OR_TAG + '/' + term + BOUNDARY)
+ elif '/' in term:
+ new.append(BOUNDARY + term + BOUNDARY)
+ else:
+ new.append(BOUNDARY + term + '/' + WORD_OR_TAG + BOUNDARY)
+ return join(new)
+
+def search(corpus, num=25):
+ print("Loading corpus...")
+ strings = [join(w+'/'+t for (w,t) in sent) for sent in corpus]
+ while True:
+ pattern = ""
+ while not pattern:
+ pattern = raw_input("search> ")
+ pattern = process(pattern)
+ i = 0
+ for sent in strings:
+ m = re.search(pattern, sent)
+ if m:
+ sent = ' '*35 + sent + ' '*45
+ print(sent[m.start():m.start()+80])
+ i += 1
+ if i > num:
+ break
+
+############################################
+# Wordnet Browser
+# now incorporated into NLTK as wordnet.browse
+############################################
+
+############################################
+# Mad Libs
+############################################
+
+madlib = """Britney Spears will meet up with her %(NP)s label for
+crisis talks about the future of her %(N)s this week reports Digital Spy.
+%(NP)s Records plan to tell Spears to stop %(VG)s and take more
+care of her %(J)s image if she wants to avoid being %(VD)s by the noun.
+The news %(V)s shortly after Britney posted a message on her
+website promising a new album and tour. The last couple of years
+have been quite a ride for me, the media has criticized %(P)s every
+noun %(C)s printed a skewed perception of who I really am as a human
+being, she wrote in a letter posted %(NR)s."""
+
+# mapping = {}
+# mapping['NP'] =
+# mapping['N'] =
+# mapping['VG'] =
+# mapping['J'] =
+# mapping['VD'] =
+# mapping['V'] =
+# mapping['P'] =
+# mapping['C'] =
+# mapping['NR'] =
+
+# print(madlib % mapping)
+
diff --git a/examples/school/count.py b/examples/school/count.py
new file mode 100644
index 0000000..9f89a09
--- /dev/null
+++ b/examples/school/count.py
@@ -0,0 +1,16 @@
+from words import *
+words = read_words('corpus/telephone.txt')
+counts = count_words(words)
+print_freq(counts)
+
+
+
+
+from words import *
+words = read_words('corpus/rural.txt')
+counts = count_pairs(words)
+print_freq(counts)
+
+
+
+
diff --git a/examples/school/generate.py b/examples/school/generate.py
new file mode 100644
index 0000000..37a9be8
--- /dev/null
+++ b/examples/school/generate.py
@@ -0,0 +1,14 @@
+from words import *
+
+telephone_words = read_words('corpus/telephone.txt')
+model = train(telephone_words)
+generate(model)
+
+
+
+
+
+
+
+
+
diff --git a/examples/school/parse1.py b/examples/school/parse1.py
new file mode 100644
index 0000000..4966b57
--- /dev/null
+++ b/examples/school/parse1.py
@@ -0,0 +1,14 @@
+from parser import *
+
+grammar = """
+ NP -> P | D J N
+ D -> 'a'
+ J -> 'red' | 'green'
+ N -> 'chair' | 'house'
+"""
+
+phrase = 'a red chair'
+
+parse_draw(phrase, grammar)
+
+
diff --git a/examples/school/parse2.py b/examples/school/parse2.py
new file mode 100644
index 0000000..c3498b2
--- /dev/null
+++ b/examples/school/parse2.py
@@ -0,0 +1,17 @@
+from parser import *
+
+grammar = """
+ S -> NP VP | VP
+ VP -> V NP | VP PP
+ NP -> Det N | NP PP
+ PP -> P NP
+ NP -> 'I'
+ Det -> 'the' | 'my'
+ N -> 'elephant' | 'pajamas'
+ V -> 'shot'
+ P -> 'in'
+"""
+
+sent = 'I shot the elephant in my pajamas'
+parse_draw(sent, grammar)
+
diff --git a/examples/school/parse3.py b/examples/school/parse3.py
new file mode 100644
index 0000000..6eebacd
--- /dev/null
+++ b/examples/school/parse3.py
@@ -0,0 +1,22 @@
+from parser import *
+
+grammar = """
+ S -> NP VP | VP
+ PP -> P NP
+ NP -> N | Det N | N N | NP PP | N VP
+ VP -> V | V NP | VP PP | VP ADVP
+ ADVP -> ADV NP
+ Det -> 'a' | 'an' | 'the'
+ N -> 'flies' | 'banana' | 'fruit' | 'arrow' | 'time'
+ V -> 'like' | 'flies' | 'time'
+ P -> 'on' | 'in' | 'by'
+ ADV -> 'like'
+"""
+
+sent = 'time flies like an arrow'
+
+parse_draw(sent, grammar)
+
+
+
+
diff --git a/examples/school/parser.py b/examples/school/parser.py
new file mode 100644
index 0000000..51a6921
--- /dev/null
+++ b/examples/school/parser.py
@@ -0,0 +1,18 @@
+from __future__ import print_function
+
+import nltk
+
+def parse(sent, grammar):
+ gr = nltk.parse_cfg(grammar)
+ parser = nltk.parse.ChartParse(gr, nltk.parse.TD_STRATEGY)
+ return parser.get_parse_list(sent.split())
+
+def parse_draw(sent, grammar):
+ trees = parse(sent, grammar)
+ nltk.draw.draw_trees(*trees)
+
+def parse_print(sent, grammar):
+ trees = parse(sent, grammar)
+ for tree in trees:
+ print(tree)
+
diff --git a/examples/school/search.py b/examples/school/search.py
new file mode 100644
index 0000000..e716e0c
--- /dev/null
+++ b/examples/school/search.py
@@ -0,0 +1,6 @@
+from words import *
+words = read_text('corpus/telephone.txt')
+concordance(" um", words)
+
+
+
diff --git a/examples/school/words.py b/examples/school/words.py
new file mode 100644
index 0000000..bf36394
--- /dev/null
+++ b/examples/school/words.py
@@ -0,0 +1,105 @@
+from __future__ import print_function
+
+import re, random
+
+from collections import defaultdict
+
+###############################################################################
+### FILE ACCESS
+###############################################################################
+
+def read_words(filename):
+ "Get the words out of the file, ignoring case and punctuation."
+ text = open(filename).read().lower()
+ return re.split('\W+', text)
+
+def read_text(filename):
+ "Load the file into a text string, normalising whitespace."
+ text = open(filename).read()
+ return re.sub('\s+', ' ', text)
+
+###############################################################################
+### SEARCHING
+###############################################################################
+
+def print_conc(pattern, text, num=25):
+ "Print segments of the file that match the pattern."
+ for i in range(num):
+ m = re.search(pattern, text)
+ if not m:
+ break
+ print(text[m.start()-30:m.start()+40])
+ text = text[m.start()+1:]
+
+###############################################################################
+### COUNTING
+###############################################################################
+
+def count_words(words):
+ "Count the number of times each word has appeared."
+ wordcounts = {}
+ for word in words:
+ if word not in wordcounts:
+ wordcounts[word] = 0
+ wordcounts[word] += 1
+ return wordcounts
+
+def print_freq(counts, num=25):
+ "Print the words and their counts, in order of decreasing frequency."
+ from operator import itemgetter
+ total = sum(counts.values())
+ cumulative = 0.0
+ sorted_word_counts = sorted(counts.items(), key=itemgetter(1), reverse=True)
+ for i in range(num):
+ word, count = sorted_word_counts[i]
+ cumulative += count * 100.0 / total
+ print("%3d %3.2d%% %s" % (i, cumulative, word))
+
+###############################################################################
+### COLLOCATIONS
+###############################################################################
+
+def count_pairs(words, num=50):
+ "Print the frequent bigrams, omitting short words"
+ paircounts = {}
+ for i in range(len(words)-1):
+ if len(words[i]) > 4 and len(words[i+1]) > 4:
+ pair = words[i] + ' ' + words[i+1]
+ if pair not in paircounts:
+ paircounts[pair] = 0
+ paircounts[pair] += 1
+ return paircounts
+
+###############################################################################
+### RANDOM TEXT GENERATION
+###############################################################################
+
+def train(words):
+ prev1 = ''
+ prev2 = ''
+ model = defaultdict(list)
+ for word in words:
+ key = (prev1, prev2)
+ if word not in model[key]:
+ model[key].append(word)
+ model[prev2].append(word)
+ prev2 = prev1
+ prev1 = word
+ return model
+
+def generate(model, num=100):
+ prev2 = ''
+ prev1 = ''
+ for i in range(num):
+ next = model[(prev1,prev2)]
+ if next:
+ word = random.choice(next)
+ else:
+ word = random.choice(model[prev2])
+ print(word, end='')
+ prev2 = prev1
+ prev1 = word
+ print()
+
+
+
diff --git a/examples/semantics/chat.db b/examples/semantics/chat.db
new file mode 100644
index 0000000..c80316f
Binary files /dev/null and b/examples/semantics/chat.db differ
diff --git a/examples/semantics/chat80.cfg b/examples/semantics/chat80.cfg
new file mode 100644
index 0000000..401451e
--- /dev/null
+++ b/examples/semantics/chat80.cfg
@@ -0,0 +1,96 @@
+## Natural Language Toolkit: chat80.cfg
+##
+##
+## Grammar used to illustrate querying the Chat-80 database.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+# ###########################
+# Grammar Rules
+# ############################
+
+S[sem=<app(?subj,?vp)>] -> NP[-pred,num=?n,sem=?subj] VP[num=?n,sem=?vp]
+
+Rel[num=?n,sem=<app(?comp,?vp)>] -> Comp[sem=?comp] VP[num=?n,sem=?vp]
+
+NP[-pred, num=pl,sem=<(\P Q. some x. ((Q x) and (P x)) ?nom)>] -> Nom[num=pl,sem=?nom]
+NP[wh=?wh,-pred,num=?n,sem=<app(?det,?nom)>] -> Det[wh=?wh, num=?n,sem=?det] Nom[num=?n,sem=?nom]
+
+
+NP[+pred,num=sg,sem=?nom] -> Det[num=sg,sem=?det] Nom[num=sg,sem=?nom]
+NP[+pred,num=pl,sem=?nom] -> Nom[num=pl,sem=?nom]
+
+NP[loc=?l,num=?n,sem=?np] -> PropN[loc=?l,num=?n,sem=?np]
+
+Nom[num=?n,sem=?nom] -> N[num=?n,sem=?nom]
+Nom[num=sg,sem=<app(?pp,?nom)>] -> N[subcat=11,num=sg,sem=?nom] PP[pform=of,sem=?pp]
+Nom[num=?n,sem=<app(?mod,?nom)>] -> Nom[num=?n,sem=?nom] Rel[num=?n,sem=?mod]
+Nom[num=?n,sem=<app(?adj,?nom)>] -> A[sem=?adj] Nom[num=?n,sem=?nom]
+
+##VP[num=?n,sem=?v] -> V[subcat=1,num=?n,sem=?v]
+VP[num=?n,sem=<app(?v,?obj)>] -> V[subcat=2, num=?n,sem=?v] NP[-pred,sem=?obj]
+VP[num=?n,sem=<app(?v,?pred)>] -> V[subcat=3, num=?n,sem=?v] NP[+pred,sem=?pred]
+
+PP[pform=?pf,sem=<app(?p,?np)>] -> P[pform=?pf, loc=?l,sem=?p] NP[loc=?l,sem=?np]
+
+
+# ############################
+# Lexical Rules
+# ############################
+
+% include chat_pnames.cfg
+
+Comp[sem=<\P Q x. ((P x) and (Q x))>] -> 'that'
+
+NP[+wh, num=sg, sem=<\P.\x.(P x)>] -> 'what'
+
+Det[-wh,num=sg,sem=<\P Q. all x. ((P x) implies (Q x))>] -> 'every'
+Det[-wh,num=pl,sem=<\P Q. all x. ((P x) implies (Q x))>] -> 'all'
+Det[-wh,sem=<\P Q. some x. ((P x) and (Q x))>] -> 'some'
+Det[-wh,num=sg,sem=<\P Q. some x. ((P x) and (Q x))>] -> 'a'
+Det[-wh,num=sg,sem=<\P Q. some x. ((P x) and (Q x))>] -> 'the'
+Det[+wh,sem=<\P Q x. ((Q x) and (P x))>] -> 'which'
+
+N[subcat=10,num=sg,sem=<city>] -> 'city'
+N[subcat=10,num=pl,sem=<city>] -> 'cities'
+N[subcat=10,num=sg,sem=<continent>] -> 'continent'
+N[subcat=10,num=pl,sem=<continent>] -> 'continents'
+N[subcat=10,num=sg,sem=<country>] -> 'country'
+N[subcat=10,num=pl,sem=<country>] -> 'countries'
+N[subcat=10,num=sg,sem=<sea>] -> 'sea'
+N[subcat=10,num=pl,sem=<sea>] -> 'seas'
+N[subcat=10,num=sg,sem=<ocean>] -> 'ocean'
+N[subcat=10,num=pl,sem=<ocean>] -> 'oceans'
+
+PL[sem=<\P Q. some x. ((P x) and (Q x))>] -> ' '
+
+N[subcat=11,num=sg,sem=<\x y. (area_of y x))>] -> 'area'
+N[subcat=11,num=sg,sem=<\x y. (capital_of y x))>] -> 'capital'
+N[subcat=11,num=sg,sem=<\x y. (currency_of y x))>] -> 'currency'
+N[subcat=11,num=sg,sem=<\x y. (region_of y x))>] -> 'region'
+N[subcat=11,num=sg,sem=<\x y. (longitude_of y x))>] -> 'longitude'
+N[subcat=11,num=sg,sem=<\x y. (latitude_of y x))>] -> 'latitude'
+N[subcat=11,num=sg,sem=<\x y. (population_of y x))>] -> 'population'
+
+
+
+## V[subcat=3,num=sg,sem=<\X y. (X \x. (x = y))>,tns=pres] -> 'is'
+## V[subcat=3,num=pl,sem=<\P. P))>,tns=pres] -> 'are'
+V[subcat=3,num=sg,sem=<\P. P>,tns=pres] -> 'is'
+V[subcat=3,num=pl,sem=<\P. P>,tns=pres] -> 'are'
+V[subcat=2,num=sg,sem=<\X y. (X \x. (border x y))>,tns=pres] -> 'borders'
+V[subcat=2,num=pl,sem=<\X y. (X \x. (border x y))>,tns=pres] -> 'border'
+V[subcat=2,num=sg,sem=<\X y. (X \x. (contain x y))>,tns=pres] -> 'contains'
+V[subcat=2,num=pl,sem=<\X y. (X \x. (contain x y))>,tns=pres] -> 'contain'
+
+A[sem=<\P x. ((contain x asia) and (P x))>] -> 'Asian'
+
+P[pform=of,sem=<\X.X>] -> 'of'
+P[+loc,sem=<\X P x. (X \y. ((P x) and (in y x)))>] -> 'in'
+P[-loc,sem=<\X P x. (X \y. ((P x) and (with y x)))>] -> 'with'
+
+
+
diff --git a/examples/semantics/chat_pnames.cfg b/examples/semantics/chat_pnames.cfg
new file mode 100644
index 0000000..00937ba
--- /dev/null
+++ b/examples/semantics/chat_pnames.cfg
@@ -0,0 +1,545 @@
+
+##################################################################
+# Lexical rules automatically generated by running 'chat80.py -x'.
+##################################################################
+
+PropN[num=sg, sem=<\P.(P abidjan)>] -> 'Abidjan'
+PropN[num=sg, sem=<\P.(P abu_dhabi)>] -> 'Abu_Dhabi'
+PropN[num=sg, sem=<\P.(P accra)>] -> 'Accra'
+PropN[num=sg, sem=<\P.(P addis_ababa)>] -> 'Addis_Ababa'
+PropN[num=sg, sem=<\P.(P aden)>] -> 'Aden'
+PropN[num=sg, sem=<\P.(P afghani)>] -> 'Afghani'
+PropN[num=sg, sem=<\P.(P afghanistan)>] -> 'Afghanistan'
+PropN[num=sg, sem=<\P.(P africa)>] -> 'Africa'
+PropN[num=sg, sem=<\P.(P albania)>] -> 'Albania'
+PropN[num=sg, sem=<\P.(P algeria)>] -> 'Algeria'
+PropN[num=sg, sem=<\P.(P algiers)>] -> 'Algiers'
+PropN[num=sg, sem=<\P.(P amazon)>] -> 'Amazon'
+PropN[num=sg, sem=<\P.(P america)>] -> 'America'
+PropN[num=sg, sem=<\P.(P amman)>] -> 'Amman'
+PropN[num=sg, sem=<\P.(P amsterdam)>] -> 'Amsterdam'
+PropN[num=sg, sem=<\P.(P amu_darya)>] -> 'Amu_Darya'
+PropN[num=sg, sem=<\P.(P amur)>] -> 'Amur'
+PropN[num=sg, sem=<\P.(P andorra)>] -> 'Andorra'
+PropN[num=sg, sem=<\P.(P andorra_la_villa)>] -> 'Andorra_La_Villa'
+PropN[num=sg, sem=<\P.(P angola)>] -> 'Angola'
+PropN[num=sg, sem=<\P.(P ankara)>] -> 'Ankara'
+PropN[num=sg, sem=<\P.(P antarctic_circle)>] -> 'Antarctic_Circle'
+PropN[num=sg, sem=<\P.(P antarctica)>] -> 'Antarctica'
+PropN[num=sg, sem=<\P.(P apia)>] -> 'Apia'
+PropN[num=sg, sem=<\P.(P arctic_circle)>] -> 'Arctic_Circle'
+PropN[num=sg, sem=<\P.(P arctic_ocean)>] -> 'Arctic_Ocean'
+PropN[num=sg, sem=<\P.(P argentina)>] -> 'Argentina'
+PropN[num=sg, sem=<\P.(P ariary)>] -> 'Ariary'
+PropN[num=sg, sem=<\P.(P asia)>] -> 'Asia'
+PropN[num=sg, sem=<\P.(P asuncion)>] -> 'Asuncion'
+PropN[num=sg, sem=<\P.(P athens)>] -> 'Athens'
+PropN[num=sg, sem=<\P.(P atlantic)>] -> 'Atlantic'
+PropN[num=sg, sem=<\P.(P australasia)>] -> 'Australasia'
+PropN[num=sg, sem=<\P.(P australia)>] -> 'Australia'
+PropN[num=sg, sem=<\P.(P australian_dollar)>] -> 'Australian_Dollar'
+PropN[num=sg, sem=<\P.(P austria)>] -> 'Austria'
+PropN[num=sg, sem=<\P.(P baghdad)>] -> 'Baghdad'
+PropN[num=sg, sem=<\P.(P bahamas)>] -> 'Bahamas'
+PropN[num=sg, sem=<\P.(P bahamian_dollar)>] -> 'Bahamian_Dollar'
+PropN[num=sg, sem=<\P.(P bahrain)>] -> 'Bahrain'
+PropN[num=sg, sem=<\P.(P baht)>] -> 'Baht'
+PropN[num=sg, sem=<\P.(P balboa)>] -> 'Balboa'
+PropN[num=sg, sem=<\P.(P baltic)>] -> 'Baltic'
+PropN[num=sg, sem=<\P.(P bamako)>] -> 'Bamako'
+PropN[num=sg, sem=<\P.(P bangkok)>] -> 'Bangkok'
+PropN[num=sg, sem=<\P.(P bangladesh)>] -> 'Bangladesh'
+PropN[num=sg, sem=<\P.(P bangui)>] -> 'Bangui'
+PropN[num=sg, sem=<\P.(P banjul)>] -> 'Banjul'
+PropN[num=sg, sem=<\P.(P barbados)>] -> 'Barbados'
+PropN[num=sg, sem=<\P.(P barcelona)>] -> 'Barcelona'
+PropN[num=sg, sem=<\P.(P beirut)>] -> 'Beirut'
+PropN[num=sg, sem=<\P.(P belgium)>] -> 'Belgium'
+PropN[num=sg, sem=<\P.(P belgrade)>] -> 'Belgrade'
+PropN[num=sg, sem=<\P.(P belize)>] -> 'Belize'
+PropN[num=sg, sem=<\P.(P belize_town)>] -> 'Belize_Town'
+PropN[num=sg, sem=<\P.(P berlin)>] -> 'Berlin'
+PropN[num=sg, sem=<\P.(P bern)>] -> 'Bern'
+PropN[num=sg, sem=<\P.(P bhutan)>] -> 'Bhutan'
+PropN[num=sg, sem=<\P.(P birmingham)>] -> 'Birmingham'
+PropN[num=sg, sem=<\P.(P bissau)>] -> 'Bissau'
+PropN[num=sg, sem=<\P.(P black_sea)>] -> 'Black_Sea'
+PropN[num=sg, sem=<\P.(P bogota)>] -> 'Bogota'
+PropN[num=sg, sem=<\P.(P bolivar)>] -> 'Bolivar'
+PropN[num=sg, sem=<\P.(P bolivia)>] -> 'Bolivia'
+PropN[num=sg, sem=<\P.(P bombay)>] -> 'Bombay'
+PropN[num=sg, sem=<\P.(P bonn)>] -> 'Bonn'
+PropN[num=sg, sem=<\P.(P botswana)>] -> 'Botswana'
+PropN[num=sg, sem=<\P.(P brahmaputra)>] -> 'Brahmaputra'
+PropN[num=sg, sem=<\P.(P brasilia)>] -> 'Brasilia'
+PropN[num=sg, sem=<\P.(P brazil)>] -> 'Brazil'
+PropN[num=sg, sem=<\P.(P brazzaville)>] -> 'Brazzaville'
+PropN[num=sg, sem=<\P.(P bridgetown)>] -> 'Bridgetown'
+PropN[num=sg, sem=<\P.(P brussels)>] -> 'Brussels'
+PropN[num=sg, sem=<\P.(P bucharest)>] -> 'Bucharest'
+PropN[num=sg, sem=<\P.(P budapest)>] -> 'Budapest'
+PropN[num=sg, sem=<\P.(P buenos_aires)>] -> 'Buenos_Aires'
+PropN[num=sg, sem=<\P.(P bujumbura)>] -> 'Bujumbura'
+PropN[num=sg, sem=<\P.(P bulgaria)>] -> 'Bulgaria'
+PropN[num=sg, sem=<\P.(P burma)>] -> 'Burma'
+PropN[num=sg, sem=<\P.(P burundi)>] -> 'Burundi'
+PropN[num=sg, sem=<\P.(P cairo)>] -> 'Cairo'
+PropN[num=sg, sem=<\P.(P calcutta)>] -> 'Calcutta'
+PropN[num=sg, sem=<\P.(P cambodia)>] -> 'Cambodia'
+PropN[num=sg, sem=<\P.(P cameroon)>] -> 'Cameroon'
+PropN[num=sg, sem=<\P.(P canada)>] -> 'Canada'
+PropN[num=sg, sem=<\P.(P canadian_dollar)>] -> 'Canadian_Dollar'
+PropN[num=sg, sem=<\P.(P canberra)>] -> 'Canberra'
+PropN[num=sg, sem=<\P.(P canton)>] -> 'Canton'
+PropN[num=sg, sem=<\P.(P caracas)>] -> 'Caracas'
+PropN[num=sg, sem=<\P.(P caribbean)>] -> 'Caribbean'
+PropN[num=sg, sem=<\P.(P caspian)>] -> 'Caspian'
+PropN[num=sg, sem=<\P.(P cayenne)>] -> 'Cayenne'
+PropN[num=sg, sem=<\P.(P cedi)>] -> 'Cedi'
+PropN[num=sg, sem=<\P.(P central_africa)>] -> 'Central_Africa'
+PropN[num=sg, sem=<\P.(P central_african_republic)>] -> 'Central_African_Republic'
+PropN[num=sg, sem=<\P.(P central_america)>] -> 'Central_America'
+PropN[num=sg, sem=<\P.(P cfa_franc)>] -> 'Cfa_Franc'
+PropN[num=sg, sem=<\P.(P chad)>] -> 'Chad'
+PropN[num=sg, sem=<\P.(P chicago)>] -> 'Chicago'
+PropN[num=sg, sem=<\P.(P chile)>] -> 'Chile'
+PropN[num=sg, sem=<\P.(P china)>] -> 'China'
+PropN[num=sg, sem=<\P.(P chungking)>] -> 'Chungking'
+PropN[num=sg, sem=<\P.(P colombia)>] -> 'Colombia'
+PropN[num=sg, sem=<\P.(P colombo)>] -> 'Colombo'
+PropN[num=sg, sem=<\P.(P colon)>] -> 'Colon'
+PropN[num=sg, sem=<\P.(P colorado)>] -> 'Colorado'
+PropN[num=sg, sem=<\P.(P conakry)>] -> 'Conakry'
+PropN[num=sg, sem=<\P.(P congo)>] -> 'Congo'
+PropN[num=sg, sem=<\P.(P congo_river)>] -> 'Congo_River'
+PropN[num=sg, sem=<\P.(P copenhagen)>] -> 'Copenhagen'
+PropN[num=sg, sem=<\P.(P cordoba)>] -> 'Cordoba'
+PropN[num=sg, sem=<\P.(P costa_rica)>] -> 'Costa_Rica'
+PropN[num=sg, sem=<\P.(P cruzeiro)>] -> 'Cruzeiro'
+PropN[num=sg, sem=<\P.(P cuba)>] -> 'Cuba'
+PropN[num=sg, sem=<\P.(P cubango)>] -> 'Cubango'
+PropN[num=sg, sem=<\P.(P cyprus)>] -> 'Cyprus'
+PropN[num=sg, sem=<\P.(P czechoslovakia)>] -> 'Czechoslovakia'
+PropN[num=sg, sem=<\P.(P dacca)>] -> 'Dacca'
+PropN[num=sg, sem=<\P.(P dahomey)>] -> 'Dahomey'
+PropN[num=sg, sem=<\P.(P dairen)>] -> 'Dairen'
+PropN[num=sg, sem=<\P.(P dakar)>] -> 'Dakar'
+PropN[num=sg, sem=<\P.(P dalasi)>] -> 'Dalasi'
+PropN[num=sg, sem=<\P.(P damascus)>] -> 'Damascus'
+PropN[num=sg, sem=<\P.(P danube)>] -> 'Danube'
+PropN[num=sg, sem=<\P.(P dar_es_salaam)>] -> 'Dar_Es_Salaam'
+PropN[num=sg, sem=<\P.(P ddr_mark)>] -> 'Ddr_Mark'
+PropN[num=sg, sem=<\P.(P delhi)>] -> 'Delhi'
+PropN[num=sg, sem=<\P.(P denmark)>] -> 'Denmark'
+PropN[num=sg, sem=<\P.(P detroit)>] -> 'Detroit'
+PropN[num=sg, sem=<\P.(P deutsche_mark)>] -> 'Deutsche_Mark'
+PropN[num=sg, sem=<\P.(P dinar)>] -> 'Dinar'
+PropN[num=sg, sem=<\P.(P dirham)>] -> 'Dirham'
+PropN[num=sg, sem=<\P.(P djibouti)>] -> 'Djibouti'
+PropN[num=sg, sem=<\P.(P doha)>] -> 'Doha'
+PropN[num=sg, sem=<\P.(P dollar)>] -> 'Dollar'
+PropN[num=sg, sem=<\P.(P dominican_republic)>] -> 'Dominican_Republic'
+PropN[num=sg, sem=<\P.(P don)>] -> 'Don'
+PropN[num=sg, sem=<\P.(P dong)>] -> 'Dong'
+PropN[num=sg, sem=<\P.(P drachma)>] -> 'Drachma'
+PropN[num=sg, sem=<\P.(P dublin)>] -> 'Dublin'
+PropN[num=sg, sem=<\P.(P east_africa)>] -> 'East_Africa'
+PropN[num=sg, sem=<\P.(P east_berlin)>] -> 'East_Berlin'
+PropN[num=sg, sem=<\P.(P east_caribbean_dollar)>] -> 'East_Caribbean_Dollar'
+PropN[num=sg, sem=<\P.(P east_carribean_dollar)>] -> 'East_Carribean_Dollar'
+PropN[num=sg, sem=<\P.(P east_germany)>] -> 'East_Germany'
+PropN[num=sg, sem=<\P.(P eastern_europe)>] -> 'Eastern_Europe'
+PropN[num=sg, sem=<\P.(P ecuador)>] -> 'Ecuador'
+PropN[num=sg, sem=<\P.(P egypt)>] -> 'Egypt'
+PropN[num=sg, sem=<\P.(P egyptian_pound)>] -> 'Egyptian_Pound'
+PropN[num=sg, sem=<\P.(P eire)>] -> 'Eire'
+PropN[num=sg, sem=<\P.(P el_salvador)>] -> 'El_Salvador'
+PropN[num=sg, sem=<\P.(P elbe)>] -> 'Elbe'
+PropN[num=sg, sem=<\P.(P equator)>] -> 'Equator'
+PropN[num=sg, sem=<\P.(P equatorial_guinea)>] -> 'Equatorial_Guinea'
+PropN[num=sg, sem=<\P.(P escudo)>] -> 'Escudo'
+PropN[num=sg, sem=<\P.(P ethiopean_dollar)>] -> 'Ethiopean_Dollar'
+PropN[num=sg, sem=<\P.(P ethiopia)>] -> 'Ethiopia'
+PropN[num=sg, sem=<\P.(P euphrates)>] -> 'Euphrates'
+PropN[num=sg, sem=<\P.(P europe)>] -> 'Europe'
+PropN[num=sg, sem=<\P.(P far_east)>] -> 'Far_East'
+PropN[num=sg, sem=<\P.(P fiji)>] -> 'Fiji'
+PropN[num=sg, sem=<\P.(P fiji_dollar)>] -> 'Fiji_Dollar'
+PropN[num=sg, sem=<\P.(P finland)>] -> 'Finland'
+PropN[num=sg, sem=<\P.(P forint)>] -> 'Forint'
+PropN[num=sg, sem=<\P.(P franc)>] -> 'Franc'
+PropN[num=sg, sem=<\P.(P franc_peseta)>] -> 'Franc_Peseta'
+PropN[num=sg, sem=<\P.(P france)>] -> 'France'
+PropN[num=sg, sem=<\P.(P freetown)>] -> 'Freetown'
+PropN[num=sg, sem=<\P.(P french_franc)>] -> 'French_Franc'
+PropN[num=sg, sem=<\P.(P french_guiana)>] -> 'French_Guiana'
+PropN[num=sg, sem=<\P.(P gabon)>] -> 'Gabon'
+PropN[num=sg, sem=<\P.(P gaborone)>] -> 'Gaborone'
+PropN[num=sg, sem=<\P.(P gambia)>] -> 'Gambia'
+PropN[num=sg, sem=<\P.(P ganges)>] -> 'Ganges'
+PropN[num=sg, sem=<\P.(P georgetown)>] -> 'Georgetown'
+PropN[num=sg, sem=<\P.(P ghana)>] -> 'Ghana'
+PropN[num=sg, sem=<\P.(P glasgow)>] -> 'Glasgow'
+PropN[num=sg, sem=<\P.(P gourde)>] -> 'Gourde'
+PropN[num=sg, sem=<\P.(P greece)>] -> 'Greece'
+PropN[num=sg, sem=<\P.(P greenland)>] -> 'Greenland'
+PropN[num=sg, sem=<\P.(P grenada)>] -> 'Grenada'
+PropN[num=sg, sem=<\P.(P guarani)>] -> 'Guarani'
+PropN[num=sg, sem=<\P.(P guatamala_city)>] -> 'Guatamala_City'
+PropN[num=sg, sem=<\P.(P guatemala)>] -> 'Guatemala'
+PropN[num=sg, sem=<\P.(P guilder)>] -> 'Guilder'
+PropN[num=sg, sem=<\P.(P guinea)>] -> 'Guinea'
+PropN[num=sg, sem=<\P.(P guinea_bissau)>] -> 'Guinea_Bissau'
+PropN[num=sg, sem=<\P.(P guyana)>] -> 'Guyana'
+PropN[num=sg, sem=<\P.(P guyana_dollar)>] -> 'Guyana_Dollar'
+PropN[num=sg, sem=<\P.(P haiti)>] -> 'Haiti'
+PropN[num=sg, sem=<\P.(P hamburg)>] -> 'Hamburg'
+PropN[num=sg, sem=<\P.(P hanoi)>] -> 'Hanoi'
+PropN[num=sg, sem=<\P.(P harbin)>] -> 'Harbin'
+PropN[num=sg, sem=<\P.(P havana)>] -> 'Havana'
+PropN[num=sg, sem=<\P.(P helsinki)>] -> 'Helsinki'
+PropN[num=sg, sem=<\P.(P honduras)>] -> 'Honduras'
+PropN[num=sg, sem=<\P.(P hongkong)>] -> 'Hongkong'
+PropN[num=sg, sem=<\P.(P hongkong_city)>] -> 'Hongkong_City'
+PropN[num=sg, sem=<\P.(P hungary)>] -> 'Hungary'
+PropN[num=sg, sem=<\P.(P hwang_ho)>] -> 'Hwang_Ho'
+PropN[num=sg, sem=<\P.(P hyderabad)>] -> 'Hyderabad'
+PropN[num=sg, sem=<\P.(P iceland)>] -> 'Iceland'
+PropN[num=sg, sem=<\P.(P india)>] -> 'India'
+PropN[num=sg, sem=<\P.(P indian_ocean)>] -> 'Indian_Ocean'
+PropN[num=sg, sem=<\P.(P indian_rupee)>] -> 'Indian_Rupee'
+PropN[num=sg, sem=<\P.(P indian_subcontinent)>] -> 'Indian_Subcontinent'
+PropN[num=sg, sem=<\P.(P indonesia)>] -> 'Indonesia'
+PropN[num=sg, sem=<\P.(P indus)>] -> 'Indus'
+PropN[num=sg, sem=<\P.(P iran)>] -> 'Iran'
+PropN[num=sg, sem=<\P.(P iraq)>] -> 'Iraq'
+PropN[num=sg, sem=<\P.(P irish_pound)>] -> 'Irish_Pound'
+PropN[num=sg, sem=<\P.(P irrawaddy)>] -> 'Irrawaddy'
+PropN[num=sg, sem=<\P.(P islamad)>] -> 'Islamad'
+PropN[num=sg, sem=<\P.(P israel)>] -> 'Israel'
+PropN[num=sg, sem=<\P.(P israeli_pound)>] -> 'Israeli_Pound'
+PropN[num=sg, sem=<\P.(P istanbul)>] -> 'Istanbul'
+PropN[num=sg, sem=<\P.(P italian_lira)>] -> 'Italian_Lira'
+PropN[num=sg, sem=<\P.(P italy)>] -> 'Italy'
+PropN[num=sg, sem=<\P.(P ivory_coast)>] -> 'Ivory_Coast'
+PropN[num=sg, sem=<\P.(P jakarta)>] -> 'Jakarta'
+PropN[num=sg, sem=<\P.(P jamaica)>] -> 'Jamaica'
+PropN[num=sg, sem=<\P.(P jamaican_dollar)>] -> 'Jamaican_Dollar'
+PropN[num=sg, sem=<\P.(P japan)>] -> 'Japan'
+PropN[num=sg, sem=<\P.(P jerusalem)>] -> 'Jerusalem'
+PropN[num=sg, sem=<\P.(P johannesburg)>] -> 'Johannesburg'
+PropN[num=sg, sem=<\P.(P jordan)>] -> 'Jordan'
+PropN[num=sg, sem=<\P.(P kabul)>] -> 'Kabul'
+PropN[num=sg, sem=<\P.(P kampala)>] -> 'Kampala'
+PropN[num=sg, sem=<\P.(P karachi)>] -> 'Karachi'
+PropN[num=sg, sem=<\P.(P katmandu)>] -> 'Katmandu'
+PropN[num=sg, sem=<\P.(P kenya)>] -> 'Kenya'
+PropN[num=sg, sem=<\P.(P kenya_shilling)>] -> 'Kenya_Shilling'
+PropN[num=sg, sem=<\P.(P khartoum)>] -> 'Khartoum'
+PropN[num=sg, sem=<\P.(P kiev)>] -> 'Kiev'
+PropN[num=sg, sem=<\P.(P kigali)>] -> 'Kigali'
+PropN[num=sg, sem=<\P.(P kingston)>] -> 'Kingston'
+PropN[num=sg, sem=<\P.(P kinshasa)>] -> 'Kinshasa'
+PropN[num=sg, sem=<\P.(P kip)>] -> 'Kip'
+PropN[num=sg, sem=<\P.(P kobe)>] -> 'Kobe'
+PropN[num=sg, sem=<\P.(P koruna)>] -> 'Koruna'
+PropN[num=sg, sem=<\P.(P kowloon)>] -> 'Kowloon'
+PropN[num=sg, sem=<\P.(P krona)>] -> 'Krona'
+PropN[num=sg, sem=<\P.(P krone)>] -> 'Krone'
+PropN[num=sg, sem=<\P.(P kuala_lumpa)>] -> 'Kuala_Lumpa'
+PropN[num=sg, sem=<\P.(P kuwait)>] -> 'Kuwait'
+PropN[num=sg, sem=<\P.(P kuwait_city)>] -> 'Kuwait_City'
+PropN[num=sg, sem=<\P.(P kuwaiti_dinar)>] -> 'Kuwaiti_Dinar'
+PropN[num=sg, sem=<\P.(P kwacha)>] -> 'Kwacha'
+PropN[num=sg, sem=<\P.(P kyat)>] -> 'Kyat'
+PropN[num=sg, sem=<\P.(P kyoto)>] -> 'Kyoto'
+PropN[num=sg, sem=<\P.(P lagos)>] -> 'Lagos'
+PropN[num=sg, sem=<\P.(P laos)>] -> 'Laos'
+PropN[num=sg, sem=<\P.(P lebanese_pound)>] -> 'Lebanese_Pound'
+PropN[num=sg, sem=<\P.(P lebanon)>] -> 'Lebanon'
+PropN[num=sg, sem=<\P.(P lek)>] -> 'Lek'
+PropN[num=sg, sem=<\P.(P lempira)>] -> 'Lempira'
+PropN[num=sg, sem=<\P.(P lena)>] -> 'Lena'
+PropN[num=sg, sem=<\P.(P leningrad)>] -> 'Leningrad'
+PropN[num=sg, sem=<\P.(P leone)>] -> 'Leone'
+PropN[num=sg, sem=<\P.(P lesotho)>] -> 'Lesotho'
+PropN[num=sg, sem=<\P.(P leu)>] -> 'Leu'
+PropN[num=sg, sem=<\P.(P lev)>] -> 'Lev'
+PropN[num=sg, sem=<\P.(P liberia)>] -> 'Liberia'
+PropN[num=sg, sem=<\P.(P libreville)>] -> 'Libreville'
+PropN[num=sg, sem=<\P.(P libya)>] -> 'Libya'
+PropN[num=sg, sem=<\P.(P libyan_dinar)>] -> 'Libyan_Dinar'
+PropN[num=sg, sem=<\P.(P liechtenstein)>] -> 'Liechtenstein'
+PropN[num=sg, sem=<\P.(P lilageru)>] -> 'Lilageru'
+PropN[num=sg, sem=<\P.(P lima)>] -> 'Lima'
+PropN[num=sg, sem=<\P.(P limpopo)>] -> 'Limpopo'
+PropN[num=sg, sem=<\P.(P lira)>] -> 'Lira'
+PropN[num=sg, sem=<\P.(P lisbon)>] -> 'Lisbon'
+PropN[num=sg, sem=<\P.(P lome)>] -> 'Lome'
+PropN[num=sg, sem=<\P.(P london)>] -> 'London'
+PropN[num=sg, sem=<\P.(P los_angeles)>] -> 'Los_Angeles'
+PropN[num=sg, sem=<\P.(P luanda)>] -> 'Luanda'
+PropN[num=sg, sem=<\P.(P lusaka)>] -> 'Lusaka'
+PropN[num=sg, sem=<\P.(P luxembourg)>] -> 'Luxembourg'
+PropN[num=sg, sem=<\P.(P luxembourg_franc)>] -> 'Luxembourg_Franc'
+PropN[num=sg, sem=<\P.(P mackenzie)>] -> 'Mackenzie'
+PropN[num=sg, sem=<\P.(P madras)>] -> 'Madras'
+PropN[num=sg, sem=<\P.(P madrid)>] -> 'Madrid'
+PropN[num=sg, sem=<\P.(P malagasy)>] -> 'Malagasy'
+PropN[num=sg, sem=<\P.(P malawi)>] -> 'Malawi'
+PropN[num=sg, sem=<\P.(P malaysia)>] -> 'Malaysia'
+PropN[num=sg, sem=<\P.(P malaysian_dollar)>] -> 'Malaysian_Dollar'
+PropN[num=sg, sem=<\P.(P maldives)>] -> 'Maldives'
+PropN[num=sg, sem=<\P.(P male)>] -> 'Male'
+PropN[num=sg, sem=<\P.(P mali)>] -> 'Mali'
+PropN[num=sg, sem=<\P.(P mali_franc)>] -> 'Mali_Franc'
+PropN[num=sg, sem=<\P.(P malta)>] -> 'Malta'
+PropN[num=sg, sem=<\P.(P managua)>] -> 'Managua'
+PropN[num=sg, sem=<\P.(P manama)>] -> 'Manama'
+PropN[num=sg, sem=<\P.(P manila)>] -> 'Manila'
+PropN[num=sg, sem=<\P.(P maputo)>] -> 'Maputo'
+PropN[num=sg, sem=<\P.(P markka)>] -> 'Markka'
+PropN[num=sg, sem=<\P.(P masero)>] -> 'Masero'
+PropN[num=sg, sem=<\P.(P mauritania)>] -> 'Mauritania'
+PropN[num=sg, sem=<\P.(P mauritius)>] -> 'Mauritius'
+PropN[num=sg, sem=<\P.(P mbabane)>] -> 'Mbabane'
+PropN[num=sg, sem=<\P.(P mediterranean)>] -> 'the_Mediterranean'
+PropN[num=sg, sem=<\P.(P mekong)>] -> 'Mekong'
+PropN[num=sg, sem=<\P.(P melbourne)>] -> 'Melbourne'
+PropN[num=sg, sem=<\P.(P mexico)>] -> 'Mexico'
+PropN[num=sg, sem=<\P.(P mexico_city)>] -> 'Mexico_City'
+PropN[num=sg, sem=<\P.(P middle_east)>] -> 'Middle_East'
+PropN[num=sg, sem=<\P.(P milan)>] -> 'Milan'
+PropN[num=sg, sem=<\P.(P mississippi)>] -> 'Mississippi'
+PropN[num=sg, sem=<\P.(P mogadishu)>] -> 'Mogadishu'
+PropN[num=sg, sem=<\P.(P monaco)>] -> 'Monaco'
+PropN[num=sg, sem=<\P.(P mongolia)>] -> 'Mongolia'
+PropN[num=sg, sem=<\P.(P monrovia)>] -> 'Monrovia'
+PropN[num=sg, sem=<\P.(P montevideo)>] -> 'Montevideo'
+PropN[num=sg, sem=<\P.(P montreal)>] -> 'Montreal'
+PropN[num=sg, sem=<\P.(P morocco)>] -> 'Morocco'
+PropN[num=sg, sem=<\P.(P moscow)>] -> 'Moscow'
+PropN[num=sg, sem=<\P.(P mozambique)>] -> 'Mozambique'
+PropN[num=sg, sem=<\P.(P mukden)>] -> 'Mukden'
+PropN[num=sg, sem=<\P.(P murray)>] -> 'Murray'
+PropN[num=sg, sem=<\P.(P muscat)>] -> 'Muscat'
+PropN[num=sg, sem=<\P.(P n_djamena)>] -> 'N_Djamena'
+PropN[num=sg, sem=<\P.(P nagoya)>] -> 'Nagoya'
+PropN[num=sg, sem=<\P.(P naira)>] -> 'Naira'
+PropN[num=sg, sem=<\P.(P nairobi)>] -> 'Nairobi'
+PropN[num=sg, sem=<\P.(P nanking)>] -> 'Nanking'
+PropN[num=sg, sem=<\P.(P naples)>] -> 'Naples'
+PropN[num=sg, sem=<\P.(P nassau)>] -> 'Nassau'
+PropN[num=sg, sem=<\P.(P nepal)>] -> 'Nepal'
+PropN[num=sg, sem=<\P.(P nepalese_rupee)>] -> 'Nepalese_Rupee'
+PropN[num=sg, sem=<\P.(P netherlands)>] -> 'Netherlands'
+PropN[num=sg, sem=<\P.(P new_delhi)>] -> 'New_Delhi'
+PropN[num=sg, sem=<\P.(P new_york)>] -> 'New_York'
+PropN[num=sg, sem=<\P.(P new_zealand)>] -> 'New_Zealand'
+PropN[num=sg, sem=<\P.(P new_zealand_dollar)>] -> 'New_Zealand_Dollar'
+PropN[num=sg, sem=<\P.(P niamey)>] -> 'Niamey'
+PropN[num=sg, sem=<\P.(P nicaragua)>] -> 'Nicaragua'
+PropN[num=sg, sem=<\P.(P nicosia)>] -> 'Nicosia'
+PropN[num=sg, sem=<\P.(P niger)>] -> 'Niger'
+PropN[num=sg, sem=<\P.(P niger_river)>] -> 'Niger_River'
+PropN[num=sg, sem=<\P.(P nigeria)>] -> 'Nigeria'
+PropN[num=sg, sem=<\P.(P nile)>] -> 'Nile'
+PropN[num=sg, sem=<\P.(P north_africa)>] -> 'North_Africa'
+PropN[num=sg, sem=<\P.(P north_america)>] -> 'North_America'
+PropN[num=sg, sem=<\P.(P north_korea)>] -> 'North_Korea'
+PropN[num=sg, sem=<\P.(P northern_asia)>] -> 'Northern_Asia'
+PropN[num=sg, sem=<\P.(P norway)>] -> 'Norway'
+PropN[num=sg, sem=<\P.(P nouakchott)>] -> 'Nouakchott'
+PropN[num=sg, sem=<\P.(P nukualofa)>] -> 'Nukualofa'
+PropN[num=sg, sem=<\P.(P ob)>] -> 'Ob'
+PropN[num=sg, sem=<\P.(P oder)>] -> 'Oder'
+PropN[num=sg, sem=<\P.(P oman)>] -> 'Oman'
+PropN[num=sg, sem=<\P.(P orange)>] -> 'Orange'
+PropN[num=sg, sem=<\P.(P orinoco)>] -> 'Orinoco'
+PropN[num=sg, sem=<\P.(P osaka)>] -> 'Osaka'
+PropN[num=sg, sem=<\P.(P oslo)>] -> 'Oslo'
+PropN[num=sg, sem=<\P.(P ottawa)>] -> 'Ottawa'
+PropN[num=sg, sem=<\P.(P ouagadougou)>] -> 'Ouagadougou'
+PropN[num=sg, sem=<\P.(P ouguiya)>] -> 'Ouguiya'
+PropN[num=sg, sem=<\P.(P pa_anga)>] -> 'Pa_Anga'
+PropN[num=sg, sem=<\P.(P pacific)>] -> 'Pacific'
+PropN[num=sg, sem=<\P.(P pakistan)>] -> 'Pakistan'
+PropN[num=sg, sem=<\P.(P panama)>] -> 'Panama'
+PropN[num=sg, sem=<\P.(P papua_new_guinea)>] -> 'Papua_New_Guinea'
+PropN[num=sg, sem=<\P.(P paraguay)>] -> 'Paraguay'
+PropN[num=sg, sem=<\P.(P paramaribo)>] -> 'Paramaribo'
+PropN[num=sg, sem=<\P.(P parana)>] -> 'Parana'
+PropN[num=sg, sem=<\P.(P paris)>] -> 'Paris'
+PropN[num=sg, sem=<\P.(P pataca)>] -> 'Pataca'
+PropN[num=sg, sem=<\P.(P peking)>] -> 'Peking'
+PropN[num=sg, sem=<\P.(P persian_gulf)>] -> 'Persian_Gulf'
+PropN[num=sg, sem=<\P.(P peru)>] -> 'Peru'
+PropN[num=sg, sem=<\P.(P peseta)>] -> 'Peseta'
+PropN[num=sg, sem=<\P.(P peso)>] -> 'Peso'
+PropN[num=sg, sem=<\P.(P peveta)>] -> 'Peveta'
+PropN[num=sg, sem=<\P.(P philadelphia)>] -> 'Philadelphia'
+PropN[num=sg, sem=<\P.(P philippines)>] -> 'Philippines'
+PropN[num=sg, sem=<\P.(P phnom_penh)>] -> 'Phnom_Penh'
+PropN[num=sg, sem=<\P.(P piso)>] -> 'Piso'
+PropN[num=sg, sem=<\P.(P poland)>] -> 'Poland'
+PropN[num=sg, sem=<\P.(P port_au_prince)>] -> 'Port_Au_Prince'
+PropN[num=sg, sem=<\P.(P port_harcourt)>] -> 'Port_Harcourt'
+PropN[num=sg, sem=<\P.(P port_louis)>] -> 'Port_Louis'
+PropN[num=sg, sem=<\P.(P port_of_spain)>] -> 'Port_Of_Spain'
+PropN[num=sg, sem=<\P.(P porto_novo)>] -> 'Porto_Novo'
+PropN[num=sg, sem=<\P.(P portugal)>] -> 'Portugal'
+PropN[num=sg, sem=<\P.(P pound)>] -> 'Pound'
+PropN[num=sg, sem=<\P.(P prague)>] -> 'Prague'
+PropN[num=sg, sem=<\P.(P pretoria)>] -> 'Pretoria'
+PropN[num=sg, sem=<\P.(P pusan)>] -> 'Pusan'
+PropN[num=sg, sem=<\P.(P pvongvang)>] -> 'Pvongvang'
+PropN[num=sg, sem=<\P.(P qatar)>] -> 'Qatar'
+PropN[num=sg, sem=<\P.(P quetzal)>] -> 'Quetzal'
+PropN[num=sg, sem=<\P.(P quezon_city)>] -> 'Quezon_City'
+PropN[num=sg, sem=<\P.(P quito)>] -> 'Quito'
+PropN[num=sg, sem=<\P.(P rabat)>] -> 'Rabat'
+PropN[num=sg, sem=<\P.(P rand)>] -> 'Rand'
+PropN[num=sg, sem=<\P.(P rangoon)>] -> 'Rangoon'
+PropN[num=sg, sem=<\P.(P red_sea)>] -> 'Red_Sea'
+PropN[num=sg, sem=<\P.(P reykjavik)>] -> 'Reykjavik'
+PropN[num=sg, sem=<\P.(P rhine)>] -> 'Rhine'
+PropN[num=sg, sem=<\P.(P rhodesian_dollar)>] -> 'Rhodesian_Dollar'
+PropN[num=sg, sem=<\P.(P rhone)>] -> 'Rhone'
+PropN[num=sg, sem=<\P.(P rial)>] -> 'Rial'
+PropN[num=sg, sem=<\P.(P riel)>] -> 'Riel'
+PropN[num=sg, sem=<\P.(P rio_de_janeiro)>] -> 'Rio_De_Janeiro'
+PropN[num=sg, sem=<\P.(P rio_grande)>] -> 'Rio_Grande'
+PropN[num=sg, sem=<\P.(P riyadh)>] -> 'Riyadh'
+PropN[num=sg, sem=<\P.(P riyal)>] -> 'Riyal'
+PropN[num=sg, sem=<\P.(P riyal_omani)>] -> 'Riyal_Omani'
+PropN[num=sg, sem=<\P.(P romania)>] -> 'Romania'
+PropN[num=sg, sem=<\P.(P rome)>] -> 'Rome'
+PropN[num=sg, sem=<\P.(P ruble)>] -> 'Ruble'
+PropN[num=sg, sem=<\P.(P rupee)>] -> 'Rupee'
+PropN[num=sg, sem=<\P.(P rupiah)>] -> 'Rupiah'
+PropN[num=sg, sem=<\P.(P rwanda)>] -> 'Rwanda'
+PropN[num=sg, sem=<\P.(P rwanda_franc)>] -> 'Rwanda_Franc'
+PropN[num=sg, sem=<\P.(P saigon)>] -> 'Saigon'
+PropN[num=sg, sem=<\P.(P salisbury)>] -> 'Salisbury'
+PropN[num=sg, sem=<\P.(P salween)>] -> 'Salween'
+PropN[num=sg, sem=<\P.(P san_jose)>] -> 'San_Jose'
+PropN[num=sg, sem=<\P.(P san_marino)>] -> 'San_Marino'
+PropN[num=sg, sem=<\P.(P san_salvador)>] -> 'San_Salvador'
+PropN[num=sg, sem=<\P.(P sana)>] -> 'Sana'
+PropN[num=sg, sem=<\P.(P santa_domingo)>] -> 'Santa_Domingo'
+PropN[num=sg, sem=<\P.(P santa_isabel)>] -> 'Santa_Isabel'
+PropN[num=sg, sem=<\P.(P santiago)>] -> 'Santiago'
+PropN[num=sg, sem=<\P.(P sao_paulo)>] -> 'Sao_Paulo'
+PropN[num=sg, sem=<\P.(P saudi_arabia)>] -> 'Saudi_Arabia'
+PropN[num=sg, sem=<\P.(P scandinavia)>] -> 'Scandinavia'
+PropN[num=sg, sem=<\P.(P schilling)>] -> 'Schilling'
+PropN[num=sg, sem=<\P.(P senegal)>] -> 'Senegal'
+PropN[num=sg, sem=<\P.(P senegal_river)>] -> 'Senegal_River'
+PropN[num=sg, sem=<\P.(P seoul)>] -> 'Seoul'
+PropN[num=sg, sem=<\P.(P seychelles)>] -> 'Seychelles'
+PropN[num=sg, sem=<\P.(P shanghai)>] -> 'Shanghai'
+PropN[num=sg, sem=<\P.(P sian)>] -> 'Sian'
+PropN[num=sg, sem=<\P.(P sierra_leone)>] -> 'Sierra_Leone'
+PropN[num=sg, sem=<\P.(P singapore)>] -> 'Singapore'
+PropN[num=sg, sem=<\P.(P singapore_city)>] -> 'Singapore_City'
+PropN[num=sg, sem=<\P.(P singapore_dollar)>] -> 'Singapore_Dollar'
+PropN[num=sg, sem=<\P.(P sofia)>] -> 'Sofia'
+PropN[num=sg, sem=<\P.(P sol)>] -> 'Sol'
+PropN[num=sg, sem=<\P.(P somali_shilling)>] -> 'Somali_Shilling'
+PropN[num=sg, sem=<\P.(P somalia)>] -> 'Somalia'
+PropN[num=sg, sem=<\P.(P south_africa)>] -> 'South_Africa'
+PropN[num=sg, sem=<\P.(P south_african_rand)>] -> 'South_African_Rand'
+PropN[num=sg, sem=<\P.(P south_america)>] -> 'South_America'
+PropN[num=sg, sem=<\P.(P south_korea)>] -> 'South_Korea'
+PropN[num=sg, sem=<\P.(P south_yemen)>] -> 'South_Yemen'
+PropN[num=sg, sem=<\P.(P southeast_east)>] -> 'Southeast_East'
+PropN[num=sg, sem=<\P.(P southern_africa)>] -> 'Southern_Africa'
+PropN[num=sg, sem=<\P.(P southern_europe)>] -> 'Southern_Europe'
+PropN[num=sg, sem=<\P.(P southern_ocean)>] -> 'Southern_Ocean'
+PropN[num=sg, sem=<\P.(P soviet_union)>] -> 'Soviet_Union'
+PropN[num=sg, sem=<\P.(P spain)>] -> 'Spain'
+PropN[num=sg, sem=<\P.(P sri_lanka)>] -> 'Sri_Lanka'
+PropN[num=sg, sem=<\P.(P st_georges)>] -> 'St_Georges'
+PropN[num=sg, sem=<\P.(P stockholm)>] -> 'Stockholm'
+PropN[num=sg, sem=<\P.(P sucre)>] -> 'Sucre'
+PropN[num=sg, sem=<\P.(P sudan)>] -> 'Sudan'
+PropN[num=sg, sem=<\P.(P surinam)>] -> 'Surinam'
+PropN[num=sg, sem=<\P.(P suva)>] -> 'Suva'
+PropN[num=sg, sem=<\P.(P swaziland)>] -> 'Swaziland'
+PropN[num=sg, sem=<\P.(P sweden)>] -> 'Sweden'
+PropN[num=sg, sem=<\P.(P swiss_franc)>] -> 'Swiss_Franc'
+PropN[num=sg, sem=<\P.(P switzerland)>] -> 'Switzerland'
+PropN[num=sg, sem=<\P.(P sydney)>] -> 'Sydney'
+PropN[num=sg, sem=<\P.(P syli)>] -> 'Syli'
+PropN[num=sg, sem=<\P.(P syria)>] -> 'Syria'
+PropN[num=sg, sem=<\P.(P syrian_pound)>] -> 'Syrian_Pound'
+PropN[num=sg, sem=<\P.(P tagus)>] -> 'Tagus'
+PropN[num=sg, sem=<\P.(P taipei)>] -> 'Taipei'
+PropN[num=sg, sem=<\P.(P taiwan)>] -> 'Taiwan'
+PropN[num=sg, sem=<\P.(P taiwan_dollar)>] -> 'Taiwan_Dollar'
+PropN[num=sg, sem=<\P.(P taka)>] -> 'Taka'
+PropN[num=sg, sem=<\P.(P tala)>] -> 'Tala'
+PropN[num=sg, sem=<\P.(P tananarive)>] -> 'Tananarive'
+PropN[num=sg, sem=<\P.(P tanzania)>] -> 'Tanzania'
+PropN[num=sg, sem=<\P.(P tanzanian_shilling)>] -> 'Tanzanian_Shilling'
+PropN[num=sg, sem=<\P.(P tegucigalpa)>] -> 'Tegucigalpa'
+PropN[num=sg, sem=<\P.(P tehran)>] -> 'Tehran'
+PropN[num=sg, sem=<\P.(P thailand)>] -> 'Thailand'
+PropN[num=sg, sem=<\P.(P thimphu)>] -> 'Thimphu'
+PropN[num=sg, sem=<\P.(P tientsin)>] -> 'Tientsin'
+PropN[num=sg, sem=<\P.(P tighrik)>] -> 'Tighrik'
+PropN[num=sg, sem=<\P.(P tirana)>] -> 'Tirana'
+PropN[num=sg, sem=<\P.(P togo)>] -> 'Togo'
+PropN[num=sg, sem=<\P.(P tokyo)>] -> 'Tokyo'
+PropN[num=sg, sem=<\P.(P tonga)>] -> 'Tonga'
+PropN[num=sg, sem=<\P.(P toronto)>] -> 'Toronto'
+PropN[num=sg, sem=<\P.(P trinidad_and_tobago)>] -> 'Trinidad_And_Tobago'
+PropN[num=sg, sem=<\P.(P trinidad_and_tobago_dollar)>] -> 'Trinidad_And_Tobago_Dollar'
+PropN[num=sg, sem=<\P.(P tripoli)>] -> 'Tripoli'
+PropN[num=sg, sem=<\P.(P tropic_of_cancer)>] -> 'Tropic_Of_Cancer'
+PropN[num=sg, sem=<\P.(P tropic_of_capricorn)>] -> 'Tropic_Of_Capricorn'
+PropN[num=sg, sem=<\P.(P tunis)>] -> 'Tunis'
+PropN[num=sg, sem=<\P.(P tunisia)>] -> 'Tunisia'
+PropN[num=sg, sem=<\P.(P turkey)>] -> 'Turkey'
+PropN[num=sg, sem=<\P.(P uganda)>] -> 'Uganda'
+PropN[num=sg, sem=<\P.(P uganda_shilling)>] -> 'Uganda_Shilling'
+PropN[num=sg, sem=<\P.(P ulan_bator)>] -> 'Ulan_Bator'
+PropN[num=sg, sem=<\P.(P united_arab_emirates)>] -> 'United_Arab_Emirates'
+PropN[num=sg, sem=<\P.(P united_kingdom)>] -> 'United_Kingdom'
+PropN[num=sg, sem=<\P.(P united_states)>] -> 'United_States'
+PropN[num=sg, sem=<\P.(P upper_volta)>] -> 'Upper_Volta'
+PropN[num=sg, sem=<\P.(P uruguay)>] -> 'Uruguay'
+PropN[num=sg, sem=<\P.(P us_dollar)>] -> 'Us_Dollar'
+PropN[num=sg, sem=<\P.(P vaduz)>] -> 'Vaduz'
+PropN[num=sg, sem=<\P.(P valetta)>] -> 'Valetta'
+PropN[num=sg, sem=<\P.(P venezuela)>] -> 'Venezuela'
+PropN[num=sg, sem=<\P.(P victoria)>] -> 'Victoria'
+PropN[num=sg, sem=<\P.(P vienna)>] -> 'Vienna'
+PropN[num=sg, sem=<\P.(P vientiane)>] -> 'Vientiane'
+PropN[num=sg, sem=<\P.(P vietnam)>] -> 'Vietnam'
+PropN[num=sg, sem=<\P.(P vistula)>] -> 'Vistula'
+PropN[num=sg, sem=<\P.(P volga)>] -> 'Volga'
+PropN[num=sg, sem=<\P.(P volta)>] -> 'Volta'
+PropN[num=sg, sem=<\P.(P warsaw)>] -> 'Warsaw'
+PropN[num=sg, sem=<\P.(P washington)>] -> 'Washington'
+PropN[num=sg, sem=<\P.(P wellington)>] -> 'Wellington'
+PropN[num=sg, sem=<\P.(P west_africa)>] -> 'West_Africa'
+PropN[num=sg, sem=<\P.(P west_germany)>] -> 'West_Germany'
+PropN[num=sg, sem=<\P.(P western_europe)>] -> 'Western_Europe'
+PropN[num=sg, sem=<\P.(P western_samoa)>] -> 'Western_Samoa'
+PropN[num=sg, sem=<\P.(P won)>] -> 'Won'
+PropN[num=sg, sem=<\P.(P yangtze)>] -> 'Yangtze'
+PropN[num=sg, sem=<\P.(P yaounde)>] -> 'Yaounde'
+PropN[num=sg, sem=<\P.(P yemen)>] -> 'Yemen'
+PropN[num=sg, sem=<\P.(P yen)>] -> 'Yen'
+PropN[num=sg, sem=<\P.(P yenisei)>] -> 'Yenisei'
+PropN[num=sg, sem=<\P.(P yokohama)>] -> 'Yokohama'
+PropN[num=sg, sem=<\P.(P yuan)>] -> 'Yuan'
+PropN[num=sg, sem=<\P.(P yugoslavia)>] -> 'Yugoslavia'
+PropN[num=sg, sem=<\P.(P yukon)>] -> 'Yukon'
+PropN[num=sg, sem=<\P.(P zaire)>] -> 'Zaire'
+PropN[num=sg, sem=<\P.(P zambesi)>] -> 'Zambesi'
+PropN[num=sg, sem=<\P.(P zambia)>] -> 'Zambia'
+PropN[num=sg, sem=<\P.(P zimbabwe)>] -> 'Zimbabwe'
+PropN[num=sg, sem=<\P.(P zloty)>] -> 'Zloty'
+PropN[num=sg, sem=<\P.(P zomba)>] -> 'Zomba'
diff --git a/examples/semantics/chat_sentences b/examples/semantics/chat_sentences
new file mode 100644
index 0000000..a6822c4
--- /dev/null
+++ b/examples/semantics/chat_sentences
@@ -0,0 +1,17 @@
+# Natural Language Toolkit: Demo Sentences
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+############################################
+# Some example sentences for the Chat-80 demo
+
+what is the capital of France
+which sea borders France
+what contains Berlin
+which Asian countries border the_Mediterranean
+
+
+
+
+
diff --git a/examples/semantics/demo_sentences b/examples/semantics/demo_sentences
new file mode 100644
index 0000000..0ec465f
--- /dev/null
+++ b/examples/semantics/demo_sentences
@@ -0,0 +1,14 @@
+# Natural Language Toolkit: Demo Sentences
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+############################################
+# Some example sentences for the sem2.cfg demo
+
+Fido sees a boy with Mary
+John sees Mary
+every girl chases a dog
+every boy chases a girl
+John walks with a girl in Noosa
+who walks
diff --git a/examples/semantics/model0.py b/examples/semantics/model0.py
new file mode 100644
index 0000000..ef73e44
--- /dev/null
+++ b/examples/semantics/model0.py
@@ -0,0 +1,44 @@
+# Natural Language Toolkit: Example Model
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+This is a sample model to accompany the U{sem2.cfg} grammar, and is
+intended to be imported as a module.
+"""
+
+from nltk.sem import *
+
+#Initialize a valuation of non-logical constants."""
+
+v = [
+ ('john', 'b1'),
+ ('mary', 'g1'),
+ ('suzie', 'g2'),
+ ('fido', 'd1'),
+ ('tess', 'd2'),
+ ('noosa', 'n'),
+ ('girl', set(['g1', 'g2'])),
+ ('boy', set(['b1', 'b2'])),
+ ('dog', set(['d1', 'd2'])),
+ ('bark', set(['d1', 'd2'])),
+ ('walk', set(['b1', 'g2', 'd1'])),
+ ('chase', set([('b1', 'g1'), ('b2', 'g1'), ('g1', 'd1'), ('g2', 'd2')])),
+ ('see', set([('b1', 'g1'), ('b2', 'd2'), ('g1', 'b1'),('d2', 'b1'), ('g2', 'n')])),
+ ('in', set([('b1', 'n'), ('b2', 'n'), ('d2', 'n')])),
+ ('with', set([('b1', 'g1'), ('g1', 'b1'), ('d1', 'b1'), ('b1', 'd1')]))
+]
+
+#Read in the data from C{v}
+val = Valuation(v)
+
+#Bind C{dom} to the C{domain} property of C{val}
+dom = val.domain
+
+#Initialize a model with parameters C{dom} and C{val}.
+m = Model(dom, val)
+
+#Initialize a variable assignment with parameter C{dom}
+g = Assignment(dom)
diff --git a/examples/semantics/model1.py b/examples/semantics/model1.py
new file mode 100644
index 0000000..3bef075
--- /dev/null
+++ b/examples/semantics/model1.py
@@ -0,0 +1,27 @@
+# Natural Language Toolkit: Example Model
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+This is a sample model to accompany the U{chat80.cfg} grammar} and is
+intended to be imported as a module.
+"""
+
+from nltk.semantics import *
+from nltk.corpora import chat80
+
+rels = chat80.rels
+concept_map = chat80.process_bundle(rels)
+concepts = concept_map.values()
+val = chat80.make_valuation(concepts, read=True)
+
+#Bind C{dom} to the C{domain} property of C{val}.
+dom = val.domain
+
+#Initialize a model with parameters C{dom} and C{val}.
+m = Model(dom, val)
+
+#Initialize a variable assignment with parameter C{dom}.
+g = Assignment(dom)
diff --git a/examples/semantics/sem0.cfg b/examples/semantics/sem0.cfg
new file mode 100644
index 0000000..5bba6a4
--- /dev/null
+++ b/examples/semantics/sem0.cfg
@@ -0,0 +1,14 @@
+## Natural Language Toolkit: sem0.cfg
+##
+## Minimal feature-based grammar with lambda semantics.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[sem = <app(?vp,?subj)>] -> NP[sem=?subj] VP[sem=?vp]
+VP[sem=?v] -> V[sem=?v]
+NP[sem=<john>] -> 'John'
+V[sem=<\x.(walk x)>] -> 'walks'
diff --git a/examples/semantics/sem1.cfg b/examples/semantics/sem1.cfg
new file mode 100644
index 0000000..2046ebe
--- /dev/null
+++ b/examples/semantics/sem1.cfg
@@ -0,0 +1,18 @@
+## Natural Language Toolkit: sem1.cfg
+##
+## Minimal feature-based grammar to illustrate the interpretation of
+## determiner phrases.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[sem = <app(?subj,?vp)>] -> NP[sem=?subj] VP[sem=?vp]
+VP[sem=?v] -> IV[sem=?v]
+NP[sem=<app(?det,?n)>] -> Det[sem=?det] N[sem=?n]
+
+Det[sem=<\Q P. some x. ((Q x) and (P x))>] -> 'a'
+N[sem=<dog>] -> 'dog'
+IV[sem=<\x.(bark x)>] -> 'barks'
diff --git a/examples/semantics/sem2.cfg b/examples/semantics/sem2.cfg
new file mode 100644
index 0000000..6c00352
--- /dev/null
+++ b/examples/semantics/sem2.cfg
@@ -0,0 +1,68 @@
+## Natural Language Toolkit: sem2.cfg
+##
+## Longer feature-based grammar with more quantifers, and illustrating
+## transitive verbs and prepositional phrases (PPs). The
+## interpretation of PPs is a bit weird and could do with further
+## work.
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+############################
+# Grammar Rules
+#############################
+
+S[sem = <app(?subj,?vp)>] -> NP[num=?n,sem=?subj] VP[num=?n,sem=?vp]
+
+NP[num=?n,sem=<app(?det,?nom)> ] -> Det[num=?n,sem=?det] Nom[num=?n,sem=?nom]
+NP[loc=?l,num=?n,sem=?np] -> PropN[loc=?l,num=?n,sem=?np]
+
+Nom[num=?n,sem=?nom] -> N[num=?n,sem=?nom]
+Nom[num=?n,sem=<app(?pp,?nom)>] -> N[num=?n,sem=?nom] PP[sem=?pp]
+
+VP[num=?n,sem=<app(?v,?obj)>] -> TV[num=?n,sem=?v] NP[sem=?obj]
+VP[num=?n,sem=?v] -> IV[num=?n,sem=?v]
+
+VP[num=?n,sem=<app(?pp,?vp)>] -> VP[num=?n,sem=?vp] PP[sem=?pp]
+
+PP[sem=<app(?p,?np)>] -> P[loc=?l,sem=?p] NP[loc=?l,sem=?np]
+
+#############################
+# Lexical Rules
+#############################
+
+PropN[-loc,num=sg,sem=<\P.(P john)>] -> 'John'
+PropN[-loc,num=sg,sem=<\P.(P mary)>] -> 'Mary'
+PropN[-loc,num=sg,sem=<\P.(P suzie)>] -> 'Suzie'
+PropN[-loc,num=sg,sem=<\P.(P fido)>] -> 'Fido'
+PropN[+loc, num=sg,sem=<\P.(P noosa)>] -> 'Noosa'
+
+NP[-loc, num=sg, sem=<\P.\x.(P x)>] -> 'who'
+
+Det[num=sg,sem=<\P Q. all x. ((P x) implies (Q x))>] -> 'every'
+Det[num=pl,sem=<\P Q. all x. ((P x) implies (Q x))>] -> 'all'
+Det[sem=<\P Q. some x. ((P x) and (Q x))>] -> 'some'
+Det[num=sg,sem=<\P Q. some x. ((P x) and (Q x))>] -> 'a'
+
+N[num=sg,sem=<boy>] -> 'boy'
+N[num=pl,sem=<boy>] -> 'boys'
+N[num=sg,sem=<girl>] -> 'girl'
+N[num=pl,sem=<girl>] -> 'girls'
+N[num=sg,sem=<dog>] -> 'dog'
+N[num=pl,sem=<dog>] -> 'dogs'
+
+TV[num=sg,sem=<\X y. (X \x. (chase x y))>,tns=pres] -> 'chases'
+TV[num=pl,sem=<\X y. (X \x. (chase x y))>,tns=pres] -> 'chase'
+TV[num=sg,sem=<\X y. (X \x. (see x y))>,tns=pres] -> 'sees'
+TV[num=pl,sem=<\X y. (X \x. (see x y))>,tns=pres] -> 'see'
+TV[num=sg,sem=<\X y. (X \x. (chase x y))>,tns=pres] -> 'chases'
+TV[num=pl,sem=<\X y. (X \x. (chase x y))>,tns=pres] -> 'chase'
+IV[num=sg,sem=<\x. (bark x)>,tns=pres] -> 'barks'
+IV[num=pl,sem=<\x. (bark x)>,tns=pres] -> 'bark'
+IV[num=sg,sem=<\x. (walk x)>,tns=pres] -> 'walks'
+IV[num=pl,sem=<\x. (walk x)>,tns=pres] -> 'walk'
+
+P[+loc,sem=<\X P x. (X \y. ((P x) and (in y x)))>] -> 'in'
+P[-loc,sem=<\X P x. (X \y. ((P x) and (with y x)))>] -> 'with'
diff --git a/examples/semantics/sem3.cfg b/examples/semantics/sem3.cfg
new file mode 100644
index 0000000..ce55962
--- /dev/null
+++ b/examples/semantics/sem3.cfg
@@ -0,0 +1,17 @@
+## Natural Language Toolkit: sem3.cfg
+##
+## First attempt at HPSG-style feature-based semantics.
+## This version doesn't work properly!
+##
+## Author: Ewan Klein <ewan at inf.ed.ac.uk>
+## URL: <http://nltk.sourceforge.net>
+## For license information, see LICENSE.TXT
+
+% start S
+
+S[sem=?vp] -> NP[sem=?np] VP[subj=?np, sem=?vp]
+VP[sem=?v, subj=?np] -> IV[sem=?v, subj=?np]
+NP[sem=[index='k',name='kim']] -> 'Kim'
+IV[sem=[rel='bark', arg=?i], subj=[sem=[index=?i]]] -> 'barks'
+#IV[fsem=[rel='bark', arg=(1)[]], subj=[fsem=[index->(1)]]] -> 'barks'
+
diff --git a/examples/semantics/syn2sem.py b/examples/semantics/syn2sem.py
new file mode 100644
index 0000000..cf93a88
--- /dev/null
+++ b/examples/semantics/syn2sem.py
@@ -0,0 +1,118 @@
+# Natural Language Toolkit: Parsers
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function
+
+"""
+Demo of how to combine the output of parsing with evaluation in a model.
+Use 'python syn2sem.py -h' to find out the various options.
+
+Note that this demo currently processes the whole input file
+before delivering any results, consequently there may be a significant initial delay.
+"""
+
+from nltk.semantics import *
+
+
+def read_sents(file):
+ sents = [l.rstrip() for l in open(file)]
+ # get rid of blank lines
+ sents = [l for l in sents if len(l) > 0]
+ sents = [l for l in sents if not l[0] == '#']
+ return sents
+
+def demo():
+ import sys
+ from optparse import OptionParser
+ description = \
+ """
+Parse and evaluate some sentences.
+ """
+
+ opts = OptionParser(description=description)
+
+ opts.set_defaults(evaluate=True, beta=True, syntrace=0,
+ semtrace=0, demo='default', grammar='', sentences='')
+
+ opts.add_option("-d", "--demo", dest="demo",
+ help="choose demo D; omit this for the default demo, or specify 'chat80'", metavar="D")
+ opts.add_option("-g", "--gram", dest="grammar",
+ help="read in grammar G", metavar="G")
+ opts.add_option("-m", "--model", dest="model",
+ help="import model M (omit '.py' suffix)", metavar="M")
+ opts.add_option("-s", "--sentences", dest="sentences",
+ help="read in a file of test sentences S", metavar="S")
+ opts.add_option("-e", "--no-eval", action="store_false", dest="evaluate",
+ help="just do a syntactic analysis")
+ opts.add_option("-b", "--no-beta-reduction", action="store_false",
+ dest="beta", help="don't carry out beta-reduction")
+ opts.add_option("-t", "--syntrace", action="count", dest="syntrace",
+ help="set syntactic tracing on; requires '-e' option")
+ opts.add_option("-T", "--semtrace", action="count", dest="semtrace",
+ help="set semantic tracing on")
+
+
+
+ (options, args) = opts.parse_args()
+
+ SPACER = '-' * 30
+
+
+
+ if options.demo == 'chat80':
+ import model1 as model
+ sentsfile = 'chat_sentences'
+ gramfile = 'chat80.cfg'
+ else:
+ import model0 as model
+ sentsfile = 'demo_sentences'
+ gramfile = 'sem2.cfg'
+
+ if options.sentences:
+ sentsfile = options.sentences
+ if options.grammar:
+ gramfile = options.grammar
+ if options.model:
+ exec "import %s as model" % options.model
+
+ sents = read_sents(sentsfile)
+
+ # NB. GrammarFile is imported indirectly via nltk.semantics
+ gram = GrammarFile.read_file(gramfile)
+
+ m = model.m
+ g = model.g
+
+ if options.evaluate:
+ evaluations = \
+ text_evaluate(sents, gram, m, g, semtrace=options.semtrace)
+ else:
+ semreps = \
+ text_interpret(sents, gram, beta_reduce=options.beta, syntrace=options.syntrace)
+
+ for sent in sents:
+ n = 1
+ print('\nSentence: %s' % sent)
+ print(SPACER)
+ if options.evaluate:
+
+ for (syntree, semrep, value) in evaluations[sent]:
+ if isinstance(value, dict):
+ value = set(value.keys())
+ print('%d: %s' % (n, semrep.infixify()))
+ print(value)
+ n += 1
+ else:
+
+ for (syntree, semrep) in semreps[sent]:
+ print('%d: %s' % (n, semrep.infixify()))
+ n += 1
+
+if __name__ == "__main__":
+ demo()
+
+
+
diff --git a/jenkins-job-config.xml b/jenkins-job-config.xml
new file mode 100644
index 0000000..d9d4119
--- /dev/null
+++ b/jenkins-job-config.xml
@@ -0,0 +1,342 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<matrix-project>
+ <actions/>
+ <description></description>
+ <keepDependencies>false</keepDependencies>
+ <properties>
+ <com.coravy.hudson.plugins.github.GithubProjectProperty plugin="github at 1.4">
+ <projectUrl>https://github.com/nltk/nltk/</projectUrl>
+ </com.coravy.hudson.plugins.github.GithubProjectProperty>
+ </properties>
+ <scm class="hudson.plugins.git.GitSCM" plugin="git at 1.1.26">
+ <configVersion>2</configVersion>
+ <userRemoteConfigs>
+ <hudson.plugins.git.UserRemoteConfig>
+ <name></name>
+ <refspec></refspec>
+ <url>git://github.com/nltk/nltk.git</url>
+ </hudson.plugins.git.UserRemoteConfig>
+ </userRemoteConfigs>
+ <branches>
+ <hudson.plugins.git.BranchSpec>
+ <name>origin/develop</name>
+ </hudson.plugins.git.BranchSpec>
+ </branches>
+ <disableSubmodules>false</disableSubmodules>
+ <recursiveSubmodules>false</recursiveSubmodules>
+ <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+ <authorOrCommitter>false</authorOrCommitter>
+ <clean>false</clean>
+ <wipeOutWorkspace>false</wipeOutWorkspace>
+ <pruneBranches>false</pruneBranches>
+ <remotePoll>false</remotePoll>
+ <ignoreNotifyCommit>false</ignoreNotifyCommit>
+ <useShallowClone>false</useShallowClone>
+ <buildChooser class="hudson.plugins.git.util.DefaultBuildChooser"/>
+ <gitTool>Default</gitTool>
+ <submoduleCfg class="list"/>
+ <relativeTargetDir></relativeTargetDir>
+ <reference></reference>
+ <excludedRegions></excludedRegions>
+ <excludedUsers></excludedUsers>
+ <gitConfigName></gitConfigName>
+ <gitConfigEmail></gitConfigEmail>
+ <skipTag>false</skipTag>
+ <includedRegions></includedRegions>
+ <scmName></scmName>
+ </scm>
+ <canRoam>true</canRoam>
+ <disabled>false</disabled>
+ <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+ <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+ <triggers class="vector">
+ <com.cloudbees.jenkins.GitHubPushTrigger plugin="github at 1.4">
+ <spec></spec>
+ </com.cloudbees.jenkins.GitHubPushTrigger>
+ <hudson.triggers.SCMTrigger>
+ <spec>H/5 * * * *</spec>
+ <ignorePostCommitHooks>false</ignorePostCommitHooks>
+ </hudson.triggers.SCMTrigger>
+ </triggers>
+ <concurrentBuild>false</concurrentBuild>
+ <axes>
+ <jenkins.plugins.shiningpanda.matrix.ToxAxis plugin="shiningpanda at 0.16">
+ <name>TOXENV</name>
+ <values>
+ <string>py26-jenkins</string>
+ <string>py32-jenkins</string>
+ </values>
+ </jenkins.plugins.shiningpanda.matrix.ToxAxis>
+ </axes>
+ <builders>
+ <jenkins.plugins.shiningpanda.builders.ToxBuilder plugin="shiningpanda at 0.16">
+ <toxIni>tox.ini</toxIni>
+ <recreate>false</recreate>
+ </jenkins.plugins.shiningpanda.builders.ToxBuilder>
+ </builders>
+ <publishers>
+ <hudson.plugins.cobertura.CoberturaPublisher plugin="cobertura at 1.7.1">
+ <coberturaReportFile>**/coverage.xml</coberturaReportFile>
+ <onlyStable>false</onlyStable>
+ <failUnhealthy>false</failUnhealthy>
+ <failUnstable>false</failUnstable>
+ <autoUpdateHealth>false</autoUpdateHealth>
+ <autoUpdateStability>false</autoUpdateStability>
+ <healthyTarget>
+ <targets class="enum-map" enum-type="hudson.plugins.cobertura.targets.CoverageMetric">
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>CONDITIONAL</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>7000000</int>
+ </entry>
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>LINE</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>8000000</int>
+ </entry>
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>METHOD</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>8000000</int>
+ </entry>
+ </targets>
+ </healthyTarget>
+ <unhealthyTarget>
+ <targets class="enum-map" enum-type="hudson.plugins.cobertura.targets.CoverageMetric">
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>CONDITIONAL</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>0</int>
+ </entry>
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>LINE</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>0</int>
+ </entry>
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>METHOD</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>0</int>
+ </entry>
+ </targets>
+ </unhealthyTarget>
+ <failingTarget>
+ <targets class="enum-map" enum-type="hudson.plugins.cobertura.targets.CoverageMetric">
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>CONDITIONAL</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>0</int>
+ </entry>
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>LINE</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>0</int>
+ </entry>
+ <entry>
+ <hudson.plugins.cobertura.targets.CoverageMetric>METHOD</hudson.plugins.cobertura.targets.CoverageMetric>
+ <int>0</int>
+ </entry>
+ </targets>
+ </failingTarget>
+ <sourceEncoding>ASCII</sourceEncoding>
+ </hudson.plugins.cobertura.CoberturaPublisher>
+ <hudson.tasks.junit.JUnitResultArchiver>
+ <testResults>**/nosetests_scrubbed.xml</testResults>
+ <keepLongStdio>false</keepLongStdio>
+ <testDataPublishers/>
+ </hudson.tasks.junit.JUnitResultArchiver>
+ <hudson.plugins.violations.ViolationsPublisher plugin="violations at 0.7.11">
+ <config>
+ <suppressions class="tree-set">
+ <no-comparator/>
+ </suppressions>
+ <typeConfigs>
+ <no-comparator/>
+ <entry>
+ <string>checkstyle</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>checkstyle</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>codenarc</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>codenarc</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>cpd</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>cpd</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>cpplint</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>cpplint</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>csslint</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>csslint</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>findbugs</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>findbugs</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>fxcop</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>fxcop</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>gendarme</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>gendarme</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>jcreport</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>jcreport</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>jslint</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>jslint</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>pep8</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>pep8</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>perlcritic</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>perlcritic</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>pmd</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>pmd</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>pylint</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>pylint</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern>**/pylintoutput</pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>simian</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>simian</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ <entry>
+ <string>stylecop</string>
+ <hudson.plugins.violations.TypeConfig>
+ <type>stylecop</type>
+ <min>10</min>
+ <max>999</max>
+ <unstable>999</unstable>
+ <usePattern>false</usePattern>
+ <pattern></pattern>
+ </hudson.plugins.violations.TypeConfig>
+ </entry>
+ </typeConfigs>
+ <limit>100</limit>
+ <sourcePathPattern></sourcePathPattern>
+ <fauxProjectPath></fauxProjectPath>
+ <encoding>default</encoding>
+ </config>
+ </hudson.plugins.violations.ViolationsPublisher>
+ </publishers>
+ <buildWrappers>
+ <EnvInjectBuildWrapper plugin="envinject at 1.73">
+ <info>
+ <propertiesContent>PATH=/opt/local/bin:/opt/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin</propertiesContent>
+ <loadFilesFromMaster>false</loadFilesFromMaster>
+ </info>
+ </EnvInjectBuildWrapper>
+ </buildWrappers>
+ <executionStrategy class="hudson.matrix.DefaultMatrixExecutionStrategyImpl">
+ <runSequentially>false</runSequentially>
+ </executionStrategy>
+</matrix-project>
diff --git a/jenkins.sh b/jenkins.sh
new file mode 100755
index 0000000..8902591
--- /dev/null
+++ b/jenkins.sh
@@ -0,0 +1,48 @@
+#!/usr/bin/bash
+
+cd `dirname $0`
+
+#download nltk data packages
+python -c "import nltk; nltk.download('all')" || echo "NLTK data download failed: $?"
+
+#download nltk python dependencies
+pip install --upgrade -r pip-req.txt --allow-external matplotlib --allow-unverified matplotlib
+
+#download external dependencies
+pushd ${HOME}
+[[ ! -d 'third' ]] && mkdir 'third'
+pushd 'third'
+
+#download nltk stanford dependencies
+stanford_parser_package_zip_name=$(curl -s 'http://nlp.stanford.edu/software/lex-parser.shtml' | grep -o 'stanford-parser-full-.*\.zip' | head -n1)
+[[ ${stanford_parser_package_zip_name} =~ (.+)\.zip ]]
+stanford_parser_package_name=${BASH_REMATCH[1]}
+if [[ ! -d ${stanford_parser_package_name} ]]; then
+ wget -nv "http://nlp.stanford.edu/software/$stanford_parser_package_zip_name"
+ unzip ${stanford_parser_package_zip_name}
+ rm ${stanford_parser_package_zip_name}
+ ln -s ${stanford_parser_package_name} 'stanford-parser'
+fi
+
+stanford_tagger_package_zip_name=$(curl -s 'http://nlp.stanford.edu/downloads/tagger.shtml' | grep -o 'stanford-postagger-full-.*\.zip' | head -n1)
+[[ ${stanford_tagger_package_zip_name} =~ (.+)\.zip ]]
+stanford_tagger_package_name=${BASH_REMATCH[1]}
+if [[ ! -d ${stanford_tagger_package_name} ]]; then
+ wget -nv "http://nlp.stanford.edu/software/$stanford_tagger_package_zip_name"
+ unzip ${stanford_tagger_package_zip_name}
+ rm ${stanford_tagger_package_zip_name}
+ ln -s ${stanford_tagger_package_name} 'stanford-postagger'
+fi
+
+popd
+popd
+
+#coverage
+coverage erase
+coverage run --source=nltk nltk/test/runtests.py --with-xunit
+coverage xml --omit=nltk/test/*
+iconv -c -f utf-8 -t utf-8 nosetests.xml > nosetests_scrubbed.xml
+pylint -f parseable nltk > pylintoutput
+
+#script always succeeds
+true
diff --git a/nltk/VERSION b/nltk/VERSION
new file mode 100644
index 0000000..2daa89b
--- /dev/null
+++ b/nltk/VERSION
@@ -0,0 +1 @@
+3.0.0b1
diff --git a/nltk/__init__.py b/nltk/__init__.py
new file mode 100644
index 0000000..85f862a
--- /dev/null
+++ b/nltk/__init__.py
@@ -0,0 +1,173 @@
+# Natural Language Toolkit (NLTK)
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Authors: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+The Natural Language Toolkit (NLTK) is an open source Python library
+for Natural Language Processing. A free online book is available.
+(If you use the library for academic research, please cite the book.)
+
+Steven Bird, Ewan Klein, and Edward Loper (2009).
+Natural Language Processing with Python. O'Reilly Media Inc.
+http://nltk.org/book
+"""
+from __future__ import print_function, absolute_import
+
+import os
+
+##//////////////////////////////////////////////////////
+## Metadata
+##//////////////////////////////////////////////////////
+
+# Version. For each new release, the version number should be updated
+# in the file VERSION.
+try:
+ # If a VERSION file exists, use it!
+ version_file = os.path.join(os.path.dirname(__file__), 'VERSION')
+ with open(version_file, 'r') as infile:
+ __version__ = infile.read().strip()
+except NameError:
+ __version__ = 'unknown (running code interactively?)'
+except IOError as ex:
+ __version__ = "unknown (%s)" % ex
+
+if __doc__ is not None: # fix for the ``python -OO``
+ __doc__ += '\n at version: ' + __version__
+
+
+# Copyright notice
+__copyright__ = """\
+Copyright (C) 2001-2014 NLTK Project.
+
+Distributed and Licensed under the Apache License, Version 2.0,
+which is included by reference.
+"""
+
+__license__ = "Apache License, Version 2.0"
+# Description of the toolkit, keywords, and the project's primary URL.
+__longdescr__ = """\
+The Natural Language Toolkit (NLTK) is a Python package for
+natural language processing. NLTK requires Python 2.6 or higher."""
+__keywords__ = ['NLP', 'CL', 'natural language processing',
+ 'computational linguistics', 'parsing', 'tagging',
+ 'tokenizing', 'syntax', 'linguistics', 'language',
+ 'natural language', 'text analytics']
+__url__ = "http://nltk.org/"
+
+# Maintainer, contributors, etc.
+__maintainer__ = "Steven Bird, Edward Loper, Ewan Klein"
+__maintainer_email__ = "stevenbird1 at gmail.com"
+__author__ = __maintainer__
+__author_email__ = __maintainer_email__
+
+# "Trove" classifiers for Python Package Index.
+__classifiers__ = [
+ 'Development Status :: 5 - Production/Stable',
+ 'Intended Audience :: Developers',
+ 'Intended Audience :: Education',
+ 'Intended Audience :: Information Technology',
+ 'Intended Audience :: Science/Research',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python :: 2.6',
+ 'Programming Language :: Python :: 2.7',
+ 'Topic :: Scientific/Engineering',
+ 'Topic :: Scientific/Engineering :: Artificial Intelligence',
+ 'Topic :: Scientific/Engineering :: Human Machine Interfaces',
+ 'Topic :: Scientific/Engineering :: Information Analysis',
+ 'Topic :: Text Processing',
+ 'Topic :: Text Processing :: Filters',
+ 'Topic :: Text Processing :: General',
+ 'Topic :: Text Processing :: Indexing',
+ 'Topic :: Text Processing :: Linguistic',
+ ]
+
+from nltk.internals import config_java
+
+# support numpy from pypy
+try:
+ import numpypy
+except ImportError:
+ pass
+
+###########################################################
+# TOP-LEVEL MODULES
+###########################################################
+
+# Import top-level functionality into top-level namespace
+
+from nltk.collocations import *
+from nltk.decorators import decorator, memoize
+from nltk.featstruct import *
+from nltk.grammar import *
+from nltk.probability import *
+from nltk.text import *
+from nltk.tree import *
+from nltk.util import *
+from nltk.jsontags import *
+
+###########################################################
+# PACKAGES
+###########################################################
+
+from nltk.align import *
+from nltk.chunk import *
+from nltk.classify import *
+from nltk.inference import *
+from nltk.metrics import *
+from nltk.parse import *
+from nltk.tag import *
+from nltk.tokenize import *
+from nltk.sem import *
+from nltk.stem import *
+
+# Packages which can be lazily imported
+# (a) we don't import *
+# (b) they're slow to import or have run-time dependencies
+# that can safely fail at run time
+
+from nltk import lazyimport
+app = lazyimport.LazyModule('nltk.app', locals(), globals())
+chat = lazyimport.LazyModule('nltk.chat', locals(), globals())
+corpus = lazyimport.LazyModule('nltk.corpus', locals(), globals())
+draw = lazyimport.LazyModule('nltk.draw', locals(), globals())
+toolbox = lazyimport.LazyModule('nltk.toolbox', locals(), globals())
+
+# Optional loading
+
+try:
+ import numpy
+except ImportError:
+ pass
+else:
+ from nltk import cluster; from .cluster import *
+
+from nltk.downloader import download, download_shell
+try:
+ import tkinter
+except ImportError:
+ pass
+else:
+ try:
+ from nltk.downloader import download_gui
+ except RuntimeError as e:
+ import warnings
+ warnings.warn("Corpus downloader GUI not loaded "
+ "(RuntimeError during import: %s)" % str(e))
+
+# explicitly import all top-level modules (ensuring
+# they override the same names inadvertently imported
+# from a subpackage)
+
+from nltk import align, ccg, chunk, classify, collocations
+from nltk import data, featstruct, grammar, help, inference, metrics
+from nltk import misc, parse, probability, sem, stem, wsd
+from nltk import tag, tbl, text, tokenize, tree, treetransforms, util
+
+# override any accidentally imported demo
+def demo():
+ print("To run the demo code for a module, type nltk.module.demo()")
diff --git a/nltk/align/__init__.py b/nltk/align/__init__.py
new file mode 100644
index 0000000..6db9d6a
--- /dev/null
+++ b/nltk/align/__init__.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Aligners
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Experimental functionality for bitext alignment.
+These interfaces are prone to change.
+"""
+
+from nltk.align.api import AlignedSent, Alignment
+from nltk.align.ibm1 import IBMModel1
+from nltk.align.ibm2 import IBMModel2
+from nltk.align.ibm3 import IBMModel3
+
+
diff --git a/nltk/align/api.py b/nltk/align/api.py
new file mode 100644
index 0000000..33bd35d
--- /dev/null
+++ b/nltk/align/api.py
@@ -0,0 +1,298 @@
+# Natural Language Toolkit: Aligned Sentences
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Will Zhang <wilzzha at gmail.com>
+# Guan Gui <ggui at student.unimelb.edu.au>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function, unicode_literals
+
+from nltk.compat import python_2_unicode_compatible, string_types
+from nltk.metrics import precision, recall
+
+ at python_2_unicode_compatible
+class AlignedSent(object):
+ """
+ Return an aligned sentence object, which encapsulates two sentences
+ along with an ``Alignment`` between them.
+
+ >>> from nltk.align import AlignedSent
+ >>> algnsent = AlignedSent(['klein', 'ist', 'das', 'Haus'],
+ ... ['the', 'house', 'is', 'small'], '0-2 1-3 2-1 3-0')
+ >>> algnsent.words
+ ['klein', 'ist', 'das', 'Haus']
+ >>> algnsent.mots
+ ['the', 'house', 'is', 'small']
+ >>> algnsent.alignment
+ Alignment([(0, 2), (1, 3), (2, 1), (3, 0)])
+ >>> algnsent.precision('0-2 1-3 2-1 3-3')
+ 0.75
+ >>> from nltk.corpus import comtrans
+ >>> print(comtrans.aligned_sents()[54])
+ <AlignedSent: 'Weshalb also sollten...' -> 'So why should EU arm...'>
+ >>> print(comtrans.aligned_sents()[54].alignment)
+ 0-0 0-1 1-0 2-2 3-4 3-5 4-7 5-8 6-3 7-9 8-9 9-10 9-11 10-12 11-6 12-6 13-13
+
+ :param words: source language words
+ :type words: list(str)
+ :param mots: target language words
+ :type mots: list(str)
+ :param alignment: the word-level alignments between the source
+ and target language
+ :type alignment: Alignment
+ """
+
+ def __init__(self, words=[], mots=[], alignment='', encoding='utf8'):
+ self._words = words
+ self._mots = mots
+ self.alignment = alignment
+
+ @property
+ def words(self):
+ return self._words
+
+ @property
+ def mots(self):
+ return self._mots
+
+ def _get_alignment(self):
+ return self._alignment
+ def _set_alignment(self, alignment):
+ if not isinstance(alignment, Alignment):
+ alignment = Alignment(alignment)
+ self._check_align(alignment)
+ self._alignment = alignment
+ alignment = property(_get_alignment, _set_alignment)
+
+ def _check_align(self, a):
+ """
+ Check whether the alignments are legal.
+
+ :param a: alignment to be checked
+ :raise IndexError: if alignment is out of sentence boundary
+ :rtype: boolean
+ """
+ if not all(0 <= p[0] < len(self._words) for p in a):
+ raise IndexError("Alignment is outside boundary of words")
+ if not all(0 <= p[1] < len(self._mots) for p in a):
+ raise IndexError("Alignment is outside boundary of mots")
+ return True
+
+ def __repr__(self):
+ """
+ Return a string representation for this ``AlignedSent``.
+
+ :rtype: str
+ """
+ words = "[%s]" % (", ".join("'%s'" % w for w in self._words))
+ mots = "[%s]" % (", ".join("'%s'" % w for w in self._mots))
+
+ return "AlignedSent(%s, %s, %r)" % (words, mots, self._alignment)
+
+ def __str__(self):
+ """
+ Return a human-readable string representation for this ``AlignedSent``.
+
+ :rtype: str
+ """
+ source = " ".join(self._words)[:20] + "..."
+ target = " ".join(self._mots)[:20] + "..."
+ return "<AlignedSent: '%s' -> '%s'>" % (source, target)
+
+ def invert(self):
+ """
+ Return the aligned sentence pair, reversing the directionality
+
+ :rtype: AlignedSent
+ """
+ return AlignedSent(self._mots, self._words,
+ self._alignment.invert())
+
+ def precision(self, reference):
+ """
+ Return the precision of an aligned sentence with respect to a
+ "gold standard" reference ``AlignedSent``.
+
+ :type reference: AlignedSent or Alignment
+ :param reference: A "gold standard" reference aligned sentence.
+ :rtype: float or None
+ """
+ # Get alignments in set of 2-tuples form
+ # The "possible" precision is used since it doesn't penalize for finding
+ # an alignment that was marked as "possible" (NAACL corpus)
+
+ align = self.alignment
+ if isinstance(reference, AlignedSent):
+ possible = reference.alignment
+ else:
+ possible = Alignment(reference)
+
+ return precision(possible, align)
+
+
+ def recall(self, reference):
+ """
+ Return the recall of an aligned sentence with respect to a
+ "gold standard" reference ``AlignedSent``.
+
+ :type reference: AlignedSent or Alignment
+ :param reference: A "gold standard" reference aligned sentence.
+ :rtype: float or None
+ """
+ # Get alignments in set of 2-tuples form
+ # The "sure" recall is used so we don't penalize for missing an
+ # alignment that was only marked as "possible".
+
+ align = self.alignment
+ if isinstance(reference, AlignedSent):
+ sure = reference.alignment
+ else:
+ sure = Alignment(reference)
+
+ # Call NLTKs existing functions for recall
+ return recall(sure, align)
+
+
+ def alignment_error_rate(self, reference, possible=None):
+ """
+ Return the Alignment Error Rate (AER) of an aligned sentence
+ with respect to a "gold standard" reference ``AlignedSent``.
+
+ Return an error rate between 0.0 (perfect alignment) and 1.0 (no
+ alignment).
+
+ >>> from nltk.align import AlignedSent
+ >>> s = AlignedSent(["the", "cat"], ["le", "chat"], [(0, 0), (1, 1)])
+ >>> s.alignment_error_rate(s)
+ 0.0
+
+ :type reference: AlignedSent or Alignment
+ :param reference: A "gold standard" reference aligned sentence.
+ :type possible: AlignedSent or Alignment or None
+ :param possible: A "gold standard" reference of possible alignments
+ (defaults to *reference* if None)
+ :rtype: float or None
+ """
+ # Get alignments in set of 2-tuples form
+ align = self.alignment
+ if isinstance(reference, AlignedSent):
+ sure = reference.alignment
+ else:
+ sure = Alignment(reference)
+
+ if possible is not None:
+ # Set possible alignment
+ if isinstance(possible, AlignedSent):
+ possible = possible.alignment
+ else:
+ possible = Alignment(possible)
+ else:
+ # Possible alignment is just sure alignment
+ possible = sure
+
+ # Sanity check
+ assert(sure.issubset(possible))
+
+ # Return the Alignment Error Rate
+ return (1.0 - float(len(align & sure) + len(align & possible)) /
+ float(len(align) + len(sure)))
+
+
+ at python_2_unicode_compatible
+class Alignment(frozenset):
+ """
+ A storage class for representing alignment between two sequences, s1, s2.
+ In general, an alignment is a set of tuples of the form (i, j, ...)
+ representing an alignment between the i-th element of s1 and the
+ j-th element of s2. Tuples are extensible (they might contain
+ additional data, such as a boolean to indicate sure vs possible alignments).
+
+ >>> from nltk.align import Alignment
+ >>> a = Alignment([(0, 0), (0, 1), (1, 2), (2, 2)])
+ >>> a.invert()
+ Alignment([(0, 0), (1, 0), (2, 1), (2, 2)])
+ >>> print(a.invert())
+ 0-0 1-0 2-1 2-2
+ >>> a[0]
+ [(0, 1), (0, 0)]
+ >>> a.invert()[2]
+ [(2, 1), (2, 2)]
+ >>> b = Alignment([(0, 0), (0, 1)])
+ >>> b.issubset(a)
+ True
+ >>> c = Alignment('0-0 0-1')
+ >>> b == c
+ True
+ """
+
+ def __new__(cls, string_or_pairs):
+ if isinstance(string_or_pairs, string_types):
+ string_or_pairs = [_giza2pair(p) for p in string_or_pairs.split()]
+ self = frozenset.__new__(cls, string_or_pairs)
+ self._len = (max(p[0] for p in self) if self != frozenset([]) else 0)
+ self._index = None
+ return self
+
+ def __getitem__(self, key):
+ """
+ Look up the alignments that map from a given index or slice.
+ """
+ if not self._index:
+ self._build_index()
+ return self._index.__getitem__(key)
+
+ def invert(self):
+ """
+ Return an Alignment object, being the inverted mapping.
+ """
+ return Alignment(((p[1], p[0]) + p[2:]) for p in self)
+
+ def range(self, positions=None):
+ """
+ Work out the range of the mapping from the given positions.
+ If no positions are specified, compute the range of the entire mapping.
+ """
+ image = set()
+ if not self._index:
+ self._build_index()
+ if not positions:
+ positions = list(range(len(self._index)))
+ for p in positions:
+ image.update(f for _,f in self._index[p])
+ return sorted(image)
+
+ def __repr__(self):
+ """
+ Produce a Giza-formatted string representing the alignment.
+ """
+ return "Alignment(%r)" % sorted(self)
+
+ def __str__(self):
+ """
+ Produce a Giza-formatted string representing the alignment.
+ """
+ return " ".join("%d-%d" % p[:2] for p in sorted(self))
+
+ def _build_index(self):
+ """
+ Build a list self._index such that self._index[i] is a list
+ of the alignments originating from word i.
+ """
+ self._index = [[] for _ in range(self._len + 1)]
+ for p in self:
+ self._index[p[0]].append(p)
+
+
+def _giza2pair(pair_string):
+ i, j = pair_string.split("-")
+ return int(i), int(j)
+
+def _naacl2pair(pair_string):
+ i, j, p = pair_string.split("-")
+ return int(i), int(j)
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/align/bleu.py b/nltk/align/bleu.py
new file mode 100644
index 0000000..8a869df
--- /dev/null
+++ b/nltk/align/bleu.py
@@ -0,0 +1,148 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: BLEU
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import division
+
+from nltk.util import ngrams
+from nltk import word_tokenize
+
+import math
+
+class BLEU(object):
+ """
+ This class implements the BLEU method, which is used to evaluate
+ the quality of machine translation.
+
+ The BLEU method mainly consists of two parts:
+
+ Part 1 - modified n-gram precision
+
+ The normal precision method may lead to some wrong translations with
+ high-precision, e.g., the translation, in which a word of reference
+ repeats several times, has very high precision. So in the modified
+ n-gram precision, a reference word will be considered exhausted after
+ a matching candidate word is identified.
+
+ Part 2 - brevity penalty
+
+ As the modified n-gram precision stil has the problem from the short
+ length sentence, brevity penalty is used to modify the overall BLEU
+ score according to length.
+
+ 1. Test with an instance:
+
+ >>> weights = [0.25, 0.25, 0.25, 0.25]
+ >>> candidate1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'which',
+ ... 'ensures', 'that', 'the', 'military', 'always',
+ ... 'obeys', 'the', 'commands', 'of', 'the', 'party', '.']
+
+ >>> candidate2 = ['It', 'is', 'to', 'insure', 'the', 'troops',
+ ... 'forever', 'hearing', 'the', 'activity', 'guidebook',
+ ... 'that', 'party', 'direct', '.']
+
+ >>> reference1 = ['It', 'is', 'a', 'guide', 'to', 'action', 'that',
+ ... 'ensures', 'that', 'the', 'military', 'will', 'forever',
+ ... 'heed', 'Party', 'commands', '.']
+
+ >>> reference2 = ['It', 'is', 'the', 'guiding', 'principle', 'which',
+ ... 'guarantees', 'the', 'military', 'forces', 'always',
+ ... 'being', 'under', 'the', 'command', 'of', 'the',
+ ... 'Party', '.']
+
+ >>> reference3 = ['It', 'is', 'the', 'practical', 'guide', 'for', 'the',
+ ... 'army', 'always', 'to', 'heed', 'the', 'directions',
+ ... 'of', 'the', 'party', '.']
+
+ >>> BLEU.compute(candidate1, [reference1, reference2, reference3], weights)
+ 0.0555662774619807
+
+ >>> BLEU.compute(candidate2, [reference1, reference2, reference3], weights)
+ 0.04211415110983725
+
+ 2. Test with two corpus that one is a reference and another is
+ an output from translation system:
+
+ >>> weights = [0.25, 0.25, 0.25, 0.25]
+ >>> ref_file = open('newstest2012-ref.en')
+ >>> candidate_file = open('newstest2012.fr-en.cmu-avenue')
+
+ >>> total = 0.0
+ >>> count = 0
+
+ >>> for candi_raw in candidate_file:
+ ... ref_raw = ref_file.readline()
+ ... ref_tokens = word_tokenize(ref_raw)
+ ... candi_tokens = word_tokenize(candi_raw)
+ ... total = BLEU.compute(candi_tokens, [ref_tokens], weights)
+ ... count += 1
+
+ >>> total/count
+ 2.787504437460048e-05
+
+ """
+
+ @staticmethod
+ def compute(candidate, references, weights):
+
+ candidate = list(map(lambda x: x.lower(), candidate))
+ references = list(map(lambda x: [c.lower() for c in x], references))
+
+ n = len(weights)
+
+ bp = BLEU.brevity_penalty(candidate, references)
+
+ s = 0.0
+ i = 1
+ for weight in weights:
+ p_n = BLEU.modified_precision(candidate, references, i)
+ if p_n != 0:
+ s += weight * math.log(p_n)
+ i += 1
+
+ return bp * math.exp(s)
+
+ @staticmethod
+ def modified_precision(candidate, references, n):
+
+ candidate_ngrams = list(ngrams(candidate, n))
+
+ if len(candidate_ngrams) == 0:
+ return 0
+
+ c_words = set(candidate_ngrams)
+
+ for word in c_words:
+ count_w = candidate_ngrams.count(word) + 1
+
+ count_max = 0
+ for reference in references:
+ reference_ngrams = list(ngrams(reference, n))
+
+ count = reference_ngrams.count(word) + 1
+ if count > count_max:
+ count_max = count
+
+ return min(count_w, count_max) / (len(candidate) + len(c_words))
+
+ @staticmethod
+ def brevity_penalty(candidate, references):
+ c = len(candidate)
+
+ lengthes_ref = map(lambda x: abs(len(x) - c), references)
+
+ r = min(*lengthes_ref)
+
+ if c > r:
+ return 1
+ else:
+ return math.exp(1 - r/c)
+
+# run doctests
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
diff --git a/nltk/align/gale_church.py b/nltk/align/gale_church.py
new file mode 100644
index 0000000..9d1c3a7
--- /dev/null
+++ b/nltk/align/gale_church.py
@@ -0,0 +1,238 @@
+# -*- coding: utf-8 -*-
+
+# Natural Language Toolkit: Gale-Church Aligner
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Author: Torsten Marek <marek at ifi.uzh.ch>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+
+A port of the Gale-Church Aligner.
+
+Gale & Church (1993), A Program for Aligning Sentences in Bilingual Corpora.
+http://aclweb.org/anthology/J93-1004.pdf
+
+"""
+
+from __future__ import division
+import math
+
+try:
+ from scipy.stats import norm
+ from norm import logsf as norm_logsf
+except ImportError:
+ def erfcc(x):
+ """Complementary error function."""
+ z = abs(x)
+ t = 1 / (1 + 0.5 * z)
+ r = t * math.exp(-z * z -
+ 1.26551223 + t *
+ (1.00002368 + t *
+ (.37409196 + t *
+ (.09678418 + t *
+ (-.18628806 + t *
+ (.27886807 + t *
+ (-1.13520398 + t *
+ (1.48851587 + t *
+ (-.82215223 + t * .17087277)))))))))
+ if x >= 0.:
+ return r
+ else:
+ return 2. - r
+
+
+ def norm_cdf(x):
+ """Return the area under the normal distribution from M{-∞..x}."""
+ return 1 - 0.5 * erfcc(x / math.sqrt(2))
+
+
+ def norm_logsf(x):
+ try:
+ return math.log(1 - norm_cdf(x))
+ except ValueError:
+ return float('-inf')
+
+
+LOG2 = math.log(2)
+
+
+class LanguageIndependent(object):
+ # These are the language-independent probabilities and parameters
+ # given in Gale & Church
+
+ # for the computation, l_1 is always the language with less characters
+ PRIORS = {
+ (1, 0): 0.0099,
+ (0, 1): 0.0099,
+ (1, 1): 0.89,
+ (2, 1): 0.089,
+ (1, 2): 0.089,
+ (2, 2): 0.011,
+ }
+
+ AVERAGE_CHARACTERS = 1
+ VARIANCE_CHARACTERS = 6.8
+
+
+def trace(backlinks, source, target):
+ links = []
+ pos = (len(source), len(target))
+
+ while pos != (0, 0):
+ s, t = backlinks[pos]
+ for i in range(s):
+ for j in range(t):
+ links.append((pos[0] - i - 1, pos[1] - j - 1))
+ pos = (pos[0] - s, pos[1] - t)
+
+ return links[::-1]
+
+
+def align_log_prob(i, j, source_sents, target_sents, alignment, params):
+ """Returns the log probability of the two sentences C{source_sents[i]}, C{target_sents[j]}
+ being aligned with a specific C{alignment}.
+
+ @param i: The offset of the source sentence.
+ @param j: The offset of the target sentence.
+ @param source_sents: The list of source sentence lengths.
+ @param target_sents: The list of target sentence lengths.
+ @param alignment: The alignment type, a tuple of two integers.
+ @param params: The sentence alignment parameters.
+
+ @returns: The log probability of a specific alignment between the two sentences, given the parameters.
+ """
+ l_s = sum(source_sents[i - offset - 1] for offset in range(alignment[0]))
+ l_t = sum(target_sents[j - offset - 1] for offset in range(alignment[1]))
+ try:
+ # actually, the paper says l_s * params.VARIANCE_CHARACTERS, this is based on the C
+ # reference implementation. With l_s in the denominator, insertions are impossible.
+ m = (l_s + l_t / params.AVERAGE_CHARACTERS) / 2
+ delta = (l_s * params.AVERAGE_CHARACTERS - l_t) / math.sqrt(m * params.VARIANCE_CHARACTERS)
+ except ZeroDivisionError:
+ return float('-inf')
+
+ return - (LOG2 + norm_logsf(abs(delta)) + math.log(params.PRIORS[alignment]))
+
+
+def align_blocks(source_sents, target_sents, params = LanguageIndependent):
+ """Return the sentence alignment of two text blocks (usually paragraphs).
+
+ >>> align_blocks([5,5,5], [7,7,7])
+ [(0, 0), (1, 1), (2, 2)]
+ >>> align_blocks([10,5,5], [12,20])
+ [(0, 0), (1, 1), (2, 1)]
+ >>> align_blocks([12,20], [10,5,5])
+ [(0, 0), (1, 1), (1, 2)]
+ >>> align_blocks([10,2,10,10,2,10], [12,3,20,3,12])
+ [(0, 0), (1, 1), (2, 2), (3, 2), (4, 3), (5, 4)]
+
+ @param source_sents: The list of source sentence lengths.
+ @param target_sents: The list of target sentence lengths.
+ @param params: the sentence alignment parameters.
+ @return: The sentence alignments, a list of index pairs.
+ """
+
+ alignment_types = list(params.PRIORS.keys())
+
+ # there are always three rows in the history (with the last of them being filled)
+ D = [[]]
+
+ backlinks = {}
+
+ for i in range(len(source_sents) + 1):
+ for j in range(len(target_sents) + 1):
+ min_dist = float('inf')
+ min_align = None
+ for a in alignment_types:
+ prev_i = - 1 - a[0]
+ prev_j = j - a[1]
+ if prev_i < -len(D) or prev_j < 0:
+ continue
+ p = D[prev_i][prev_j] + align_log_prob(i, j, source_sents, target_sents, a, params)
+ if p < min_dist:
+ min_dist = p
+ min_align = a
+
+ if min_dist == float('inf'):
+ min_dist = 0
+
+ backlinks[(i, j)] = min_align
+ D[-1].append(min_dist)
+
+ if len(D) > 2:
+ D.pop(0)
+ D.append([])
+
+ return trace(backlinks, source_sents, target_sents)
+
+
+def align_texts(source_blocks, target_blocks, params = LanguageIndependent):
+ """Creates the sentence alignment of two texts.
+
+ Texts can consist of several blocks. Block boundaries cannot be crossed by sentence
+ alignment links.
+
+ Each block consists of a list that contains the lengths (in characters) of the sentences
+ in this block.
+
+ @param source_blocks: The list of blocks in the source text.
+ @param target_blocks: The list of blocks in the target text.
+ @param params: the sentence alignment parameters.
+
+ @returns: A list of sentence alignment lists
+ """
+ if len(source_blocks) != len(target_blocks):
+ raise ValueError("Source and target texts do not have the same number of blocks.")
+
+ return [align_blocks(source_block, target_block, params)
+ for source_block, target_block in zip(source_blocks, target_blocks)]
+
+
+# File I/O functions; may belong in a corpus reader
+
+def split_at(it, split_value):
+ """Splits an iterator C{it} at values of C{split_value}.
+
+ Each instance of C{split_value} is swallowed. The iterator produces
+ subiterators which need to be consumed fully before the next subiterator
+ can be used.
+ """
+ def _chunk_iterator(first):
+ v = first
+ while v != split_value:
+ yield v
+ v = it.next()
+
+ while True:
+ yield _chunk_iterator(it.next())
+
+
+def parse_token_stream(stream, soft_delimiter, hard_delimiter):
+ """Parses a stream of tokens and splits it into sentences (using C{soft_delimiter} tokens)
+ and blocks (using C{hard_delimiter} tokens) for use with the L{align_texts} function.
+ """
+ return [
+ [sum(len(token) for token in sentence_it)
+ for sentence_it in split_at(block_it, soft_delimiter)]
+ for block_it in split_at(stream, hard_delimiter)]
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
+
+# Code for test files in nltk_contrib/align/data/*.tok
+# import sys
+# from contextlib import nested
+# with nested(open(sys.argv[1], "r"), open(sys.argv[2], "r")) as (s, t):
+# source = parse_token_stream((l.strip() for l in s), ".EOS", ".EOP")
+# target = parse_token_stream((l.strip() for l in t), ".EOS", ".EOP")
+# print align_texts(source, target)
+
+
+
+
+
diff --git a/nltk/align/ibm1.py b/nltk/align/ibm1.py
new file mode 100644
index 0000000..e88fe10
--- /dev/null
+++ b/nltk/align/ibm1.py
@@ -0,0 +1,135 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: IBM Model 1
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Author: Chin Yee Lee <c.lee32 at student.unimelb.edu.au>
+# Hengfeng Li <hengfeng12345 at gmail.com>
+# Ruxin Hou <r.hou at student.unimelb.edu.au>
+# Calvin Tanujaya Lim <c.tanujayalim at gmail.com>
+# Based on earlier version by:
+# Will Zhang <wilzzha at gmail.com>
+# Guan Gui <ggui at student.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import division
+from collections import defaultdict
+from nltk.align import AlignedSent
+from nltk.corpus import comtrans
+
+class IBMModel1(object):
+ """
+ This class implements the algorithm of Expectation Maximization for
+ the IBM Model 1.
+
+ Step 1 - Collect the evidence of a English word being translated by a
+ foreign language word.
+
+ Step 2 - Estimate the probability of translation according to the
+ evidence from Step 1.
+
+ >>> bitexts = comtrans.aligned_sents()[:100]
+ >>> ibm = IBMModel1(bitexts, 20)
+
+ >>> aligned_sent = ibm.align(bitexts[6])
+ >>> aligned_sent.alignment
+ Alignment([(0, 0), (1, 1), (2, 2), (3, 7), (4, 7), (5, 8)])
+ >>> print('{0:.3f}'.format(bitexts[6].precision(aligned_sent)))
+ 0.556
+ >>> print('{0:.3f}'.format(bitexts[6].recall(aligned_sent)))
+ 0.833
+ >>> print('{0:.3f}'.format(bitexts[6].alignment_error_rate(aligned_sent)))
+ 0.333
+
+ """
+ def __init__(self, align_sents, num_iter):
+ self.probabilities = self.train(align_sents, num_iter)
+
+ def train(self, align_sents, num_iter):
+ """
+ Return the translation probability model trained by IBM model 1.
+
+ Arguments:
+ align_sents -- A list of instances of AlignedSent class, which
+ contains sentence pairs.
+ num_iter -- The number of iterations.
+
+ Returns:
+ t_ef -- A dictionary of translation probabilities.
+ """
+
+ # Vocabulary of each language
+ fr_vocab = set()
+ en_vocab = set()
+ for alignSent in align_sents:
+ en_vocab.update(alignSent.words)
+ fr_vocab.update(alignSent.mots)
+ # Add the Null token
+ fr_vocab.add(None)
+
+ # Initial probability
+ init_prob = 1 / len(en_vocab)
+
+ # Create the translation model with initial probability
+ t_ef = defaultdict(lambda: defaultdict(lambda: init_prob))
+
+ total_e = defaultdict(lambda: 0.0)
+
+ for i in range(0, num_iter):
+ count_ef = defaultdict(lambda: defaultdict(lambda: 0.0))
+ total_f = defaultdict(lambda: 0.0)
+
+ for alignSent in align_sents:
+ en_set = alignSent.words
+ fr_set = [None] + alignSent.mots
+
+ # Compute normalization
+ for e in en_set:
+ total_e[e] = 0.0
+ for f in fr_set:
+ total_e[e] += t_ef[e][f]
+
+ # Collect counts
+ for e in en_set:
+ for f in fr_set:
+ c = t_ef[e][f] / total_e[e]
+ count_ef[e][f] += c
+ total_f[f] += c
+
+ # Compute the estimate probabilities
+ for f in fr_vocab:
+ for e in en_vocab:
+ t_ef[e][f] = count_ef[e][f] / total_f[f]
+
+ return t_ef
+
+ def align(self, align_sent):
+ """
+ Returns the alignment result for one sentence pair.
+ """
+
+ if self.probabilities is None:
+ raise ValueError("The model does not train.")
+
+ alignment = []
+
+ for j, en_word in enumerate(align_sent.words):
+
+ # Initialize the maximum probability with Null token
+ max_align_prob = (self.probabilities[en_word][None], None)
+ for i, fr_word in enumerate(align_sent.mots):
+ # Find out the maximum probability
+ max_align_prob = max(max_align_prob,
+ (self.probabilities[en_word][fr_word], i))
+
+ # If the maximum probability is not Null token,
+ # then append it to the alignment.
+ if max_align_prob[1] is not None:
+ alignment.append((j, max_align_prob[1]))
+
+ return AlignedSent(align_sent.words, align_sent.mots, alignment)
+
+# run doctests
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
diff --git a/nltk/align/ibm2.py b/nltk/align/ibm2.py
new file mode 100644
index 0000000..1aca80c
--- /dev/null
+++ b/nltk/align/ibm2.py
@@ -0,0 +1,200 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: IBM Model 2
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import division
+from collections import defaultdict
+from nltk.align import AlignedSent
+from nltk.corpus import comtrans
+from nltk.align.ibm1 import IBMModel1
+
+class IBMModel2(object):
+ """
+ This class implements the algorithm of Expectation Maximization for
+ the IBM Model 2.
+
+ Step 1 - Run a number of iterations of IBM Model 1 and get the initial
+ distribution of translation probability.
+
+ Step 2 - Collect the evidence of a English word being translated by a
+ foreign language word.
+
+ Step 3 - Estimate the probability of translation and alignment according
+ to the evidence from Step 2.
+
+ >>> bitexts = comtrans.aligned_sents()[:100]
+ >>> ibm = IBMModel2(bitexts, 5)
+ >>> aligned_sent = ibm.align(bitexts[0])
+ >>> aligned_sent.words
+ ['Wiederaufnahme', 'der', 'Sitzungsperiode']
+ >>> aligned_sent.mots
+ ['Resumption', 'of', 'the', 'session']
+ >>> aligned_sent.alignment
+ Alignment([(0, 0), (1, 2), (2, 3)])
+ >>> bitexts[0].precision(aligned_sent)
+ 0.75
+ >>> bitexts[0].recall(aligned_sent)
+ 1.0
+ >>> bitexts[0].alignment_error_rate(aligned_sent)
+ 0.1428571428571429
+
+ """
+ def __init__(self, align_sents, num_iter):
+ self.probabilities, self.alignments = self.train(align_sents, num_iter)
+
+ def train(self, align_sents, num_iter):
+ """
+ Return the translation and alignment probability distributions
+ trained by the Expectation Maximization algorithm for IBM Model 2.
+
+ Arguments:
+ align_sents -- A list contains some sentence pairs.
+ num_iter -- The number of iterations.
+
+ Returns:
+ t_ef -- A distribution of translation probabilities.
+ align -- A distribution of alignment probabilities.
+ """
+
+ # Get initial translation probability distribution
+ # from a few iterations of Model 1 training.
+ ibm1 = IBMModel1(align_sents, 10)
+ t_ef = ibm1.probabilities
+
+ # Vocabulary of each language
+ fr_vocab = set()
+ en_vocab = set()
+ for alignSent in align_sents:
+ en_vocab.update(alignSent.words)
+ fr_vocab.update(alignSent.mots)
+ fr_vocab.add(None)
+
+ align = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: float))))
+
+ # Initialize the distribution of alignment probability,
+ # a(i|j,l_e, l_f) = 1/(l_f + 1)
+ for alignSent in align_sents:
+ en_set = alignSent.words
+ fr_set = [None] + alignSent.mots
+ l_f = len(fr_set) - 1
+ l_e = len(en_set)
+ initial_value = 1 / (l_f + 1)
+ for i in range(0, l_f+1):
+ for j in range(1, l_e+1):
+ align[i][j][l_e][l_f] = initial_value
+
+
+ for i in range(0, num_iter):
+ count_ef = defaultdict(lambda: defaultdict(float))
+ total_f = defaultdict(float)
+
+ count_align = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0))))
+ total_align = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+
+ total_e = defaultdict(float)
+
+ for alignSent in align_sents:
+ en_set = alignSent.words
+ fr_set = [None] + alignSent.mots
+ l_f = len(fr_set) - 1
+ l_e = len(en_set)
+
+ # compute normalization
+ for j in range(1, l_e+1):
+ en_word = en_set[j-1]
+ total_e[en_word] = 0
+ for i in range(0, l_f+1):
+ total_e[en_word] += t_ef[en_word][fr_set[i]] * align[i][j][l_e][l_f]
+
+ # collect counts
+ for j in range(1, l_e+1):
+ en_word = en_set[j-1]
+ for i in range(0, l_f+1):
+ fr_word = fr_set[i]
+ c = t_ef[en_word][fr_word] * align[i][j][l_e][l_f] / total_e[en_word]
+ count_ef[en_word][fr_word] += c
+ total_f[fr_word] += c
+ count_align[i][j][l_e][l_f] += c
+ total_align[j][l_e][l_f] += c
+
+ # estimate probabilities
+ t_ef = defaultdict(lambda: defaultdict(lambda: 0.0))
+ align = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0))))
+
+ # Smoothing the counts for alignments
+ for alignSent in align_sents:
+ en_set = alignSent.words
+ fr_set = [None] + alignSent.mots
+ l_f = len(fr_set) - 1
+ l_e = len(en_set)
+
+ laplace = 1.0
+ for i in range(0, l_f+1):
+ for j in range(1, l_e+1):
+ value = count_align[i][j][l_e][l_f]
+ if 0 < value < laplace:
+ laplace = value
+
+ laplace *= 0.5
+ for i in range(0, l_f+1):
+ for j in range(1, l_e+1):
+ count_align[i][j][l_e][l_f] += laplace
+
+ initial_value = laplace * l_e
+ for j in range(1, l_e+1):
+ total_align[j][l_e][l_f] += initial_value
+
+ # Estimate the new lexical translation probabilities
+ for f in fr_vocab:
+ for e in en_vocab:
+ t_ef[e][f] = count_ef[e][f] / total_f[f]
+
+ # Estimate the new alignment probabilities
+ for alignSent in align_sents:
+ en_set = alignSent.words
+ fr_set = [None] + alignSent.mots
+ l_f = len(fr_set) - 1
+ l_e = len(en_set)
+ for i in range(0, l_f+1):
+ for j in range(1, l_e+1):
+ align[i][j][l_e][l_f] = count_align[i][j][l_e][l_f] / total_align[j][l_e][l_f]
+
+ return t_ef, align
+
+ def align(self, align_sent):
+ """
+ Returns the alignment result for one sentence pair.
+ """
+
+ if self.probabilities is None or self.alignments is None:
+ raise ValueError("The model does not train.")
+
+ alignment = []
+
+ l_e = len(align_sent.words)
+ l_f = len(align_sent.mots)
+
+ for j, en_word in enumerate(align_sent.words):
+
+ # Initialize the maximum probability with Null token
+ max_align_prob = (self.probabilities[en_word][None]*self.alignments[0][j+1][l_e][l_f], None)
+ for i, fr_word in enumerate(align_sent.mots):
+ # Find out the maximum probability
+ max_align_prob = max(max_align_prob,
+ (self.probabilities[en_word][fr_word]*self.alignments[i+1][j+1][l_e][l_f], i))
+
+ # If the maximum probability is not Null token,
+ # then append it to the alignment.
+ if max_align_prob[1] is not None:
+ alignment.append((j, max_align_prob[1]))
+
+ return AlignedSent(align_sent.words, align_sent.mots, alignment)
+
+# run doctests
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
diff --git a/nltk/align/ibm3.py b/nltk/align/ibm3.py
new file mode 100644
index 0000000..ec54fb4
--- /dev/null
+++ b/nltk/align/ibm3.py
@@ -0,0 +1,403 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: IBM Model 3
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Authors: Chin Yee Lee, Hengfeng Li, Ruxin Hou, Calvin Tanujaya Lim
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import division
+from collections import defaultdict
+from nltk.align import AlignedSent
+from nltk.align.ibm2 import IBMModel2
+from math import factorial
+
+class HashableDict(dict):
+ """
+ This class implements a hashable dict, which can be
+ put into a set.
+ """
+ def __key(self):
+ return tuple((k,self[k]) for k in sorted(self))
+
+ def __hash__(self):
+ return hash(self.__key())
+
+ def __eq__(self, other):
+ return self.__key() == other.__key()
+
+class IBMModel3(object):
+ """
+ This class implements the algorithm of Expectation Maximization for
+ the IBM Model 3.
+
+ Step 1 - Run a number of iterations of IBM Model 2 and get the initial
+ distribution of translation probability.
+
+ Step 2 - Sample the alignment spaces by using the hillclimb approach.
+
+ Step 3 - Collect the evidence of translation probabilities, distortion,
+ the probability of null insertion, and fertility.
+
+ Step 4 - Estimate the new probabilities according to the evidence from
+ Step 3.
+
+ >>> align_sents = []
+ >>> align_sents.append(AlignedSent(['klein', 'ist', 'das', 'Haus'], ['the', 'house', 'is', 'small']))
+ >>> align_sents.append(AlignedSent(['das', 'Haus'], ['the', 'house']))
+ >>> align_sents.append(AlignedSent(['das', 'Buch'], ['the', 'book']))
+ >>> align_sents.append(AlignedSent(['ein', 'Buch'], ['a', 'book']))
+
+ >>> ibm3 = IBMModel3(align_sents, 5)
+
+ >>> print('{0:.1f}'.format(ibm3.probabilities['Buch']['book']))
+ 1.0
+ >>> print('{0:.1f}'.format(ibm3.probabilities['das']['book']))
+ 0.0
+ >>> print('{0:.1f}'.format(ibm3.probabilities[None]['book']))
+ 0.0
+
+ >>> aligned_sent = ibm3.align(align_sents[0])
+ >>> aligned_sent.words
+ ['klein', 'ist', 'das', 'Haus']
+ >>> aligned_sent.mots
+ ['the', 'house', 'is', 'small']
+ >>> aligned_sent.alignment
+ Alignment([(0, 2), (1, 3), (2, 0), (3, 1)])
+
+ """
+
+ def __init__(self, align_sents, num_iter):
+ # If there is not an initial value, it throws an exception of
+ # the number divided by zero. And the value of computing
+ # probability will be always zero.
+ self.PROB_SMOOTH = 0.1
+
+ self.train(align_sents, num_iter)
+
+
+ def train(self, align_sents, num_iter):
+ """
+ This function is the main process of training model, which
+ initialize all the probability distributions and executes
+ a specific number of iterations.
+ """
+ # Get the translation and alignment probabilities from IBM model 2
+ ibm2 = IBMModel2(align_sents, num_iter)
+ self.probabilities, self.align_table = ibm2.probabilities, ibm2.alignments
+
+ fr_vocab = set()
+ en_vocab = set()
+ for alignSent in align_sents:
+ en_vocab.update(alignSent.words)
+ fr_vocab.update(alignSent.mots)
+ fr_vocab.add(None)
+
+ # Initial probability of null insertion.
+ self.null_insertion = 0.5
+
+ self.fertility = defaultdict(lambda: defaultdict(lambda: self.PROB_SMOOTH))
+ self.distortion = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: self.PROB_SMOOTH))))
+
+ for k in range(0, num_iter):
+ max_fert = 0
+ # Set all count* and total* to 0
+ count_t = defaultdict(lambda: defaultdict(lambda: 0.0))
+ total_t = defaultdict(lambda: 0.0)
+
+ count_d = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0))))
+ total_d = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0)))
+
+ count_p0 = 0.0
+ count_p1 = 0.0
+
+ count_f = defaultdict(lambda: defaultdict(lambda: 0.0))
+ total_f = defaultdict(lambda: 0.0)
+
+ for alignSent in align_sents:
+
+ en_set = alignSent.words
+ fr_set = [None] + alignSent.mots
+ l_f = len(fr_set) - 1
+ l_e = len(en_set)
+
+ # Sample the alignment space
+ A = self.sample( en_set, fr_set)
+
+ # Collect counts
+ c_total = 0.0
+
+ for (a, fert) in A:
+ c_total += self.probability(a, en_set, fr_set, fert)
+
+ for (a, fert) in A:
+ c = self.probability(a, en_set, fr_set, fert)/c_total
+ null = 0
+
+ for j in range(1, l_e+1):
+ en_word = en_set[j-1]
+ fr_word = fr_set[a[j]]
+
+ # Lexical translation
+ count_t[en_word][fr_word] += c
+ total_t[fr_word] += c
+
+ # Distortion
+ count_d[j][a[j]][l_e][l_f] += c
+ total_d[a[j]][l_e][l_f] += c
+
+ if a[j] == 0:
+ null += 1
+
+ # Collect the counts of null insetion
+ count_p1 += null * c
+ count_p0 += (l_e - 2 * null) * c
+
+ # Collect the counts of fertility
+ for i in range(0, l_f+1):
+ fertility = 0
+
+ for j in range(1, l_e+1):
+ if i == a[j]:
+ fertility += 1
+
+ fr_word = fr_set[i]
+ count_f[fertility][fr_word] += c
+ total_f[fr_word] += c
+
+ if fertility > max_fert:
+ max_fert = fertility
+
+
+ self.probabilities = defaultdict(lambda: defaultdict(lambda: 0.0))
+ self.distortion = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 0.0))))
+ self.fertility = defaultdict(lambda: defaultdict(lambda: 0.0))
+
+ # Estimate translation probability distribution
+ for f in fr_vocab:
+ for e in en_vocab:
+ self.probabilities[e][f] = count_t[e][f] / total_t[f]
+
+ # Estimate distortion
+ for alignSent in align_sents:
+ en_set = alignSent.words
+ fr_set = [None] + alignSent.mots
+ l_f = len(fr_set) - 1
+ l_e = len(en_set)
+
+ for i in range(0, l_f+1):
+ for j in range(1, l_e+1):
+ self.distortion[j][i][l_e][l_f] = count_d[j][i][l_e][l_f] / total_d[i][l_e][l_f]
+
+ # Estimate the fertility, n(Fertility | input word)
+ for ferti in range(0, max_fert+1):
+ for fr_word in fr_vocab:
+ self.fertility[ferti][fr_word] = count_f[ferti][fr_word] / total_f[fr_word]
+
+ # Estimate the probability of null insertion
+ p1 = count_p1 / (count_p1+count_p0)
+ self.null_insertion = 1 - p1
+
+ def sample(self, e, f):
+ """
+ This function returns a sample from the entire alignment space.
+ First, it pegs one alignment point and finds out the best alignment
+ through the IBM model 2. Then, using the hillclimb approach, it
+ finds out the best alignment on local and returns all its neighborings,
+ which are swapped or moved one distance from the best alignment.
+ """
+ A = set()
+
+ le = len(e)
+ lf = len(f) - 1
+
+ # Compute Normalization
+ for i in range(0, lf+1):
+ for j in range(1, le+1):
+ a = HashableDict()
+ fert = HashableDict()
+ # Initialize all fertility to zero
+ for ii in range(0, lf+1):
+ fert[ii] = 0
+
+ # Pegging one alignment point
+ a[j] = i
+ fert[i] = 1
+
+ for jj in range(1, le+1):
+ if jj != j:
+ # Find the best alignment according to model 2
+ maxalignment = 0
+ besti = 1
+
+ for ii in range(0, lf+1):
+ # Notice that the probabilities returned by IBM model 2,
+ # which is not distortion, is alignment.
+ #
+ # The alignment probability predicts foreign input word
+ # positions conditioned on English output word positions.
+ # However, the distortion probability in a reverse direction
+ # predicts the output word position based on input word
+ # position.
+ #
+ # Actually, you cannot just change the index to get a
+ # distortion from alignment table, because its process of
+ # collecting evidence is different from each other.
+ alignment = self.probabilities[e[jj-1]][f[ii]] * self.align_table[ii][jj][le][lf]
+ if alignment > maxalignment:
+ maxalignment = alignment
+ besti = ii
+
+ a[jj] = besti
+ fert[besti] += 1
+
+ a = self.hillclimb(a, j, e, f, fert)
+ neighbor = self.neighboring(a, j, e, f, fert)
+ A.update(neighbor)
+
+ return A
+
+ def hillclimb(self, a, j_pegged, es, fs, fert):
+ """
+ This function returns the best alignment on local. It gets
+ some neighboring alignments and finds out the alignment with
+ highest probability in those alignment spaces. If the current
+ alignment recorded has the highest probability, then stop the
+ search loop. If not, then continue the search loop until it
+ finds out the highest probability of alignment in local.
+ """
+ so_far_fert = fert
+
+ while True:
+ a_old = a
+
+ for (a_nerghbor, neighbor_Fert) in self.neighboring(a, j_pegged, es, fs, so_far_fert):
+ if self.probability(a_nerghbor, es, fs, neighbor_Fert) > self.probability(a, es, fs, so_far_fert):
+ # If the probability of an alignment is higher than
+ # the current alignment recorded, then replace the
+ # current one.
+ a = a_nerghbor
+ so_far_fert = neighbor_Fert
+
+ if a == a_old:
+ # Until this alignment is the highest one in local
+ break
+
+ return a
+
+ def probability(self, a, es, fs, Fert):
+ """
+ This function returns the probability given an alignment.
+ The Fert variable is math syntax 'Phi' in the fomula, which
+ represents the fertility according to the current alignment,
+ which records how many output words are generated by each
+ input word.
+ """
+ l_e = len(es)
+ l_f = len(fs) - 1
+ p1 = 1 - self.null_insertion
+
+ total = 1.0
+
+ # Compute the NULL insertation
+ total *= pow(p1, Fert[0]) * pow(self.null_insertion, l_e - 2 * Fert[0])
+ if total == 0:
+ return total
+
+ # Compute the combination (l_e - Fert[0]) choose Fert[0]
+ for i in range(1, Fert[0]+1):
+ total *= (l_e - Fert[0] - i + 1) / i
+ if total == 0:
+ return total
+
+ # Compute fertilities term
+ for i in range(1, l_f+1):
+ total *= factorial(Fert[i]) * self.fertility[Fert[i]][fs[i]]
+ if total == 0:
+ return total
+
+ # Multiply the lexical and distortion probabilities
+ for j in range(1, l_e+1):
+ en_word = es[j-1]
+ fr_word = fs[a[j]]
+
+ total *= self.probabilities[en_word][fr_word]
+ total *= self.distortion[j][a[j]][l_e][l_f]
+ if total == 0:
+ return total
+
+ return total
+
+ def neighboring(self, a, j_pegged, es, fs, fert):
+ """
+ This function returns the neighboring alignments from
+ the given alignment by moving or swapping one distance.
+ """
+ N = set()
+
+ l_e = len(es)
+ l_f = len(fs) - 1
+
+ for j in range(1, l_e+1):
+ if j != j_pegged:
+ # Moves
+ for i in range(0, l_f+1):
+ new_align = HashableDict(a)
+ new_align[j] = i
+
+ new_fert = fert
+ if new_fert[a[j]] > 0:
+ new_fert = HashableDict(fert)
+ new_fert[a[j]] -= 1
+ new_fert[i] += 1
+
+ N.update([(new_align, new_fert)])
+
+
+ for j_one in range(1, l_e+1):
+ if j_one != j_pegged:
+ # Swaps
+ for j_two in range(1, l_e+1):
+ if j_two != j_pegged and j_two != j_one:
+ new_align = HashableDict(a)
+ new_fert = fert
+ new_align[j_one] = a[j_two]
+ new_align[j_two] = a[j_one]
+
+ N.update([(new_align, new_fert)])
+
+ return N
+
+ def align(self, align_sent):
+ """
+ Returns the alignment result for one sentence pair.
+ """
+
+ if self.probabilities is None or self.distortion is None:
+ raise ValueError("The model does not train.")
+
+ alignment = []
+
+ l_e = len(align_sent.words)
+ l_f = len(align_sent.mots)
+
+ for j, en_word in enumerate(align_sent.words):
+
+ # Initialize the maximum probability with Null token
+ max_align_prob = (self.probabilities[en_word][None]*self.distortion[j+1][0][l_e][l_f], 0)
+ for i, fr_word in enumerate(align_sent.mots):
+ # Find out the maximum probability
+ max_align_prob = max(max_align_prob,
+ (self.probabilities[en_word][fr_word]*self.distortion[j+1][i+1][l_e][l_f], i))
+
+ # If the maximum probability is not Null token,
+ # then append it to the alignment.
+ if max_align_prob[1] is not None:
+ alignment.append((j, max_align_prob[1]))
+
+ return AlignedSent(align_sent.words, align_sent.mots, alignment)
+
+# run doctests
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod()
diff --git a/nltk/align/util.py b/nltk/align/util.py
new file mode 100644
index 0000000..f70d060
--- /dev/null
+++ b/nltk/align/util.py
@@ -0,0 +1,7 @@
+# Natural Language Toolkit: Aligner Utilities
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Author:
+# URL: <http://www.nltk.org/>
+# For license information, see LICENSE.TXT
+
diff --git a/nltk/app/__init__.py b/nltk/app/__init__.py
new file mode 100644
index 0000000..437d25e
--- /dev/null
+++ b/nltk/app/__init__.py
@@ -0,0 +1,53 @@
+# Natural Language Toolkit: Applications package
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interactive NLTK Applications:
+
+chartparser: Chart Parser
+chunkparser: Regular-Expression Chunk Parser
+collocations: Find collocations in text
+concordance: Part-of-speech concordancer
+nemo: Finding (and Replacing) Nemo regular expression tool
+rdparser: Recursive Descent Parser
+srparser: Shift-Reduce Parser
+wordnet: WordNet Browser
+"""
+
+
+# Import Tkinter-based modules if Tkinter is installed
+import nltk.compat
+try:
+ import tkinter
+except ImportError:
+ import warnings
+ warnings.warn("nltk.app package not loaded "
+ "(please install Tkinter library).")
+else:
+ from nltk.app.chartparser_app import app as chartparser
+ from nltk.app.chunkparser_app import app as chunkparser
+ from nltk.app.collocations_app import app as collocations
+ from nltk.app.concordance_app import app as concordance
+ from nltk.app.nemo_app import app as nemo
+ from nltk.app.rdparser_app import app as rdparser
+ from nltk.app.srparser_app import app as srparser
+ from nltk.app.wordnet_app import app as wordnet
+
+ try:
+ import pylab
+ except ImportError:
+ import warnings
+ warnings.warn("nltk.app.wordfreq not loaded "
+ "(requires the pylab library).")
+ else:
+ from nltk.app.wordfreq_app import app as wordfreq
+
+# skip doctests from this package
+def setup_module(module):
+ from nose import SkipTest
+ raise SkipTest("nltk.app examples are not doctests")
diff --git a/nltk/app/chartparser_app.py b/nltk/app/chartparser_app.py
new file mode 100644
index 0000000..17d2f99
--- /dev/null
+++ b/nltk/app/chartparser_app.py
@@ -0,0 +1,2276 @@
+# Natural Language Toolkit: Chart Parser Application
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Jean Mark Gawron <gawron at mail.sdsu.edu>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A graphical tool for exploring chart parsing.
+
+Chart parsing is a flexible parsing algorithm that uses a data
+structure called a "chart" to record hypotheses about syntactic
+constituents. Each hypothesis is represented by a single "edge" on
+the chart. A set of "chart rules" determine when new edges can be
+added to the chart. This set of rules controls the overall behavior
+of the parser (e.g. whether it parses top-down or bottom-up).
+
+The chart parsing tool demonstrates the process of parsing a single
+sentence, with a given grammar and lexicon. Its display is divided
+into three sections: the bottom section displays the chart; the middle
+section displays the sentence; and the top section displays the
+partial syntax tree corresponding to the selected edge. Buttons along
+the bottom of the window are used to control the execution of the
+algorithm.
+
+The chart parsing tool allows for flexible control of the parsing
+algorithm. At each step of the algorithm, you can select which rule
+or strategy you wish to apply. This allows you to experiment with
+mixing different strategies (e.g. top-down and bottom-up). You can
+exercise fine-grained control over the algorithm by selecting which
+edge you wish to apply a rule to.
+"""
+
+# At some point, we should rewrite this tool to use the new canvas
+# widget system.
+
+
+
+import nltk.compat
+import pickle
+from tkinter.filedialog import asksaveasfilename, askopenfilename
+import tkinter
+import math
+import os.path
+import tkinter.font, tkinter.messagebox
+
+from nltk.parse.chart import (BottomUpPredictCombineRule, BottomUpPredictRule,
+ Chart, LeafEdge, LeafInitRule, SingleEdgeFundamentalRule,
+ SteppingChartParser, TopDownInitRule, TopDownPredictRule,
+ TreeEdge)
+from nltk.tree import Tree
+from nltk.grammar import Nonterminal, CFG
+from nltk.util import in_idle
+from nltk.draw.util import (CanvasFrame, ColorizedList,
+ EntryDialog, MutableOptionMenu,
+ ShowText, SymbolWidget)
+from nltk.draw import CFGEditor, tree_to_treesegment, TreeSegmentWidget
+
+# Known bug: ChartView doesn't handle edges generated by epsilon
+# productions (e.g., [Production: PP -> ]) very well.
+
+#######################################################################
+# Edge List
+#######################################################################
+
+class EdgeList(ColorizedList):
+ ARROW = SymbolWidget.SYMBOLS['rightarrow']
+
+ def _init_colortags(self, textwidget, options):
+ textwidget.tag_config('terminal', foreground='#006000')
+ textwidget.tag_config('arrow', font='symbol', underline='0')
+ textwidget.tag_config('dot', foreground = '#000000')
+ textwidget.tag_config('nonterminal', foreground='blue',
+ font=('helvetica', -12, 'bold'))
+
+ def _item_repr(self, item):
+ contents = []
+ contents.append(('%s\t' % item.lhs(), 'nonterminal'))
+ contents.append((self.ARROW, 'arrow'))
+ for i, elt in enumerate(item.rhs()):
+ if i == item.dot():
+ contents.append((' *', 'dot'))
+ if isinstance(elt, Nonterminal):
+ contents.append((' %s' % elt.symbol(), 'nonterminal'))
+ else:
+ contents.append((' %r' % elt, 'terminal'))
+ if item.is_complete():
+ contents.append((' *', 'dot'))
+ return contents
+
+#######################################################################
+# Chart Matrix View
+#######################################################################
+
+class ChartMatrixView(object):
+ """
+ A view of a chart that displays the contents of the corresponding matrix.
+ """
+ def __init__(self, parent, chart, toplevel=True, title='Chart Matrix',
+ show_numedges=False):
+ self._chart = chart
+ self._cells = []
+ self._marks = []
+
+ self._selected_cell = None
+
+ if toplevel:
+ self._root = tkinter.Toplevel(parent)
+ self._root.title(title)
+ self._root.bind('<Control-q>', self.destroy)
+ self._init_quit(self._root)
+ else:
+ self._root = tkinter.Frame(parent)
+
+ self._init_matrix(self._root)
+ self._init_list(self._root)
+ if show_numedges:
+ self._init_numedges(self._root)
+ else:
+ self._numedges_label = None
+
+ self._callbacks = {}
+
+ self._num_edges = 0
+
+ self.draw()
+
+ def _init_quit(self, root):
+ quit = tkinter.Button(root, text='Quit', command=self.destroy)
+ quit.pack(side='bottom', expand=0, fill='none')
+
+ def _init_matrix(self, root):
+ cframe = tkinter.Frame(root, border=2, relief='sunken')
+ cframe.pack(expand=0, fill='none', padx=1, pady=3, side='top')
+ self._canvas = tkinter.Canvas(cframe, width=200, height=200,
+ background='white')
+ self._canvas.pack(expand=0, fill='none')
+
+ def _init_numedges(self, root):
+ self._numedges_label = tkinter.Label(root, text='0 edges')
+ self._numedges_label.pack(expand=0, fill='none', side='top')
+
+ def _init_list(self, root):
+ self._list = EdgeList(root, [], width=20, height=5)
+ self._list.pack(side='top', expand=1, fill='both', pady=3)
+ def cb(edge, self=self): self._fire_callbacks('select', edge)
+ self._list.add_callback('select', cb)
+ self._list.focus()
+
+ def destroy(self, *e):
+ if self._root is None: return
+ try: self._root.destroy()
+ except: pass
+ self._root = None
+
+ def set_chart(self, chart):
+ if chart is not self._chart:
+ self._chart = chart
+ self._num_edges = 0
+ self.draw()
+
+ def update(self):
+ if self._root is None: return
+
+ # Count the edges in each cell
+ N = len(self._cells)
+ cell_edges = [[0 for i in range(N)] for j in range(N)]
+ for edge in self._chart:
+ cell_edges[edge.start()][edge.end()] += 1
+
+ # Color the cells correspondingly.
+ for i in range(N):
+ for j in range(i, N):
+ if cell_edges[i][j] == 0:
+ color = 'gray20'
+ else:
+ color = ('#00%02x%02x' %
+ (min(255, 50+128*cell_edges[i][j]/10),
+ max(0, 128-128*cell_edges[i][j]/10)))
+ cell_tag = self._cells[i][j]
+ self._canvas.itemconfig(cell_tag, fill=color)
+ if (i,j) == self._selected_cell:
+ self._canvas.itemconfig(cell_tag, outline='#00ffff',
+ width=3)
+ self._canvas.tag_raise(cell_tag)
+ else:
+ self._canvas.itemconfig(cell_tag, outline='black',
+ width=1)
+
+ # Update the edge list.
+ edges = list(self._chart.select(span=self._selected_cell))
+ self._list.set(edges)
+
+ # Update our edge count.
+ self._num_edges = self._chart.num_edges()
+ if self._numedges_label is not None:
+ self._numedges_label['text'] = '%d edges' % self._num_edges
+
+ def activate(self):
+ self._canvas.itemconfig('inactivebox', state='hidden')
+ self.update()
+
+ def inactivate(self):
+ self._canvas.itemconfig('inactivebox', state='normal')
+ self.update()
+
+ def add_callback(self, event, func):
+ self._callbacks.setdefault(event,{})[func] = 1
+
+ def remove_callback(self, event, func=None):
+ if func is None: del self._callbacks[event]
+ else:
+ try: del self._callbacks[event][func]
+ except: pass
+
+ def _fire_callbacks(self, event, *args):
+ if event not in self._callbacks: return
+ for cb_func in list(self._callbacks[event].keys()): cb_func(*args)
+
+ def select_cell(self, i, j):
+ if self._root is None: return
+
+ # If the cell is already selected (and the chart contents
+ # haven't changed), then do nothing.
+ if ((i,j) == self._selected_cell and
+ self._chart.num_edges() == self._num_edges): return
+
+ self._selected_cell = (i,j)
+ self.update()
+
+ # Fire the callback.
+ self._fire_callbacks('select_cell', i, j)
+
+ def deselect_cell(self):
+ if self._root is None: return
+ self._selected_cell = None
+ self._list.set([])
+ self.update()
+
+ def _click_cell(self, i, j):
+ if self._selected_cell == (i,j):
+ self.deselect_cell()
+ else:
+ self.select_cell(i, j)
+
+ def view_edge(self, edge):
+ self.select_cell(*edge.span())
+ self._list.view(edge)
+
+ def mark_edge(self, edge):
+ if self._root is None: return
+ self.select_cell(*edge.span())
+ self._list.mark(edge)
+
+ def unmark_edge(self, edge=None):
+ if self._root is None: return
+ self._list.unmark(edge)
+
+ def markonly_edge(self, edge):
+ if self._root is None: return
+ self.select_cell(*edge.span())
+ self._list.markonly(edge)
+
+ def draw(self):
+ if self._root is None: return
+ LEFT_MARGIN = BOT_MARGIN = 15
+ TOP_MARGIN = 5
+ c = self._canvas
+ c.delete('all')
+ N = self._chart.num_leaves()+1
+ dx = (int(c['width'])-LEFT_MARGIN)/N
+ dy = (int(c['height'])-TOP_MARGIN-BOT_MARGIN)/N
+
+ c.delete('all')
+
+ # Labels and dotted lines
+ for i in range(N):
+ c.create_text(LEFT_MARGIN-2, i*dy+dy/2+TOP_MARGIN,
+ text=repr(i), anchor='e')
+ c.create_text(i*dx+dx/2+LEFT_MARGIN, N*dy+TOP_MARGIN+1,
+ text=repr(i), anchor='n')
+ c.create_line(LEFT_MARGIN, dy*(i+1)+TOP_MARGIN,
+ dx*N+LEFT_MARGIN, dy*(i+1)+TOP_MARGIN, dash='.')
+ c.create_line(dx*i+LEFT_MARGIN, TOP_MARGIN,
+ dx*i+LEFT_MARGIN, dy*N+TOP_MARGIN, dash='.')
+
+ # A box around the whole thing
+ c.create_rectangle(LEFT_MARGIN, TOP_MARGIN,
+ LEFT_MARGIN+dx*N, dy*N+TOP_MARGIN,
+ width=2)
+
+ # Cells
+ self._cells = [[None for i in range(N)] for j in range(N)]
+ for i in range(N):
+ for j in range(i, N):
+ t = c.create_rectangle(j*dx+LEFT_MARGIN, i*dy+TOP_MARGIN,
+ (j+1)*dx+LEFT_MARGIN,
+ (i+1)*dy+TOP_MARGIN,
+ fill='gray20')
+ self._cells[i][j] = t
+ def cb(event, self=self, i=i, j=j): self._click_cell(i,j)
+ c.tag_bind(t, '<Button-1>', cb)
+
+ # Inactive box
+ xmax, ymax = int(c['width']), int(c['height'])
+ t = c.create_rectangle(-100, -100, xmax+100, ymax+100,
+ fill='gray50', state='hidden',
+ tag='inactivebox')
+ c.tag_lower(t)
+
+ # Update the cells.
+ self.update()
+
+ def pack(self, *args, **kwargs):
+ self._root.pack(*args, **kwargs)
+
+#######################################################################
+# Chart Results View
+#######################################################################
+
+class ChartResultsView(object):
+ def __init__(self, parent, chart, grammar, toplevel=True):
+ self._chart = chart
+ self._grammar = grammar
+ self._trees = []
+ self._y = 10
+ self._treewidgets = []
+ self._selection = None
+ self._selectbox = None
+
+ if toplevel:
+ self._root = tkinter.Toplevel(parent)
+ self._root.title('Chart Parser Application: Results')
+ self._root.bind('<Control-q>', self.destroy)
+ else:
+ self._root = tkinter.Frame(parent)
+
+ # Buttons
+ if toplevel:
+ buttons = tkinter.Frame(self._root)
+ buttons.pack(side='bottom', expand=0, fill='x')
+ tkinter.Button(buttons, text='Quit',
+ command=self.destroy).pack(side='right')
+ tkinter.Button(buttons, text='Print All',
+ command=self.print_all).pack(side='left')
+ tkinter.Button(buttons, text='Print Selection',
+ command=self.print_selection).pack(side='left')
+
+ # Canvas frame.
+ self._cframe = CanvasFrame(self._root, closeenough=20)
+ self._cframe.pack(side='top', expand=1, fill='both')
+
+ # Initial update
+ self.update()
+
+ def update(self, edge=None):
+ if self._root is None: return
+ # If the edge isn't a parse edge, do nothing.
+ if edge is not None:
+ if edge.lhs() != self._grammar.start(): return
+ if edge.span() != (0, self._chart.num_leaves()): return
+
+ for parse in self._chart.parses(self._grammar.start()):
+ if parse not in self._trees:
+ self._add(parse)
+
+ def _add(self, parse):
+ # Add it to self._trees.
+ self._trees.append(parse)
+
+ # Create a widget for it.
+ c = self._cframe.canvas()
+ treewidget = tree_to_treesegment(c, parse)
+
+ # Add it to the canvas frame.
+ self._treewidgets.append(treewidget)
+ self._cframe.add_widget(treewidget, 10, self._y)
+
+ # Register callbacks.
+ treewidget.bind_click(self._click)
+
+ # Update y.
+ self._y = treewidget.bbox()[3] + 10
+
+ def _click(self, widget):
+ c = self._cframe.canvas()
+ if self._selection is not None:
+ c.delete(self._selectbox)
+ self._selection = widget
+ (x1, y1, x2, y2) = widget.bbox()
+ self._selectbox = c.create_rectangle(x1, y1, x2, y2,
+ width=2, outline='#088')
+
+ def _color(self, treewidget, color):
+ treewidget.label()['color'] = color
+ for child in treewidget.subtrees():
+ if isinstance(child, TreeSegmentWidget):
+ self._color(child, color)
+ else:
+ child['color'] = color
+
+ def print_all(self, *e):
+ if self._root is None: return
+ self._cframe.print_to_file()
+
+ def print_selection(self, *e):
+ if self._root is None: return
+ if self._selection is None:
+ tkinter.messagebox.showerror('Print Error', 'No tree selected')
+ else:
+ c = self._cframe.canvas()
+ for widget in self._treewidgets:
+ if widget is not self._selection:
+ self._cframe.destroy_widget(widget)
+ c.delete(self._selectbox)
+ (x1,y1,x2,y2) = self._selection.bbox()
+ self._selection.move(10-x1,10-y1)
+ c['scrollregion'] = '0 0 %s %s' % (x2-x1+20, y2-y1+20)
+ self._cframe.print_to_file()
+
+ # Restore our state.
+ self._treewidgets = [self._selection]
+ self.clear()
+ self.update()
+
+ def clear(self):
+ if self._root is None: return
+ for treewidget in self._treewidgets:
+ self._cframe.destroy_widget(treewidget)
+ self._trees = []
+ self._treewidgets = []
+ if self._selection is not None:
+ self._cframe.canvas().delete(self._selectbox)
+ self._selection = None
+ self._y = 10
+
+ def set_chart(self, chart):
+ self.clear()
+ self._chart = chart
+ self.update()
+
+ def set_grammar(self, grammar):
+ self.clear()
+ self._grammar = grammar
+ self.update()
+
+ def destroy(self, *e):
+ if self._root is None: return
+ try: self._root.destroy()
+ except: pass
+ self._root = None
+
+ def pack(self, *args, **kwargs):
+ self._root.pack(*args, **kwargs)
+
+#######################################################################
+# Chart Comparer
+#######################################################################
+
+class ChartComparer(object):
+ """
+
+ :ivar _root: The root window
+
+ :ivar _charts: A dictionary mapping names to charts. When
+ charts are loaded, they are added to this dictionary.
+
+ :ivar _left_chart: The left ``Chart``.
+ :ivar _left_name: The name ``_left_chart`` (derived from filename)
+ :ivar _left_matrix: The ``ChartMatrixView`` for ``_left_chart``
+ :ivar _left_selector: The drop-down ``MutableOptionsMenu`` used
+ to select ``_left_chart``.
+
+ :ivar _right_chart: The right ``Chart``.
+ :ivar _right_name: The name ``_right_chart`` (derived from filename)
+ :ivar _right_matrix: The ``ChartMatrixView`` for ``_right_chart``
+ :ivar _right_selector: The drop-down ``MutableOptionsMenu`` used
+ to select ``_right_chart``.
+
+ :ivar _out_chart: The out ``Chart``.
+ :ivar _out_name: The name ``_out_chart`` (derived from filename)
+ :ivar _out_matrix: The ``ChartMatrixView`` for ``_out_chart``
+ :ivar _out_label: The label for ``_out_chart``.
+
+ :ivar _op_label: A Label containing the most recent operation.
+ """
+
+ _OPSYMBOL = {'-': '-',
+ 'and': SymbolWidget.SYMBOLS['intersection'],
+ 'or': SymbolWidget.SYMBOLS['union']}
+
+ def __init__(self, *chart_filenames):
+ # This chart is displayed when we don't have a value (eg
+ # before any chart is loaded).
+ faketok = [''] * 8
+ self._emptychart = Chart(faketok)
+
+ # The left & right charts start out empty.
+ self._left_name = 'None'
+ self._right_name = 'None'
+ self._left_chart = self._emptychart
+ self._right_chart = self._emptychart
+
+ # The charts that have been loaded.
+ self._charts = {'None': self._emptychart}
+
+ # The output chart.
+ self._out_chart = self._emptychart
+
+ # The most recent operation
+ self._operator = None
+
+ # Set up the root window.
+ self._root = tkinter.Tk()
+ self._root.title('Chart Comparison')
+ self._root.bind('<Control-q>', self.destroy)
+ self._root.bind('<Control-x>', self.destroy)
+
+ # Initialize all widgets, etc.
+ self._init_menubar(self._root)
+ self._init_chartviews(self._root)
+ self._init_divider(self._root)
+ self._init_buttons(self._root)
+ self._init_bindings(self._root)
+
+ # Load any specified charts.
+ for filename in chart_filenames:
+ self.load_chart(filename)
+
+ def destroy(self, *e):
+ if self._root is None: return
+ try: self._root.destroy()
+ except: pass
+ self._root = None
+
+ def mainloop(self, *args, **kwargs):
+ return
+ self._root.mainloop(*args, **kwargs)
+
+ #////////////////////////////////////////////////////////////
+ # Initialization
+ #////////////////////////////////////////////////////////////
+
+ def _init_menubar(self, root):
+ menubar = tkinter.Menu(root)
+
+ # File menu
+ filemenu = tkinter.Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Load Chart', accelerator='Ctrl-o',
+ underline=0, command=self.load_chart_dialog)
+ filemenu.add_command(label='Save Output', accelerator='Ctrl-s',
+ underline=0, command=self.save_chart_dialog)
+ filemenu.add_separator()
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-x')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ # Compare menu
+ opmenu = tkinter.Menu(menubar, tearoff=0)
+ opmenu.add_command(label='Intersection',
+ command=self._intersection,
+ accelerator='+')
+ opmenu.add_command(label='Union',
+ command=self._union,
+ accelerator='*')
+ opmenu.add_command(label='Difference',
+ command=self._difference,
+ accelerator='-')
+ opmenu.add_separator()
+ opmenu.add_command(label='Swap Charts',
+ command=self._swapcharts)
+ menubar.add_cascade(label='Compare', underline=0, menu=opmenu)
+
+ # Add the menu
+ self._root.config(menu=menubar)
+
+ def _init_divider(self, root):
+ divider = tkinter.Frame(root, border=2, relief='sunken')
+ divider.pack(side='top', fill='x', ipady=2)
+
+ def _init_chartviews(self, root):
+ opfont=('symbol', -36) # Font for operator.
+ eqfont=('helvetica', -36) # Font for equals sign.
+
+ frame = tkinter.Frame(root, background='#c0c0c0')
+ frame.pack(side='top', expand=1, fill='both')
+
+ # The left matrix.
+ cv1_frame = tkinter.Frame(frame, border=3, relief='groove')
+ cv1_frame.pack(side='left', padx=8, pady=7, expand=1, fill='both')
+ self._left_selector = MutableOptionMenu(
+ cv1_frame, list(self._charts.keys()), command=self._select_left)
+ self._left_selector.pack(side='top', pady=5, fill='x')
+ self._left_matrix = ChartMatrixView(cv1_frame, self._emptychart,
+ toplevel=False,
+ show_numedges=True)
+ self._left_matrix.pack(side='bottom', padx=5, pady=5,
+ expand=1, fill='both')
+ self._left_matrix.add_callback('select', self.select_edge)
+ self._left_matrix.add_callback('select_cell', self.select_cell)
+ self._left_matrix.inactivate()
+
+ # The operator.
+ self._op_label = tkinter.Label(frame, text=' ', width=3,
+ background='#c0c0c0', font=opfont)
+ self._op_label.pack(side='left', padx=5, pady=5)
+
+ # The right matrix.
+ cv2_frame = tkinter.Frame(frame, border=3, relief='groove')
+ cv2_frame.pack(side='left', padx=8, pady=7, expand=1, fill='both')
+ self._right_selector = MutableOptionMenu(
+ cv2_frame, list(self._charts.keys()), command=self._select_right)
+ self._right_selector.pack(side='top', pady=5, fill='x')
+ self._right_matrix = ChartMatrixView(cv2_frame, self._emptychart,
+ toplevel=False,
+ show_numedges=True)
+ self._right_matrix.pack(side='bottom', padx=5, pady=5,
+ expand=1, fill='both')
+ self._right_matrix.add_callback('select', self.select_edge)
+ self._right_matrix.add_callback('select_cell', self.select_cell)
+ self._right_matrix.inactivate()
+
+ # The equals sign
+ tkinter.Label(frame, text='=', width=3, background='#c0c0c0',
+ font=eqfont).pack(side='left', padx=5, pady=5)
+
+ # The output matrix.
+ out_frame = tkinter.Frame(frame, border=3, relief='groove')
+ out_frame.pack(side='left', padx=8, pady=7, expand=1, fill='both')
+ self._out_label = tkinter.Label(out_frame, text='Output')
+ self._out_label.pack(side='top', pady=9)
+ self._out_matrix = ChartMatrixView(out_frame, self._emptychart,
+ toplevel=False,
+ show_numedges=True)
+ self._out_matrix.pack(side='bottom', padx=5, pady=5,
+ expand=1, fill='both')
+ self._out_matrix.add_callback('select', self.select_edge)
+ self._out_matrix.add_callback('select_cell', self.select_cell)
+ self._out_matrix.inactivate()
+
+ def _init_buttons(self, root):
+ buttons = tkinter.Frame(root)
+ buttons.pack(side='bottom', pady=5, fill='x', expand=0)
+ tkinter.Button(buttons, text='Intersection',
+ command=self._intersection).pack(side='left')
+ tkinter.Button(buttons, text='Union',
+ command=self._union).pack(side='left')
+ tkinter.Button(buttons, text='Difference',
+ command=self._difference).pack(side='left')
+ tkinter.Frame(buttons, width=20).pack(side='left')
+ tkinter.Button(buttons, text='Swap Charts',
+ command=self._swapcharts).pack(side='left')
+
+ tkinter.Button(buttons, text='Detatch Output',
+ command=self._detatch_out).pack(side='right')
+
+ def _init_bindings(self, root):
+ #root.bind('<Control-s>', self.save_chart)
+ root.bind('<Control-o>', self.load_chart_dialog)
+ #root.bind('<Control-r>', self.reset)
+
+ #////////////////////////////////////////////////////////////
+ # Input Handling
+ #////////////////////////////////////////////////////////////
+
+ def _select_left(self, name):
+ self._left_name = name
+ self._left_chart = self._charts[name]
+ self._left_matrix.set_chart(self._left_chart)
+ if name == 'None': self._left_matrix.inactivate()
+ self._apply_op()
+
+ def _select_right(self, name):
+ self._right_name = name
+ self._right_chart = self._charts[name]
+ self._right_matrix.set_chart(self._right_chart)
+ if name == 'None': self._right_matrix.inactivate()
+ self._apply_op()
+
+ def _apply_op(self):
+ if self._operator == '-': self._difference()
+ elif self._operator == 'or': self._union()
+ elif self._operator == 'and': self._intersection()
+
+
+ #////////////////////////////////////////////////////////////
+ # File
+ #////////////////////////////////////////////////////////////
+ CHART_FILE_TYPES = [('Pickle file', '.pickle'),
+ ('All files', '*')]
+
+ def save_chart_dialog(self, *args):
+ filename = asksaveasfilename(filetypes=self.CHART_FILE_TYPES,
+ defaultextension='.pickle')
+ if not filename: return
+ try:
+ with open(filename, 'wb') as outfile:
+ pickle.dump(self._out_chart, outfile)
+ except Exception as e:
+ tkinter.messagebox.showerror('Error Saving Chart',
+ 'Unable to open file: %r\n%s' %
+ (filename, e))
+
+ def load_chart_dialog(self, *args):
+ filename = askopenfilename(filetypes=self.CHART_FILE_TYPES,
+ defaultextension='.pickle')
+ if not filename: return
+ try: self.load_chart(filename)
+ except Exception as e:
+ tkinter.messagebox.showerror('Error Loading Chart',
+ 'Unable to open file: %r\n%s' %
+ (filename, e))
+
+ def load_chart(self, filename):
+ with open(filename, 'rb') as infile:
+ chart = pickle.load(infile)
+ name = os.path.basename(filename)
+ if name.endswith('.pickle'): name = name[:-7]
+ if name.endswith('.chart'): name = name[:-6]
+ self._charts[name] = chart
+ self._left_selector.add(name)
+ self._right_selector.add(name)
+
+ # If either left_matrix or right_matrix is empty, then
+ # display the new chart.
+ if self._left_chart is self._emptychart:
+ self._left_selector.set(name)
+ elif self._right_chart is self._emptychart:
+ self._right_selector.set(name)
+
+ def _update_chartviews(self):
+ self._left_matrix.update()
+ self._right_matrix.update()
+ self._out_matrix.update()
+
+ #////////////////////////////////////////////////////////////
+ # Selection
+ #////////////////////////////////////////////////////////////
+
+ def select_edge(self, edge):
+ if edge in self._left_chart:
+ self._left_matrix.markonly_edge(edge)
+ else:
+ self._left_matrix.unmark_edge()
+ if edge in self._right_chart:
+ self._right_matrix.markonly_edge(edge)
+ else:
+ self._right_matrix.unmark_edge()
+ if edge in self._out_chart:
+ self._out_matrix.markonly_edge(edge)
+ else:
+ self._out_matrix.unmark_edge()
+
+ def select_cell(self, i, j):
+ self._left_matrix.select_cell(i, j)
+ self._right_matrix.select_cell(i, j)
+ self._out_matrix.select_cell(i, j)
+
+ #////////////////////////////////////////////////////////////
+ # Operations
+ #////////////////////////////////////////////////////////////
+
+ def _difference(self):
+ if not self._checkcompat(): return
+
+ out_chart = Chart(self._left_chart.tokens())
+ for edge in self._left_chart:
+ if edge not in self._right_chart:
+ out_chart.insert(edge, [])
+
+ self._update('-', out_chart)
+
+ def _intersection(self):
+ if not self._checkcompat(): return
+
+ out_chart = Chart(self._left_chart.tokens())
+ for edge in self._left_chart:
+ if edge in self._right_chart:
+ out_chart.insert(edge, [])
+
+ self._update('and', out_chart)
+
+ def _union(self):
+ if not self._checkcompat(): return
+
+ out_chart = Chart(self._left_chart.tokens())
+ for edge in self._left_chart:
+ out_chart.insert(edge, [])
+ for edge in self._right_chart:
+ out_chart.insert(edge, [])
+
+ self._update('or', out_chart)
+
+ def _swapcharts(self):
+ left, right = self._left_name, self._right_name
+ self._left_selector.set(right)
+ self._right_selector.set(left)
+
+ def _checkcompat(self):
+ if (self._left_chart.tokens() != self._right_chart.tokens() or
+ self._left_chart.property_names() !=
+ self._right_chart.property_names() or
+ self._left_chart == self._emptychart or
+ self._right_chart == self._emptychart):
+ # Clear & inactivate the output chart.
+ self._out_chart = self._emptychart
+ self._out_matrix.set_chart(self._out_chart)
+ self._out_matrix.inactivate()
+ self._out_label['text'] = 'Output'
+ # Issue some other warning?
+ return False
+ else:
+ return True
+
+ def _update(self, operator, out_chart):
+ self._operator = operator
+ self._op_label['text'] = self._OPSYMBOL[operator]
+ self._out_chart = out_chart
+ self._out_matrix.set_chart(out_chart)
+ self._out_label['text'] = '%s %s %s' % (self._left_name,
+ self._operator,
+ self._right_name)
+
+ def _clear_out_chart(self):
+ self._out_chart = self._emptychart
+ self._out_matrix.set_chart(self._out_chart)
+ self._op_label['text'] = ' '
+ self._out_matrix.inactivate()
+
+ def _detatch_out(self):
+ ChartMatrixView(self._root, self._out_chart,
+ title=self._out_label['text'])
+
+
+
+
+
+
+
+
+#######################################################################
+# Chart View
+#######################################################################
+
+class ChartView(object):
+ """
+ A component for viewing charts. This is used by ``ChartParserApp`` to
+ allow students to interactively experiment with various chart
+ parsing techniques. It is also used by ``Chart.draw()``.
+
+ :ivar _chart: The chart that we are giving a view of. This chart
+ may be modified; after it is modified, you should call
+ ``update``.
+ :ivar _sentence: The list of tokens that the chart spans.
+
+ :ivar _root: The root window.
+ :ivar _chart_canvas: The canvas we're using to display the chart
+ itself.
+ :ivar _tree_canvas: The canvas we're using to display the tree
+ that each edge spans. May be None, if we're not displaying
+ trees.
+ :ivar _sentence_canvas: The canvas we're using to display the sentence
+ text. May be None, if we're not displaying the sentence text.
+ :ivar _edgetags: A dictionary mapping from edges to the tags of
+ the canvas elements (lines, etc) used to display that edge.
+ The values of this dictionary have the form
+ ``(linetag, rhstag1, dottag, rhstag2, lhstag)``.
+ :ivar _treetags: A list of all the tags that make up the tree;
+ used to erase the tree (without erasing the loclines).
+ :ivar _chart_height: The height of the chart canvas.
+ :ivar _sentence_height: The height of the sentence canvas.
+ :ivar _tree_height: The height of the tree
+
+ :ivar _text_height: The height of a text string (in the normal
+ font).
+
+ :ivar _edgelevels: A list of edges at each level of the chart (the
+ top level is the 0th element). This list is used to remember
+ where edges should be drawn; and to make sure that no edges
+ are overlapping on the chart view.
+
+ :ivar _unitsize: Pixel size of one unit (from the location). This
+ is determined by the span of the chart's location, and the
+ width of the chart display canvas.
+
+ :ivar _fontsize: The current font size
+
+ :ivar _marks: A dictionary from edges to marks. Marks are
+ strings, specifying colors (e.g. 'green').
+ """
+
+ _LEAF_SPACING = 10
+ _MARGIN = 10
+ _TREE_LEVEL_SIZE = 12
+ _CHART_LEVEL_SIZE = 40
+
+ def __init__(self, chart, root=None, **kw):
+ """
+ Construct a new ``Chart`` display.
+ """
+ # Process keyword args.
+ draw_tree = kw.get('draw_tree', 0)
+ draw_sentence = kw.get('draw_sentence', 1)
+ self._fontsize = kw.get('fontsize', -12)
+
+ # The chart!
+ self._chart = chart
+
+ # Callback functions
+ self._callbacks = {}
+
+ # Keep track of drawn edges
+ self._edgelevels = []
+ self._edgetags = {}
+
+ # Keep track of which edges are marked.
+ self._marks = {}
+
+ # These are used to keep track of the set of tree tokens
+ # currently displayed in the tree canvas.
+ self._treetoks = []
+ self._treetoks_edge = None
+ self._treetoks_index = 0
+
+ # Keep track of the tags used to draw the tree
+ self._tree_tags = []
+
+ # Put multiple edges on each level?
+ self._compact = 0
+
+ # If they didn't provide a main window, then set one up.
+ if root is None:
+ top = tkinter.Tk()
+ top.title('Chart View')
+ def destroy1(e, top=top): top.destroy()
+ def destroy2(top=top): top.destroy()
+ top.bind('q', destroy1)
+ b = tkinter.Button(top, text='Done', command=destroy2)
+ b.pack(side='bottom')
+ self._root = top
+ else:
+ self._root = root
+
+ # Create some fonts.
+ self._init_fonts(root)
+
+ # Create the chart canvas.
+ (self._chart_sb, self._chart_canvas) = self._sb_canvas(self._root)
+ self._chart_canvas['height'] = 300
+ self._chart_canvas['closeenough'] = 15
+
+ # Create the sentence canvas.
+ if draw_sentence:
+ cframe = tkinter.Frame(self._root, relief='sunk', border=2)
+ cframe.pack(fill='both', side='bottom')
+ self._sentence_canvas = tkinter.Canvas(cframe, height=50)
+ self._sentence_canvas['background'] = '#e0e0e0'
+ self._sentence_canvas.pack(fill='both')
+ #self._sentence_canvas['height'] = self._sentence_height
+ else:
+ self._sentence_canvas = None
+
+ # Create the tree canvas.
+ if draw_tree:
+ (sb, canvas) = self._sb_canvas(self._root, 'n', 'x')
+ (self._tree_sb, self._tree_canvas) = (sb, canvas)
+ self._tree_canvas['height'] = 200
+ else:
+ self._tree_canvas = None
+
+ # Do some analysis to figure out how big the window should be
+ self._analyze()
+ self.draw()
+ self._resize()
+ self._grow()
+
+ # Set up the configure callback, which will be called whenever
+ # the window is resized.
+ self._chart_canvas.bind('<Configure>', self._configure)
+
+ def _init_fonts(self, root):
+ self._boldfont = tkinter.font.Font(family='helvetica', weight='bold',
+ size=self._fontsize)
+ self._font = tkinter.font.Font(family='helvetica',
+ size=self._fontsize)
+ # See: <http://www.astro.washington.edu/owen/ROTKFolklore.html>
+ self._sysfont = tkinter.font.Font(font=tkinter.Button()["font"])
+ root.option_add("*Font", self._sysfont)
+
+ def _sb_canvas(self, root, expand='y',
+ fill='both', side='bottom'):
+ """
+ Helper for __init__: construct a canvas with a scrollbar.
+ """
+ cframe =tkinter.Frame(root, relief='sunk', border=2)
+ cframe.pack(fill=fill, expand=expand, side=side)
+ canvas = tkinter.Canvas(cframe, background='#e0e0e0')
+
+ # Give the canvas a scrollbar.
+ sb = tkinter.Scrollbar(cframe, orient='vertical')
+ sb.pack(side='right', fill='y')
+ canvas.pack(side='left', fill=fill, expand='yes')
+
+ # Connect the scrollbars to the canvas.
+ sb['command']= canvas.yview
+ canvas['yscrollcommand'] = sb.set
+
+ return (sb, canvas)
+
+ def scroll_up(self, *e):
+ self._chart_canvas.yview('scroll', -1, 'units')
+
+ def scroll_down(self, *e):
+ self._chart_canvas.yview('scroll', 1, 'units')
+
+ def page_up(self, *e):
+ self._chart_canvas.yview('scroll', -1, 'pages')
+
+ def page_down(self, *e):
+ self._chart_canvas.yview('scroll', 1, 'pages')
+
+ def _grow(self):
+ """
+ Grow the window, if necessary
+ """
+ # Grow, if need-be
+ N = self._chart.num_leaves()
+ width = max(int(self._chart_canvas['width']),
+ N * self._unitsize + ChartView._MARGIN * 2 )
+
+ # It won't resize without the second (height) line, but I
+ # don't understand why not.
+ self._chart_canvas.configure(width=width)
+ self._chart_canvas.configure(height=self._chart_canvas['height'])
+
+ self._unitsize = (width - 2*ChartView._MARGIN) / N
+
+ # Reset the height for the sentence window.
+ if self._sentence_canvas is not None:
+ self._sentence_canvas['height'] = self._sentence_height
+
+ def set_font_size(self, size):
+ self._font.configure(size=-abs(size))
+ self._boldfont.configure(size=-abs(size))
+ self._sysfont.configure(size=-abs(size))
+ self._analyze()
+ self._grow()
+ self.draw()
+
+ def get_font_size(self):
+ return abs(self._fontsize)
+
+ def _configure(self, e):
+ """
+ The configure callback. This is called whenever the window is
+ resized. It is also called when the window is first mapped.
+ It figures out the unit size, and redraws the contents of each
+ canvas.
+ """
+ N = self._chart.num_leaves()
+ self._unitsize = (e.width - 2*ChartView._MARGIN) / N
+ self.draw()
+
+ def update(self, chart=None):
+ """
+ Draw any edges that have not been drawn. This is typically
+ called when a after modifies the canvas that a CanvasView is
+ displaying. ``update`` will cause any edges that have been
+ added to the chart to be drawn.
+
+ If update is given a ``chart`` argument, then it will replace
+ the current chart with the given chart.
+ """
+ if chart is not None:
+ self._chart = chart
+ self._edgelevels = []
+ self._marks = {}
+ self._analyze()
+ self._grow()
+ self.draw()
+ self.erase_tree()
+ self._resize()
+ else:
+ for edge in self._chart:
+ if edge not in self._edgetags:
+ self._add_edge(edge)
+ self._resize()
+
+
+ def _edge_conflict(self, edge, lvl):
+ """
+ Return True if the given edge overlaps with any edge on the given
+ level. This is used by _add_edge to figure out what level a
+ new edge should be added to.
+ """
+ (s1, e1) = edge.span()
+ for otheredge in self._edgelevels[lvl]:
+ (s2, e2) = otheredge.span()
+ if (s1 <= s2 < e1) or (s2 <= s1 < e2) or (s1==s2==e1==e2):
+ return True
+ return False
+
+ def _analyze_edge(self, edge):
+ """
+ Given a new edge, recalculate:
+
+ - _text_height
+ - _unitsize (if the edge text is too big for the current
+ _unitsize, then increase _unitsize)
+ """
+ c = self._chart_canvas
+
+ if isinstance(edge, TreeEdge):
+ lhs = edge.lhs()
+ rhselts = []
+ for elt in edge.rhs():
+ if isinstance(elt, Nonterminal):
+ rhselts.append(str(elt.symbol()))
+ else:
+ rhselts.append(repr(elt))
+ rhs = " ".join(rhselts)
+ else:
+ lhs = edge.lhs()
+ rhs = ''
+
+ for s in (lhs, rhs):
+ tag = c.create_text(0,0, text=s,
+ font=self._boldfont,
+ anchor='nw', justify='left')
+ bbox = c.bbox(tag)
+ c.delete(tag)
+ width = bbox[2] #+ ChartView._LEAF_SPACING
+ edgelen = max(edge.length(), 1)
+ self._unitsize = max(self._unitsize, width/edgelen)
+ self._text_height = max(self._text_height, bbox[3] - bbox[1])
+
+ def _add_edge(self, edge, minlvl=0):
+ """
+ Add a single edge to the ChartView:
+
+ - Call analyze_edge to recalculate display parameters
+ - Find an available level
+ - Call _draw_edge
+ """
+ # Do NOT show leaf edges in the chart.
+ if isinstance(edge, LeafEdge): return
+
+ if edge in self._edgetags: return
+ self._analyze_edge(edge)
+ self._grow()
+
+ if not self._compact:
+ self._edgelevels.append([edge])
+ lvl = len(self._edgelevels)-1
+ self._draw_edge(edge, lvl)
+ self._resize()
+ return
+
+ # Figure out what level to draw the edge on.
+ lvl = 0
+ while True:
+ # If this level doesn't exist yet, create it.
+ while lvl >= len(self._edgelevels):
+ self._edgelevels.append([])
+ self._resize()
+
+ # Check if we can fit the edge in this level.
+ if lvl>=minlvl and not self._edge_conflict(edge, lvl):
+ # Go ahead and draw it.
+ self._edgelevels[lvl].append(edge)
+ break
+
+ # Try the next level.
+ lvl += 1
+
+ self._draw_edge(edge, lvl)
+
+ def view_edge(self, edge):
+ level = None
+ for i in range(len(self._edgelevels)):
+ if edge in self._edgelevels[i]:
+ level = i
+ break
+ if level is None: return
+ # Try to view the new edge..
+ y = (level+1) * self._chart_level_size
+ dy = self._text_height + 10
+ self._chart_canvas.yview('moveto', 1.0)
+ if self._chart_height != 0:
+ self._chart_canvas.yview('moveto',
+ float(y-dy)/self._chart_height)
+
+ def _draw_edge(self, edge, lvl):
+ """
+ Draw a single edge on the ChartView.
+ """
+ c = self._chart_canvas
+
+ # Draw the arrow.
+ x1 = (edge.start() * self._unitsize + ChartView._MARGIN)
+ x2 = (edge.end() * self._unitsize + ChartView._MARGIN)
+ if x2 == x1: x2 += max(4, self._unitsize/5)
+ y = (lvl+1) * self._chart_level_size
+ linetag = c.create_line(x1, y, x2, y, arrow='last', width=3)
+
+ # Draw a label for the edge.
+ if isinstance(edge, TreeEdge):
+ rhs = []
+ for elt in edge.rhs():
+ if isinstance(elt, Nonterminal):
+ rhs.append(str(elt.symbol()))
+ else:
+ rhs.append(repr(elt))
+ pos = edge.dot()
+ else:
+ rhs = []
+ pos = 0
+
+ rhs1 = " ".join(rhs[:pos])
+ rhs2 = " ".join(rhs[pos:])
+ rhstag1 = c.create_text(x1+3, y, text=rhs1,
+ font=self._font,
+ anchor='nw')
+ dotx = c.bbox(rhstag1)[2] + 6
+ doty = (c.bbox(rhstag1)[1]+c.bbox(rhstag1)[3])/2
+ dottag = c.create_oval(dotx-2, doty-2, dotx+2, doty+2)
+ rhstag2 = c.create_text(dotx+6, y, text=rhs2,
+ font=self._font,
+ anchor='nw')
+ lhstag = c.create_text((x1+x2)/2, y, text=str(edge.lhs()),
+ anchor='s',
+ font=self._boldfont)
+
+ # Keep track of the edge's tags.
+ self._edgetags[edge] = (linetag, rhstag1,
+ dottag, rhstag2, lhstag)
+
+ # Register a callback for clicking on the edge.
+ def cb(event, self=self, edge=edge):
+ self._fire_callbacks('select', edge)
+ c.tag_bind(rhstag1, '<Button-1>', cb)
+ c.tag_bind(rhstag2, '<Button-1>', cb)
+ c.tag_bind(linetag, '<Button-1>', cb)
+ c.tag_bind(dottag, '<Button-1>', cb)
+ c.tag_bind(lhstag, '<Button-1>', cb)
+
+ self._color_edge(edge)
+
+ def _color_edge(self, edge, linecolor=None, textcolor=None):
+ """
+ Color in an edge with the given colors.
+ If no colors are specified, use intelligent defaults
+ (dependent on selection, etc.)
+ """
+ if edge not in self._edgetags: return
+ c = self._chart_canvas
+
+ if linecolor is not None and textcolor is not None:
+ if edge in self._marks:
+ linecolor = self._marks[edge]
+ tags = self._edgetags[edge]
+ c.itemconfig(tags[0], fill=linecolor)
+ c.itemconfig(tags[1], fill=textcolor)
+ c.itemconfig(tags[2], fill=textcolor,
+ outline=textcolor)
+ c.itemconfig(tags[3], fill=textcolor)
+ c.itemconfig(tags[4], fill=textcolor)
+ return
+ else:
+ N = self._chart.num_leaves()
+ if edge in self._marks:
+ self._color_edge(self._marks[edge])
+ if (edge.is_complete() and edge.span() == (0, N)):
+ self._color_edge(edge, '#084', '#042')
+ elif isinstance(edge, LeafEdge):
+ self._color_edge(edge, '#48c', '#246')
+ else:
+ self._color_edge(edge, '#00f', '#008')
+
+ def mark_edge(self, edge, mark='#0df'):
+ """
+ Mark an edge
+ """
+ self._marks[edge] = mark
+ self._color_edge(edge)
+
+ def unmark_edge(self, edge=None):
+ """
+ Unmark an edge (or all edges)
+ """
+ if edge is None:
+ old_marked_edges = list(self._marks.keys())
+ self._marks = {}
+ for edge in old_marked_edges:
+ self._color_edge(edge)
+ else:
+ del self._marks[edge]
+ self._color_edge(edge)
+
+ def markonly_edge(self, edge, mark='#0df'):
+ self.unmark_edge()
+ self.mark_edge(edge, mark)
+
+ def _analyze(self):
+ """
+ Analyze the sentence string, to figure out how big a unit needs
+ to be, How big the tree should be, etc.
+ """
+ # Figure out the text height and the unit size.
+ unitsize = 70 # min unitsize
+ text_height = 0
+ c = self._chart_canvas
+
+ # Check against all tokens
+ for leaf in self._chart.leaves():
+ tag = c.create_text(0,0, text=repr(leaf),
+ font=self._font,
+ anchor='nw', justify='left')
+ bbox = c.bbox(tag)
+ c.delete(tag)
+ width = bbox[2] + ChartView._LEAF_SPACING
+ unitsize = max(width, unitsize)
+ text_height = max(text_height, bbox[3] - bbox[1])
+
+ self._unitsize = unitsize
+ self._text_height = text_height
+ self._sentence_height = (self._text_height +
+ 2*ChartView._MARGIN)
+
+ # Check against edges.
+ for edge in self._chart.edges():
+ self._analyze_edge(edge)
+
+ # Size of chart levels
+ self._chart_level_size = self._text_height * 2
+
+ # Default tree size..
+ self._tree_height = (3 * (ChartView._TREE_LEVEL_SIZE +
+ self._text_height))
+
+ # Resize the scrollregions.
+ self._resize()
+
+ def _resize(self):
+ """
+ Update the scroll-regions for each canvas. This ensures that
+ everything is within a scroll-region, so the user can use the
+ scrollbars to view the entire display. This does *not*
+ resize the window.
+ """
+ c = self._chart_canvas
+
+ # Reset the chart scroll region
+ width = ( self._chart.num_leaves() * self._unitsize +
+ ChartView._MARGIN * 2 )
+
+ levels = len(self._edgelevels)
+ self._chart_height = (levels+2)*self._chart_level_size
+ c['scrollregion']=(0,0,width,self._chart_height)
+
+ # Reset the tree scroll region
+ if self._tree_canvas:
+ self._tree_canvas['scrollregion'] = (0, 0, width,
+ self._tree_height)
+
+ def _draw_loclines(self):
+ """
+ Draw location lines. These are vertical gridlines used to
+ show where each location unit is.
+ """
+ BOTTOM = 50000
+ c1 = self._tree_canvas
+ c2 = self._sentence_canvas
+ c3 = self._chart_canvas
+ margin = ChartView._MARGIN
+ self._loclines = []
+ for i in range(0, self._chart.num_leaves()+1):
+ x = i*self._unitsize + margin
+
+ if c1:
+ t1=c1.create_line(x, 0, x, BOTTOM)
+ c1.tag_lower(t1)
+ if c2:
+ t2=c2.create_line(x, 0, x, self._sentence_height)
+ c2.tag_lower(t2)
+ t3=c3.create_line(x, 0, x, BOTTOM)
+ c3.tag_lower(t3)
+ t4=c3.create_text(x+2, 0, text=repr(i), anchor='nw',
+ font=self._font)
+ c3.tag_lower(t4)
+ #if i % 4 == 0:
+ # if c1: c1.itemconfig(t1, width=2, fill='gray60')
+ # if c2: c2.itemconfig(t2, width=2, fill='gray60')
+ # c3.itemconfig(t3, width=2, fill='gray60')
+ if i % 2 == 0:
+ if c1: c1.itemconfig(t1, fill='gray60')
+ if c2: c2.itemconfig(t2, fill='gray60')
+ c3.itemconfig(t3, fill='gray60')
+ else:
+ if c1: c1.itemconfig(t1, fill='gray80')
+ if c2: c2.itemconfig(t2, fill='gray80')
+ c3.itemconfig(t3, fill='gray80')
+
+ def _draw_sentence(self):
+ """Draw the sentence string."""
+ if self._chart.num_leaves() == 0: return
+ c = self._sentence_canvas
+ margin = ChartView._MARGIN
+ y = ChartView._MARGIN
+
+ for i, leaf in enumerate(self._chart.leaves()):
+ x1 = i * self._unitsize + margin
+ x2 = x1 + self._unitsize
+ x = (x1+x2)/2
+ tag = c.create_text(x, y, text=repr(leaf),
+ font=self._font,
+ anchor='n', justify='left')
+ bbox = c.bbox(tag)
+ rt=c.create_rectangle(x1+2, bbox[1]-(ChartView._LEAF_SPACING/2),
+ x2-2, bbox[3]+(ChartView._LEAF_SPACING/2),
+ fill='#f0f0f0', outline='#f0f0f0')
+ c.tag_lower(rt)
+
+ def erase_tree(self):
+ for tag in self._tree_tags: self._tree_canvas.delete(tag)
+ self._treetoks = []
+ self._treetoks_edge = None
+ self._treetoks_index = 0
+
+ def draw_tree(self, edge=None):
+ if edge is None and self._treetoks_edge is None: return
+ if edge is None: edge = self._treetoks_edge
+
+ # If it's a new edge, then get a new list of treetoks.
+ if self._treetoks_edge != edge:
+ self._treetoks = [t for t in self._chart.trees(edge)
+ if isinstance(t, Tree)]
+ self._treetoks_edge = edge
+ self._treetoks_index = 0
+
+ # Make sure there's something to draw.
+ if len(self._treetoks) == 0: return
+
+ # Erase the old tree.
+ for tag in self._tree_tags: self._tree_canvas.delete(tag)
+
+ # Draw the new tree.
+ tree = self._treetoks[self._treetoks_index]
+ self._draw_treetok(tree, edge.start())
+
+ # Show how many trees are available for the edge.
+ self._draw_treecycle()
+
+ # Update the scroll region.
+ w = self._chart.num_leaves()*self._unitsize+2*ChartView._MARGIN
+ h = tree.height() * (ChartView._TREE_LEVEL_SIZE+self._text_height)
+ self._tree_canvas['scrollregion'] = (0, 0, w, h)
+
+ def cycle_tree(self):
+ self._treetoks_index = (self._treetoks_index+1)%len(self._treetoks)
+ self.draw_tree(self._treetoks_edge)
+
+ def _draw_treecycle(self):
+ if len(self._treetoks) <= 1: return
+
+ # Draw the label.
+ label = '%d Trees' % len(self._treetoks)
+ c = self._tree_canvas
+ margin = ChartView._MARGIN
+ right = self._chart.num_leaves()*self._unitsize+margin-2
+ tag = c.create_text(right, 2, anchor='ne', text=label,
+ font=self._boldfont)
+ self._tree_tags.append(tag)
+ _, _, _, y = c.bbox(tag)
+
+ # Draw the triangles.
+ for i in range(len(self._treetoks)):
+ x = right - 20*(len(self._treetoks)-i-1)
+ if i == self._treetoks_index: fill = '#084'
+ else: fill = '#fff'
+ tag = c.create_polygon(x, y+10, x-5, y, x-10, y+10,
+ fill=fill, outline='black')
+ self._tree_tags.append(tag)
+
+ # Set up a callback: show the tree if they click on its
+ # triangle.
+ def cb(event, self=self, i=i):
+ self._treetoks_index = i
+ self.draw_tree()
+ c.tag_bind(tag, '<Button-1>', cb)
+
+ def _draw_treetok(self, treetok, index, depth=0):
+ """
+ :param index: The index of the first leaf in the tree.
+ :return: The index of the first leaf after the tree.
+ """
+ c = self._tree_canvas
+ margin = ChartView._MARGIN
+
+ # Draw the children
+ child_xs = []
+ for child in treetok:
+ if isinstance(child, Tree):
+ child_x, index = self._draw_treetok(child, index, depth+1)
+ child_xs.append(child_x)
+ else:
+ child_xs.append((2*index+1)*self._unitsize/2 + margin)
+ index += 1
+
+ # If we have children, then get the node's x by averaging their
+ # node x's. Otherwise, make room for ourselves.
+ if child_xs:
+ nodex = sum(child_xs)/len(child_xs)
+ else:
+ # [XX] breaks for null productions.
+ nodex = (2*index+1)*self._unitsize/2 + margin
+ index += 1
+
+ # Draw the node
+ nodey = depth * (ChartView._TREE_LEVEL_SIZE + self._text_height)
+ tag = c.create_text(nodex, nodey, anchor='n', justify='center',
+ text=str(treetok.label()), fill='#042',
+ font=self._boldfont)
+ self._tree_tags.append(tag)
+
+ # Draw lines to the children.
+ childy = nodey + ChartView._TREE_LEVEL_SIZE + self._text_height
+ for childx, child in zip(child_xs, treetok):
+ if isinstance(child, Tree) and child:
+ # A "real" tree token:
+ tag = c.create_line(nodex, nodey + self._text_height,
+ childx, childy, width=2, fill='#084')
+ self._tree_tags.append(tag)
+ if isinstance(child, Tree) and not child:
+ # An unexpanded tree token:
+ tag = c.create_line(nodex, nodey + self._text_height,
+ childx, childy, width=2,
+ fill='#048', dash='2 3')
+ self._tree_tags.append(tag)
+ if not isinstance(child, Tree):
+ # A leaf:
+ tag = c.create_line(nodex, nodey + self._text_height,
+ childx, 10000, width=2, fill='#084')
+ self._tree_tags.append(tag)
+
+ return nodex, index
+
+ def draw(self):
+ """
+ Draw everything (from scratch).
+ """
+ if self._tree_canvas:
+ self._tree_canvas.delete('all')
+ self.draw_tree()
+
+ if self._sentence_canvas:
+ self._sentence_canvas.delete('all')
+ self._draw_sentence()
+
+ self._chart_canvas.delete('all')
+ self._edgetags = {}
+
+ # Redraw any edges we erased.
+ for lvl in range(len(self._edgelevels)):
+ for edge in self._edgelevels[lvl]:
+ self._draw_edge(edge, lvl)
+
+ for edge in self._chart:
+ self._add_edge(edge)
+
+ self._draw_loclines()
+
+ def add_callback(self, event, func):
+ self._callbacks.setdefault(event,{})[func] = 1
+
+ def remove_callback(self, event, func=None):
+ if func is None: del self._callbacks[event]
+ else:
+ try: del self._callbacks[event][func]
+ except: pass
+
+ def _fire_callbacks(self, event, *args):
+ if event not in self._callbacks: return
+ for cb_func in list(self._callbacks[event].keys()): cb_func(*args)
+
+#######################################################################
+# Edge Rules
+#######################################################################
+# These version of the chart rules only apply to a specific edge.
+# This lets the user select an edge, and then apply a rule.
+
+class EdgeRule(object):
+ """
+ To create an edge rule, make an empty base class that uses
+ EdgeRule as the first base class, and the basic rule as the
+ second base class. (Order matters!)
+ """
+ def __init__(self, edge):
+ super = self.__class__.__bases__[1]
+ self._edge = edge
+ self.NUM_EDGES = super.NUM_EDGES-1
+ def apply(self, chart, grammar, *edges):
+ super = self.__class__.__bases__[1]
+ edges += (self._edge,)
+ for e in super.apply(self, chart, grammar, *edges): yield e
+ def __str__(self):
+ super = self.__class__.__bases__[1]
+ return super.__str__(self)
+
+class TopDownPredictEdgeRule(EdgeRule, TopDownPredictRule):
+ pass
+class BottomUpEdgeRule(EdgeRule, BottomUpPredictRule):
+ pass
+class BottomUpLeftCornerEdgeRule(EdgeRule, BottomUpPredictCombineRule):
+ pass
+class FundamentalEdgeRule(EdgeRule, SingleEdgeFundamentalRule):
+ pass
+
+#######################################################################
+# Chart Parser Application
+#######################################################################
+
+class ChartParserApp(object):
+ def __init__(self, grammar, tokens, title='Chart Parser Application'):
+ # Initialize the parser
+ self._init_parser(grammar, tokens)
+
+ self._root = None
+ try:
+ # Create the root window.
+ self._root = tkinter.Tk()
+ self._root.title(title)
+ self._root.bind('<Control-q>', self.destroy)
+
+ # Set up some frames.
+ frame3 = tkinter.Frame(self._root)
+ frame2 = tkinter.Frame(self._root)
+ frame1 = tkinter.Frame(self._root)
+ frame3.pack(side='bottom', fill='none')
+ frame2.pack(side='bottom', fill='x')
+ frame1.pack(side='bottom', fill='both', expand=1)
+
+ self._init_fonts(self._root)
+ self._init_animation()
+ self._init_chartview(frame1)
+ self._init_rulelabel(frame2)
+ self._init_buttons(frame3)
+ self._init_menubar()
+
+ self._matrix = None
+ self._results = None
+
+ # Set up keyboard bindings.
+ self._init_bindings()
+
+ except:
+ print('Error creating Tree View')
+ self.destroy()
+ raise
+
+ def destroy(self, *args):
+ if self._root is None: return
+ self._root.destroy()
+ self._root = None
+
+ def mainloop(self, *args, **kwargs):
+ """
+ Enter the Tkinter mainloop. This function must be called if
+ this demo is created from a non-interactive program (e.g.
+ from a secript); otherwise, the demo will close as soon as
+ the script completes.
+ """
+ if in_idle(): return
+ self._root.mainloop(*args, **kwargs)
+
+ #////////////////////////////////////////////////////////////
+ # Initialization Helpers
+ #////////////////////////////////////////////////////////////
+
+ def _init_parser(self, grammar, tokens):
+ self._grammar = grammar
+ self._tokens = tokens
+ self._reset_parser()
+
+ def _reset_parser(self):
+ self._cp = SteppingChartParser(self._grammar)
+ self._cp.initialize(self._tokens)
+ self._chart = self._cp.chart()
+
+ # Insert LeafEdges before the parsing starts.
+ for _new_edge in LeafInitRule().apply(self._chart, self._grammar):
+ pass
+
+ # The step iterator -- use this to generate new edges
+ self._cpstep = self._cp.step()
+
+ # The currently selected edge
+ self._selection = None
+
+ def _init_fonts(self, root):
+ # See: <http://www.astro.washington.edu/owen/ROTKFolklore.html>
+ self._sysfont = tkinter.font.Font(font=tkinter.Button()["font"])
+ root.option_add("*Font", self._sysfont)
+
+ # TWhat's our font size (default=same as sysfont)
+ self._size = tkinter.IntVar(root)
+ self._size.set(self._sysfont.cget('size'))
+
+ self._boldfont = tkinter.font.Font(family='helvetica', weight='bold',
+ size=self._size.get())
+ self._font = tkinter.font.Font(family='helvetica',
+ size=self._size.get())
+
+ def _init_animation(self):
+ # Are we stepping? (default=yes)
+ self._step = tkinter.IntVar(self._root)
+ self._step.set(1)
+
+ # What's our animation speed (default=fast)
+ self._animate = tkinter.IntVar(self._root)
+ self._animate.set(3) # Default speed = fast
+
+ # Are we currently animating?
+ self._animating = 0
+
+ def _init_chartview(self, parent):
+ self._cv = ChartView(self._chart, parent,
+ draw_tree=1, draw_sentence=1)
+ self._cv.add_callback('select', self._click_cv_edge)
+
+ def _init_rulelabel(self, parent):
+ ruletxt = 'Last edge generated by:'
+
+ self._rulelabel1 = tkinter.Label(parent,text=ruletxt,
+ font=self._boldfont)
+ self._rulelabel2 = tkinter.Label(parent, width=40,
+ relief='groove', anchor='w',
+ font=self._boldfont)
+ self._rulelabel1.pack(side='left')
+ self._rulelabel2.pack(side='left')
+ step = tkinter.Checkbutton(parent, variable=self._step,
+ text='Step')
+ step.pack(side='right')
+
+ def _init_buttons(self, parent):
+ frame1 = tkinter.Frame(parent)
+ frame2 = tkinter.Frame(parent)
+ frame1.pack(side='bottom', fill='x')
+ frame2.pack(side='top', fill='none')
+
+ tkinter.Button(frame1, text='Reset\nParser',
+ background='#90c0d0', foreground='black',
+ command=self.reset).pack(side='right')
+ #Tkinter.Button(frame1, text='Pause',
+ # background='#90c0d0', foreground='black',
+ # command=self.pause).pack(side='left')
+
+ tkinter.Button(frame1, text='Top Down\nStrategy',
+ background='#90c0d0', foreground='black',
+ command=self.top_down_strategy).pack(side='left')
+ tkinter.Button(frame1, text='Bottom Up\nStrategy',
+ background='#90c0d0', foreground='black',
+ command=self.bottom_up_strategy).pack(side='left')
+ tkinter.Button(frame1, text='Bottom Up\nLeft-Corner Strategy',
+ background='#90c0d0', foreground='black',
+ command=self.bottom_up_leftcorner_strategy).pack(side='left')
+
+ tkinter.Button(frame2, text='Top Down Init\nRule',
+ background='#90f090', foreground='black',
+ command=self.top_down_init).pack(side='left')
+ tkinter.Button(frame2, text='Top Down Predict\nRule',
+ background='#90f090', foreground='black',
+ command=self.top_down_predict).pack(side='left')
+ tkinter.Frame(frame2, width=20).pack(side='left')
+
+ tkinter.Button(frame2, text='Bottom Up Predict\nRule',
+ background='#90f090', foreground='black',
+ command=self.bottom_up).pack(side='left')
+ tkinter.Frame(frame2, width=20).pack(side='left')
+
+ tkinter.Button(frame2, text='Bottom Up Left-Corner\nPredict Rule',
+ background='#90f090', foreground='black',
+ command=self.bottom_up_leftcorner).pack(side='left')
+ tkinter.Frame(frame2, width=20).pack(side='left')
+
+ tkinter.Button(frame2, text='Fundamental\nRule',
+ background='#90f090', foreground='black',
+ command=self.fundamental).pack(side='left')
+
+ def _init_bindings(self):
+ self._root.bind('<Up>', self._cv.scroll_up)
+ self._root.bind('<Down>', self._cv.scroll_down)
+ self._root.bind('<Prior>', self._cv.page_up)
+ self._root.bind('<Next>', self._cv.page_down)
+ self._root.bind('<Control-q>', self.destroy)
+ self._root.bind('<Control-x>', self.destroy)
+ self._root.bind('<F1>', self.help)
+
+ self._root.bind('<Control-s>', self.save_chart)
+ self._root.bind('<Control-o>', self.load_chart)
+ self._root.bind('<Control-r>', self.reset)
+
+ self._root.bind('t', self.top_down_strategy)
+ self._root.bind('b', self.bottom_up_strategy)
+ self._root.bind('c', self.bottom_up_leftcorner_strategy)
+ self._root.bind('<space>', self._stop_animation)
+
+ self._root.bind('<Control-g>', self.edit_grammar)
+ self._root.bind('<Control-t>', self.edit_sentence)
+
+ # Animation speed control
+ self._root.bind('-', lambda e,a=self._animate:a.set(1))
+ self._root.bind('=', lambda e,a=self._animate:a.set(2))
+ self._root.bind('+', lambda e,a=self._animate:a.set(3))
+
+ # Step control
+ self._root.bind('s', lambda e,s=self._step:s.set(not s.get()))
+
+ def _init_menubar(self):
+ menubar = tkinter.Menu(self._root)
+
+ filemenu = tkinter.Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Save Chart', underline=0,
+ command=self.save_chart, accelerator='Ctrl-s')
+ filemenu.add_command(label='Load Chart', underline=0,
+ command=self.load_chart, accelerator='Ctrl-o')
+ filemenu.add_command(label='Reset Chart', underline=0,
+ command=self.reset, accelerator='Ctrl-r')
+ filemenu.add_separator()
+ filemenu.add_command(label='Save Grammar',
+ command=self.save_grammar)
+ filemenu.add_command(label='Load Grammar',
+ command=self.load_grammar)
+ filemenu.add_separator()
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-x')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ editmenu = tkinter.Menu(menubar, tearoff=0)
+ editmenu.add_command(label='Edit Grammar', underline=5,
+ command=self.edit_grammar,
+ accelerator='Ctrl-g')
+ editmenu.add_command(label='Edit Text', underline=5,
+ command=self.edit_sentence,
+ accelerator='Ctrl-t')
+ menubar.add_cascade(label='Edit', underline=0, menu=editmenu)
+
+ viewmenu = tkinter.Menu(menubar, tearoff=0)
+ viewmenu.add_command(label='Chart Matrix', underline=6,
+ command=self.view_matrix)
+ viewmenu.add_command(label='Results', underline=0,
+ command=self.view_results)
+ menubar.add_cascade(label='View', underline=0, menu=viewmenu)
+
+ rulemenu = tkinter.Menu(menubar, tearoff=0)
+ rulemenu.add_command(label='Top Down Strategy', underline=0,
+ command=self.top_down_strategy,
+ accelerator='t')
+ rulemenu.add_command(label='Bottom Up Strategy', underline=0,
+ command=self.bottom_up_strategy,
+ accelerator='b')
+ rulemenu.add_command(label='Bottom Up Left-Corner Strategy', underline=0,
+ command=self.bottom_up_leftcorner_strategy,
+ accelerator='c')
+ rulemenu.add_separator()
+ rulemenu.add_command(label='Bottom Up Rule',
+ command=self.bottom_up)
+ rulemenu.add_command(label='Bottom Up Left-Corner Rule',
+ command=self.bottom_up_leftcorner)
+ rulemenu.add_command(label='Top Down Init Rule',
+ command=self.top_down_init)
+ rulemenu.add_command(label='Top Down Predict Rule',
+ command=self.top_down_predict)
+ rulemenu.add_command(label='Fundamental Rule',
+ command=self.fundamental)
+ menubar.add_cascade(label='Apply', underline=0, menu=rulemenu)
+
+ animatemenu = tkinter.Menu(menubar, tearoff=0)
+ animatemenu.add_checkbutton(label="Step", underline=0,
+ variable=self._step,
+ accelerator='s')
+ animatemenu.add_separator()
+ animatemenu.add_radiobutton(label="No Animation", underline=0,
+ variable=self._animate, value=0)
+ animatemenu.add_radiobutton(label="Slow Animation", underline=0,
+ variable=self._animate, value=1,
+ accelerator='-')
+ animatemenu.add_radiobutton(label="Normal Animation", underline=0,
+ variable=self._animate, value=2,
+ accelerator='=')
+ animatemenu.add_radiobutton(label="Fast Animation", underline=0,
+ variable=self._animate, value=3,
+ accelerator='+')
+ menubar.add_cascade(label="Animate", underline=1, menu=animatemenu)
+
+ zoommenu = tkinter.Menu(menubar, tearoff=0)
+ zoommenu.add_radiobutton(label='Tiny', variable=self._size,
+ underline=0, value=10, command=self.resize)
+ zoommenu.add_radiobutton(label='Small', variable=self._size,
+ underline=0, value=12, command=self.resize)
+ zoommenu.add_radiobutton(label='Medium', variable=self._size,
+ underline=0, value=14, command=self.resize)
+ zoommenu.add_radiobutton(label='Large', variable=self._size,
+ underline=0, value=18, command=self.resize)
+ zoommenu.add_radiobutton(label='Huge', variable=self._size,
+ underline=0, value=24, command=self.resize)
+ menubar.add_cascade(label='Zoom', underline=0, menu=zoommenu)
+
+ helpmenu = tkinter.Menu(menubar, tearoff=0)
+ helpmenu.add_command(label='About', underline=0,
+ command=self.about)
+ helpmenu.add_command(label='Instructions', underline=0,
+ command=self.help, accelerator='F1')
+ menubar.add_cascade(label='Help', underline=0, menu=helpmenu)
+
+ self._root.config(menu=menubar)
+
+ #////////////////////////////////////////////////////////////
+ # Selection Handling
+ #////////////////////////////////////////////////////////////
+
+ def _click_cv_edge(self, edge):
+ if edge != self._selection:
+ # Clicking on a new edge selects it.
+ self._select_edge(edge)
+ else:
+ # Repeated clicks on one edge cycle its trees.
+ self._cv.cycle_tree()
+ # [XX] this can get confused if animation is running
+ # faster than the callbacks...
+
+ def _select_matrix_edge(self, edge):
+ self._select_edge(edge)
+ self._cv.view_edge(edge)
+
+ def _select_edge(self, edge):
+ self._selection = edge
+ # Update the chart view.
+ self._cv.markonly_edge(edge, '#f00')
+ self._cv.draw_tree(edge)
+ # Update the matrix view.
+ if self._matrix: self._matrix.markonly_edge(edge)
+ if self._matrix: self._matrix.view_edge(edge)
+
+ def _deselect_edge(self):
+ self._selection = None
+ # Update the chart view.
+ self._cv.unmark_edge()
+ self._cv.erase_tree()
+ # Update the matrix view
+ if self._matrix: self._matrix.unmark_edge()
+
+ def _show_new_edge(self, edge):
+ self._display_rule(self._cp.current_chartrule())
+ # Update the chart view.
+ self._cv.update()
+ self._cv.draw_tree(edge)
+ self._cv.markonly_edge(edge, '#0df')
+ self._cv.view_edge(edge)
+ # Update the matrix view.
+ if self._matrix: self._matrix.update()
+ if self._matrix: self._matrix.markonly_edge(edge)
+ if self._matrix: self._matrix.view_edge(edge)
+ # Update the results view.
+ if self._results: self._results.update(edge)
+
+ #////////////////////////////////////////////////////////////
+ # Help/usage
+ #////////////////////////////////////////////////////////////
+
+ def help(self, *e):
+ self._animating = 0
+ # The default font's not very legible; try using 'fixed' instead.
+ try:
+ ShowText(self._root, 'Help: Chart Parser Application',
+ (__doc__ or '').strip(), width=75, font='fixed')
+ except:
+ ShowText(self._root, 'Help: Chart Parser Application',
+ (__doc__ or '').strip(), width=75)
+
+ def about(self, *e):
+ ABOUT = ("NLTK Chart Parser Application\n"+
+ "Written by Edward Loper")
+ tkinter.messagebox.showinfo('About: Chart Parser Application', ABOUT)
+
+ #////////////////////////////////////////////////////////////
+ # File Menu
+ #////////////////////////////////////////////////////////////
+
+ CHART_FILE_TYPES = [('Pickle file', '.pickle'),
+ ('All files', '*')]
+ GRAMMAR_FILE_TYPES = [('Plaintext grammar file', '.cfg'),
+ ('Pickle file', '.pickle'),
+ ('All files', '*')]
+
+ def load_chart(self, *args):
+ "Load a chart from a pickle file"
+ filename = askopenfilename(filetypes=self.CHART_FILE_TYPES,
+ defaultextension='.pickle')
+ if not filename: return
+ try:
+ with open(filename, 'rb') as infile:
+ chart = pickle.load(infile)
+ self._chart = chart
+ self._cv.update(chart)
+ if self._matrix: self._matrix.set_chart(chart)
+ if self._matrix: self._matrix.deselect_cell()
+ if self._results: self._results.set_chart(chart)
+ self._cp.set_chart(chart)
+ except Exception as e:
+ raise
+ tkinter.messagebox.showerror('Error Loading Chart',
+ 'Unable to open file: %r' % filename)
+
+ def save_chart(self, *args):
+ "Save a chart to a pickle file"
+ filename = asksaveasfilename(filetypes=self.CHART_FILE_TYPES,
+ defaultextension='.pickle')
+ if not filename: return
+ try:
+ with open(filename, 'wb') as outfile:
+ pickle.dump(self._chart, outfile)
+ except Exception as e:
+ raise
+ tkinter.messagebox.showerror('Error Saving Chart',
+ 'Unable to open file: %r' % filename)
+
+ def load_grammar(self, *args):
+ "Load a grammar from a pickle file"
+ filename = askopenfilename(filetypes=self.GRAMMAR_FILE_TYPES,
+ defaultextension='.cfg')
+ if not filename: return
+ try:
+ if filename.endswith('.pickle'):
+ with open(filename, 'rb') as infile:
+ grammar = pickle.load(infile)
+ else:
+ with open(filename, 'r') as infile:
+ grammar = CFG.fromstring(infile.read())
+ self.set_grammar(grammar)
+ except Exception as e:
+ tkinter.messagebox.showerror('Error Loading Grammar',
+ 'Unable to open file: %r' % filename)
+
+ def save_grammar(self, *args):
+ filename = asksaveasfilename(filetypes=self.GRAMMAR_FILE_TYPES,
+ defaultextension='.cfg')
+ if not filename: return
+ try:
+ if filename.endswith('.pickle'):
+ with open(filename, 'wb') as outfile:
+ pickle.dump((self._chart, self._tokens), outfile)
+ else:
+ with open(filename, 'w') as outfile:
+ prods = self._grammar.productions()
+ start = [p for p in prods if p.lhs() == self._grammar.start()]
+ rest = [p for p in prods if p.lhs() != self._grammar.start()]
+ for prod in start: outfile.write('%s\n' % prod)
+ for prod in rest: outfile.write('%s\n' % prod)
+ except Exception as e:
+ tkinter.messagebox.showerror('Error Saving Grammar',
+ 'Unable to open file: %r' % filename)
+
+ def reset(self, *args):
+ self._animating = 0
+ self._reset_parser()
+ self._cv.update(self._chart)
+ if self._matrix: self._matrix.set_chart(self._chart)
+ if self._matrix: self._matrix.deselect_cell()
+ if self._results: self._results.set_chart(self._chart)
+
+ #////////////////////////////////////////////////////////////
+ # Edit
+ #////////////////////////////////////////////////////////////
+
+ def edit_grammar(self, *e):
+ CFGEditor(self._root, self._grammar, self.set_grammar)
+
+ def set_grammar(self, grammar):
+ self._grammar = grammar
+ self._cp.set_grammar(grammar)
+ if self._results: self._results.set_grammar(grammar)
+
+ def edit_sentence(self, *e):
+ sentence = " ".join(self._tokens)
+ title = 'Edit Text'
+ instr = 'Enter a new sentence to parse.'
+ EntryDialog(self._root, sentence, instr, self.set_sentence, title)
+
+ def set_sentence(self, sentence):
+ self._tokens = list(sentence.split())
+ self.reset()
+
+ #////////////////////////////////////////////////////////////
+ # View Menu
+ #////////////////////////////////////////////////////////////
+
+ def view_matrix(self, *e):
+ if self._matrix is not None: self._matrix.destroy()
+ self._matrix = ChartMatrixView(self._root, self._chart)
+ self._matrix.add_callback('select', self._select_matrix_edge)
+
+ def view_results(self, *e):
+ if self._results is not None: self._results.destroy()
+ self._results = ChartResultsView(self._root, self._chart,
+ self._grammar)
+
+ #////////////////////////////////////////////////////////////
+ # Zoom Menu
+ #////////////////////////////////////////////////////////////
+
+ def resize(self):
+ self._animating = 0
+ self.set_font_size(self._size.get())
+
+ def set_font_size(self, size):
+ self._cv.set_font_size(size)
+ self._font.configure(size=-abs(size))
+ self._boldfont.configure(size=-abs(size))
+ self._sysfont.configure(size=-abs(size))
+
+ def get_font_size(self):
+ return abs(self._size.get())
+
+ #////////////////////////////////////////////////////////////
+ # Parsing
+ #////////////////////////////////////////////////////////////
+
+ def apply_strategy(self, strategy, edge_strategy=None):
+ # If we're animating, then stop.
+ if self._animating:
+ self._animating = 0
+ return
+
+ # Clear the rule display & mark.
+ self._display_rule(None)
+ #self._cv.unmark_edge()
+
+ if self._step.get():
+ selection = self._selection
+ if (selection is not None) and (edge_strategy is not None):
+ # Apply the given strategy to the selected edge.
+ self._cp.set_strategy([edge_strategy(selection)])
+ newedge = self._apply_strategy()
+
+ # If it failed, then clear the selection.
+ if newedge is None:
+ self._cv.unmark_edge()
+ self._selection = None
+ else:
+ self._cp.set_strategy(strategy)
+ self._apply_strategy()
+
+ else:
+ self._cp.set_strategy(strategy)
+ if self._animate.get():
+ self._animating = 1
+ self._animate_strategy()
+ else:
+ for edge in self._cpstep:
+ if edge is None: break
+ self._cv.update()
+ if self._matrix: self._matrix.update()
+ if self._results: self._results.update()
+
+ def _stop_animation(self, *e):
+ self._animating = 0
+
+ def _animate_strategy(self, speed=1):
+ if self._animating == 0: return
+ if self._apply_strategy() is not None:
+ if self._animate.get() == 0 or self._step.get() == 1:
+ return
+ if self._animate.get() == 1:
+ self._root.after(3000, self._animate_strategy)
+ elif self._animate.get() == 2:
+ self._root.after(1000, self._animate_strategy)
+ else:
+ self._root.after(20, self._animate_strategy)
+
+ def _apply_strategy(self):
+ new_edge = next(self._cpstep)
+
+ if new_edge is not None:
+ self._show_new_edge(new_edge)
+ return new_edge
+
+ def _display_rule(self, rule):
+ if rule is None:
+ self._rulelabel2['text'] = ''
+ else:
+ name = str(rule)
+ self._rulelabel2['text'] = name
+ size = self._cv.get_font_size()
+
+ #////////////////////////////////////////////////////////////
+ # Parsing Strategies
+ #////////////////////////////////////////////////////////////
+
+ # Basic rules:
+ _TD_INIT = [TopDownInitRule()]
+ _TD_PREDICT = [TopDownPredictRule()]
+ _BU_RULE = [BottomUpPredictRule()]
+ _BU_LC_RULE = [BottomUpPredictCombineRule()]
+ _FUNDAMENTAL = [SingleEdgeFundamentalRule()]
+
+ # Complete strategies:
+ _TD_STRATEGY = _TD_INIT + _TD_PREDICT + _FUNDAMENTAL
+ _BU_STRATEGY = _BU_RULE + _FUNDAMENTAL
+ _BU_LC_STRATEGY = _BU_LC_RULE + _FUNDAMENTAL
+
+ # Button callback functions:
+ def top_down_init(self, *e):
+ self.apply_strategy(self._TD_INIT, None)
+ def top_down_predict(self, *e):
+ self.apply_strategy(self._TD_PREDICT, TopDownPredictEdgeRule)
+ def bottom_up(self, *e):
+ self.apply_strategy(self._BU_RULE, BottomUpEdgeRule)
+ def bottom_up_leftcorner(self, *e):
+ self.apply_strategy(self._BU_LC_RULE, BottomUpLeftCornerEdgeRule)
+ def fundamental(self, *e):
+ self.apply_strategy(self._FUNDAMENTAL, FundamentalEdgeRule)
+ def bottom_up_strategy(self, *e):
+ self.apply_strategy(self._BU_STRATEGY, BottomUpEdgeRule)
+ def bottom_up_leftcorner_strategy(self, *e):
+ self.apply_strategy(self._BU_LC_STRATEGY, BottomUpLeftCornerEdgeRule)
+ def top_down_strategy(self, *e):
+ self.apply_strategy(self._TD_STRATEGY, TopDownPredictEdgeRule)
+
+def app():
+ grammar = CFG.fromstring("""
+ # Grammatical productions.
+ S -> NP VP
+ VP -> VP PP | V NP | V
+ NP -> Det N | NP PP
+ PP -> P NP
+ # Lexical productions.
+ NP -> 'John' | 'I'
+ Det -> 'the' | 'my' | 'a'
+ N -> 'dog' | 'cookie' | 'table' | 'cake' | 'fork'
+ V -> 'ate' | 'saw'
+ P -> 'on' | 'under' | 'with'
+ """)
+
+ sent = 'John ate the cake on the table with a fork'
+ sent = 'John ate the cake on the table'
+ tokens = list(sent.split())
+
+ print('grammar= (')
+ for rule in grammar.productions():
+ print((' ', repr(rule)+','))
+ print(')')
+ print(('tokens = %r' % tokens))
+ print('Calling "ChartParserApp(grammar, tokens)"...')
+ ChartParserApp(grammar, tokens).mainloop()
+
+if __name__ == '__main__':
+ app()
+
+ # Chart comparer:
+ #charts = ['/tmp/earley.pickle',
+ # '/tmp/topdown.pickle',
+ # '/tmp/bottomup.pickle']
+ #ChartComparer(*charts).mainloop()
+
+ #import profile
+ #profile.run('demo2()', '/tmp/profile.out')
+ #import pstats
+ #p = pstats.Stats('/tmp/profile.out')
+ #p.strip_dirs().sort_stats('time', 'cum').print_stats(60)
+ #p.strip_dirs().sort_stats('cum', 'time').print_stats(60)
+
+__all__ = ['app']
+
diff --git a/nltk/app/chunkparser_app.py b/nltk/app/chunkparser_app.py
new file mode 100644
index 0000000..f0b3c4b
--- /dev/null
+++ b/nltk/app/chunkparser_app.py
@@ -0,0 +1,1263 @@
+# Natural Language Toolkit: Regexp Chunk Parser Application
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A graphical tool for exploring the regular expression based chunk
+parser ``nltk.chunk.RegexpChunkParser``.
+"""
+
+# Todo: Add a way to select the development set from the menubar. This
+# might just need to be a selection box (conll vs treebank etc) plus
+# configuration parameters to select what's being chunked (eg VP vs NP)
+# and what part of the data is being used as the development set.
+
+import nltk.compat
+import time
+import textwrap
+import re
+import random
+import tkinter.filedialog, tkinter.font
+
+from tkinter import (Button, Canvas, Checkbutton,
+ Frame, IntVar, Label, Menu,
+ Scrollbar, Text, Tk)
+
+from nltk.tree import Tree
+from nltk.util import in_idle
+from nltk.draw.util import ShowText
+from nltk.corpus import conll2000, treebank_chunk
+from nltk.chunk import ChunkScore, RegexpChunkParser
+from nltk.chunk.regexp import RegexpChunkRule
+
+class RegexpChunkApp(object):
+ """
+ A graphical tool for exploring the regular expression based chunk
+ parser ``nltk.chunk.RegexpChunkParser``.
+
+ See ``HELP`` for instructional text.
+ """
+
+ ##/////////////////////////////////////////////////////////////////
+ ## Help Text
+ ##/////////////////////////////////////////////////////////////////
+
+ #: A dictionary mapping from part of speech tags to descriptions,
+ #: which is used in the help text. (This should probably live with
+ #: the conll and/or treebank corpus instead.)
+ TAGSET = {
+ 'CC': 'Coordinating conjunction', 'PRP$': 'Possessive pronoun',
+ 'CD': 'Cardinal number', 'RB': 'Adverb',
+ 'DT': 'Determiner', 'RBR': 'Adverb, comparative',
+ 'EX': 'Existential there', 'RBS': 'Adverb, superlative',
+ 'FW': 'Foreign word', 'RP': 'Particle',
+ 'JJ': 'Adjective', 'TO': 'to',
+ 'JJR': 'Adjective, comparative', 'UH': 'Interjection',
+ 'JJS': 'Adjective, superlative', 'VB': 'Verb, base form',
+ 'LS': 'List item marker', 'VBD': 'Verb, past tense',
+ 'MD': 'Modal', 'NNS': 'Noun, plural',
+ 'NN': 'Noun, singular or masps', 'VBN': 'Verb, past participle',
+ 'VBZ': 'Verb,3rd ps. sing. present', 'NNP': 'Proper noun, singular',
+ 'NNPS': 'Proper noun plural', 'WDT': 'wh-determiner',
+ 'PDT': 'Predeterminer', 'WP': 'wh-pronoun',
+ 'POS': 'Possessive ending', 'WP$': 'Possessive wh-pronoun',
+ 'PRP': 'Personal pronoun', 'WRB': 'wh-adverb',
+ '(': 'open parenthesis', ')': 'close parenthesis',
+ '``': 'open quote', ',': 'comma',
+ "''": 'close quote', '.': 'period',
+ '#': 'pound sign (currency marker)',
+ '$': 'dollar sign (currency marker)',
+ 'IN': 'Preposition/subord. conjunction',
+ 'SYM': 'Symbol (mathematical or scientific)',
+ 'VBG': 'Verb, gerund/present participle',
+ 'VBP': 'Verb, non-3rd ps. sing. present',
+ ':': 'colon',
+ }
+
+ #: Contents for the help box. This is a list of tuples, one for
+ #: each help page, where each tuple has four elements:
+ #: - A title (displayed as a tab)
+ #: - A string description of tabstops (see Tkinter.Text for details)
+ #: - The text contents for the help page. You can use expressions
+ #: like <red>...</red> to colorize the text; see ``HELP_AUTOTAG``
+ #: for a list of tags you can use for colorizing.
+ HELP = [
+ ('Help', '20',
+ "Welcome to the regular expression chunk-parser grammar editor. "
+ "You can use this editor to develop and test chunk parser grammars "
+ "based on NLTK's RegexpChunkParser class.\n\n"
+ # Help box.
+ "Use this box ('Help') to learn more about the editor; click on the "
+ "tabs for help on specific topics:"
+ "<indent>\n"
+ "Rules: grammar rule types\n"
+ "Regexps: regular expression syntax\n"
+ "Tags: part of speech tags\n</indent>\n"
+ # Grammar.
+ "Use the upper-left box ('Grammar') to edit your grammar. "
+ "Each line of your grammar specifies a single 'rule', "
+ "which performs an action such as creating a chunk or merging "
+ "two chunks.\n\n"
+ # Dev set.
+ "The lower-left box ('Development Set') runs your grammar on the "
+ "development set, and displays the results. "
+ "Your grammar's chunks are <highlight>highlighted</highlight>, and "
+ "the correct (gold standard) chunks are "
+ "<underline>underlined</underline>. If they "
+ "match, they are displayed in <green>green</green>; otherwise, "
+ "they are displayed in <red>red</red>. The box displays a single "
+ "sentence from the development set at a time; use the scrollbar or "
+ "the next/previous buttons view additional sentences.\n\n"
+ # Performance
+ "The lower-right box ('Evaluation') tracks the performance of "
+ "your grammar on the development set. The 'precision' axis "
+ "indicates how many of your grammar's chunks are correct; and "
+ "the 'recall' axis indicates how many of the gold standard "
+ "chunks your system generated. Typically, you should try to "
+ "design a grammar that scores high on both metrics. The "
+ "exact precision and recall of the current grammar, as well "
+ "as their harmonic mean (the 'f-score'), are displayed in "
+ "the status bar at the bottom of the window."
+ ),
+ ('Rules', '10',
+ "<h1>{...regexp...}</h1>"
+ "<indent>\nChunk rule: creates new chunks from words matching "
+ "regexp.</indent>\n\n"
+ "<h1>}...regexp...{</h1>"
+ "<indent>\nChink rule: removes words matching regexp from existing "
+ "chunks.</indent>\n\n"
+ "<h1>...regexp1...}{...regexp2...</h1>"
+ "<indent>\nSplit rule: splits chunks that match regexp1 followed by "
+ "regexp2 in two.</indent>\n\n"
+ "<h1>...regexp...{}...regexp...</h1>"
+ "<indent>\nMerge rule: joins consecutive chunks that match regexp1 "
+ "and regexp2</indent>\n"
+ ),
+ ('Regexps', '10 60',
+ #"Regular Expression Syntax Summary:\n\n"
+ "<h1>Pattern\t\tMatches...</h1>\n"
+ "<hangindent>"
+ "\t<<var>T</var>>\ta word with tag <var>T</var> "
+ "(where <var>T</var> may be a regexp).\n"
+ "\t<var>x</var>?\tan optional <var>x</var>\n"
+ "\t<var>x</var>+\ta sequence of 1 or more <var>x</var>'s\n"
+ "\t<var>x</var>*\ta sequence of 0 or more <var>x</var>'s\n"
+ "\t<var>x</var>|<var>y</var>\t<var>x</var> or <var>y</var>\n"
+ "\t.\tmatches any character\n"
+ "\t(<var>x</var>)\tTreats <var>x</var> as a group\n"
+ "\t# <var>x...</var>\tTreats <var>x...</var> "
+ "(to the end of the line) as a comment\n"
+ "\t\\<var>C</var>\tmatches character <var>C</var> "
+ "(useful when <var>C</var> is a special character "
+ "like + or #)\n"
+ "</hangindent>"
+ "\n<h1>Examples:</h1>\n"
+ "<hangindent>"
+ '\t<regexp><NN></regexp>\n'
+ '\t\tMatches <match>"cow/NN"</match>\n'
+ '\t\tMatches <match>"green/NN"</match>\n'
+ '\t<regexp><VB.*></regexp>\n'
+ '\t\tMatches <match>"eating/VBG"</match>\n'
+ '\t\tMatches <match>"ate/VBD"</match>\n'
+ '\t<regexp><IN><DT><NN></regexp>\n'
+ '\t\tMatches <match>"on/IN the/DT car/NN"</match>\n'
+ '\t<regexp><RB>?<VBD></regexp>\n'
+ '\t\tMatches <match>"ran/VBD"</match>\n'
+ '\t\tMatches <match>"slowly/RB ate/VBD"</match>\n'
+ '\t<regexp><\#><CD> # This is a comment...</regexp>\n'
+ '\t\tMatches <match>"#/# 100/CD"</match>\n'
+ "</hangindent>"
+ ),
+ ('Tags', '10 60',
+ "<h1>Part of Speech Tags:</h1>\n" +
+ '<hangindent>' +
+ '<<TAGSET>>' + # this gets auto-substituted w/ self.TAGSET
+ '</hangindent>\n')
+ ]
+
+ HELP_AUTOTAG = [
+ ('red', dict(foreground='#a00')),
+ ('green', dict(foreground='#080')),
+ ('highlight', dict(background='#ddd')),
+ ('underline', dict(underline=True)),
+ ('h1', dict(underline=True)),
+ ('indent', dict(lmargin1=20, lmargin2=20)),
+ ('hangindent', dict(lmargin1=0, lmargin2=60)),
+ ('var', dict(foreground='#88f')),
+ ('regexp', dict(foreground='#ba7')),
+ ('match', dict(foreground='#6a6')),
+ ]
+
+ ##/////////////////////////////////////////////////////////////////
+ ## Config Parmeters
+ ##/////////////////////////////////////////////////////////////////
+
+ _EVAL_DELAY = 1
+ """If the user has not pressed any key for this amount of time (in
+ seconds), and the current grammar has not been evaluated, then
+ the eval demon will evaluate it."""
+
+ _EVAL_CHUNK = 15
+ """The number of sentences that should be evaluated by the eval
+ demon each time it runs."""
+ _EVAL_FREQ = 0.2
+ """The frequency (in seconds) at which the eval demon is run"""
+ _EVAL_DEMON_MIN = .02
+ """The minimum amount of time that the eval demon should take each time
+ it runs -- if it takes less than this time, _EVAL_CHUNK will be
+ modified upwards."""
+ _EVAL_DEMON_MAX = .04
+ """The maximum amount of time that the eval demon should take each time
+ it runs -- if it takes more than this time, _EVAL_CHUNK will be
+ modified downwards."""
+
+ _GRAMMARBOX_PARAMS = dict(
+ width=40, height=12, background='#efe', highlightbackground='#efe',
+ highlightthickness=1, relief='groove', border=2, wrap='word')
+ _HELPBOX_PARAMS = dict(
+ width=15, height=15, background='#efe', highlightbackground='#efe',
+ foreground='#555',
+ highlightthickness=1, relief='groove', border=2, wrap='word')
+ _DEVSETBOX_PARAMS = dict(
+ width=70, height=10, background='#eef', highlightbackground='#eef',
+ highlightthickness=1, relief='groove', border=2, wrap='word',
+ tabs=(30,))
+ _STATUS_PARAMS = dict(
+ background='#9bb', relief='groove', border=2)
+ _FONT_PARAMS = dict(
+ family='helvetica', size=-20)
+ _FRAME_PARAMS = dict(
+ background='#777', padx=2, pady=2, border=3)
+ _EVALBOX_PARAMS = dict(
+ background='#eef', highlightbackground='#eef',
+ highlightthickness=1, relief='groove', border=2,
+ width=300, height=280)
+ _BUTTON_PARAMS = dict(
+ background='#777', activebackground='#777',
+ highlightbackground='#777')
+ _HELPTAB_BG_COLOR = '#aba'
+ _HELPTAB_FG_COLOR = '#efe'
+
+ _HELPTAB_FG_PARAMS = dict(background='#efe')
+ _HELPTAB_BG_PARAMS = dict(background='#aba')
+ _HELPTAB_SPACER = 6
+
+ def normalize_grammar(self, grammar):
+ # Strip comments
+ grammar = re.sub(r'((\\.|[^#])*)(#.*)?', r'\1', grammar)
+ # Normalize whitespace
+ grammar = re.sub(' +', ' ', grammar)
+ grammar = re.sub('\n\s+', '\n', grammar)
+ grammar = grammar.strip()
+ # [xx] Hack: automatically backslash $!
+ grammar = re.sub(r'([^\\])\$', r'\1\\$', grammar)
+ return grammar
+
+ def __init__(self, devset_name='conll2000', devset=None,
+ grammar = '', chunk_label='NP', tagset=None):
+ """
+ :param devset_name: The name of the development set; used for
+ display & for save files. If either the name 'treebank'
+ or the name 'conll2000' is used, and devset is None, then
+ devset will be set automatically.
+ :param devset: A list of chunked sentences
+ :param grammar: The initial grammar to display.
+ :param tagset: Dictionary from tags to string descriptions, used
+ for the help page. Defaults to ``self.TAGSET``.
+ """
+ self._chunk_label = chunk_label
+
+ if tagset is None: tagset = self.TAGSET
+ self.tagset = tagset
+
+ # Named development sets:
+ if devset is None:
+ if devset_name == 'conll2000':
+ devset = conll2000.chunked_sents('train.txt')#[:100]
+ elif devset == 'treebank':
+ devset = treebank_chunk.chunked_sents()#[:100]
+ else:
+ raise ValueError('Unknown development set %s' % devset_name)
+
+ self.chunker = None
+ """The chunker built from the grammar string"""
+
+ self.grammar = grammar
+ """The unparsed grammar string"""
+
+ self.normalized_grammar = None
+ """A normalized version of ``self.grammar``."""
+
+ self.grammar_changed = 0
+ """The last time() that the grammar was changed."""
+
+ self.devset = devset
+ """The development set -- a list of chunked sentences."""
+
+ self.devset_name = devset_name
+ """The name of the development set (for save files)."""
+
+ self.devset_index = -1
+ """The index into the development set of the first instance
+ that's currently being viewed."""
+
+ self._last_keypress = 0
+ """The time() when a key was most recently pressed"""
+
+ self._history = []
+ """A list of (grammar, precision, recall, fscore) tuples for
+ grammars that the user has already tried."""
+
+ self._history_index = 0
+ """When the user is scrolling through previous grammars, this
+ is used to keep track of which grammar they're looking at."""
+
+ self._eval_grammar = None
+ """The grammar that is being currently evaluated by the eval
+ demon."""
+
+ self._eval_normalized_grammar = None
+ """A normalized copy of ``_eval_grammar``."""
+
+ self._eval_index = 0
+ """The index of the next sentence in the development set that
+ should be looked at by the eval demon."""
+
+ self._eval_score = ChunkScore(chunk_label=chunk_label)
+ """The ``ChunkScore`` object that's used to keep track of the score
+ of the current grammar on the development set."""
+
+ # Set up the main window.
+ top = self.top = Tk()
+ top.geometry('+50+50')
+ top.title('Regexp Chunk Parser App')
+ top.bind('<Control-q>', self.destroy)
+
+ # Varaible that restricts how much of the devset we look at.
+ self._devset_size = IntVar(top)
+ self._devset_size.set(100)
+
+ # Set up all the tkinter widgets
+ self._init_fonts(top)
+ self._init_widgets(top)
+ self._init_bindings(top)
+ self._init_menubar(top)
+ self.grammarbox.focus()
+
+
+ # If a grammar was given, then display it.
+ if grammar:
+ self.grammarbox.insert('end', grammar+'\n')
+ self.grammarbox.mark_set('insert', '1.0')
+
+ # Display the first item in the development set
+ self.show_devset(0)
+ self.update()
+
+ def _init_bindings(self, top):
+ top.bind('<Control-n>', self._devset_next)
+ top.bind('<Control-p>', self._devset_prev)
+ top.bind('<Control-t>', self.toggle_show_trace)
+ top.bind('<KeyPress>', self.update)
+ top.bind('<Control-s>', lambda e: self.save_grammar())
+ top.bind('<Control-o>', lambda e: self.load_grammar())
+ self.grammarbox.bind('<Control-t>', self.toggle_show_trace)
+ self.grammarbox.bind('<Control-n>', self._devset_next)
+ self.grammarbox.bind('<Control-p>', self._devset_prev)
+
+ # Redraw the eval graph when the window size changes
+ self.evalbox.bind('<Configure>', self._eval_plot)
+
+ def _init_fonts(self, top):
+ # TWhat's our font size (default=same as sysfont)
+ self._size = IntVar(top)
+ self._size.set(20)
+ self._font = tkinter.font.Font(family='helvetica',
+ size=-self._size.get())
+ self._smallfont = tkinter.font.Font(family='helvetica',
+ size=-(int(self._size.get()*14/20)))
+
+ def _init_menubar(self, parent):
+ menubar = Menu(parent)
+
+ filemenu = Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Reset Application', underline=0,
+ command=self.reset)
+ filemenu.add_command(label='Save Current Grammar', underline=0,
+ accelerator='Ctrl-s',
+ command=self.save_grammar)
+ filemenu.add_command(label='Load Grammar', underline=0,
+ accelerator='Ctrl-o',
+ command=self.load_grammar)
+
+ filemenu.add_command(label='Save Grammar History', underline=13,
+ command=self.save_history)
+
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-q')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ viewmenu = Menu(menubar, tearoff=0)
+ viewmenu.add_radiobutton(label='Tiny', variable=self._size,
+ underline=0, value=10, command=self.resize)
+ viewmenu.add_radiobutton(label='Small', variable=self._size,
+ underline=0, value=16, command=self.resize)
+ viewmenu.add_radiobutton(label='Medium', variable=self._size,
+ underline=0, value=20, command=self.resize)
+ viewmenu.add_radiobutton(label='Large', variable=self._size,
+ underline=0, value=24, command=self.resize)
+ viewmenu.add_radiobutton(label='Huge', variable=self._size,
+ underline=0, value=34, command=self.resize)
+ menubar.add_cascade(label='View', underline=0, menu=viewmenu)
+
+ devsetmenu = Menu(menubar, tearoff=0)
+ devsetmenu.add_radiobutton(label='50 sentences',
+ variable=self._devset_size,
+ value=50, command=self.set_devset_size)
+ devsetmenu.add_radiobutton(label='100 sentences',
+ variable=self._devset_size,
+ value=100, command=self.set_devset_size)
+ devsetmenu.add_radiobutton(label='200 sentences',
+ variable=self._devset_size,
+ value=200, command=self.set_devset_size)
+ devsetmenu.add_radiobutton(label='500 sentences',
+ variable=self._devset_size,
+ value=500, command=self.set_devset_size)
+ menubar.add_cascade(label='Development-Set', underline=0,
+ menu=devsetmenu)
+
+ helpmenu = Menu(menubar, tearoff=0)
+ helpmenu.add_command(label='About', underline=0,
+ command=self.about)
+ menubar.add_cascade(label='Help', underline=0, menu=helpmenu)
+
+ parent.config(menu=menubar)
+
+ def toggle_show_trace(self, *e):
+ if self._showing_trace:
+ self.show_devset()
+ else:
+ self.show_trace()
+ return 'break'
+
+
+ _SCALE_N = 5 # center on the last 5 examples.
+ _DRAW_LINES = False
+ def _eval_plot(self, *e, **config):
+ width = config.get('width', self.evalbox.winfo_width())
+ height = config.get('height', self.evalbox.winfo_height())
+
+ # Clear the canvas
+ self.evalbox.delete('all')
+
+ # Draw the precision & recall labels.
+ tag = self.evalbox.create_text(10, height/2-10, justify='left',
+ anchor='w', text='Precision')
+ left, right = self.evalbox.bbox(tag)[2] + 5, width-10
+ tag = self.evalbox.create_text(left + (width-left)/2, height-10,
+ anchor='s', text='Recall', justify='center')
+ top, bot = 10, self.evalbox.bbox(tag)[1]-10
+
+ # Draw masks for clipping the plot.
+ bg = self._EVALBOX_PARAMS['background']
+ self.evalbox.lower(self.evalbox.create_rectangle(0, 0, left-1, 5000,
+ fill=bg, outline=bg))
+ self.evalbox.lower(self.evalbox.create_rectangle(0, bot+1, 5000, 5000,
+ fill=bg, outline=bg))
+
+ # Calculate the plot's scale.
+ if self._autoscale.get() and len(self._history) > 1:
+ max_precision = max_recall = 0
+ min_precision = min_recall = 1
+ for i in range(1, min(len(self._history), self._SCALE_N+1)):
+ grammar, precision, recall, fmeasure = self._history[-i]
+ min_precision = min(precision, min_precision)
+ min_recall = min(recall, min_recall)
+ max_precision = max(precision, max_precision)
+ max_recall = max(recall, max_recall)
+# if max_precision-min_precision > max_recall-min_recall:
+# min_recall -= (max_precision-min_precision)/2
+# max_recall += (max_precision-min_precision)/2
+# else:
+# min_precision -= (max_recall-min_recall)/2
+# max_precision += (max_recall-min_recall)/2
+# if min_recall < 0:
+# max_recall -= min_recall
+# min_recall = 0
+# if min_precision < 0:
+# max_precision -= min_precision
+# min_precision = 0
+ min_precision = max(min_precision-.01, 0)
+ min_recall = max(min_recall-.01, 0)
+ max_precision = min(max_precision+.01, 1)
+ max_recall = min(max_recall+.01, 1)
+ else:
+ min_precision = min_recall = 0
+ max_precision = max_recall = 1
+
+ # Draw the axis lines & grid lines
+ for i in range(11):
+ x = left + (right-left)*((i/10.-min_recall)/
+ (max_recall-min_recall))
+ y = bot - (bot-top)*((i/10.-min_precision)/
+ (max_precision-min_precision))
+ if left < x < right:
+ self.evalbox.create_line(x, top, x, bot, fill='#888')
+ if top < y < bot:
+ self.evalbox.create_line(left, y, right, y, fill='#888')
+ self.evalbox.create_line(left, top, left, bot)
+ self.evalbox.create_line(left, bot, right, bot)
+
+ # Display the plot's scale
+ self.evalbox.create_text(
+ left-3, bot, justify='right', anchor='se',
+ text='%d%%' % (100*min_precision))
+ self.evalbox.create_text(
+ left-3, top, justify='right', anchor='ne',
+ text='%d%%' % (100*max_precision))
+ self.evalbox.create_text(
+ left, bot+3, justify='center', anchor='nw',
+ text='%d%%' % (100*min_recall))
+ self.evalbox.create_text(
+ right, bot+3, justify='center', anchor='ne',
+ text='%d%%' % (100*max_recall))
+
+ # Display the scores.
+ prev_x = prev_y = None
+ for i, (_, precision, recall, fscore) in enumerate(self._history):
+ x = left + (right-left) * ((recall-min_recall) /
+ (max_recall-min_recall))
+ y = bot - (bot-top) * ((precision-min_precision) /
+ (max_precision-min_precision))
+ if i == self._history_index:
+ self.evalbox.create_oval(x-2,y-2,x+2,y+2,
+ fill='#0f0', outline='#000')
+ self.status['text'] = (
+ 'Precision: %.2f%%\t' % (precision*100)+
+ 'Recall: %.2f%%\t' % (recall*100)+
+ 'F-score: %.2f%%' % (fscore*100))
+ else:
+ self.evalbox.lower(
+ self.evalbox.create_oval(x-2,y-2,x+2,y+2,
+ fill='#afa', outline='#8c8'))
+ if prev_x is not None and self._eval_lines.get():
+ self.evalbox.lower(
+ self.evalbox.create_line(prev_x, prev_y, x, y,
+ fill='#8c8'))
+ prev_x, prev_y = x, y
+
+ _eval_demon_running = False
+ def _eval_demon(self):
+ if self.top is None: return
+ if self.chunker is None:
+ self._eval_demon_running = False
+ return
+
+ # Note our starting time.
+ t0 = time.time()
+
+ # If are still typing, then wait for them to finish.
+ if (time.time()-self._last_keypress < self._EVAL_DELAY and
+ self.normalized_grammar != self._eval_normalized_grammar):
+ self._eval_demon_running = True
+ return self.top.after(int(self._EVAL_FREQ*1000), self._eval_demon)
+
+ # If the grammar changed, restart the evaluation.
+ if self.normalized_grammar != self._eval_normalized_grammar:
+ # Check if we've seen this grammar already. If so, then
+ # just use the old evaluation values.
+ for (g, p, r, f) in self._history:
+ if self.normalized_grammar == self.normalize_grammar(g):
+ self._history.append( (g, p, r, f) )
+ self._history_index = len(self._history) - 1
+ self._eval_plot()
+ self._eval_demon_running = False
+ self._eval_normalized_grammar = None
+ return
+ self._eval_index = 0
+ self._eval_score = ChunkScore(chunk_label=self._chunk_label)
+ self._eval_grammar = self.grammar
+ self._eval_normalized_grammar = self.normalized_grammar
+
+ # If the grammar is empty, the don't bother evaluating it, or
+ # recording it in history -- the score will just be 0.
+ if self.normalized_grammar.strip() == '':
+ #self._eval_index = self._devset_size.get()
+ self._eval_demon_running = False
+ return
+
+ # Score the next set of examples
+ for gold in self.devset[self._eval_index:
+ min(self._eval_index+self._EVAL_CHUNK,
+ self._devset_size.get())]:
+ guess = self._chunkparse(gold.leaves())
+ self._eval_score.score(gold, guess)
+
+ # update our index in the devset.
+ self._eval_index += self._EVAL_CHUNK
+
+ # Check if we're done
+ if self._eval_index >= self._devset_size.get():
+ self._history.append( (self._eval_grammar,
+ self._eval_score.precision(),
+ self._eval_score.recall(),
+ self._eval_score.f_measure()) )
+ self._history_index = len(self._history)-1
+ self._eval_plot()
+ self._eval_demon_running = False
+ self._eval_normalized_grammar = None
+ else:
+ progress = 100*self._eval_index/self._devset_size.get()
+ self.status['text'] = ('Evaluating on Development Set (%d%%)' %
+ progress)
+ self._eval_demon_running = True
+ self._adaptively_modify_eval_chunk(time.time() - t0)
+ self.top.after(int(self._EVAL_FREQ*1000), self._eval_demon)
+
+ def _adaptively_modify_eval_chunk(self, t):
+ """
+ Modify _EVAL_CHUNK to try to keep the amount of time that the
+ eval demon takes between _EVAL_DEMON_MIN and _EVAL_DEMON_MAX.
+
+ :param t: The amount of time that the eval demon took.
+ """
+ if t > self._EVAL_DEMON_MAX and self._EVAL_CHUNK > 5:
+ self._EVAL_CHUNK = min(self._EVAL_CHUNK-1,
+ max(int(self._EVAL_CHUNK*(self._EVAL_DEMON_MAX/t)),
+ self._EVAL_CHUNK-10))
+ elif t < self._EVAL_DEMON_MIN:
+ self._EVAL_CHUNK = max(self._EVAL_CHUNK+1,
+ min(int(self._EVAL_CHUNK*(self._EVAL_DEMON_MIN/t)),
+ self._EVAL_CHUNK+10))
+
+ def _init_widgets(self, top):
+ frame0 = Frame(top, **self._FRAME_PARAMS)
+ frame0.grid_columnconfigure(0, weight=4)
+ frame0.grid_columnconfigure(3, weight=2)
+ frame0.grid_rowconfigure(1, weight=1)
+ frame0.grid_rowconfigure(5, weight=1)
+
+ # The grammar
+ self.grammarbox = Text(frame0, font=self._font,
+ **self._GRAMMARBOX_PARAMS)
+ self.grammarlabel = Label(frame0, font=self._font, text='Grammar:',
+ highlightcolor='black',
+ background=self._GRAMMARBOX_PARAMS['background'])
+ self.grammarlabel.grid(column=0, row=0, sticky='SW')
+ self.grammarbox.grid(column=0, row=1, sticky='NEWS')
+
+ # Scroll bar for grammar
+ grammar_scrollbar = Scrollbar(frame0, command=self.grammarbox.yview)
+ grammar_scrollbar.grid(column=1, row=1, sticky='NWS')
+ self.grammarbox.config(yscrollcommand=grammar_scrollbar.set)
+
+ # grammar buttons
+ bg = self._FRAME_PARAMS['background']
+ frame3 = Frame(frame0, background=bg)
+ frame3.grid(column=0, row=2, sticky='EW')
+ Button(frame3, text='Prev Grammar', command=self._history_prev,
+ **self._BUTTON_PARAMS).pack(side='left')
+ Button(frame3, text='Next Grammar', command=self._history_next,
+ **self._BUTTON_PARAMS).pack(side='left')
+
+ # Help box
+ self.helpbox = Text(frame0, font=self._smallfont,
+ **self._HELPBOX_PARAMS)
+ self.helpbox.grid(column=3, row=1, sticky='NEWS')
+ self.helptabs = {}
+ bg = self._FRAME_PARAMS['background']
+ helptab_frame = Frame(frame0, background=bg)
+ helptab_frame.grid(column=3, row=0, sticky='SW')
+ for i, (tab, tabstops, text) in enumerate(self.HELP):
+ label = Label(helptab_frame, text=tab, font=self._smallfont)
+ label.grid(column=i*2, row=0, sticky='S')
+ #help_frame.grid_columnconfigure(i, weight=1)
+ #label.pack(side='left')
+ label.bind('<ButtonPress>', lambda e, tab=tab: self.show_help(tab))
+ self.helptabs[tab] = label
+ Frame(helptab_frame, height=1, width=self._HELPTAB_SPACER,
+ background=bg).grid(column=i*2+1, row=0)
+ self.helptabs[self.HELP[0][0]].configure(font=self._font)
+ self.helpbox.tag_config('elide', elide=True)
+ for (tag, params) in self.HELP_AUTOTAG:
+ self.helpbox.tag_config('tag-%s' % tag, **params)
+ self.show_help(self.HELP[0][0])
+
+ # Scroll bar for helpbox
+ help_scrollbar = Scrollbar(frame0, command=self.helpbox.yview)
+ self.helpbox.config(yscrollcommand=help_scrollbar.set)
+ help_scrollbar.grid(column=4, row=1, sticky='NWS')
+
+ # The dev set
+ frame4 = Frame(frame0, background=self._FRAME_PARAMS['background'])
+ self.devsetbox = Text(frame4, font=self._font,
+ **self._DEVSETBOX_PARAMS)
+ self.devsetbox.pack(expand=True, fill='both')
+ self.devsetlabel = Label(frame0, font=self._font,
+ text='Development Set:', justify='right',
+ background=self._DEVSETBOX_PARAMS['background'])
+ self.devsetlabel.grid(column=0, row=4, sticky='SW')
+ frame4.grid(column=0, row=5, sticky='NEWS')
+
+ # dev set scrollbars
+ self.devset_scroll = Scrollbar(frame0, command=self._devset_scroll)
+ self.devset_scroll.grid(column=1, row=5, sticky='NWS')
+ self.devset_xscroll = Scrollbar(frame4, command=self.devsetbox.xview,
+ orient='horiz')
+ self.devsetbox['xscrollcommand'] = self.devset_xscroll.set
+ self.devset_xscroll.pack(side='bottom', fill='x')
+
+ # dev set buttons
+ bg = self._FRAME_PARAMS['background']
+ frame1 = Frame(frame0, background=bg)
+ frame1.grid(column=0, row=7, sticky='EW')
+ Button(frame1, text='Prev Example (Ctrl-p)',
+ command=self._devset_prev,
+ **self._BUTTON_PARAMS).pack(side='left')
+ Button(frame1, text='Next Example (Ctrl-n)',
+ command=self._devset_next,
+ **self._BUTTON_PARAMS).pack(side='left')
+ self.devset_button = Button(frame1, text='Show example',
+ command=self.show_devset,
+ state='disabled',
+ **self._BUTTON_PARAMS)
+ self.devset_button.pack(side='right')
+ self.trace_button = Button(frame1, text='Show trace',
+ command=self.show_trace,
+ **self._BUTTON_PARAMS)
+ self.trace_button.pack(side='right')
+
+
+ # evaluation box
+ self.evalbox = Canvas(frame0, **self._EVALBOX_PARAMS)
+ label = Label(frame0, font=self._font, text='Evaluation:',
+ justify='right', background=self._EVALBOX_PARAMS['background'])
+ label.grid(column=3, row=4, sticky='SW')
+ self.evalbox.grid(column=3, row=5, sticky='NEWS', columnspan=2)
+
+ # evaluation box buttons
+ bg = self._FRAME_PARAMS['background']
+ frame2 = Frame(frame0, background=bg)
+ frame2.grid(column=3, row=7, sticky='EW')
+ self._autoscale = IntVar(self.top)
+ self._autoscale.set(False)
+ Checkbutton(frame2, variable=self._autoscale, command=self._eval_plot,
+ text='Zoom', **self._BUTTON_PARAMS).pack(side='left')
+ self._eval_lines = IntVar(self.top)
+ self._eval_lines.set(False)
+ Checkbutton(frame2, variable=self._eval_lines, command=self._eval_plot,
+ text='Lines', **self._BUTTON_PARAMS).pack(side='left')
+ Button(frame2, text='History',
+ **self._BUTTON_PARAMS).pack(side='right')
+
+ # The status label
+ self.status = Label(frame0, font=self._font, **self._STATUS_PARAMS)
+ self.status.grid(column=0, row=9, sticky='NEW', padx=3, pady=2,
+ columnspan=5)
+
+ # Help box & devset box can't be edited.
+ self.helpbox['state'] = 'disabled'
+ self.devsetbox['state'] = 'disabled'
+
+ # Spacers
+ bg = self._FRAME_PARAMS['background']
+ Frame(frame0, height=10, width=0, background=bg).grid(column=0, row=3)
+ Frame(frame0, height=0, width=10, background=bg).grid(column=2, row=0)
+ Frame(frame0, height=6, width=0, background=bg).grid(column=0, row=8)
+
+ # pack the frame.
+ frame0.pack(fill='both', expand=True)
+
+ # Set up colors for the devset box
+ self.devsetbox.tag_config('true-pos', background='#afa',
+ underline='True')
+ self.devsetbox.tag_config('false-neg', underline='True',
+ foreground='#800')
+ self.devsetbox.tag_config('false-pos', background='#faa')
+ self.devsetbox.tag_config('trace', foreground='#666', wrap='none')
+ self.devsetbox.tag_config('wrapindent', lmargin2=30, wrap='none')
+ self.devsetbox.tag_config('error', foreground='#800')
+
+ # And for the grammarbox
+ self.grammarbox.tag_config('error', background='#fec')
+ self.grammarbox.tag_config('comment', foreground='#840')
+ self.grammarbox.tag_config('angle', foreground='#00f')
+ self.grammarbox.tag_config('brace', foreground='#0a0')
+ self.grammarbox.tag_config('hangindent', lmargin1=0, lmargin2=40)
+
+ _showing_trace = False
+ def show_trace(self, *e):
+ self._showing_trace = True
+ self.trace_button['state'] = 'disabled'
+ self.devset_button['state'] = 'normal'
+
+ self.devsetbox['state'] = 'normal'
+ #self.devsetbox['wrap'] = 'none'
+ self.devsetbox.delete('1.0', 'end')
+ self.devsetlabel['text']='Development Set (%d/%d)' % (
+ (self.devset_index+1, self._devset_size.get()))
+
+ if self.chunker is None:
+ self.devsetbox.insert('1.0', 'Trace: waiting for a valid grammar.')
+ self.devsetbox.tag_add('error', '1.0', 'end')
+ return # can't do anything more
+
+ gold_tree = self.devset[self.devset_index]
+ rules = self.chunker.rules()
+
+ # Calculate the tag sequence
+ tagseq = '\t'
+ charnum = [1]
+ for wordnum, (word, pos) in enumerate(gold_tree.leaves()):
+ tagseq += '%s ' % pos
+ charnum.append(len(tagseq))
+ self.charnum = dict(((i, j), charnum[j])
+ for i in range(len(rules)+1)
+ for j in range(len(charnum)))
+ self.linenum = dict((i,i*2+2) for i in range(len(rules)+1))
+
+ for i in range(len(rules)+1):
+ if i == 0:
+ self.devsetbox.insert('end', 'Start:\n')
+ self.devsetbox.tag_add('trace', 'end -2c linestart', 'end -2c')
+ else:
+ self.devsetbox.insert('end', 'Apply %s:\n' % rules[i-1])
+ self.devsetbox.tag_add('trace', 'end -2c linestart', 'end -2c')
+ # Display the tag sequence.
+ self.devsetbox.insert('end', tagseq+'\n')
+ self.devsetbox.tag_add('wrapindent','end -2c linestart','end -2c')
+ # Run a partial parser, and extract gold & test chunks
+ chunker = RegexpChunkParser(rules[:i])
+ test_tree = self._chunkparse(gold_tree.leaves())
+ gold_chunks = self._chunks(gold_tree)
+ test_chunks = self._chunks(test_tree)
+ # Compare them.
+ for chunk in gold_chunks.intersection(test_chunks):
+ self._color_chunk(i, chunk, 'true-pos')
+ for chunk in gold_chunks - test_chunks:
+ self._color_chunk(i, chunk, 'false-neg')
+ for chunk in test_chunks - gold_chunks:
+ self._color_chunk(i, chunk, 'false-pos')
+ self.devsetbox.insert('end', 'Finished.\n')
+ self.devsetbox.tag_add('trace', 'end -2c linestart', 'end -2c')
+
+ # This is a hack, because the x-scrollbar isn't updating its
+ # position right -- I'm not sure what the underlying cause is
+ # though. (This is on OS X w/ python 2.5)
+ self.top.after(100, self.devset_xscroll.set, 0, .3)
+
+ def show_help(self, tab):
+ self.helpbox['state'] = 'normal'
+ self.helpbox.delete('1.0', 'end')
+ for (name, tabstops, text) in self.HELP:
+ if name == tab:
+ text = text.replace('<<TAGSET>>', '\n'.join(
+ ('\t%s\t%s' % item for item in sorted(list(self.tagset.items()),
+ key=lambda t_w:re.match('\w+',t_w[0]) and (0,t_w[0]) or (1,t_w[0])))))
+
+ self.helptabs[name].config(**self._HELPTAB_FG_PARAMS)
+ self.helpbox.config(tabs=tabstops)
+ self.helpbox.insert('1.0', text+'\n'*20)
+ C = '1.0 + %d chars'
+ for (tag, params) in self.HELP_AUTOTAG:
+ pattern = '(?s)(<%s>)(.*?)(</%s>)' % (tag, tag)
+ for m in re.finditer(pattern, text):
+ self.helpbox.tag_add('elide',
+ C % m.start(1), C % m.end(1))
+ self.helpbox.tag_add('tag-%s' % tag,
+ C % m.start(2), C % m.end(2))
+ self.helpbox.tag_add('elide',
+ C % m.start(3), C % m.end(3))
+ else:
+ self.helptabs[name].config(**self._HELPTAB_BG_PARAMS)
+ self.helpbox['state'] = 'disabled'
+
+ def _history_prev(self, *e):
+ self._view_history(self._history_index-1)
+ return 'break'
+
+ def _history_next(self, *e):
+ self._view_history(self._history_index+1)
+ return 'break'
+
+ def _view_history(self, index):
+ # Bounds & sanity checking:
+ index = max(0, min(len(self._history)-1, index))
+ if not self._history: return
+ # Already viewing the requested history item?
+ if index == self._history_index:
+ return
+ # Show the requested grammar. It will get added to _history
+ # only if they edit it (causing self.update() to get run.)
+ self.grammarbox['state'] = 'normal'
+ self.grammarbox.delete('1.0', 'end')
+ self.grammarbox.insert('end', self._history[index][0])
+ self.grammarbox.mark_set('insert', '1.0')
+ self._history_index = index
+ self._syntax_highlight_grammar(self._history[index][0])
+ # Record the normalized grammar & regenerate the chunker.
+ self.normalized_grammar = self.normalize_grammar(
+ self._history[index][0])
+ if self.normalized_grammar:
+ rules = [RegexpChunkRule.parse(line)
+ for line in self.normalized_grammar.split('\n')]
+ else:
+ rules = []
+ self.chunker = RegexpChunkParser(rules)
+ # Show the score.
+ self._eval_plot()
+ # Update the devset box
+ self._highlight_devset()
+ if self._showing_trace: self.show_trace()
+ # Update the grammar label
+ if self._history_index < len(self._history)-1:
+ self.grammarlabel['text'] = 'Grammar %s/%s:' % (
+ self._history_index+1, len(self._history))
+ else:
+ self.grammarlabel['text'] = 'Grammar:'
+
+ def _devset_next(self, *e):
+ self._devset_scroll('scroll', 1, 'page')
+ return 'break'
+
+ def _devset_prev(self, *e):
+ self._devset_scroll('scroll', -1, 'page')
+ return 'break'
+
+ def destroy(self, *e):
+ if self.top is None: return
+ self.top.destroy()
+ self.top = None
+
+ def _devset_scroll(self, command, *args):
+ N = 1 # size of a page -- one sentence.
+ showing_trace = self._showing_trace
+ if command == 'scroll' and args[1].startswith('unit'):
+ self.show_devset(self.devset_index+int(args[0]))
+ elif command == 'scroll' and args[1].startswith('page'):
+ self.show_devset(self.devset_index+N*int(args[0]))
+ elif command == 'moveto':
+ self.show_devset(int(float(args[0])*self._devset_size.get()))
+ else:
+ assert 0, 'bad scroll command %s %s' % (command, args)
+ if showing_trace:
+ self.show_trace()
+
+ def show_devset(self, index=None):
+ if index is None: index = self.devset_index
+
+ # Bounds checking
+ index = min(max(0, index), self._devset_size.get()-1)
+
+ if index == self.devset_index and not self._showing_trace: return
+ self.devset_index = index
+
+ self._showing_trace = False
+ self.trace_button['state'] = 'normal'
+ self.devset_button['state'] = 'disabled'
+
+ # Clear the text box.
+ self.devsetbox['state'] = 'normal'
+ self.devsetbox['wrap'] = 'word'
+ self.devsetbox.delete('1.0', 'end')
+ self.devsetlabel['text']='Development Set (%d/%d)' % (
+ (self.devset_index+1, self._devset_size.get()))
+
+ # Add the sentences
+ sample = self.devset[self.devset_index:self.devset_index+1]
+ self.charnum = {}
+ self.linenum = {0:1}
+ for sentnum, sent in enumerate(sample):
+ linestr = ''
+ for wordnum, (word, pos) in enumerate(sent.leaves()):
+ self.charnum[sentnum, wordnum] = len(linestr)
+ linestr += '%s/%s ' % (word, pos)
+ self.charnum[sentnum, wordnum+1] = len(linestr)
+ self.devsetbox.insert('end', linestr[:-1]+'\n\n')
+
+ # Highlight chunks in the dev set
+ if self.chunker is not None:
+ self._highlight_devset()
+ self.devsetbox['state'] = 'disabled'
+
+ # Update the scrollbar
+ first = float(self.devset_index)/self._devset_size.get()
+ last = float(self.devset_index+2)/self._devset_size.get()
+ self.devset_scroll.set(first, last)
+
+ def _chunks(self, tree):
+ chunks = set()
+ wordnum = 0
+ for child in tree:
+ if isinstance(child, Tree):
+ if child.label() == self._chunk_label:
+ chunks.add( (wordnum, wordnum+len(child)) )
+ wordnum += len(child)
+ else:
+ wordnum += 1
+ return chunks
+
+ def _syntax_highlight_grammar(self, grammar):
+ if self.top is None: return
+ self.grammarbox.tag_remove('comment', '1.0', 'end')
+ self.grammarbox.tag_remove('angle', '1.0', 'end')
+ self.grammarbox.tag_remove('brace', '1.0', 'end')
+ self.grammarbox.tag_add('hangindent', '1.0', 'end')
+ for lineno, line in enumerate(grammar.split('\n')):
+ if not line.strip(): continue
+ m = re.match(r'(\\.|[^#])*(#.*)?', line)
+ comment_start = None
+ if m.group(2):
+ comment_start = m.start(2)
+ s = '%d.%d' % (lineno+1, m.start(2))
+ e = '%d.%d' % (lineno+1, m.end(2))
+ self.grammarbox.tag_add('comment', s, e)
+ for m in re.finditer('[<>{}]', line):
+ if comment_start is not None and m.start() >= comment_start:
+ break
+ s = '%d.%d' % (lineno+1, m.start())
+ e = '%d.%d' % (lineno+1, m.end())
+ if m.group() in '<>':
+ self.grammarbox.tag_add('angle', s, e)
+ else:
+ self.grammarbox.tag_add('brace', s, e)
+
+
+ def _grammarcheck(self, grammar):
+ if self.top is None: return
+ self.grammarbox.tag_remove('error', '1.0', 'end')
+ self._grammarcheck_errs = []
+ for lineno, line in enumerate(grammar.split('\n')):
+ line = re.sub(r'((\\.|[^#])*)(#.*)?', r'\1', line)
+ line = line.strip()
+ if line:
+ try:
+ RegexpChunkRule.parse(line)
+ except ValueError as e:
+ self.grammarbox.tag_add('error', '%s.0' % (lineno+1),
+ '%s.0 lineend' % (lineno+1))
+ self.status['text'] = ''
+
+ def update(self, *event):
+ # Record when update was called (for grammarcheck)
+ if event:
+ self._last_keypress = time.time()
+
+ # Read the grammar from the Text box.
+ self.grammar = grammar = self.grammarbox.get('1.0', 'end')
+
+ # If the grammar hasn't changed, do nothing:
+ normalized_grammar = self.normalize_grammar(grammar)
+ if normalized_grammar == self.normalized_grammar:
+ return
+ else:
+ self.normalized_grammar = normalized_grammar
+
+ # If the grammar has changed, and we're looking at history,
+ # then stop looking at history.
+ if self._history_index < len(self._history)-1:
+ self.grammarlabel['text'] = 'Grammar:'
+
+ self._syntax_highlight_grammar(grammar)
+
+ # The grammar has changed; try parsing it. If it doesn't
+ # parse, do nothing. (flag error location?)
+ try:
+ # Note: the normalized grammar has no blank lines.
+ if normalized_grammar:
+ rules = [RegexpChunkRule.parse(line)
+ for line in normalized_grammar.split('\n')]
+ else:
+ rules = []
+ except ValueError as e:
+ # Use the un-normalized grammar for error highlighting.
+ self._grammarcheck(grammar)
+ self.chunker = None
+ return
+
+ self.chunker = RegexpChunkParser(rules)
+ self.grammarbox.tag_remove('error', '1.0', 'end')
+ self.grammar_changed = time.time()
+ # Display the results
+ if self._showing_trace:
+ self.show_trace()
+ else:
+ self._highlight_devset()
+ # Start the eval demon
+ if not self._eval_demon_running:
+ self._eval_demon()
+
+ def _highlight_devset(self, sample=None):
+ if sample is None:
+ sample = self.devset[self.devset_index:self.devset_index+1]
+
+ self.devsetbox.tag_remove('true-pos', '1.0', 'end')
+ self.devsetbox.tag_remove('false-neg', '1.0', 'end')
+ self.devsetbox.tag_remove('false-pos', '1.0', 'end')
+
+ # Run the grammar on the test cases.
+ for sentnum, gold_tree in enumerate(sample):
+ # Run the chunk parser
+ test_tree = self._chunkparse(gold_tree.leaves())
+ # Extract gold & test chunks
+ gold_chunks = self._chunks(gold_tree)
+ test_chunks = self._chunks(test_tree)
+ # Compare them.
+ for chunk in gold_chunks.intersection(test_chunks):
+ self._color_chunk(sentnum, chunk, 'true-pos')
+ for chunk in gold_chunks - test_chunks:
+ self._color_chunk(sentnum, chunk, 'false-neg')
+ for chunk in test_chunks - gold_chunks:
+ self._color_chunk(sentnum, chunk, 'false-pos')
+
+ def _chunkparse(self, words):
+ try:
+ return self.chunker.parse(words)
+ except (ValueError, IndexError) as e:
+ # There's an error somewhere in the grammar, but we're not sure
+ # exactly where, so just mark the whole grammar as bad.
+ # E.g., this is caused by: "({<NN>})"
+ self.grammarbox.tag_add('error', '1.0', 'end')
+ # Treat it as tagging nothing:
+ return words
+
+ def _color_chunk(self, sentnum, chunk, tag):
+ start, end = chunk
+ self.devsetbox.tag_add(tag,
+ '%s.%s' % (self.linenum[sentnum], self.charnum[sentnum, start]),
+ '%s.%s' % (self.linenum[sentnum], self.charnum[sentnum, end]-1))
+
+ def reset(self):
+ # Clear various variables
+ self.chunker = None
+ self.grammar = None
+ self.normalized_grammar = None
+ self.grammar_changed = 0
+ self._history = []
+ self._history_index = 0
+ # Update the on-screen display.
+ self.grammarbox.delete('1.0', 'end')
+ self.show_devset(0)
+ self.update()
+ #self._eval_plot()
+
+ SAVE_GRAMMAR_TEMPLATE = (
+ '# Regexp Chunk Parsing Grammar\n'
+ '# Saved %(date)s\n'
+ '#\n'
+ '# Development set: %(devset)s\n'
+ '# Precision: %(precision)s\n'
+ '# Recall: %(recall)s\n'
+ '# F-score: %(fscore)s\n\n'
+ '%(grammar)s\n')
+
+ def save_grammar(self, filename=None):
+ if not filename:
+ ftypes = [('Chunk Gramamr', '.chunk'),
+ ('All files', '*')]
+ filename = tkinter.filedialog.asksaveasfilename(filetypes=ftypes,
+ defaultextension='.chunk')
+ if not filename: return
+ if (self._history and self.normalized_grammar ==
+ self.normalize_grammar(self._history[-1][0])):
+ precision, recall, fscore = ['%.2f%%' % (100*v) for v in
+ self._history[-1][1:]]
+ elif self.chunker is None:
+ precision = recall = fscore = 'Grammar not well formed'
+ else:
+ precision = recall = fscore = 'Not finished evaluation yet'
+
+ with open(filename, 'w') as outfile:
+ outfile.write(self.SAVE_GRAMMAR_TEMPLATE % dict(
+ date=time.ctime(), devset=self.devset_name,
+ precision=precision, recall=recall, fscore=fscore,
+ grammar=self.grammar.strip()))
+
+ def load_grammar(self, filename=None):
+ if not filename:
+ ftypes = [('Chunk Gramamr', '.chunk'),
+ ('All files', '*')]
+ filename = tkinter.filedialog.askopenfilename(filetypes=ftypes,
+ defaultextension='.chunk')
+ if not filename: return
+ self.grammarbox.delete('1.0', 'end')
+ self.update()
+ with open(filename, 'r') as infile:
+ grammar = infile.read()
+ grammar = re.sub('^\# Regexp Chunk Parsing Grammar[\s\S]*'
+ 'F-score:.*\n', '', grammar).lstrip()
+ self.grammarbox.insert('1.0', grammar)
+ self.update()
+
+ def save_history(self, filename=None):
+ if not filename:
+ ftypes = [('Chunk Gramamr History', '.txt'),
+ ('All files', '*')]
+ filename = tkinter.filedialog.asksaveasfilename(filetypes=ftypes,
+ defaultextension='.txt')
+ if not filename: return
+
+ with open(filename, 'w') as outfile:
+ outfile.write('# Regexp Chunk Parsing Grammar History\n')
+ outfile.write('# Saved %s\n' % time.ctime())
+ outfile.write('# Development set: %s\n' % self.devset_name)
+ for i, (g, p, r, f) in enumerate(self._history):
+ hdr = ('Grammar %d/%d (precision=%.2f%%, recall=%.2f%%, '
+ 'fscore=%.2f%%)' % (i+1, len(self._history),
+ p*100, r*100, f*100))
+ outfile.write('\n%s\n' % hdr)
+ outfile.write(''.join(' %s\n' % line for line in g.strip().split()))
+
+ if not (self._history and self.normalized_grammar ==
+ self.normalize_grammar(self._history[-1][0])):
+ if self.chunker is None:
+ outfile.write('\nCurrent Grammar (not well-formed)\n')
+ else:
+ outfile.write('\nCurrent Grammar (not evaluated)\n')
+ outfile.write(''.join(' %s\n' % line for line
+ in self.grammar.strip().split()))
+
+ def about(self, *e):
+ ABOUT = ("NLTK RegExp Chunk Parser Application\n"+
+ "Written by Edward Loper")
+ TITLE = 'About: Regular Expression Chunk Parser Application'
+ try:
+ from tkinter.messagebox import Message
+ Message(message=ABOUT, title=TITLE).show()
+ except:
+ ShowText(self.top, TITLE, ABOUT)
+
+ def set_devset_size(self, size=None):
+ if size is not None: self._devset_size.set(size)
+ self._devset_size.set(min(len(self.devset), self._devset_size.get()))
+ self.show_devset(1)
+ self.show_devset(0)
+ # what about history? Evaluated at diff dev set sizes!
+
+ def resize(self, size=None):
+ if size is not None: self._size.set(size)
+ size = self._size.get()
+ self._font.configure(size=-(abs(size)))
+ self._smallfont.configure(size=min(-10, -(abs(size))*14/20))
+
+ def mainloop(self, *args, **kwargs):
+ """
+ Enter the Tkinter mainloop. This function must be called if
+ this demo is created from a non-interactive program (e.g.
+ from a secript); otherwise, the demo will close as soon as
+ the script completes.
+ """
+ if in_idle(): return
+ self.top.mainloop(*args, **kwargs)
+
+def app():
+ RegexpChunkApp().mainloop()
+
+if __name__ == '__main__':
+ app()
+
+__all__ = ['app']
+
diff --git a/nltk/app/collocations_app.py b/nltk/app/collocations_app.py
new file mode 100644
index 0000000..adf9444
--- /dev/null
+++ b/nltk/app/collocations_app.py
@@ -0,0 +1,348 @@
+# Natural Language Toolkit: Collocations Application
+# Much of the GUI code is imported from concordance.py; We intend to merge these tools together
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Sumukh Ghodke <sghodke at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+
+import nltk.compat
+import threading
+import tkinter.font
+if nltk.compat.PY3:
+ import queue as q
+else:
+ import Queue as q
+from tkinter import (Button, END, Frame, IntVar, LEFT, Label, Menu,
+ OptionMenu, SUNKEN, Scrollbar, StringVar,
+ Text, Tk)
+
+from nltk.corpus import (cess_cat, brown, nps_chat, treebank, sinica_treebank, alpino,
+ indian, floresta, mac_morpho, machado, cess_esp)
+from nltk.util import in_idle
+from nltk.probability import FreqDist
+
+
+CORPUS_LOADED_EVENT = '<<CL_EVENT>>'
+ERROR_LOADING_CORPUS_EVENT = '<<ELC_EVENT>>'
+POLL_INTERVAL = 100
+
+_DEFAULT = 'English: Brown Corpus (Humor)'
+_CORPORA = {
+ 'Catalan: CESS-CAT Corpus':
+ lambda: cess_cat.words(),
+ 'English: Brown Corpus':
+ lambda: brown.words(),
+ 'English: Brown Corpus (Press)':
+ lambda: brown.words(categories=['news', 'editorial', 'reviews']),
+ 'English: Brown Corpus (Religion)':
+ lambda: brown.words(categories='religion'),
+ 'English: Brown Corpus (Learned)':
+ lambda: brown.words(categories='learned'),
+ 'English: Brown Corpus (Science Fiction)':
+ lambda: brown.words(categories='science_fiction'),
+ 'English: Brown Corpus (Romance)':
+ lambda: brown.words(categories='romance'),
+ 'English: Brown Corpus (Humor)':
+ lambda: brown.words(categories='humor'),
+ 'English: NPS Chat Corpus':
+ lambda: nps_chat.words(),
+ 'English: Wall Street Journal Corpus':
+ lambda: treebank.words(),
+ 'Chinese: Sinica Corpus':
+ lambda: sinica_treebank.words(),
+ 'Dutch: Alpino Corpus':
+ lambda: alpino.words(),
+ 'Hindi: Indian Languages Corpus':
+ lambda: indian.words(files='hindi.pos'),
+ 'Portuguese: Floresta Corpus (Portugal)':
+ lambda: floresta.words(),
+ 'Portuguese: MAC-MORPHO Corpus (Brazil)':
+ lambda: mac_morpho.words(),
+ 'Portuguese: Machado Corpus (Brazil)':
+ lambda: machado.words(),
+ 'Spanish: CESS-ESP Corpus':
+ lambda: cess_esp.words()
+ }
+
+class CollocationsView:
+ _BACKGROUND_COLOUR='#FFF' #white
+
+ def __init__(self):
+ self.queue = q.Queue()
+ self.model = CollocationsModel(self.queue)
+ self.top = Tk()
+ self._init_top(self.top)
+ self._init_menubar()
+ self._init_widgets(self.top)
+ self.load_corpus(self.model.DEFAULT_CORPUS)
+ self.after = self.top.after(POLL_INTERVAL, self._poll)
+
+ def _init_top(self, top):
+ top.geometry('550x650+50+50')
+ top.title('NLTK Collocations List')
+ top.bind('<Control-q>', self.destroy)
+ top.protocol('WM_DELETE_WINDOW', self.destroy)
+ top.minsize(550,650)
+
+ def _init_widgets(self, parent):
+ self.main_frame = Frame(parent, dict(background=self._BACKGROUND_COLOUR, padx=1, pady=1, border=1))
+ self._init_corpus_select(self.main_frame)
+ self._init_results_box(self.main_frame)
+ self._init_paging(self.main_frame)
+ self._init_status(self.main_frame)
+ self.main_frame.pack(fill='both', expand=True)
+
+ def _init_corpus_select(self, parent):
+ innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+ self.var = StringVar(innerframe)
+ self.var.set(self.model.DEFAULT_CORPUS)
+ Label(innerframe, justify=LEFT, text=' Corpus: ', background=self._BACKGROUND_COLOUR, padx = 2, pady = 1, border = 0).pack(side='left')
+
+ other_corpora = list(self.model.CORPORA.keys()).remove(self.model.DEFAULT_CORPUS)
+ om = OptionMenu(innerframe, self.var, self.model.DEFAULT_CORPUS, command=self.corpus_selected, *self.model.non_default_corpora())
+ om['borderwidth'] = 0
+ om['highlightthickness'] = 1
+ om.pack(side='left')
+ innerframe.pack(side='top', fill='x', anchor='n')
+
+ def _init_status(self, parent):
+ self.status = Label(parent, justify=LEFT, relief=SUNKEN, background=self._BACKGROUND_COLOUR, border=0, padx = 1, pady = 0)
+ self.status.pack(side='top', anchor='sw')
+
+ def _init_menubar(self):
+ self._result_size = IntVar(self.top)
+ menubar = Menu(self.top)
+
+ filemenu = Menu(menubar, tearoff=0, borderwidth=0)
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-q')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ editmenu = Menu(menubar, tearoff=0)
+ rescntmenu = Menu(editmenu, tearoff=0)
+ rescntmenu.add_radiobutton(label='20', variable=self._result_size,
+ underline=0, value=20, command=self.set_result_size)
+ rescntmenu.add_radiobutton(label='50', variable=self._result_size,
+ underline=0, value=50, command=self.set_result_size)
+ rescntmenu.add_radiobutton(label='100', variable=self._result_size,
+ underline=0, value=100, command=self.set_result_size)
+ rescntmenu.invoke(1)
+ editmenu.add_cascade(label='Result Count', underline=0, menu=rescntmenu)
+
+ menubar.add_cascade(label='Edit', underline=0, menu=editmenu)
+ self.top.config(menu=menubar)
+
+ def set_result_size(self, **kwargs):
+ self.model.result_count = self._result_size.get()
+
+ def _init_results_box(self, parent):
+ innerframe = Frame(parent)
+ i1 = Frame(innerframe)
+ i2 = Frame(innerframe)
+ vscrollbar = Scrollbar(i1, borderwidth=1)
+ hscrollbar = Scrollbar(i2, borderwidth=1, orient='horiz')
+ self.results_box = Text(i1,
+ font=tkinter.font.Font(family='courier', size='16'),
+ state='disabled', borderwidth=1,
+ yscrollcommand=vscrollbar.set,
+ xscrollcommand=hscrollbar.set, wrap='none', width='40', height = '20', exportselection=1)
+ self.results_box.pack(side='left', fill='both', expand=True)
+ vscrollbar.pack(side='left', fill='y', anchor='e')
+ vscrollbar.config(command=self.results_box.yview)
+ hscrollbar.pack(side='left', fill='x', expand=True, anchor='w')
+ hscrollbar.config(command=self.results_box.xview)
+ #there is no other way of avoiding the overlap of scrollbars while using pack layout manager!!!
+ Label(i2, text=' ', background=self._BACKGROUND_COLOUR).pack(side='left', anchor='e')
+ i1.pack(side='top', fill='both', expand=True, anchor='n')
+ i2.pack(side='bottom', fill='x', anchor='s')
+ innerframe.pack(side='top', fill='both', expand=True)
+
+ def _init_paging(self, parent):
+ innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+ self.prev = prev = Button(innerframe, text='Previous', command=self.previous, width='10', borderwidth=1, highlightthickness=1, state='disabled')
+ prev.pack(side='left', anchor='center')
+ self.next = next = Button(innerframe, text='Next', command=self.__next__, width='10', borderwidth=1, highlightthickness=1, state='disabled')
+ next.pack(side='right', anchor='center')
+ innerframe.pack(side='top', fill='y')
+ self.reset_current_page()
+
+ def reset_current_page(self):
+ self.current_page = -1
+
+ def _poll(self):
+ try:
+ event = self.queue.get(block=False)
+ except q.Empty:
+ pass
+ else:
+ if event == CORPUS_LOADED_EVENT:
+ self.handle_corpus_loaded(event)
+ elif event == ERROR_LOADING_CORPUS_EVENT:
+ self.handle_error_loading_corpus(event)
+ self.after = self.top.after(POLL_INTERVAL, self._poll)
+
+ def handle_error_loading_corpus(self, event):
+ self.status['text'] = 'Error in loading ' + self.var.get()
+ self.unfreeze_editable()
+ self.clear_results_box()
+ self.freeze_editable()
+ self.reset_current_page()
+
+ def handle_corpus_loaded(self, event):
+ self.status['text'] = self.var.get() + ' is loaded'
+ self.unfreeze_editable()
+ self.clear_results_box()
+ self.reset_current_page()
+ #self.next()
+ collocations = self.model.next(self.current_page + 1)
+ self.write_results(collocations)
+ self.current_page += 1
+
+ def corpus_selected(self, *args):
+ new_selection = self.var.get()
+ self.load_corpus(new_selection)
+
+ def previous(self):
+ self.freeze_editable()
+ collocations = self.model.prev(self.current_page - 1)
+ self.current_page= self.current_page - 1
+ self.clear_results_box()
+ self.write_results(collocations)
+ self.unfreeze_editable()
+
+ def __next__(self):
+ self.freeze_editable()
+ collocations = self.model.next(self.current_page + 1)
+ self.clear_results_box()
+ self.write_results(collocations)
+ self.current_page += 1
+ self.unfreeze_editable()
+
+ def load_corpus(self, selection):
+ if self.model.selected_corpus != selection:
+ self.status['text'] = 'Loading ' + selection + '...'
+ self.freeze_editable()
+ self.model.load_corpus(selection)
+
+ def freeze_editable(self):
+ self.prev['state'] = 'disabled'
+ self.next['state'] = 'disabled'
+
+ def clear_results_box(self):
+ self.results_box['state'] = 'normal'
+ self.results_box.delete("1.0", END)
+ self.results_box['state'] = 'disabled'
+
+ def fire_event(self, event):
+ #Firing an event so that rendering of widgets happen in the mainloop thread
+ self.top.event_generate(event, when='tail')
+
+ def destroy(self, *e):
+ if self.top is None: return
+ self.top.after_cancel(self.after)
+ self.top.destroy()
+ self.top = None
+
+ def mainloop(self, *args, **kwargs):
+ if in_idle(): return
+ self.top.mainloop(*args, **kwargs)
+
+ def unfreeze_editable(self):
+ self.set_paging_button_states()
+
+ def set_paging_button_states(self):
+ if self.current_page == -1 or self.current_page == 0:
+ self.prev['state'] = 'disabled'
+ else:
+ self.prev['state'] = 'normal'
+ if self.model.is_last_page(self.current_page):
+ self.next['state'] = 'disabled'
+ else:
+ self.next['state'] = 'normal'
+
+ def write_results(self, results):
+ self.results_box['state'] = 'normal'
+ row = 1
+ for each in results:
+ self.results_box.insert(str(row) + '.0', each[0] + " " + each[1] + "\n")
+ row += 1
+ self.results_box['state'] = 'disabled'
+
+class CollocationsModel:
+ def __init__(self, queue):
+ self.result_count = None
+ self.selected_corpus = None
+ self.collocations = None
+ self.CORPORA = _CORPORA
+ self.DEFAULT_CORPUS = _DEFAULT
+ self.queue = queue
+ self.reset_results()
+
+ def reset_results(self):
+ self.result_pages = []
+ self.results_returned = 0
+
+ def load_corpus(self, name):
+ self.selected_corpus = name
+ self.collocations = None
+ runner_thread = self.LoadCorpus(name, self)
+ runner_thread.start()
+ self.reset_results()
+
+ def non_default_corpora(self):
+ copy = []
+ copy.extend(list(self.CORPORA.keys()))
+ copy.remove(self.DEFAULT_CORPUS)
+ copy.sort()
+ return copy
+
+ def is_last_page(self, number):
+ if number < len(self.result_pages):
+ return False
+ return self.results_returned + (number - len(self.result_pages)) * self.result_count >= len(self.collocations)
+
+ def next(self, page):
+ if (len(self.result_pages) - 1) < page:
+ for i in range(page - (len(self.result_pages) - 1)):
+ self.result_pages.append(self.collocations[self.results_returned:self.results_returned+self.result_count])
+ self.results_returned += self.result_count
+ return self.result_pages[page]
+
+ def prev(self, page):
+ if page == -1:
+ return []
+ return self.result_pages[page]
+
+ class LoadCorpus(threading.Thread):
+ def __init__(self, name, model):
+ threading.Thread.__init__(self)
+ self.model, self.name = model, name
+
+ def run(self):
+ try:
+ words = self.model.CORPORA[self.name]()
+ from operator import itemgetter
+ text = [w for w in words if len(w) > 2]
+ fd = FreqDist(tuple(text[i:i+2]) for i in range(len(text)-1))
+ vocab = FreqDist(text)
+ scored = [((w1,w2), fd[(w1,w2)] ** 3 / float(vocab[w1] * vocab[w2])) for w1, w2 in fd]
+ scored.sort(key=itemgetter(1), reverse=True)
+ self.model.collocations = list(map(itemgetter(0), scored))
+ self.model.queue.put(CORPUS_LOADED_EVENT)
+ except Exception as e:
+ print(e)
+ self.model.queue.put(ERROR_LOADING_CORPUS_EVENT)
+
+#def collocations():
+# colloc_strings = [w1 + ' ' + w2 for w1, w2 in self._collocations[:num]]
+
+def app():
+ c = CollocationsView()
+ c.mainloop()
+
+if __name__ == '__main__':
+ app()
+
+__all__ = ['app']
diff --git a/nltk/app/concordance_app.py b/nltk/app/concordance_app.py
new file mode 100755
index 0000000..5e58a96
--- /dev/null
+++ b/nltk/app/concordance_app.py
@@ -0,0 +1,570 @@
+# Natural Language Toolkit: Concordance Application
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Sumukh Ghodke <sghodke at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+
+import nltk.compat
+import re
+import threading
+if nltk.compat.PY3:
+ import queue as q
+else:
+ import Queue as q
+import tkinter.font
+from tkinter import (Tk, Button, END, Entry, Frame, IntVar, LEFT,
+ Label, Menu, OptionMenu, SUNKEN, Scrollbar,
+ StringVar, Text)
+
+from nltk.corpus import (cess_cat, brown, nps_chat, treebank, sinica_treebank,
+ alpino, indian, floresta, mac_morpho, cess_esp)
+from nltk.util import in_idle
+from nltk.draw.util import ShowText
+
+WORD_OR_TAG = '[^/ ]+'
+BOUNDARY = r'\b'
+
+CORPUS_LOADED_EVENT = '<<CL_EVENT>>'
+SEARCH_TERMINATED_EVENT = '<<ST_EVENT>>'
+SEARCH_ERROR_EVENT = '<<SE_EVENT>>'
+ERROR_LOADING_CORPUS_EVENT = '<<ELC_EVENT>>'
+
+POLL_INTERVAL = 50
+
+# NB All corpora must be specified in a lambda expression so as not to be
+# loaded when the module is imported.
+
+_DEFAULT = 'English: Brown Corpus (Humor, simplified)'
+_CORPORA = {
+ 'Catalan: CESS-CAT Corpus (simplified)':
+ lambda: cess_cat.tagged_sents(tagset='universal'),
+ 'English: Brown Corpus':
+ lambda: brown.tagged_sents(),
+ 'English: Brown Corpus (simplified)':
+ lambda: brown.tagged_sents(tagset='universal'),
+ 'English: Brown Corpus (Press, simplified)':
+ lambda: brown.tagged_sents(categories=['news', 'editorial', 'reviews'], tagset='universal'),
+ 'English: Brown Corpus (Religion, simplified)':
+ lambda: brown.tagged_sents(categories='religion', tagset='universal'),
+ 'English: Brown Corpus (Learned, simplified)':
+ lambda: brown.tagged_sents(categories='learned', tagset='universal'),
+ 'English: Brown Corpus (Science Fiction, simplified)':
+ lambda: brown.tagged_sents(categories='science_fiction', tagset='universal'),
+ 'English: Brown Corpus (Romance, simplified)':
+ lambda: brown.tagged_sents(categories='romance', tagset='universal'),
+ 'English: Brown Corpus (Humor, simplified)':
+ lambda: brown.tagged_sents(categories='humor', tagset='universal'),
+ 'English: NPS Chat Corpus':
+ lambda: nps_chat.tagged_posts(),
+ 'English: NPS Chat Corpus (simplified)':
+ lambda: nps_chat.tagged_posts(tagset='universal'),
+ 'English: Wall Street Journal Corpus':
+ lambda: treebank.tagged_sents(),
+ 'English: Wall Street Journal Corpus (simplified)':
+ lambda: treebank.tagged_sents(tagset='universal'),
+ 'Chinese: Sinica Corpus':
+ lambda: sinica_treebank.tagged_sents(),
+ 'Chinese: Sinica Corpus (simplified)':
+ lambda: sinica_treebank.tagged_sents(tagset='universal'),
+ 'Dutch: Alpino Corpus':
+ lambda: alpino.tagged_sents(),
+ 'Dutch: Alpino Corpus (simplified)':
+ lambda: alpino.tagged_sents(tagset='universal'),
+ 'Hindi: Indian Languages Corpus':
+ lambda: indian.tagged_sents(files='hindi.pos'),
+ 'Hindi: Indian Languages Corpus (simplified)':
+ lambda: indian.tagged_sents(files='hindi.pos', tagset='universal'),
+ 'Portuguese: Floresta Corpus (Portugal)':
+ lambda: floresta.tagged_sents(),
+ 'Portuguese: Floresta Corpus (Portugal, simplified)':
+ lambda: floresta.tagged_sents(tagset='universal'),
+ 'Portuguese: MAC-MORPHO Corpus (Brazil)':
+ lambda: mac_morpho.tagged_sents(),
+ 'Portuguese: MAC-MORPHO Corpus (Brazil, simplified)':
+ lambda: mac_morpho.tagged_sents(tagset='universal'),
+ 'Spanish: CESS-ESP Corpus (simplified)':
+ lambda: cess_esp.tagged_sents(tagset='universal'),
+ }
+
+class ConcordanceSearchView(object):
+ _BACKGROUND_COLOUR='#FFF' #white
+
+ #Colour of highlighted results
+ _HIGHLIGHT_WORD_COLOUR='#F00' #red
+ _HIGHLIGHT_WORD_TAG='HL_WRD_TAG'
+
+ _HIGHLIGHT_LABEL_COLOUR='#C0C0C0' # dark grey
+ _HIGHLIGHT_LABEL_TAG='HL_LBL_TAG'
+
+
+ #Percentage of text left of the scrollbar position
+ _FRACTION_LEFT_TEXT=0.30
+
+ def __init__(self):
+ self.queue = q.Queue()
+ self.model = ConcordanceSearchModel(self.queue)
+ self.top = Tk()
+ self._init_top(self.top)
+ self._init_menubar()
+ self._init_widgets(self.top)
+ self.load_corpus(self.model.DEFAULT_CORPUS)
+ self.after = self.top.after(POLL_INTERVAL, self._poll)
+
+ def _init_top(self, top):
+ top.geometry('950x680+50+50')
+ top.title('NLTK Concordance Search')
+ top.bind('<Control-q>', self.destroy)
+ top.protocol('WM_DELETE_WINDOW', self.destroy)
+ top.minsize(950,680)
+
+ def _init_widgets(self, parent):
+ self.main_frame = Frame(parent, dict(background=self._BACKGROUND_COLOUR, padx=1, pady=1, border=1))
+ self._init_corpus_select(self.main_frame)
+ self._init_query_box(self.main_frame)
+ self._init_results_box(self.main_frame)
+ self._init_paging(self.main_frame)
+ self._init_status(self.main_frame)
+ self.main_frame.pack(fill='both', expand=True)
+
+ def _init_menubar(self):
+ self._result_size = IntVar(self.top)
+ self._cntx_bf_len = IntVar(self.top)
+ self._cntx_af_len = IntVar(self.top)
+ menubar = Menu(self.top)
+
+ filemenu = Menu(menubar, tearoff=0, borderwidth=0)
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-q')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ editmenu = Menu(menubar, tearoff=0)
+ rescntmenu = Menu(editmenu, tearoff=0)
+ rescntmenu.add_radiobutton(label='20', variable=self._result_size,
+ underline=0, value=20,
+ command=self.set_result_size)
+ rescntmenu.add_radiobutton(label='50', variable=self._result_size,
+ underline=0, value=50,
+ command=self.set_result_size)
+ rescntmenu.add_radiobutton(label='100', variable=self._result_size,
+ underline=0, value=100,
+ command=self.set_result_size)
+ rescntmenu.invoke(1)
+ editmenu.add_cascade(label='Result Count', underline=0, menu=rescntmenu)
+
+ cntxmenu = Menu(editmenu, tearoff=0)
+ cntxbfmenu = Menu(cntxmenu, tearoff=0)
+ cntxbfmenu.add_radiobutton(label='60 characters',
+ variable=self._cntx_bf_len,
+ underline=0, value=60,
+ command=self.set_cntx_bf_len)
+ cntxbfmenu.add_radiobutton(label='80 characters',
+ variable=self._cntx_bf_len,
+ underline=0, value=80,
+ command=self.set_cntx_bf_len)
+ cntxbfmenu.add_radiobutton(label='100 characters',
+ variable=self._cntx_bf_len,
+ underline=0, value=100,
+ command=self.set_cntx_bf_len)
+ cntxbfmenu.invoke(1)
+ cntxmenu.add_cascade(label='Before', underline=0, menu=cntxbfmenu)
+
+ cntxafmenu = Menu(cntxmenu, tearoff=0)
+ cntxafmenu.add_radiobutton(label='70 characters',
+ variable=self._cntx_af_len,
+ underline=0, value=70,
+ command=self.set_cntx_af_len)
+ cntxafmenu.add_radiobutton(label='90 characters',
+ variable=self._cntx_af_len,
+ underline=0, value=90,
+ command=self.set_cntx_af_len)
+ cntxafmenu.add_radiobutton(label='110 characters',
+ variable=self._cntx_af_len,
+ underline=0, value=110,
+ command=self.set_cntx_af_len)
+ cntxafmenu.invoke(1)
+ cntxmenu.add_cascade(label='After', underline=0, menu=cntxafmenu)
+
+ editmenu.add_cascade(label='Context', underline=0, menu=cntxmenu)
+
+ menubar.add_cascade(label='Edit', underline=0, menu=editmenu)
+
+ self.top.config(menu=menubar)
+
+ def set_result_size(self, **kwargs):
+ self.model.result_count = self._result_size.get()
+
+ def set_cntx_af_len(self, **kwargs):
+ self._char_after = self._cntx_af_len.get()
+
+ def set_cntx_bf_len(self, **kwargs):
+ self._char_before = self._cntx_bf_len.get()
+
+ def _init_corpus_select(self, parent):
+ innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+ self.var = StringVar(innerframe)
+ self.var.set(self.model.DEFAULT_CORPUS)
+ Label(innerframe, justify=LEFT, text=' Corpus: ',
+ background=self._BACKGROUND_COLOUR, padx = 2, pady = 1, border = 0).pack(side='left')
+
+ other_corpora = list(self.model.CORPORA.keys()).remove(self.model.DEFAULT_CORPUS)
+ om = OptionMenu(innerframe, self.var, self.model.DEFAULT_CORPUS, command=self.corpus_selected, *self.model.non_default_corpora())
+ om['borderwidth'] = 0
+ om['highlightthickness'] = 1
+ om.pack(side='left')
+ innerframe.pack(side='top', fill='x', anchor='n')
+
+ def _init_status(self, parent):
+ self.status = Label(parent, justify=LEFT, relief=SUNKEN, background=self._BACKGROUND_COLOUR, border=0, padx = 1, pady = 0)
+ self.status.pack(side='top', anchor='sw')
+
+ def _init_query_box(self, parent):
+ innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+ another = Frame(innerframe, background=self._BACKGROUND_COLOUR)
+ self.query_box = Entry(another, width=60)
+ self.query_box.pack(side='left', fill='x', pady=25, anchor='center')
+ self.search_button = Button(another, text='Search', command=self.search, borderwidth=1, highlightthickness=1)
+ self.search_button.pack(side='left', fill='x', pady=25, anchor='center')
+ self.query_box.bind('<KeyPress-Return>', self.search_enter_keypress_handler)
+ another.pack()
+ innerframe.pack(side='top', fill='x', anchor='n')
+
+ def search_enter_keypress_handler(self, *event):
+ self.search()
+
+ def _init_results_box(self, parent):
+ innerframe = Frame(parent)
+ i1 = Frame(innerframe)
+ i2 = Frame(innerframe)
+ vscrollbar = Scrollbar(i1, borderwidth=1)
+ hscrollbar = Scrollbar(i2, borderwidth=1, orient='horiz')
+ self.results_box = Text(i1,
+ font=tkinter.font.Font(family='courier', size='16'),
+ state='disabled', borderwidth=1,
+ yscrollcommand=vscrollbar.set,
+ xscrollcommand=hscrollbar.set, wrap='none', width='40', height = '20', exportselection=1)
+ self.results_box.pack(side='left', fill='both', expand=True)
+ self.results_box.tag_config(self._HIGHLIGHT_WORD_TAG, foreground=self._HIGHLIGHT_WORD_COLOUR)
+ self.results_box.tag_config(self._HIGHLIGHT_LABEL_TAG, foreground=self._HIGHLIGHT_LABEL_COLOUR)
+ vscrollbar.pack(side='left', fill='y', anchor='e')
+ vscrollbar.config(command=self.results_box.yview)
+ hscrollbar.pack(side='left', fill='x', expand=True, anchor='w')
+ hscrollbar.config(command=self.results_box.xview)
+ #there is no other way of avoiding the overlap of scrollbars while using pack layout manager!!!
+ Label(i2, text=' ', background=self._BACKGROUND_COLOUR).pack(side='left', anchor='e')
+ i1.pack(side='top', fill='both', expand=True, anchor='n')
+ i2.pack(side='bottom', fill='x', anchor='s')
+ innerframe.pack(side='top', fill='both', expand=True)
+
+ def _init_paging(self, parent):
+ innerframe = Frame(parent, background=self._BACKGROUND_COLOUR)
+ self.prev = prev = Button(innerframe, text='Previous', command=self.previous, width='10', borderwidth=1, highlightthickness=1, state='disabled')
+ prev.pack(side='left', anchor='center')
+ self.next = next = Button(innerframe, text='Next', command=self.__next__, width='10', borderwidth=1, highlightthickness=1, state='disabled')
+ next.pack(side='right', anchor='center')
+ innerframe.pack(side='top', fill='y')
+ self.current_page = 0
+
+ def previous(self):
+ self.clear_results_box()
+ self.freeze_editable()
+ self.model.prev(self.current_page - 1)
+
+ def __next__(self):
+ self.clear_results_box()
+ self.freeze_editable()
+ self.model.next(self.current_page + 1)
+
+ def about(self, *e):
+ ABOUT = ("NLTK Concordance Search Demo\n")
+ TITLE = 'About: NLTK Concordance Search Demo'
+ try:
+ from tkinter.messagebox import Message
+ Message(message=ABOUT, title=TITLE, parent=self.main_frame).show()
+ except:
+ ShowText(self.top, TITLE, ABOUT)
+
+ def _bind_event_handlers(self):
+ self.top.bind(CORPUS_LOADED_EVENT, self.handle_corpus_loaded)
+ self.top.bind(SEARCH_TERMINATED_EVENT, self.handle_search_terminated)
+ self.top.bind(SEARCH_ERROR_EVENT, self.handle_search_error)
+ self.top.bind(ERROR_LOADING_CORPUS_EVENT, self.handle_error_loading_corpus)
+
+ def _poll(self):
+ try:
+ event = self.queue.get(block=False)
+ except q.Empty:
+ pass
+ else:
+ if event == CORPUS_LOADED_EVENT:
+ self.handle_corpus_loaded(event)
+ elif event == SEARCH_TERMINATED_EVENT:
+ self.handle_search_terminated(event)
+ elif event == SEARCH_ERROR_EVENT:
+ self.handle_search_error(event)
+ elif event == ERROR_LOADING_CORPUS_EVENT:
+ self.handle_error_loading_corpus(event)
+ self.after = self.top.after(POLL_INTERVAL, self._poll)
+
+ def handle_error_loading_corpus(self, event):
+ self.status['text'] = 'Error in loading ' + self.var.get()
+ self.unfreeze_editable()
+ self.clear_all()
+ self.freeze_editable()
+
+ def handle_corpus_loaded(self, event):
+ self.status['text'] = self.var.get() + ' is loaded'
+ self.unfreeze_editable()
+ self.clear_all()
+ self.query_box.focus_set()
+
+ def handle_search_terminated(self, event):
+ #todo: refactor the model such that it is less state sensitive
+ results = self.model.get_results()
+ self.write_results(results)
+ self.status['text'] = ''
+ if len(results) == 0:
+ self.status['text'] = 'No results found for ' + self.model.query
+ else:
+ self.current_page = self.model.last_requested_page
+ self.unfreeze_editable()
+ self.results_box.xview_moveto(self._FRACTION_LEFT_TEXT)
+
+ def handle_search_error(self, event):
+ self.status['text'] = 'Error in query ' + self.model.query
+ self.unfreeze_editable()
+
+ def corpus_selected(self, *args):
+ new_selection = self.var.get()
+ self.load_corpus(new_selection)
+
+ def load_corpus(self, selection):
+ if self.model.selected_corpus != selection:
+ self.status['text'] = 'Loading ' + selection + '...'
+ self.freeze_editable()
+ self.model.load_corpus(selection)
+
+ def search(self):
+ self.current_page = 0
+ self.clear_results_box()
+ self.model.reset_results()
+ query = self.query_box.get()
+ if (len(query.strip()) == 0): return
+ self.status['text'] = 'Searching for ' + query
+ self.freeze_editable()
+ self.model.search(query, self.current_page + 1, )
+
+
+ def write_results(self, results):
+ self.results_box['state'] = 'normal'
+ row = 1
+ for each in results:
+ sent, pos1, pos2 = each[0].strip(), each[1], each[2]
+ if len(sent) != 0:
+ if (pos1 < self._char_before):
+ sent, pos1, pos2 = self.pad(sent, pos1, pos2)
+ sentence = sent[pos1-self._char_before:pos1+self._char_after]
+ if not row == len(results):
+ sentence += '\n'
+ self.results_box.insert(str(row) + '.0', sentence)
+ word_markers, label_markers = self.words_and_labels(sent, pos1, pos2)
+ for marker in word_markers: self.results_box.tag_add(self._HIGHLIGHT_WORD_TAG, str(row) + '.' + str(marker[0]), str(row) + '.' + str(marker[1]))
+ for marker in label_markers: self.results_box.tag_add(self._HIGHLIGHT_LABEL_TAG, str(row) + '.' + str(marker[0]), str(row) + '.' + str(marker[1]))
+ row += 1
+ self.results_box['state'] = 'disabled'
+
+ def words_and_labels(self, sentence, pos1, pos2):
+ search_exp = sentence[pos1:pos2]
+ words, labels = [], []
+ labeled_words = search_exp.split(' ')
+ index = 0
+ for each in labeled_words:
+ if each == '':
+ index += 1
+ else:
+ word, label = each.split('/')
+ words.append((self._char_before + index, self._char_before + index + len(word)))
+ index += len(word) + 1
+ labels.append((self._char_before + index, self._char_before + index + len(label)))
+ index += len(label)
+ index += 1
+ return words, labels
+
+ def pad(self, sent, hstart, hend):
+ if hstart >= self._char_before:
+ return sent, hstart, hend
+ d = self._char_before - hstart
+ sent = ''.join([' '] * d) + sent
+ return sent, hstart + d, hend + d
+
+ def destroy(self, *e):
+ if self.top is None: return
+ self.top.after_cancel(self.after)
+ self.top.destroy()
+ self.top = None
+
+ def clear_all(self):
+ self.query_box.delete(0, END)
+ self.model.reset_query()
+ self.clear_results_box()
+
+ def clear_results_box(self):
+ self.results_box['state'] = 'normal'
+ self.results_box.delete("1.0", END)
+ self.results_box['state'] = 'disabled'
+
+ def freeze_editable(self):
+ self.query_box['state'] = 'disabled'
+ self.search_button['state'] = 'disabled'
+ self.prev['state'] = 'disabled'
+ self.next['state'] = 'disabled'
+
+ def unfreeze_editable(self):
+ self.query_box['state'] = 'normal'
+ self.search_button['state'] = 'normal'
+ self.set_paging_button_states()
+
+ def set_paging_button_states(self):
+ if self.current_page == 0 or self.current_page == 1:
+ self.prev['state'] = 'disabled'
+ else:
+ self.prev['state'] = 'normal'
+ if self.model.has_more_pages(self.current_page):
+ self.next['state'] = 'normal'
+ else:
+ self.next['state'] = 'disabled'
+
+ def fire_event(self, event):
+ #Firing an event so that rendering of widgets happen in the mainloop thread
+ self.top.event_generate(event, when='tail')
+
+ def mainloop(self, *args, **kwargs):
+ if in_idle(): return
+ self.top.mainloop(*args, **kwargs)
+
+class ConcordanceSearchModel(object):
+ def __init__(self, queue):
+ self.queue = queue
+ self.CORPORA = _CORPORA
+ self.DEFAULT_CORPUS = _DEFAULT
+ self.selected_corpus = None
+ self.reset_query()
+ self.reset_results()
+ self.result_count = None
+ self.last_sent_searched = 0
+
+ def non_default_corpora(self):
+ copy = []
+ copy.extend(list(self.CORPORA.keys()))
+ copy.remove(self.DEFAULT_CORPUS)
+ copy.sort()
+ return copy
+
+ def load_corpus(self, name):
+ self.selected_corpus = name
+ self.tagged_sents = []
+ runner_thread = self.LoadCorpus(name, self)
+ runner_thread.start()
+
+ def search(self, query, page):
+ self.query = query
+ self.last_requested_page = page
+ self.SearchCorpus(self, page, self.result_count).start()
+
+ def next(self, page):
+ self.last_requested_page = page
+ if len(self.results) < page:
+ self.search(self.query, page)
+ else:
+ self.queue.put(SEARCH_TERMINATED_EVENT)
+
+ def prev(self, page):
+ self.last_requested_page = page
+ self.queue.put(SEARCH_TERMINATED_EVENT)
+
+ def reset_results(self):
+ self.last_sent_searched = 0
+ self.results = []
+ self.last_page = None
+
+ def reset_query(self):
+ self.query = None
+
+ def set_results(self, page, resultset):
+ self.results.insert(page - 1, resultset)
+
+ def get_results(self):
+ return self.results[self.last_requested_page - 1]
+
+ def has_more_pages(self, page):
+ if self.results == [] or self.results[0] == []:
+ return False
+ if self.last_page is None:
+ return True
+ return page < self.last_page
+
+ class LoadCorpus(threading.Thread):
+ def __init__(self, name, model):
+ threading.Thread.__init__(self)
+ self.model, self.name = model, name
+
+ def run(self):
+ try:
+ ts = self.model.CORPORA[self.name]()
+ self.model.tagged_sents = [' '.join(w+'/'+t for (w,t) in sent) for sent in ts]
+ self.model.queue.put(CORPUS_LOADED_EVENT)
+ except Exception as e:
+ print(e)
+ self.model.queue.put(ERROR_LOADING_CORPUS_EVENT)
+
+ class SearchCorpus(threading.Thread):
+ def __init__(self, model, page, count):
+ self.model, self.count, self.page = model, count, page
+ threading.Thread.__init__(self)
+
+ def run(self):
+ q = self.processed_query()
+ sent_pos, i, sent_count = [], 0, 0
+ for sent in self.model.tagged_sents[self.model.last_sent_searched:]:
+ try:
+ m = re.search(q, sent)
+ except re.error:
+ self.model.reset_results()
+ self.model.queue.put(SEARCH_ERROR_EVENT)
+ return
+ if m:
+ sent_pos.append((sent, m.start(), m.end()))
+ i += 1
+ if i > self.count:
+ self.model.last_sent_searched += sent_count - 1
+ break
+ sent_count += 1
+ if (self.count >= len(sent_pos)):
+ self.model.last_sent_searched += sent_count - 1
+ self.model.last_page = self.page
+ self.model.set_results(self.page, sent_pos)
+ else:
+ self.model.set_results(self.page, sent_pos[:-1])
+ self.model.queue.put(SEARCH_TERMINATED_EVENT)
+
+ def processed_query(self):
+ new = []
+ for term in self.model.query.split():
+ term = re.sub(r'\.', r'[^/ ]', term)
+ if re.match('[A-Z]+$', term):
+ new.append(BOUNDARY + WORD_OR_TAG + '/' + term + BOUNDARY)
+ elif '/' in term:
+ new.append(BOUNDARY + term + BOUNDARY)
+ else:
+ new.append(BOUNDARY + term + '/' + WORD_OR_TAG + BOUNDARY)
+ return ' '.join(new)
+
+def app():
+ d = ConcordanceSearchView()
+ d.mainloop()
+
+if __name__ == '__main__':
+ app()
+
+__all__ = ['app']
diff --git a/nltk/app/nemo_app.py b/nltk/app/nemo_app.py
new file mode 100755
index 0000000..0767539
--- /dev/null
+++ b/nltk/app/nemo_app.py
@@ -0,0 +1,155 @@
+# Finding (and Replacing) Nemo, Version 1.1, Aristide Grange 2006/06/06
+# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/496783
+
+"""
+Finding (and Replacing) Nemo
+
+Instant Regular Expressions
+Created by Aristide Grange
+"""
+import nltk.compat
+import tkinter as tk
+import re
+import itertools
+
+windowTitle = "Finding (and Replacing) Nemo"
+initialFind = r"n(.*?)e(.*?)m(.*?)o"
+initialRepl = r"M\1A\2K\3I"
+initialText = """\
+Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.
+Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.
+Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+"""
+images = {
+ "FIND":"R0lGODlhMAAiAPcAMf/////37//35//n1v97Off///f/9/f37/fexvfOvfeEQvd7QvdrQvdrKfdaKfdSMfdSIe/v9+/v7+/v5+/n3u/e1u/Wxu/Gre+1lO+tnO+thO+Ua+97Y+97Oe97Me9rOe9rMe9jOe9jMe9jIe9aMefe5+fe3ufezuece+eEWudzQudaIedSIedKMedKIedCKedCId7e1t7Wzt7Oxt7Gvd69vd69rd61pd6ljN6UjN6Ue96EY95zY95rUt5rQt5jMd5SId5KIdbn59be3tbGztbGvda1rdaEa9Z7a9Z7WtZzQtZzOdZzMdZjMdZaQtZSOdZSMdZKMdZCKdZCGNY5Ic7W1s7Oxs7Gtc69xs69tc69rc6tpc6llM6clM6cjM6Ue86EY85zWs5rSs5SKc5KKc5KGMa1tcatrcalvcalnMaUpcZ7c8ZzMcZrUsZrOcZrMcZaQsZSOcZSMcZK [...]
+ "find":"R0lGODlhMAAiAPQAMf////f39+/v7+fn597e3tbW1s7OzsbGxr29vbW1ta2traWlpZycnJSUlIyMjISEhHt7e3Nzc2tra2NjY1paWlJSUkpKSkJCQjk5OSkpKRgYGAAAAAAAAAAAAAAAAAAAACH+AS4ALAAAAAAwACIAAAX/ICCOZGmeaKquY2AGLiuvMCAUBuHWc48Kh0iFInEYCb4kSQCxPBiMxkMigRQEgJiSFVBYHNGG0RiZOHjblWAiiY4fkDhEYoBp06dAWfyAQyKAgAwDaHgnB0RwgYASgQ0IhDuGJDAIFhMRVFSLEX8QCJJ4AQM5AgQHTZqqjBAOCQQEkWkCDRMUFQsICQ4Vm5maEwwHOAsPDTpKMAsUDlO4CssTcb+2DAp8YGCyNFoCEsZwFQ3QDRTTVBRS0g1QbgsCd5QAAwgIBwYFAwStzQ8UEdCKVchky0yVBw7YuXkAKt4IAg74vXHVagqF [...]
+ "REPL":"R0lGODlhMAAjAPcAMf/////3//+lOf+UKf+MEPf///f39/f35/fv7/ecQvecOfecKfeUIfeUGPeUEPeUCPeMAO/37+/v9+/v3u/n3u/n1u+9jO+9c++1hO+ta++tY++tWu+tUu+tSu+lUu+lQu+lMe+UMe+UKe+UGO+UEO+UAO+MCOfv5+fvxufn7+fn5+fnzue9lOe9c+e1jOe1e+e1c+e1a+etWuetUuelQuecOeeUUueUCN7e597e3t7e1t7ezt7evd7Wzt7Oxt7Ovd7Otd7Opd7OnN7Gtd7Gpd69lN61hN6ta96lStbextberdbW3tbWztbWxtbOvdbOrda1hNalUtaECM7W1s7Ozs7Oxs7Otc7Gxs7Gvc69tc69rc69pc61jM6lc8bWlMbOvcbGxsbGpca9tca9pca1nMaMAL3OhL3Gtb21vb21tb2tpb2tnL2tlLW9tbW9pbW9e7W1pbWtjLWcKa21 [...]
+ "repl":"R0lGODlhMAAjAPQAMf////f39+/v7+fn597e3tbW1s7OzsbGxr29vbW1ta2traWlpZycnJSUlIyMjISEhHt7e3Nzc2tra2NjY1paWlJSUkpKSkJCQjk5OTExMSkpKSEhIRgYGBAQEAgICAAAACH+AS4ALAAAAAAwACMAAAX/ICCOZGmeaKqubOu+gCDANBkIQ1EMQhAghFptYEAkEgjEwXBo7ISvweGgWCwUysPjwTgEoCafTySYIhYMxgLBjEQgCULvCw0QdAZdoVhUIJUFChISEAxYeQM1N1OMTAp+UwZ5eA4TEhFbDWYFdC4ECVMJjwl5BwsQa0umEhUVlhESDgqlBp0rAn5nVpBMDxeZDRQbHBgWFBSWDgtLBnFjKwRYCI9VqQsPs0YKEcMXFq0UEalFDWx4BAO2IwPjppAKDkrTWKYUGd7fEJJFEZpM00cOzCgh4EE8SaoWxKNixQooBRMyZMBwAYIR [...]
+}
+colors = ["#FF7B39","#80F121"]
+emphColors = ["#DAFC33","#F42548"]
+fieldParams = {
+ "height":3,
+ "width":70,
+ "font":("monaco",14),
+ "highlightthickness":0,
+ "borderwidth":0,
+ "background":"white",
+}
+textParams = {
+ "bg":"#F7E0D4",
+ "fg":"#2321F1",
+ "highlightthickness":0,
+ "width":1,
+ "height":10,
+ "font":("verdana",16),
+ "wrap":"word",
+}
+
+
+class Zone:
+ def __init__(self, image, initialField, initialText):
+ frm = tk.Frame(root)
+ frm.config(background="white")
+ self.image = tk.PhotoImage(format='gif',data=images[image.upper()])
+ self.imageDimmed = tk.PhotoImage(format='gif',data=images[image])
+ self.img = tk.Label(frm)
+ self.img.config(borderwidth=0)
+ self.img.pack(side = "left")
+ self.fld = tk.Text(frm, **fieldParams)
+ self.initScrollText(frm,self.fld,initialField)
+ frm = tk.Frame(root)
+ self.txt = tk.Text(frm, **textParams)
+ self.initScrollText(frm,self.txt,initialText)
+ for i in range(2):
+ self.txt.tag_config(colors[i], background = colors[i])
+ self.txt.tag_config("emph"+colors[i], foreground = emphColors[i])
+ def initScrollText(self,frm,txt,contents):
+ scl = tk.Scrollbar(frm)
+ scl.config(command = txt.yview)
+ scl.pack(side="right",fill="y")
+ txt.pack(side = "left", expand=True, fill="x")
+ txt.config(yscrollcommand = scl.set)
+ txt.insert("1.0",contents)
+ frm.pack(fill = "x")
+ tk.Frame(height=2, bd=1, relief="ridge").pack(fill="x")
+ def refresh(self):
+ self.colorCycle = itertools.cycle(colors)
+ try:
+ self.substitute()
+ self.img.config(image = self.image)
+ except re.error:
+ self.img.config(image = self.imageDimmed)
+
+
+class FindZone(Zone):
+ def addTags(self,m):
+ color = next(self.colorCycle)
+ self.txt.tag_add(color,"1.0+%sc"%m.start(),"1.0+%sc"%m.end())
+ try:
+ self.txt.tag_add("emph"+color,"1.0+%sc"%m.start("emph"),
+ "1.0+%sc"%m.end("emph"))
+ except:
+ pass
+ def substitute(self,*args):
+ for color in colors:
+ self.txt.tag_remove(color,"1.0","end")
+ self.txt.tag_remove("emph"+color,"1.0","end")
+ self.rex = re.compile("") # default value in case of misformed regexp
+ self.rex = re.compile(self.fld.get("1.0","end")[:-1],re.MULTILINE)
+ try:
+ re.compile("(?P<emph>%s)" % self.fld.get(tk.SEL_FIRST,
+ tk.SEL_LAST))
+ self.rexSel = re.compile("%s(?P<emph>%s)%s" % (
+ self.fld.get("1.0",tk.SEL_FIRST),
+ self.fld.get(tk.SEL_FIRST,tk.SEL_LAST),
+ self.fld.get(tk.SEL_LAST,"end")[:-1],
+ ),re.MULTILINE)
+ except:
+ self.rexSel = self.rex
+ self.rexSel.sub(self.addTags,self.txt.get("1.0","end"))
+
+
+class ReplaceZone(Zone):
+ def addTags(self,m):
+ s = sz.rex.sub(self.repl,m.group())
+ self.txt.delete("1.0+%sc"%(m.start()+self.diff),
+ "1.0+%sc"%(m.end()+self.diff))
+ self.txt.insert("1.0+%sc"%(m.start()+self.diff),s,
+ next(self.colorCycle))
+ self.diff += len(s) - (m.end() - m.start())
+ def substitute(self):
+ self.txt.delete("1.0","end")
+ self.txt.insert("1.0",sz.txt.get("1.0","end")[:-1])
+ self.diff = 0
+ self.repl = rex0.sub(r"\\g<\1>",self.fld.get("1.0","end")[:-1])
+ sz.rex.sub(self.addTags,sz.txt.get("1.0","end")[:-1])
+
+
+def launchRefresh(_):
+ sz.fld.after_idle(sz.refresh)
+ rz.fld.after_idle(rz.refresh)
+
+
+def app():
+ global root, sz, rz, rex0
+ root = tk.Tk()
+ root.resizable(height=False,width=True)
+ root.title(windowTitle)
+ root.minsize(width=250,height=0)
+ sz = FindZone("find",initialFind,initialText)
+ sz.fld.bind("<Button-1>",launchRefresh)
+ sz.fld.bind("<ButtonRelease-1>",launchRefresh)
+ sz.fld.bind("<B1-Motion>",launchRefresh)
+ sz.rexSel = re.compile("")
+ rz = ReplaceZone("repl",initialRepl,"")
+ rex0 = re.compile(r"(?<!\\)\\([0-9]+)")
+ root.bind_all("<Key>",launchRefresh)
+ launchRefresh(None)
+ root.mainloop()
+
+if __name__ == '__main__':
+ app()
+
+__all__ = ['app']
diff --git a/nltk/app/rdparser_app.py b/nltk/app/rdparser_app.py
new file mode 100644
index 0000000..d5f3526
--- /dev/null
+++ b/nltk/app/rdparser_app.py
@@ -0,0 +1,894 @@
+# Natural Language Toolkit: Recursive Descent Parser Application
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A graphical tool for exploring the recursive descent parser.
+
+The recursive descent parser maintains a tree, which records the
+structure of the portion of the text that has been parsed. It uses
+CFG productions to expand the fringe of the tree, and matches its
+leaves against the text. Initially, the tree contains the start
+symbol ("S"). It is shown in the main canvas, to the right of the
+list of available expansions.
+
+The parser builds up a tree structure for the text using three
+operations:
+
+ - "expand" uses a CFG production to add children to a node on the
+ fringe of the tree.
+ - "match" compares a leaf in the tree to a text token.
+ - "backtrack" returns the tree to its state before the most recent
+ expand or match operation.
+
+The parser maintains a list of tree locations called a "frontier" to
+remember which nodes have not yet been expanded and which leaves have
+not yet been matched against the text. The leftmost frontier node is
+shown in green, and the other frontier nodes are shown in blue. The
+parser always performs expand and match operations on the leftmost
+element of the frontier.
+
+You can control the parser's operation by using the "expand," "match,"
+and "backtrack" buttons; or you can use the "step" button to let the
+parser automatically decide which operation to apply. The parser uses
+the following rules to decide which operation to apply:
+
+ - If the leftmost frontier element is a token, try matching it.
+ - If the leftmost frontier element is a node, try expanding it with
+ the first untried expansion.
+ - Otherwise, backtrack.
+
+The "expand" button applies the untried expansion whose CFG production
+is listed earliest in the grammar. To manually choose which expansion
+to apply, click on a CFG production from the list of available
+expansions, on the left side of the main window.
+
+The "autostep" button will let the parser continue applying
+applications to the tree until it reaches a complete parse. You can
+cancel an autostep in progress at any time by clicking on the
+"autostep" button again.
+
+Keyboard Shortcuts::
+ [Space]\t Perform the next expand, match, or backtrack operation
+ [a]\t Step through operations until the next complete parse
+ [e]\t Perform an expand operation
+ [m]\t Perform a match operation
+ [b]\t Perform a backtrack operation
+ [Delete]\t Reset the parser
+ [g]\t Show/hide available expansions list
+ [h]\t Help
+ [Ctrl-p]\t Print
+ [q]\t Quit
+"""
+
+import nltk.compat
+import tkinter.font
+from tkinter import (Listbox, IntVar, Button,
+ Frame, Label, Menu, Scrollbar, Tk)
+
+from nltk.tree import Tree
+from nltk.util import in_idle
+from nltk.parse import SteppingRecursiveDescentParser
+from nltk.draw.util import TextWidget, ShowText, CanvasFrame, EntryDialog
+from nltk.draw import CFGEditor, TreeSegmentWidget, tree_to_treesegment
+
+class RecursiveDescentApp(object):
+ """
+ A graphical tool for exploring the recursive descent parser. The tool
+ displays the parser's tree and the remaining text, and allows the
+ user to control the parser's operation. In particular, the user
+ can expand subtrees on the frontier, match tokens on the frontier
+ against the text, and backtrack. A "step" button simply steps
+ through the parsing process, performing the operations that
+ ``RecursiveDescentParser`` would use.
+ """
+ def __init__(self, grammar, sent, trace=0):
+ self._sent = sent
+ self._parser = SteppingRecursiveDescentParser(grammar, trace)
+
+ # Set up the main window.
+ self._top = Tk()
+ self._top.title('Recursive Descent Parser Application')
+
+ # Set up key bindings.
+ self._init_bindings()
+
+ # Initialize the fonts.
+ self._init_fonts(self._top)
+
+ # Animations. animating_lock is a lock to prevent the demo
+ # from performing new operations while it's animating.
+ self._animation_frames = IntVar(self._top)
+ self._animation_frames.set(5)
+ self._animating_lock = 0
+ self._autostep = 0
+
+ # The user can hide the grammar.
+ self._show_grammar = IntVar(self._top)
+ self._show_grammar.set(1)
+
+ # Create the basic frames.
+ self._init_menubar(self._top)
+ self._init_buttons(self._top)
+ self._init_feedback(self._top)
+ self._init_grammar(self._top)
+ self._init_canvas(self._top)
+
+ # Initialize the parser.
+ self._parser.initialize(self._sent)
+
+ # Resize callback
+ self._canvas.bind('<Configure>', self._configure)
+
+ #########################################
+ ## Initialization Helpers
+ #########################################
+
+ def _init_fonts(self, root):
+ # See: <http://www.astro.washington.edu/owen/ROTKFolklore.html>
+ self._sysfont = tkinter.font.Font(font=Button()["font"])
+ root.option_add("*Font", self._sysfont)
+
+ # TWhat's our font size (default=same as sysfont)
+ self._size = IntVar(root)
+ self._size.set(self._sysfont.cget('size'))
+
+ self._boldfont = tkinter.font.Font(family='helvetica', weight='bold',
+ size=self._size.get())
+ self._font = tkinter.font.Font(family='helvetica',
+ size=self._size.get())
+ if self._size.get() < 0: big = self._size.get()-2
+ else: big = self._size.get()+2
+ self._bigfont = tkinter.font.Font(family='helvetica', weight='bold',
+ size=big)
+
+ def _init_grammar(self, parent):
+ # Grammar view.
+ self._prodframe = listframe = Frame(parent)
+ self._prodframe.pack(fill='both', side='left', padx=2)
+ self._prodlist_label = Label(self._prodframe, font=self._boldfont,
+ text='Available Expansions')
+ self._prodlist_label.pack()
+ self._prodlist = Listbox(self._prodframe, selectmode='single',
+ relief='groove', background='white',
+ foreground='#909090', font=self._font,
+ selectforeground='#004040',
+ selectbackground='#c0f0c0')
+
+ self._prodlist.pack(side='right', fill='both', expand=1)
+
+ self._productions = list(self._parser.grammar().productions())
+ for production in self._productions:
+ self._prodlist.insert('end', (' %s' % production))
+ self._prodlist.config(height=min(len(self._productions), 25))
+
+ # Add a scrollbar if there are more than 25 productions.
+ if len(self._productions) > 25:
+ listscroll = Scrollbar(self._prodframe,
+ orient='vertical')
+ self._prodlist.config(yscrollcommand = listscroll.set)
+ listscroll.config(command=self._prodlist.yview)
+ listscroll.pack(side='left', fill='y')
+
+ # If they select a production, apply it.
+ self._prodlist.bind('<<ListboxSelect>>', self._prodlist_select)
+
+ def _init_bindings(self):
+ # Key bindings are a good thing.
+ self._top.bind('<Control-q>', self.destroy)
+ self._top.bind('<Control-x>', self.destroy)
+ self._top.bind('<Escape>', self.destroy)
+ self._top.bind('e', self.expand)
+ #self._top.bind('<Alt-e>', self.expand)
+ #self._top.bind('<Control-e>', self.expand)
+ self._top.bind('m', self.match)
+ self._top.bind('<Alt-m>', self.match)
+ self._top.bind('<Control-m>', self.match)
+ self._top.bind('b', self.backtrack)
+ self._top.bind('<Alt-b>', self.backtrack)
+ self._top.bind('<Control-b>', self.backtrack)
+ self._top.bind('<Control-z>', self.backtrack)
+ self._top.bind('<BackSpace>', self.backtrack)
+ self._top.bind('a', self.autostep)
+ #self._top.bind('<Control-a>', self.autostep)
+ self._top.bind('<Control-space>', self.autostep)
+ self._top.bind('<Control-c>', self.cancel_autostep)
+ self._top.bind('<space>', self.step)
+ self._top.bind('<Delete>', self.reset)
+ self._top.bind('<Control-p>', self.postscript)
+ #self._top.bind('<h>', self.help)
+ #self._top.bind('<Alt-h>', self.help)
+ self._top.bind('<Control-h>', self.help)
+ self._top.bind('<F1>', self.help)
+ #self._top.bind('<g>', self.toggle_grammar)
+ #self._top.bind('<Alt-g>', self.toggle_grammar)
+ #self._top.bind('<Control-g>', self.toggle_grammar)
+ self._top.bind('<Control-g>', self.edit_grammar)
+ self._top.bind('<Control-t>', self.edit_sentence)
+
+ def _init_buttons(self, parent):
+ # Set up the frames.
+ self._buttonframe = buttonframe = Frame(parent)
+ buttonframe.pack(fill='none', side='bottom', padx=3, pady=2)
+ Button(buttonframe, text='Step',
+ background='#90c0d0', foreground='black',
+ command=self.step,).pack(side='left')
+ Button(buttonframe, text='Autostep',
+ background='#90c0d0', foreground='black',
+ command=self.autostep,).pack(side='left')
+ Button(buttonframe, text='Expand', underline=0,
+ background='#90f090', foreground='black',
+ command=self.expand).pack(side='left')
+ Button(buttonframe, text='Match', underline=0,
+ background='#90f090', foreground='black',
+ command=self.match).pack(side='left')
+ Button(buttonframe, text='Backtrack', underline=0,
+ background='#f0a0a0', foreground='black',
+ command=self.backtrack).pack(side='left')
+ # Replace autostep...
+# self._autostep_button = Button(buttonframe, text='Autostep',
+# underline=0, command=self.autostep)
+# self._autostep_button.pack(side='left')
+
+ def _configure(self, event):
+ self._autostep = 0
+ (x1, y1, x2, y2) = self._cframe.scrollregion()
+ y2 = event.height - 6
+ self._canvas['scrollregion'] = '%d %d %d %d' % (x1,y1,x2,y2)
+ self._redraw()
+
+ def _init_feedback(self, parent):
+ self._feedbackframe = feedbackframe = Frame(parent)
+ feedbackframe.pack(fill='x', side='bottom', padx=3, pady=3)
+ self._lastoper_label = Label(feedbackframe, text='Last Operation:',
+ font=self._font)
+ self._lastoper_label.pack(side='left')
+ lastoperframe = Frame(feedbackframe, relief='sunken', border=1)
+ lastoperframe.pack(fill='x', side='right', expand=1, padx=5)
+ self._lastoper1 = Label(lastoperframe, foreground='#007070',
+ background='#f0f0f0', font=self._font)
+ self._lastoper2 = Label(lastoperframe, anchor='w', width=30,
+ foreground='#004040', background='#f0f0f0',
+ font=self._font)
+ self._lastoper1.pack(side='left')
+ self._lastoper2.pack(side='left', fill='x', expand=1)
+
+ def _init_canvas(self, parent):
+ self._cframe = CanvasFrame(parent, background='white',
+ #width=525, height=250,
+ closeenough=10,
+ border=2, relief='sunken')
+ self._cframe.pack(expand=1, fill='both', side='top', pady=2)
+ canvas = self._canvas = self._cframe.canvas()
+
+ # Initially, there's no tree or text
+ self._tree = None
+ self._textwidgets = []
+ self._textline = None
+
+ def _init_menubar(self, parent):
+ menubar = Menu(parent)
+
+ filemenu = Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Reset Parser', underline=0,
+ command=self.reset, accelerator='Del')
+ filemenu.add_command(label='Print to Postscript', underline=0,
+ command=self.postscript, accelerator='Ctrl-p')
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-x')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ editmenu = Menu(menubar, tearoff=0)
+ editmenu.add_command(label='Edit Grammar', underline=5,
+ command=self.edit_grammar,
+ accelerator='Ctrl-g')
+ editmenu.add_command(label='Edit Text', underline=5,
+ command=self.edit_sentence,
+ accelerator='Ctrl-t')
+ menubar.add_cascade(label='Edit', underline=0, menu=editmenu)
+
+ rulemenu = Menu(menubar, tearoff=0)
+ rulemenu.add_command(label='Step', underline=1,
+ command=self.step, accelerator='Space')
+ rulemenu.add_separator()
+ rulemenu.add_command(label='Match', underline=0,
+ command=self.match, accelerator='Ctrl-m')
+ rulemenu.add_command(label='Expand', underline=0,
+ command=self.expand, accelerator='Ctrl-e')
+ rulemenu.add_separator()
+ rulemenu.add_command(label='Backtrack', underline=0,
+ command=self.backtrack, accelerator='Ctrl-b')
+ menubar.add_cascade(label='Apply', underline=0, menu=rulemenu)
+
+ viewmenu = Menu(menubar, tearoff=0)
+ viewmenu.add_checkbutton(label="Show Grammar", underline=0,
+ variable=self._show_grammar,
+ command=self._toggle_grammar)
+ viewmenu.add_separator()
+ viewmenu.add_radiobutton(label='Tiny', variable=self._size,
+ underline=0, value=10, command=self.resize)
+ viewmenu.add_radiobutton(label='Small', variable=self._size,
+ underline=0, value=12, command=self.resize)
+ viewmenu.add_radiobutton(label='Medium', variable=self._size,
+ underline=0, value=14, command=self.resize)
+ viewmenu.add_radiobutton(label='Large', variable=self._size,
+ underline=0, value=18, command=self.resize)
+ viewmenu.add_radiobutton(label='Huge', variable=self._size,
+ underline=0, value=24, command=self.resize)
+ menubar.add_cascade(label='View', underline=0, menu=viewmenu)
+
+ animatemenu = Menu(menubar, tearoff=0)
+ animatemenu.add_radiobutton(label="No Animation", underline=0,
+ variable=self._animation_frames,
+ value=0)
+ animatemenu.add_radiobutton(label="Slow Animation", underline=0,
+ variable=self._animation_frames,
+ value=10, accelerator='-')
+ animatemenu.add_radiobutton(label="Normal Animation", underline=0,
+ variable=self._animation_frames,
+ value=5, accelerator='=')
+ animatemenu.add_radiobutton(label="Fast Animation", underline=0,
+ variable=self._animation_frames,
+ value=2, accelerator='+')
+ menubar.add_cascade(label="Animate", underline=1, menu=animatemenu)
+
+
+ helpmenu = Menu(menubar, tearoff=0)
+ helpmenu.add_command(label='About', underline=0,
+ command=self.about)
+ helpmenu.add_command(label='Instructions', underline=0,
+ command=self.help, accelerator='F1')
+ menubar.add_cascade(label='Help', underline=0, menu=helpmenu)
+
+ parent.config(menu=menubar)
+
+ #########################################
+ ## Helper
+ #########################################
+
+ def _get(self, widget, treeloc):
+ for i in treeloc: widget = widget.subtrees()[i]
+ if isinstance(widget, TreeSegmentWidget):
+ widget = widget.label()
+ return widget
+
+ #########################################
+ ## Main draw procedure
+ #########################################
+
+ def _redraw(self):
+ canvas = self._canvas
+
+ # Delete the old tree, widgets, etc.
+ if self._tree is not None:
+ self._cframe.destroy_widget(self._tree)
+ for twidget in self._textwidgets:
+ self._cframe.destroy_widget(twidget)
+ if self._textline is not None:
+ self._canvas.delete(self._textline)
+
+ # Draw the tree.
+ helv = ('helvetica', -self._size.get())
+ bold = ('helvetica', -self._size.get(), 'bold')
+ attribs = {'tree_color': '#000000', 'tree_width': 2,
+ 'node_font': bold, 'leaf_font': helv,}
+ tree = self._parser.tree()
+ self._tree = tree_to_treesegment(canvas, tree, **attribs)
+ self._cframe.add_widget(self._tree, 30, 5)
+
+ # Draw the text.
+ helv = ('helvetica', -self._size.get())
+ bottom = y = self._cframe.scrollregion()[3]
+ self._textwidgets = [TextWidget(canvas, word, font=self._font)
+ for word in self._sent]
+ for twidget in self._textwidgets:
+ self._cframe.add_widget(twidget, 0, 0)
+ twidget.move(0, bottom-twidget.bbox()[3]-5)
+ y = min(y, twidget.bbox()[1])
+
+ # Draw a line over the text, to separate it from the tree.
+ self._textline = canvas.create_line(-5000, y-5, 5000, y-5, dash='.')
+
+ # Highlight appropriate nodes.
+ self._highlight_nodes()
+ self._highlight_prodlist()
+
+ # Make sure the text lines up.
+ self._position_text()
+
+
+ def _redraw_quick(self):
+ # This should be more-or-less sufficient after an animation.
+ self._highlight_nodes()
+ self._highlight_prodlist()
+ self._position_text()
+
+ def _highlight_nodes(self):
+ # Highlight the list of nodes to be checked.
+ bold = ('helvetica', -self._size.get(), 'bold')
+ for treeloc in self._parser.frontier()[:1]:
+ self._get(self._tree, treeloc)['color'] = '#20a050'
+ self._get(self._tree, treeloc)['font'] = bold
+ for treeloc in self._parser.frontier()[1:]:
+ self._get(self._tree, treeloc)['color'] = '#008080'
+
+ def _highlight_prodlist(self):
+ # Highlight the productions that can be expanded.
+ # Boy, too bad tkinter doesn't implement Listbox.itemconfig;
+ # that would be pretty useful here.
+ self._prodlist.delete(0, 'end')
+ expandable = self._parser.expandable_productions()
+ untried = self._parser.untried_expandable_productions()
+ productions = self._productions
+ for index in range(len(productions)):
+ if productions[index] in expandable:
+ if productions[index] in untried:
+ self._prodlist.insert(index, ' %s' % productions[index])
+ else:
+ self._prodlist.insert(index, ' %s (TRIED)' %
+ productions[index])
+ self._prodlist.selection_set(index)
+ else:
+ self._prodlist.insert(index, ' %s' % productions[index])
+
+ def _position_text(self):
+ # Line up the text widgets that are matched against the tree
+ numwords = len(self._sent)
+ num_matched = numwords - len(self._parser.remaining_text())
+ leaves = self._tree_leaves()[:num_matched]
+ xmax = self._tree.bbox()[0]
+ for i in range(0, len(leaves)):
+ widget = self._textwidgets[i]
+ leaf = leaves[i]
+ widget['color'] = '#006040'
+ leaf['color'] = '#006040'
+ widget.move(leaf.bbox()[0] - widget.bbox()[0], 0)
+ xmax = widget.bbox()[2] + 10
+
+ # Line up the text widgets that are not matched against the tree.
+ for i in range(len(leaves), numwords):
+ widget = self._textwidgets[i]
+ widget['color'] = '#a0a0a0'
+ widget.move(xmax - widget.bbox()[0], 0)
+ xmax = widget.bbox()[2] + 10
+
+ # If we have a complete parse, make everything green :)
+ if self._parser.currently_complete():
+ for twidget in self._textwidgets:
+ twidget['color'] = '#00a000'
+
+ # Move the matched leaves down to the text.
+ for i in range(0, len(leaves)):
+ widget = self._textwidgets[i]
+ leaf = leaves[i]
+ dy = widget.bbox()[1] - leaf.bbox()[3] - 10.0
+ dy = max(dy, leaf.parent().label().bbox()[3] - leaf.bbox()[3] + 10)
+ leaf.move(0, dy)
+
+ def _tree_leaves(self, tree=None):
+ if tree is None: tree = self._tree
+ if isinstance(tree, TreeSegmentWidget):
+ leaves = []
+ for child in tree.subtrees(): leaves += self._tree_leaves(child)
+ return leaves
+ else:
+ return [tree]
+
+ #########################################
+ ## Button Callbacks
+ #########################################
+
+ def destroy(self, *e):
+ self._autostep = 0
+ if self._top is None: return
+ self._top.destroy()
+ self._top = None
+
+ def reset(self, *e):
+ self._autostep = 0
+ self._parser.initialize(self._sent)
+ self._lastoper1['text'] = 'Reset Application'
+ self._lastoper2['text'] = ''
+ self._redraw()
+
+ def autostep(self, *e):
+ if self._animation_frames.get() == 0:
+ self._animation_frames.set(2)
+ if self._autostep:
+ self._autostep = 0
+ else:
+ self._autostep = 1
+ self._step()
+
+ def cancel_autostep(self, *e):
+ #self._autostep_button['text'] = 'Autostep'
+ self._autostep = 0
+
+ # Make sure to stop auto-stepping if we get any user input.
+ def step(self, *e): self._autostep = 0; self._step()
+ def match(self, *e): self._autostep = 0; self._match()
+ def expand(self, *e): self._autostep = 0; self._expand()
+ def backtrack(self, *e): self._autostep = 0; self._backtrack()
+
+ def _step(self):
+ if self._animating_lock: return
+
+ # Try expanding, matching, and backtracking (in that order)
+ if self._expand(): pass
+ elif self._parser.untried_match() and self._match(): pass
+ elif self._backtrack(): pass
+ else:
+ self._lastoper1['text'] = 'Finished'
+ self._lastoper2['text'] = ''
+ self._autostep = 0
+
+ # Check if we just completed a parse.
+ if self._parser.currently_complete():
+ self._autostep = 0
+ self._lastoper2['text'] += ' [COMPLETE PARSE]'
+
+ def _expand(self, *e):
+ if self._animating_lock: return
+ old_frontier = self._parser.frontier()
+ rv = self._parser.expand()
+ if rv is not None:
+ self._lastoper1['text'] = 'Expand:'
+ self._lastoper2['text'] = rv
+ self._prodlist.selection_clear(0, 'end')
+ index = self._productions.index(rv)
+ self._prodlist.selection_set(index)
+ self._animate_expand(old_frontier[0])
+ return True
+ else:
+ self._lastoper1['text'] = 'Expand:'
+ self._lastoper2['text'] = '(all expansions tried)'
+ return False
+
+ def _match(self, *e):
+ if self._animating_lock: return
+ old_frontier = self._parser.frontier()
+ rv = self._parser.match()
+ if rv is not None:
+ self._lastoper1['text'] = 'Match:'
+ self._lastoper2['text'] = rv
+ self._animate_match(old_frontier[0])
+ return True
+ else:
+ self._lastoper1['text'] = 'Match:'
+ self._lastoper2['text'] = '(failed)'
+ return False
+
+ def _backtrack(self, *e):
+ if self._animating_lock: return
+ if self._parser.backtrack():
+ elt = self._parser.tree()
+ for i in self._parser.frontier()[0]:
+ elt = elt[i]
+ self._lastoper1['text'] = 'Backtrack'
+ self._lastoper2['text'] = ''
+ if isinstance(elt, Tree):
+ self._animate_backtrack(self._parser.frontier()[0])
+ else:
+ self._animate_match_backtrack(self._parser.frontier()[0])
+ return True
+ else:
+ self._autostep = 0
+ self._lastoper1['text'] = 'Finished'
+ self._lastoper2['text'] = ''
+ return False
+
+ def about(self, *e):
+ ABOUT = ("NLTK Recursive Descent Parser Application\n"+
+ "Written by Edward Loper")
+ TITLE = 'About: Recursive Descent Parser Application'
+ try:
+ from tkinter.messagebox import Message
+ Message(message=ABOUT, title=TITLE).show()
+ except:
+ ShowText(self._top, TITLE, ABOUT)
+
+ def help(self, *e):
+ self._autostep = 0
+ # The default font's not very legible; try using 'fixed' instead.
+ try:
+ ShowText(self._top, 'Help: Recursive Descent Parser Application',
+ (__doc__ or '').strip(), width=75, font='fixed')
+ except:
+ ShowText(self._top, 'Help: Recursive Descent Parser Application',
+ (__doc__ or '').strip(), width=75)
+
+ def postscript(self, *e):
+ self._autostep = 0
+ self._cframe.print_to_file()
+
+ def mainloop(self, *args, **kwargs):
+ """
+ Enter the Tkinter mainloop. This function must be called if
+ this demo is created from a non-interactive program (e.g.
+ from a secript); otherwise, the demo will close as soon as
+ the script completes.
+ """
+ if in_idle(): return
+ self._top.mainloop(*args, **kwargs)
+
+ def resize(self, size=None):
+ if size is not None: self._size.set(size)
+ size = self._size.get()
+ self._font.configure(size=-(abs(size)))
+ self._boldfont.configure(size=-(abs(size)))
+ self._sysfont.configure(size=-(abs(size)))
+ self._bigfont.configure(size=-(abs(size+2)))
+ self._redraw()
+
+ #########################################
+ ## Expand Production Selection
+ #########################################
+
+ def _toggle_grammar(self, *e):
+ if self._show_grammar.get():
+ self._prodframe.pack(fill='both', side='left', padx=2,
+ after=self._feedbackframe)
+ self._lastoper1['text'] = 'Show Grammar'
+ else:
+ self._prodframe.pack_forget()
+ self._lastoper1['text'] = 'Hide Grammar'
+ self._lastoper2['text'] = ''
+
+# def toggle_grammar(self, *e):
+# self._show_grammar = not self._show_grammar
+# if self._show_grammar:
+# self._prodframe.pack(fill='both', expand='y', side='left',
+# after=self._feedbackframe)
+# self._lastoper1['text'] = 'Show Grammar'
+# else:
+# self._prodframe.pack_forget()
+# self._lastoper1['text'] = 'Hide Grammar'
+# self._lastoper2['text'] = ''
+
+ def _prodlist_select(self, event):
+ selection = self._prodlist.curselection()
+ if len(selection) != 1: return
+ index = int(selection[0])
+ old_frontier = self._parser.frontier()
+ production = self._parser.expand(self._productions[index])
+
+ if production:
+ self._lastoper1['text'] = 'Expand:'
+ self._lastoper2['text'] = production
+ self._prodlist.selection_clear(0, 'end')
+ self._prodlist.selection_set(index)
+ self._animate_expand(old_frontier[0])
+ else:
+ # Reset the production selections.
+ self._prodlist.selection_clear(0, 'end')
+ for prod in self._parser.expandable_productions():
+ index = self._productions.index(prod)
+ self._prodlist.selection_set(index)
+
+ #########################################
+ ## Animation
+ #########################################
+
+ def _animate_expand(self, treeloc):
+ oldwidget = self._get(self._tree, treeloc)
+ oldtree = oldwidget.parent()
+ top = not isinstance(oldtree.parent(), TreeSegmentWidget)
+
+ tree = self._parser.tree()
+ for i in treeloc:
+ tree = tree[i]
+
+ widget = tree_to_treesegment(self._canvas, tree,
+ node_font=self._boldfont,
+ leaf_color='white',
+ tree_width=2, tree_color='white',
+ node_color='white',
+ leaf_font=self._font)
+ widget.label()['color'] = '#20a050'
+
+ (oldx, oldy) = oldtree.label().bbox()[:2]
+ (newx, newy) = widget.label().bbox()[:2]
+ widget.move(oldx-newx, oldy-newy)
+
+ if top:
+ self._cframe.add_widget(widget, 0, 5)
+ widget.move(30-widget.label().bbox()[0], 0)
+ self._tree = widget
+ else:
+ oldtree.parent().replace_child(oldtree, widget)
+
+ # Move the children over so they don't overlap.
+ # Line the children up in a strange way.
+ if widget.subtrees():
+ dx = (oldx + widget.label().width()/2 -
+ widget.subtrees()[0].bbox()[0]/2 -
+ widget.subtrees()[0].bbox()[2]/2)
+ for subtree in widget.subtrees(): subtree.move(dx, 0)
+
+ self._makeroom(widget)
+
+ if top:
+ self._cframe.destroy_widget(oldtree)
+ else:
+ oldtree.destroy()
+
+ colors = ['gray%d' % (10*int(10*x/self._animation_frames.get()))
+ for x in range(self._animation_frames.get(),0,-1)]
+
+ # Move the text string down, if necessary.
+ dy = widget.bbox()[3] + 30 - self._canvas.coords(self._textline)[1]
+ if dy > 0:
+ for twidget in self._textwidgets: twidget.move(0, dy)
+ self._canvas.move(self._textline, 0, dy)
+
+ self._animate_expand_frame(widget, colors)
+
+ def _makeroom(self, treeseg):
+ """
+ Make sure that no sibling tree bbox's overlap.
+ """
+ parent = treeseg.parent()
+ if not isinstance(parent, TreeSegmentWidget): return
+
+ index = parent.subtrees().index(treeseg)
+
+ # Handle siblings to the right
+ rsiblings = parent.subtrees()[index+1:]
+ if rsiblings:
+ dx = treeseg.bbox()[2] - rsiblings[0].bbox()[0] + 10
+ for sibling in rsiblings: sibling.move(dx, 0)
+
+ # Handle siblings to the left
+ if index > 0:
+ lsibling = parent.subtrees()[index-1]
+ dx = max(0, lsibling.bbox()[2] - treeseg.bbox()[0] + 10)
+ treeseg.move(dx, 0)
+
+ # Keep working up the tree.
+ self._makeroom(parent)
+
+ def _animate_expand_frame(self, widget, colors):
+ if len(colors) > 0:
+ self._animating_lock = 1
+ widget['color'] = colors[0]
+ for subtree in widget.subtrees():
+ if isinstance(subtree, TreeSegmentWidget):
+ subtree.label()['color'] = colors[0]
+ else:
+ subtree['color'] = colors[0]
+ self._top.after(50, self._animate_expand_frame,
+ widget, colors[1:])
+ else:
+ widget['color'] = 'black'
+ for subtree in widget.subtrees():
+ if isinstance(subtree, TreeSegmentWidget):
+ subtree.label()['color'] = 'black'
+ else:
+ subtree['color'] = 'black'
+ self._redraw_quick()
+ widget.label()['color'] = 'black'
+ self._animating_lock = 0
+ if self._autostep: self._step()
+
+ def _animate_backtrack(self, treeloc):
+ # Flash red first, if we're animating.
+ if self._animation_frames.get() == 0: colors = []
+ else: colors = ['#a00000', '#000000', '#a00000']
+ colors += ['gray%d' % (10*int(10*x/(self._animation_frames.get())))
+ for x in range(1, self._animation_frames.get()+1)]
+
+ widgets = [self._get(self._tree, treeloc).parent()]
+ for subtree in widgets[0].subtrees():
+ if isinstance(subtree, TreeSegmentWidget):
+ widgets.append(subtree.label())
+ else:
+ widgets.append(subtree)
+
+ self._animate_backtrack_frame(widgets, colors)
+
+ def _animate_backtrack_frame(self, widgets, colors):
+ if len(colors) > 0:
+ self._animating_lock = 1
+ for widget in widgets: widget['color'] = colors[0]
+ self._top.after(50, self._animate_backtrack_frame,
+ widgets, colors[1:])
+ else:
+ for widget in widgets[0].subtrees():
+ widgets[0].remove_child(widget)
+ widget.destroy()
+ self._redraw_quick()
+ self._animating_lock = 0
+ if self._autostep: self._step()
+
+ def _animate_match_backtrack(self, treeloc):
+ widget = self._get(self._tree, treeloc)
+ node = widget.parent().label()
+ dy = (1.0 * (node.bbox()[3] - widget.bbox()[1] + 14) /
+ max(1, self._animation_frames.get()))
+ self._animate_match_backtrack_frame(self._animation_frames.get(),
+ widget, dy)
+
+ def _animate_match(self, treeloc):
+ widget = self._get(self._tree, treeloc)
+
+ dy = ((self._textwidgets[0].bbox()[1] - widget.bbox()[3] - 10.0) /
+ max(1, self._animation_frames.get()))
+ self._animate_match_frame(self._animation_frames.get(), widget, dy)
+
+ def _animate_match_frame(self, frame, widget, dy):
+ if frame > 0:
+ self._animating_lock = 1
+ widget.move(0, dy)
+ self._top.after(10, self._animate_match_frame,
+ frame-1, widget, dy)
+ else:
+ widget['color'] = '#006040'
+ self._redraw_quick()
+ self._animating_lock = 0
+ if self._autostep: self._step()
+
+ def _animate_match_backtrack_frame(self, frame, widget, dy):
+ if frame > 0:
+ self._animating_lock = 1
+ widget.move(0, dy)
+ self._top.after(10, self._animate_match_backtrack_frame,
+ frame-1, widget, dy)
+ else:
+ widget.parent().remove_child(widget)
+ widget.destroy()
+ self._animating_lock = 0
+ if self._autostep: self._step()
+
+ def edit_grammar(self, *e):
+ CFGEditor(self._top, self._parser.grammar(), self.set_grammar)
+
+ def set_grammar(self, grammar):
+ self._parser.set_grammar(grammar)
+ self._productions = list(grammar.productions())
+ self._prodlist.delete(0, 'end')
+ for production in self._productions:
+ self._prodlist.insert('end', (' %s' % production))
+
+ def edit_sentence(self, *e):
+ sentence = " ".join(self._sent)
+ title = 'Edit Text'
+ instr = 'Enter a new sentence to parse.'
+ EntryDialog(self._top, sentence, instr, self.set_sentence, title)
+
+ def set_sentence(self, sentence):
+ self._sent = sentence.split() #[XX] use tagged?
+ self.reset()
+
+def app():
+ """
+ Create a recursive descent parser demo, using a simple grammar and
+ text.
+ """
+ from nltk.grammar import CFG
+ grammar = CFG.fromstring("""
+ # Grammatical productions.
+ S -> NP VP
+ NP -> Det N PP | Det N
+ VP -> V NP PP | V NP | V
+ PP -> P NP
+ # Lexical productions.
+ NP -> 'I'
+ Det -> 'the' | 'a'
+ N -> 'man' | 'park' | 'dog' | 'telescope'
+ V -> 'ate' | 'saw'
+ P -> 'in' | 'under' | 'with'
+ """)
+
+ sent = 'the dog saw a man in the park'.split()
+
+ RecursiveDescentApp(grammar, sent).mainloop()
+
+if __name__ == '__main__':
+ app()
+
+__all__ = ['app']
+
+
diff --git a/nltk/app/srparser_app.py b/nltk/app/srparser_app.py
new file mode 100644
index 0000000..b3ff44a
--- /dev/null
+++ b/nltk/app/srparser_app.py
@@ -0,0 +1,809 @@
+# Natural Language Toolkit: Shift-Reduce Parser Application
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A graphical tool for exploring the shift-reduce parser.
+
+The shift-reduce parser maintains a stack, which records the structure
+of the portion of the text that has been parsed. The stack is
+initially empty. Its contents are shown on the left side of the main
+canvas.
+
+On the right side of the main canvas is the remaining text. This is
+the portion of the text which has not yet been considered by the
+parser.
+
+The parser builds up a tree structure for the text using two
+operations:
+
+ - "shift" moves the first token from the remaining text to the top
+ of the stack. In the demo, the top of the stack is its right-hand
+ side.
+ - "reduce" uses a grammar production to combine the rightmost stack
+ elements into a single tree token.
+
+You can control the parser's operation by using the "shift" and
+"reduce" buttons; or you can use the "step" button to let the parser
+automatically decide which operation to apply. The parser uses the
+following rules to decide which operation to apply:
+
+ - Only shift if no reductions are available.
+ - If multiple reductions are available, then apply the reduction
+ whose CFG production is listed earliest in the grammar.
+
+The "reduce" button applies the reduction whose CFG production is
+listed earliest in the grammar. There are two ways to manually choose
+which reduction to apply:
+
+ - Click on a CFG production from the list of available reductions,
+ on the left side of the main window. The reduction based on that
+ production will be applied to the top of the stack.
+ - Click on one of the stack elements. A popup window will appear,
+ containing all available reductions. Select one, and it will be
+ applied to the top of the stack.
+
+Note that reductions can only be applied to the top of the stack.
+
+Keyboard Shortcuts::
+ [Space]\t Perform the next shift or reduce operation
+ [s]\t Perform a shift operation
+ [r]\t Perform a reduction operation
+ [Ctrl-z]\t Undo most recent operation
+ [Delete]\t Reset the parser
+ [g]\t Show/hide available production list
+ [Ctrl-a]\t Toggle animations
+ [h]\t Help
+ [Ctrl-p]\t Print
+ [q]\t Quit
+"""
+
+"""
+Possible future improvements:
+ - button/window to change and/or select text. Just pop up a window
+ with an entry, and let them modify the text; and then retokenize
+ it? Maybe give a warning if it contains tokens whose types are
+ not in the grammar.
+ - button/window to change and/or select grammar. Select from
+ several alternative grammars? Or actually change the grammar? If
+ the later, then I'd want to define nltk.draw.cfg, which would be
+ responsible for that.
+"""
+
+import nltk.compat
+import tkinter.font
+from tkinter import (IntVar, Listbox, Button, Frame, Label, Menu,
+ Scrollbar, Tk)
+
+from nltk.tree import Tree
+from nltk.parse import SteppingShiftReduceParser
+from nltk.util import in_idle
+from nltk.draw.util import CanvasFrame, EntryDialog, ShowText, TextWidget
+from nltk.draw import CFGEditor, TreeSegmentWidget, tree_to_treesegment
+
+class ShiftReduceApp(object):
+ """
+ A graphical tool for exploring the shift-reduce parser. The tool
+ displays the parser's stack and the remaining text, and allows the
+ user to control the parser's operation. In particular, the user
+ can shift tokens onto the stack, and can perform reductions on the
+ top elements of the stack. A "step" button simply steps through
+ the parsing process, performing the operations that
+ ``nltk.parse.ShiftReduceParser`` would use.
+ """
+ def __init__(self, grammar, sent, trace=0):
+ self._sent = sent
+ self._parser = SteppingShiftReduceParser(grammar, trace)
+
+ # Set up the main window.
+ self._top = Tk()
+ self._top.title('Shift Reduce Parser Application')
+
+ # Animations. animating_lock is a lock to prevent the demo
+ # from performing new operations while it's animating.
+ self._animating_lock = 0
+ self._animate = IntVar(self._top)
+ self._animate.set(10) # = medium
+
+ # The user can hide the grammar.
+ self._show_grammar = IntVar(self._top)
+ self._show_grammar.set(1)
+
+ # Initialize fonts.
+ self._init_fonts(self._top)
+
+ # Set up key bindings.
+ self._init_bindings()
+
+ # Create the basic frames.
+ self._init_menubar(self._top)
+ self._init_buttons(self._top)
+ self._init_feedback(self._top)
+ self._init_grammar(self._top)
+ self._init_canvas(self._top)
+
+ # A popup menu for reducing.
+ self._reduce_menu = Menu(self._canvas, tearoff=0)
+
+ # Reset the demo, and set the feedback frame to empty.
+ self.reset()
+ self._lastoper1['text'] = ''
+
+ #########################################
+ ## Initialization Helpers
+ #########################################
+
+ def _init_fonts(self, root):
+ # See: <http://www.astro.washington.edu/owen/ROTKFolklore.html>
+ self._sysfont = tkinter.font.Font(font=Button()["font"])
+ root.option_add("*Font", self._sysfont)
+
+ # TWhat's our font size (default=same as sysfont)
+ self._size = IntVar(root)
+ self._size.set(self._sysfont.cget('size'))
+
+ self._boldfont = tkinter.font.Font(family='helvetica', weight='bold',
+ size=self._size.get())
+ self._font = tkinter.font.Font(family='helvetica',
+ size=self._size.get())
+
+ def _init_grammar(self, parent):
+ # Grammar view.
+ self._prodframe = listframe = Frame(parent)
+ self._prodframe.pack(fill='both', side='left', padx=2)
+ self._prodlist_label = Label(self._prodframe,
+ font=self._boldfont,
+ text='Available Reductions')
+ self._prodlist_label.pack()
+ self._prodlist = Listbox(self._prodframe, selectmode='single',
+ relief='groove', background='white',
+ foreground='#909090',
+ font=self._font,
+ selectforeground='#004040',
+ selectbackground='#c0f0c0')
+
+ self._prodlist.pack(side='right', fill='both', expand=1)
+
+ self._productions = list(self._parser.grammar().productions())
+ for production in self._productions:
+ self._prodlist.insert('end', (' %s' % production))
+ self._prodlist.config(height=min(len(self._productions), 25))
+
+ # Add a scrollbar if there are more than 25 productions.
+ if 1:#len(self._productions) > 25:
+ listscroll = Scrollbar(self._prodframe,
+ orient='vertical')
+ self._prodlist.config(yscrollcommand = listscroll.set)
+ listscroll.config(command=self._prodlist.yview)
+ listscroll.pack(side='left', fill='y')
+
+ # If they select a production, apply it.
+ self._prodlist.bind('<<ListboxSelect>>', self._prodlist_select)
+
+ # When they hover over a production, highlight it.
+ self._hover = -1
+ self._prodlist.bind('<Motion>', self._highlight_hover)
+ self._prodlist.bind('<Leave>', self._clear_hover)
+
+ def _init_bindings(self):
+ # Quit
+ self._top.bind('<Control-q>', self.destroy)
+ self._top.bind('<Control-x>', self.destroy)
+ self._top.bind('<Alt-q>', self.destroy)
+ self._top.bind('<Alt-x>', self.destroy)
+
+ # Ops (step, shift, reduce, undo)
+ self._top.bind('<space>', self.step)
+ self._top.bind('<s>', self.shift)
+ self._top.bind('<Alt-s>', self.shift)
+ self._top.bind('<Control-s>', self.shift)
+ self._top.bind('<r>', self.reduce)
+ self._top.bind('<Alt-r>', self.reduce)
+ self._top.bind('<Control-r>', self.reduce)
+ self._top.bind('<Delete>', self.reset)
+ self._top.bind('<u>', self.undo)
+ self._top.bind('<Alt-u>', self.undo)
+ self._top.bind('<Control-u>', self.undo)
+ self._top.bind('<Control-z>', self.undo)
+ self._top.bind('<BackSpace>', self.undo)
+
+ # Misc
+ self._top.bind('<Control-p>', self.postscript)
+ self._top.bind('<Control-h>', self.help)
+ self._top.bind('<F1>', self.help)
+ self._top.bind('<Control-g>', self.edit_grammar)
+ self._top.bind('<Control-t>', self.edit_sentence)
+
+ # Animation speed control
+ self._top.bind('-', lambda e,a=self._animate:a.set(20))
+ self._top.bind('=', lambda e,a=self._animate:a.set(10))
+ self._top.bind('+', lambda e,a=self._animate:a.set(4))
+
+ def _init_buttons(self, parent):
+ # Set up the frames.
+ self._buttonframe = buttonframe = Frame(parent)
+ buttonframe.pack(fill='none', side='bottom')
+ Button(buttonframe, text='Step',
+ background='#90c0d0', foreground='black',
+ command=self.step,).pack(side='left')
+ Button(buttonframe, text='Shift', underline=0,
+ background='#90f090', foreground='black',
+ command=self.shift).pack(side='left')
+ Button(buttonframe, text='Reduce', underline=0,
+ background='#90f090', foreground='black',
+ command=self.reduce).pack(side='left')
+ Button(buttonframe, text='Undo', underline=0,
+ background='#f0a0a0', foreground='black',
+ command=self.undo).pack(side='left')
+
+ def _init_menubar(self, parent):
+ menubar = Menu(parent)
+
+ filemenu = Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Reset Parser', underline=0,
+ command=self.reset, accelerator='Del')
+ filemenu.add_command(label='Print to Postscript', underline=0,
+ command=self.postscript, accelerator='Ctrl-p')
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-x')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ editmenu = Menu(menubar, tearoff=0)
+ editmenu.add_command(label='Edit Grammar', underline=5,
+ command=self.edit_grammar,
+ accelerator='Ctrl-g')
+ editmenu.add_command(label='Edit Text', underline=5,
+ command=self.edit_sentence,
+ accelerator='Ctrl-t')
+ menubar.add_cascade(label='Edit', underline=0, menu=editmenu)
+
+ rulemenu = Menu(menubar, tearoff=0)
+ rulemenu.add_command(label='Step', underline=1,
+ command=self.step, accelerator='Space')
+ rulemenu.add_separator()
+ rulemenu.add_command(label='Shift', underline=0,
+ command=self.shift, accelerator='Ctrl-s')
+ rulemenu.add_command(label='Reduce', underline=0,
+ command=self.reduce, accelerator='Ctrl-r')
+ rulemenu.add_separator()
+ rulemenu.add_command(label='Undo', underline=0,
+ command=self.undo, accelerator='Ctrl-u')
+ menubar.add_cascade(label='Apply', underline=0, menu=rulemenu)
+
+ viewmenu = Menu(menubar, tearoff=0)
+ viewmenu.add_checkbutton(label="Show Grammar", underline=0,
+ variable=self._show_grammar,
+ command=self._toggle_grammar)
+ viewmenu.add_separator()
+ viewmenu.add_radiobutton(label='Tiny', variable=self._size,
+ underline=0, value=10, command=self.resize)
+ viewmenu.add_radiobutton(label='Small', variable=self._size,
+ underline=0, value=12, command=self.resize)
+ viewmenu.add_radiobutton(label='Medium', variable=self._size,
+ underline=0, value=14, command=self.resize)
+ viewmenu.add_radiobutton(label='Large', variable=self._size,
+ underline=0, value=18, command=self.resize)
+ viewmenu.add_radiobutton(label='Huge', variable=self._size,
+ underline=0, value=24, command=self.resize)
+ menubar.add_cascade(label='View', underline=0, menu=viewmenu)
+
+ animatemenu = Menu(menubar, tearoff=0)
+ animatemenu.add_radiobutton(label="No Animation", underline=0,
+ variable=self._animate, value=0)
+ animatemenu.add_radiobutton(label="Slow Animation", underline=0,
+ variable=self._animate, value=20,
+ accelerator='-')
+ animatemenu.add_radiobutton(label="Normal Animation", underline=0,
+ variable=self._animate, value=10,
+ accelerator='=')
+ animatemenu.add_radiobutton(label="Fast Animation", underline=0,
+ variable=self._animate, value=4,
+ accelerator='+')
+ menubar.add_cascade(label="Animate", underline=1, menu=animatemenu)
+
+
+ helpmenu = Menu(menubar, tearoff=0)
+ helpmenu.add_command(label='About', underline=0,
+ command=self.about)
+ helpmenu.add_command(label='Instructions', underline=0,
+ command=self.help, accelerator='F1')
+ menubar.add_cascade(label='Help', underline=0, menu=helpmenu)
+
+ parent.config(menu=menubar)
+
+ def _init_feedback(self, parent):
+ self._feedbackframe = feedbackframe = Frame(parent)
+ feedbackframe.pack(fill='x', side='bottom', padx=3, pady=3)
+ self._lastoper_label = Label(feedbackframe, text='Last Operation:',
+ font=self._font)
+ self._lastoper_label.pack(side='left')
+ lastoperframe = Frame(feedbackframe, relief='sunken', border=1)
+ lastoperframe.pack(fill='x', side='right', expand=1, padx=5)
+ self._lastoper1 = Label(lastoperframe, foreground='#007070',
+ background='#f0f0f0', font=self._font)
+ self._lastoper2 = Label(lastoperframe, anchor='w', width=30,
+ foreground='#004040', background='#f0f0f0',
+ font=self._font)
+ self._lastoper1.pack(side='left')
+ self._lastoper2.pack(side='left', fill='x', expand=1)
+
+ def _init_canvas(self, parent):
+ self._cframe = CanvasFrame(parent, background='white',
+ width=525, closeenough=10,
+ border=2, relief='sunken')
+ self._cframe.pack(expand=1, fill='both', side='top', pady=2)
+ canvas = self._canvas = self._cframe.canvas()
+
+ self._stackwidgets = []
+ self._rtextwidgets = []
+ self._titlebar = canvas.create_rectangle(0,0,0,0, fill='#c0f0f0',
+ outline='black')
+ self._exprline = canvas.create_line(0,0,0,0, dash='.')
+ self._stacktop = canvas.create_line(0,0,0,0, fill='#408080')
+ size = self._size.get()+4
+ self._stacklabel = TextWidget(canvas, 'Stack', color='#004040',
+ font=self._boldfont)
+ self._rtextlabel = TextWidget(canvas, 'Remaining Text',
+ color='#004040', font=self._boldfont)
+ self._cframe.add_widget(self._stacklabel)
+ self._cframe.add_widget(self._rtextlabel)
+
+ #########################################
+ ## Main draw procedure
+ #########################################
+
+ def _redraw(self):
+ scrollregion = self._canvas['scrollregion'].split()
+ (cx1, cy1, cx2, cy2) = [int(c) for c in scrollregion]
+
+ # Delete the old stack & rtext widgets.
+ for stackwidget in self._stackwidgets:
+ self._cframe.destroy_widget(stackwidget)
+ self._stackwidgets = []
+ for rtextwidget in self._rtextwidgets:
+ self._cframe.destroy_widget(rtextwidget)
+ self._rtextwidgets = []
+
+ # Position the titlebar & exprline
+ (x1, y1, x2, y2) = self._stacklabel.bbox()
+ y = y2-y1+10
+ self._canvas.coords(self._titlebar, -5000, 0, 5000, y-4)
+ self._canvas.coords(self._exprline, 0, y*2-10, 5000, y*2-10)
+
+ # Position the titlebar labels..
+ (x1, y1, x2, y2) = self._stacklabel.bbox()
+ self._stacklabel.move(5-x1, 3-y1)
+ (x1, y1, x2, y2) = self._rtextlabel.bbox()
+ self._rtextlabel.move(cx2-x2-5, 3-y1)
+
+ # Draw the stack.
+ stackx = 5
+ for tok in self._parser.stack():
+ if isinstance(tok, Tree):
+ attribs = {'tree_color': '#4080a0', 'tree_width': 2,
+ 'node_font': self._boldfont,
+ 'node_color': '#006060',
+ 'leaf_color': '#006060', 'leaf_font':self._font}
+ widget = tree_to_treesegment(self._canvas, tok,
+ **attribs)
+ widget.label()['color'] = '#000000'
+ else:
+ widget = TextWidget(self._canvas, tok,
+ color='#000000', font=self._font)
+ widget.bind_click(self._popup_reduce)
+ self._stackwidgets.append(widget)
+ self._cframe.add_widget(widget, stackx, y)
+ stackx = widget.bbox()[2] + 10
+
+ # Draw the remaining text.
+ rtextwidth = 0
+ for tok in self._parser.remaining_text():
+ widget = TextWidget(self._canvas, tok,
+ color='#000000', font=self._font)
+ self._rtextwidgets.append(widget)
+ self._cframe.add_widget(widget, rtextwidth, y)
+ rtextwidth = widget.bbox()[2] + 4
+
+ # Allow enough room to shift the next token (for animations)
+ if len(self._rtextwidgets) > 0:
+ stackx += self._rtextwidgets[0].width()
+
+ # Move the remaining text to the correct location (keep it
+ # right-justified, when possible); and move the remaining text
+ # label, if necessary.
+ stackx = max(stackx, self._stacklabel.width()+25)
+ rlabelwidth = self._rtextlabel.width()+10
+ if stackx >= cx2-max(rtextwidth, rlabelwidth):
+ cx2 = stackx + max(rtextwidth, rlabelwidth)
+ for rtextwidget in self._rtextwidgets:
+ rtextwidget.move(4+cx2-rtextwidth, 0)
+ self._rtextlabel.move(cx2-self._rtextlabel.bbox()[2]-5, 0)
+
+ midx = (stackx + cx2-max(rtextwidth, rlabelwidth))/2
+ self._canvas.coords(self._stacktop, midx, 0, midx, 5000)
+ (x1, y1, x2, y2) = self._stacklabel.bbox()
+
+ # Set up binding to allow them to shift a token by dragging it.
+ if len(self._rtextwidgets) > 0:
+ def drag_shift(widget, midx=midx, self=self):
+ if widget.bbox()[0] < midx: self.shift()
+ else: self._redraw()
+ self._rtextwidgets[0].bind_drag(drag_shift)
+ self._rtextwidgets[0].bind_click(self.shift)
+
+ # Draw the stack top.
+ self._highlight_productions()
+
+ def _draw_stack_top(self, widget):
+ # hack..
+ midx = widget.bbox()[2]+50
+ self._canvas.coords(self._stacktop, midx, 0, midx, 5000)
+
+ def _highlight_productions(self):
+ # Highlight the productions that can be reduced.
+ self._prodlist.selection_clear(0, 'end')
+ for prod in self._parser.reducible_productions():
+ index = self._productions.index(prod)
+ self._prodlist.selection_set(index)
+
+ #########################################
+ ## Button Callbacks
+ #########################################
+
+ def destroy(self, *e):
+ if self._top is None: return
+ self._top.destroy()
+ self._top = None
+
+ def reset(self, *e):
+ self._parser.initialize(self._sent)
+ self._lastoper1['text'] = 'Reset App'
+ self._lastoper2['text'] = ''
+ self._redraw()
+
+ def step(self, *e):
+ if self.reduce(): return True
+ elif self.shift(): return True
+ else:
+ if list(self._parser.parses()):
+ self._lastoper1['text'] = 'Finished:'
+ self._lastoper2['text'] = 'Success'
+ else:
+ self._lastoper1['text'] = 'Finished:'
+ self._lastoper2['text'] = 'Failure'
+
+ def shift(self, *e):
+ if self._animating_lock: return
+ if self._parser.shift():
+ tok = self._parser.stack()[-1]
+ self._lastoper1['text'] = 'Shift:'
+ self._lastoper2['text'] = '%r' % tok
+ if self._animate.get():
+ self._animate_shift()
+ else:
+ self._redraw()
+ return True
+ return False
+
+ def reduce(self, *e):
+ if self._animating_lock: return
+ production = self._parser.reduce()
+ if production:
+ self._lastoper1['text'] = 'Reduce:'
+ self._lastoper2['text'] = '%s' % production
+ if self._animate.get():
+ self._animate_reduce()
+ else:
+ self._redraw()
+ return production
+
+ def undo(self, *e):
+ if self._animating_lock: return
+ if self._parser.undo():
+ self._redraw()
+
+ def postscript(self, *e):
+ self._cframe.print_to_file()
+
+ def mainloop(self, *args, **kwargs):
+ """
+ Enter the Tkinter mainloop. This function must be called if
+ this demo is created from a non-interactive program (e.g.
+ from a secript); otherwise, the demo will close as soon as
+ the script completes.
+ """
+ if in_idle(): return
+ self._top.mainloop(*args, **kwargs)
+
+ #########################################
+ ## Menubar callbacks
+ #########################################
+
+ def resize(self, size=None):
+ if size is not None: self._size.set(size)
+ size = self._size.get()
+ self._font.configure(size=-(abs(size)))
+ self._boldfont.configure(size=-(abs(size)))
+ self._sysfont.configure(size=-(abs(size)))
+
+ #self._stacklabel['font'] = ('helvetica', -size-4, 'bold')
+ #self._rtextlabel['font'] = ('helvetica', -size-4, 'bold')
+ #self._lastoper_label['font'] = ('helvetica', -size)
+ #self._lastoper1['font'] = ('helvetica', -size)
+ #self._lastoper2['font'] = ('helvetica', -size)
+ #self._prodlist['font'] = ('helvetica', -size)
+ #self._prodlist_label['font'] = ('helvetica', -size-2, 'bold')
+ self._redraw()
+
+ def help(self, *e):
+ # The default font's not very legible; try using 'fixed' instead.
+ try:
+ ShowText(self._top, 'Help: Shift-Reduce Parser Application',
+ (__doc__ or '').strip(), width=75, font='fixed')
+ except:
+ ShowText(self._top, 'Help: Shift-Reduce Parser Application',
+ (__doc__ or '').strip(), width=75)
+
+ def about(self, *e):
+ ABOUT = ("NLTK Shift-Reduce Parser Application\n"+
+ "Written by Edward Loper")
+ TITLE = 'About: Shift-Reduce Parser Application'
+ try:
+ from tkinter.messagebox import Message
+ Message(message=ABOUT, title=TITLE).show()
+ except:
+ ShowText(self._top, TITLE, ABOUT)
+
+ def edit_grammar(self, *e):
+ CFGEditor(self._top, self._parser.grammar(), self.set_grammar)
+
+ def set_grammar(self, grammar):
+ self._parser.set_grammar(grammar)
+ self._productions = list(grammar.productions())
+ self._prodlist.delete(0, 'end')
+ for production in self._productions:
+ self._prodlist.insert('end', (' %s' % production))
+
+ def edit_sentence(self, *e):
+ sentence = " ".join(self._sent)
+ title = 'Edit Text'
+ instr = 'Enter a new sentence to parse.'
+ EntryDialog(self._top, sentence, instr, self.set_sentence, title)
+
+ def set_sentence(self, sent):
+ self._sent = sent.split() #[XX] use tagged?
+ self.reset()
+
+ #########################################
+ ## Reduce Production Selection
+ #########################################
+
+ def _toggle_grammar(self, *e):
+ if self._show_grammar.get():
+ self._prodframe.pack(fill='both', side='left', padx=2,
+ after=self._feedbackframe)
+ self._lastoper1['text'] = 'Show Grammar'
+ else:
+ self._prodframe.pack_forget()
+ self._lastoper1['text'] = 'Hide Grammar'
+ self._lastoper2['text'] = ''
+
+ def _prodlist_select(self, event):
+ selection = self._prodlist.curselection()
+ if len(selection) != 1: return
+ index = int(selection[0])
+ production = self._parser.reduce(self._productions[index])
+ if production:
+ self._lastoper1['text'] = 'Reduce:'
+ self._lastoper2['text'] = '%s' % production
+ if self._animate.get():
+ self._animate_reduce()
+ else:
+ self._redraw()
+ else:
+ # Reset the production selections.
+ self._prodlist.selection_clear(0, 'end')
+ for prod in self._parser.reducible_productions():
+ index = self._productions.index(prod)
+ self._prodlist.selection_set(index)
+
+ def _popup_reduce(self, widget):
+ # Remove old commands.
+ productions = self._parser.reducible_productions()
+ if len(productions) == 0: return
+
+ self._reduce_menu.delete(0, 'end')
+ for production in productions:
+ self._reduce_menu.add_command(label=str(production),
+ command=self.reduce)
+ self._reduce_menu.post(self._canvas.winfo_pointerx(),
+ self._canvas.winfo_pointery())
+
+ #########################################
+ ## Animations
+ #########################################
+
+ def _animate_shift(self):
+ # What widget are we shifting?
+ widget = self._rtextwidgets[0]
+
+ # Where are we shifting from & to?
+ right = widget.bbox()[0]
+ if len(self._stackwidgets) == 0: left = 5
+ else: left = self._stackwidgets[-1].bbox()[2]+10
+
+ # Start animating.
+ dt = self._animate.get()
+ dx = (left-right)*1.0/dt
+ self._animate_shift_frame(dt, widget, dx)
+
+ def _animate_shift_frame(self, frame, widget, dx):
+ if frame > 0:
+ self._animating_lock = 1
+ widget.move(dx, 0)
+ self._top.after(10, self._animate_shift_frame,
+ frame-1, widget, dx)
+ else:
+ # but: stacktop??
+
+ # Shift the widget to the stack.
+ del self._rtextwidgets[0]
+ self._stackwidgets.append(widget)
+ self._animating_lock = 0
+
+ # Display the available productions.
+ self._draw_stack_top(widget)
+ self._highlight_productions()
+
+ def _animate_reduce(self):
+ # What widgets are we shifting?
+ numwidgets = len(self._parser.stack()[-1]) # number of children
+ widgets = self._stackwidgets[-numwidgets:]
+
+ # How far are we moving?
+ if isinstance(widgets[0], TreeSegmentWidget):
+ ydist = 15 + widgets[0].label().height()
+ else:
+ ydist = 15 + widgets[0].height()
+
+ # Start animating.
+ dt = self._animate.get()
+ dy = ydist*2.0/dt
+ self._animate_reduce_frame(dt/2, widgets, dy)
+
+ def _animate_reduce_frame(self, frame, widgets, dy):
+ if frame > 0:
+ self._animating_lock = 1
+ for widget in widgets: widget.move(0, dy)
+ self._top.after(10, self._animate_reduce_frame,
+ frame-1, widgets, dy)
+ else:
+ del self._stackwidgets[-len(widgets):]
+ for widget in widgets:
+ self._cframe.remove_widget(widget)
+ tok = self._parser.stack()[-1]
+ if not isinstance(tok, Tree): raise ValueError()
+ label = TextWidget(self._canvas, str(tok.label()), color='#006060',
+ font=self._boldfont)
+ widget = TreeSegmentWidget(self._canvas, label, widgets,
+ width=2)
+ (x1, y1, x2, y2) = self._stacklabel.bbox()
+ y = y2-y1+10
+ if not self._stackwidgets: x = 5
+ else: x = self._stackwidgets[-1].bbox()[2] + 10
+ self._cframe.add_widget(widget, x, y)
+ self._stackwidgets.append(widget)
+
+ # Display the available productions.
+ self._draw_stack_top(widget)
+ self._highlight_productions()
+
+# # Delete the old widgets..
+# del self._stackwidgets[-len(widgets):]
+# for widget in widgets:
+# self._cframe.destroy_widget(widget)
+#
+# # Make a new one.
+# tok = self._parser.stack()[-1]
+# if isinstance(tok, Tree):
+# attribs = {'tree_color': '#4080a0', 'tree_width': 2,
+# 'node_font': bold, 'node_color': '#006060',
+# 'leaf_color': '#006060', 'leaf_font':self._font}
+# widget = tree_to_treesegment(self._canvas, tok.type(),
+# **attribs)
+# widget.node()['color'] = '#000000'
+# else:
+# widget = TextWidget(self._canvas, tok.type(),
+# color='#000000', font=self._font)
+# widget.bind_click(self._popup_reduce)
+# (x1, y1, x2, y2) = self._stacklabel.bbox()
+# y = y2-y1+10
+# if not self._stackwidgets: x = 5
+# else: x = self._stackwidgets[-1].bbox()[2] + 10
+# self._cframe.add_widget(widget, x, y)
+# self._stackwidgets.append(widget)
+
+ #self._redraw()
+ self._animating_lock = 0
+
+ #########################################
+ ## Hovering.
+ #########################################
+
+ def _highlight_hover(self, event):
+ # What production are we hovering over?
+ index = self._prodlist.nearest(event.y)
+ if self._hover == index: return
+
+ # Clear any previous hover highlighting.
+ self._clear_hover()
+
+ # If the production corresponds to an available reduction,
+ # highlight the stack.
+ selection = [int(s) for s in self._prodlist.curselection()]
+ if index in selection:
+ rhslen = len(self._productions[index].rhs())
+ for stackwidget in self._stackwidgets[-rhslen:]:
+ if isinstance(stackwidget, TreeSegmentWidget):
+ stackwidget.label()['color'] = '#00a000'
+ else:
+ stackwidget['color'] = '#00a000'
+
+ # Remember what production we're hovering over.
+ self._hover = index
+
+ def _clear_hover(self, *event):
+ # Clear any previous hover highlighting.
+ if self._hover == -1: return
+ self._hover = -1
+ for stackwidget in self._stackwidgets:
+ if isinstance(stackwidget, TreeSegmentWidget):
+ stackwidget.label()['color'] = 'black'
+ else:
+ stackwidget['color'] = 'black'
+
+
+def app():
+ """
+ Create a shift reduce parser app, using a simple grammar and
+ text.
+ """
+
+ from nltk.grammar import Nonterminal, Production, CFG
+ nonterminals = 'S VP NP PP P N Name V Det'
+ (S, VP, NP, PP, P, N, Name, V, Det) = [Nonterminal(s)
+ for s in nonterminals.split()]
+
+ productions = (
+ # Syntactic Productions
+ Production(S, [NP, VP]),
+ Production(NP, [Det, N]),
+ Production(NP, [NP, PP]),
+ Production(VP, [VP, PP]),
+ Production(VP, [V, NP, PP]),
+ Production(VP, [V, NP]),
+ Production(PP, [P, NP]),
+
+ # Lexical Productions
+ Production(NP, ['I']), Production(Det, ['the']),
+ Production(Det, ['a']), Production(N, ['man']),
+ Production(V, ['saw']), Production(P, ['in']),
+ Production(P, ['with']), Production(N, ['park']),
+ Production(N, ['dog']), Production(N, ['statue']),
+ Production(Det, ['my']),
+ )
+
+ grammar = CFG(S, productions)
+
+ # tokenize the sentence
+ sent = 'my dog saw a man in the park with a statue'.split()
+
+ ShiftReduceApp(grammar, sent).mainloop()
+
+if __name__ == '__main__':
+ app()
+
+__all__ = ['app']
diff --git a/nltk/app/wordfreq_app.py b/nltk/app/wordfreq_app.py
new file mode 100644
index 0000000..640d5b4
--- /dev/null
+++ b/nltk/app/wordfreq_app.py
@@ -0,0 +1,32 @@
+# Natural Language Toolkit: Wordfreq Application
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Sumukh Ghodke <sghodke at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import pylab
+import nltk.text
+from nltk.corpus import gutenberg
+
+def plot_word_freq_dist(text):
+ fd = text.vocab()
+
+ samples = fd.keys()[:50]
+ values = [fd[sample] for sample in samples]
+ values = [sum(values[:i+1]) * 100.0/fd.N() for i in range(len(values))]
+ pylab.title(text.name)
+ pylab.xlabel("Samples")
+ pylab.ylabel("Cumulative Percentage")
+ pylab.plot(values)
+ pylab.xticks(range(len(samples)), [str(s) for s in samples], rotation=90)
+ pylab.show()
+
+def app():
+ t1 = nltk.Text(gutenberg.words('melville-moby_dick.txt'))
+ plot_word_freq_dist(t1)
+
+if __name__ == '__main__':
+ app()
+
+__all__ = ['app']
diff --git a/nltk/app/wordnet_app.py b/nltk/app/wordnet_app.py
new file mode 100644
index 0000000..687ba1f
--- /dev/null
+++ b/nltk/app/wordnet_app.py
@@ -0,0 +1,968 @@
+# Natural Language Toolkit: WordNet Browser Application
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Jussi Salmela <jtsalmela at users.sourceforge.net>
+# Paul Bone <pbone at students.csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A WordNet Browser application which launches the default browser
+(if it is not already running) and opens a new tab with a connection
+to http://localhost:port/ . It also starts an HTTP server on the
+specified port and begins serving browser requests. The default
+port is 8000. (For command-line help, run "python wordnet -h")
+This application requires that the user's web browser supports
+Javascript.
+
+BrowServer is a server for browsing the NLTK Wordnet database It first
+launches a browser client to be used for browsing and then starts
+serving the requests of that and maybe other clients
+
+Usage::
+
+ browserver.py -h
+ browserver.py [-s] [-p <port>]
+
+Options::
+
+ -h or --help
+ Display this help message.
+
+ -l <file> or --log-file <file>
+ Logs messages to the given file, If this option is not specified
+ messages are silently dropped.
+
+ -p <port> or --port <port>
+ Run the web server on this TCP port, defaults to 8000.
+
+ -s or --server-mode
+ Do not start a web browser, and do not allow a user to
+ shotdown the server through the web interface.
+"""
+# TODO: throughout this package variable names and docstrings need
+# modifying to be compliant with NLTK's coding standards. Tests also
+# need to be develop to ensure this continues to work in the face of
+# changes to other NLTK packages.
+from __future__ import print_function
+
+# Allow this program to run inside the NLTK source tree.
+from sys import path
+
+import os
+import sys
+from sys import argv
+from collections import defaultdict
+import webbrowser
+import datetime
+import re
+import threading
+import time
+import getopt
+import base64
+import pickle
+import copy
+
+from nltk import compat
+from nltk.corpus import wordnet as wn
+from nltk.corpus.reader.wordnet import Synset, Lemma
+
+if compat.PY3:
+ from http.server import HTTPServer, BaseHTTPRequestHandler
+else:
+ from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
+
+# now included in local file
+# from util import html_header, html_trailer, \
+# get_static_index_page, get_static_page_by_path, \
+# page_from_word, page_from_href
+
+firstClient = True
+
+# True if we're not also running a web browser. The value f server_mode
+# gets set by demo().
+server_mode = None
+
+# If set this is a file object for writting log messages.
+logfile = None
+
+
+class MyServerHandler(BaseHTTPRequestHandler):
+
+ def do_HEAD(self):
+ self.send_head()
+
+ def do_GET(self):
+ global firstClient
+ sp = self.path[1:]
+ if compat.unquote_plus(sp) == 'SHUTDOWN THE SERVER':
+ if server_mode:
+ page = "Server must be killed with SIGTERM."
+ type = "text/plain"
+ else:
+ print('Server shutting down!')
+ os._exit(0)
+
+ elif sp == '': # First request.
+ type = 'text/html'
+ if not server_mode and firstClient:
+ firstClient = False
+ page = get_static_index_page(True)
+ else:
+ page = get_static_index_page(False)
+ word = 'green'
+
+ elif sp.endswith('.html'): # Trying to fetch a HTML file TODO:
+ type = 'text/html'
+ usp = compat.unquote_plus(sp)
+ if usp == 'NLTK Wordnet Browser Database Info.html':
+ word = '* Database Info *'
+ if os.path.isfile(usp):
+ with open(usp, 'r') as infile:
+ page = infile.read()
+ else:
+ page = (html_header % word) + \
+ '<p>The database info file:'\
+ '<p><b>' + usp + '</b>' + \
+ '<p>was not found. Run this:' + \
+ '<p><b>python dbinfo_html.py</b>' + \
+ '<p>to produce it.' + html_trailer
+ else:
+ # Handle files here.
+ word = sp
+ page = get_static_page_by_path(usp)
+ elif sp.startswith("search"):
+ # This doesn't seem to work with MWEs.
+ type = 'text/html'
+ parts = (sp.split("?")[1]).split("&")
+ word = [p.split("=")[1].replace("+", " ")
+ for p in parts if p.startswith("nextWord")][0]
+ page, word = page_from_word(word)
+ elif sp.startswith("lookup_"):
+ # TODO add a variation of this that takes a non ecoded word or MWE.
+ type = 'text/html'
+ sp = sp[len("lookup_"):]
+ page, word = page_from_href(sp)
+ elif sp == "start_page":
+ # if this is the first request we should display help
+ # information, and possibly set a default word.
+ type = 'text/html'
+ page, word = page_from_word("wordnet")
+ else:
+ type = 'text/plain'
+ page = "Could not parse request: '%s'" % sp
+
+ # Send result.
+ self.send_head(type)
+ self.wfile.write(page.encode('utf8'))
+
+
+ def send_head(self, type=None):
+ self.send_response(200)
+ self.send_header('Content-type', type)
+ self.end_headers()
+
+ def log_message(self, format, *args):
+ global logfile
+
+ if logfile:
+ logfile.write(
+ "%s - - [%s] %s\n" %
+ (self.address_string(),
+ self.log_date_time_string(),
+ format%args))
+
+
+def get_unique_counter_from_url(sp):
+ """
+ Extract the unique counter from the URL if it has one. Otherwise return
+ null.
+ """
+ pos = sp.rfind('%23')
+ if pos != -1:
+ return int(sp[(pos + 3):])
+ else:
+ return None
+
+
+def wnb(port=8000, runBrowser=True, logfilename=None):
+ """
+ Run NLTK Wordnet Browser Server.
+
+ :param port: The port number for the server to listen on, defaults to
+ 8000
+ :type port: int
+
+ :param runBrowser: True to start a web browser and point it at the web
+ server.
+ :type runBrowser: bool
+ """
+ # The webbrowser module is unpredictable, typically it blocks if it uses
+ # a console web browser, and doesn't block if it uses a GUI webbrowser,
+ # so we need to force it to have a clear correct behaviour.
+ #
+ # Normally the server should run for as long as the user wants. they
+ # should idealy be able to control this from the UI by closing the
+ # window or tab. Second best would be clicking a button to say
+ # 'Shutdown' that first shutsdown the server and closes the window or
+ # tab, or exits the text-mode browser. Both of these are unfreasable.
+ #
+ # The next best alternative is to start the server, have it close when
+ # it receives SIGTERM (default), and run the browser as well. The user
+ # may have to shutdown both programs.
+ #
+ # Since webbrowser may block, and the webserver will block, we must run
+ # them in separate threads.
+ #
+ global server_mode, logfile
+ server_mode = not runBrowser
+
+ # Setup logging.
+ if logfilename:
+ try:
+ logfile = open(logfilename, "a", 1) # 1 means 'line buffering'
+ except IOError as e:
+ sys.stderr.write("Couldn't open %s for writing: %s",
+ logfilename, e)
+ sys.exit(1)
+ else:
+ logfile = None
+
+ # Compute URL and start web browser
+ url = 'http://localhost:' + str(port)
+
+ server_ready = None
+ browser_thread = None
+
+ if runBrowser:
+ server_ready = threading.Event()
+ browser_thread = startBrowser(url, server_ready)
+
+ # Start the server.
+ server = HTTPServer(('', port), MyServerHandler)
+ if logfile:
+ logfile.write(
+ 'NLTK Wordnet browser server running serving: %s\n' % url)
+ if runBrowser:
+ server_ready.set()
+
+ try:
+ server.serve_forever()
+ except KeyboardInterrupt:
+ pass
+
+ if runBrowser:
+ browser_thread.join()
+
+ if logfile:
+ logfile.close()
+
+
+def startBrowser(url, server_ready):
+ def run():
+ server_ready.wait()
+ time.sleep(1) # Wait a little bit more, there's still the chance of
+ # a race condition.
+ webbrowser.open(url, new = 2, autoraise = 1)
+ t = threading.Thread(target=run)
+ t.start()
+ return t
+
+#####################################################################
+# Utilities
+#####################################################################
+
+
+"""
+WordNet Browser Utilities.
+
+This provides a backend to both wxbrowse and browserver.py.
+"""
+
+
+################################################################################
+#
+# Main logic for wordnet browser.
+#
+
+# This is wrapped inside a function since wn is only available if the
+# WordNet corpus is installed.
+def _pos_tuples():
+ return [
+ (wn.NOUN,'N','noun'),
+ (wn.VERB,'V','verb'),
+ (wn.ADJ,'J','adj'),
+ (wn.ADV,'R','adv')]
+
+def _pos_match(pos_tuple):
+ """
+ This function returns the complete pos tuple for the partial pos
+ tuple given to it. It attempts to match it against the first
+ non-null component of the given pos tuple.
+ """
+ if pos_tuple[0] == 's':
+ pos_tuple = ('a', pos_tuple[1], pos_tuple[2])
+ for n,x in enumerate(pos_tuple):
+ if x is not None:
+ break
+ for pt in _pos_tuples():
+ if pt[n] == pos_tuple[n]: return pt
+ return None
+
+
+HYPONYM = 0
+HYPERNYM = 1
+CLASS_REGIONAL = 2
+PART_HOLONYM = 3
+PART_MERONYM = 4
+ATTRIBUTE = 5
+SUBSTANCE_HOLONYM = 6
+SUBSTANCE_MERONYM = 7
+MEMBER_HOLONYM = 8
+MEMBER_MERONYM = 9
+VERB_GROUP = 10
+INSTANCE_HYPONYM = 12
+INSTANCE_HYPERNYM = 13
+CAUSE = 14
+ALSO_SEE = 15
+SIMILAR = 16
+ENTAILMENT = 17
+ANTONYM = 18
+FRAMES = 19
+PERTAINYM = 20
+
+CLASS_CATEGORY = 21
+CLASS_USAGE = 22
+CLASS_REGIONAL = 23
+CLASS_USAGE = 24
+CLASS_CATEGORY = 11
+
+DERIVATIONALLY_RELATED_FORM = 25
+
+INDIRECT_HYPERNYMS = 26
+
+
+def lemma_property(word, synset, func):
+
+ def flattern(l):
+ if l == []:
+ return []
+ else:
+ return l[0] + flattern(l[1:])
+
+ return flattern([func(l) for l in synset.lemmas if l.name == word])
+
+
+def rebuild_tree(orig_tree):
+ node = orig_tree[0]
+ children = orig_tree[1:]
+ return (node, [rebuild_tree(t) for t in children])
+
+
+def get_relations_data(word, synset):
+ """
+ Get synset relations data for a synset. Note that this doesn't
+ yet support things such as full hyponym vs direct hyponym.
+ """
+ if synset.pos() == wn.NOUN:
+ return ((HYPONYM, 'Hyponyms',
+ synset.hyponyms()),
+ (INSTANCE_HYPONYM , 'Instance hyponyms',
+ synset.instance_hyponyms()),
+ (HYPERNYM, 'Direct hypernyms',
+ synset.hypernyms()),
+ (INDIRECT_HYPERNYMS, 'Indirect hypernyms',
+ rebuild_tree(synset.tree(lambda x: x.hypernyms()))[1]),
+# hypernyms', 'Sister terms',
+ (INSTANCE_HYPERNYM , 'Instance hypernyms',
+ synset.instance_hypernyms()),
+# (CLASS_REGIONAL, ['domain term region'], ),
+ (PART_HOLONYM, 'Part holonyms',
+ synset.part_holonyms()),
+ (PART_MERONYM, 'Part meronyms',
+ synset.part_meronyms()),
+ (SUBSTANCE_HOLONYM, 'Substance holonyms',
+ synset.substance_holonyms()),
+ (SUBSTANCE_MERONYM, 'Substance meronyms',
+ synset.substance_meronyms()),
+ (MEMBER_HOLONYM, 'Member holonyms',
+ synset.member_holonyms()),
+ (MEMBER_MERONYM, 'Member meronyms',
+ synset.member_meronyms()),
+ (ATTRIBUTE, 'Attributes',
+ synset.attributes()),
+ (ANTONYM, "Antonyms",
+ lemma_property(word, synset, lambda l: l.antonyms())),
+ (DERIVATIONALLY_RELATED_FORM, "Derivationally related form",
+ lemma_property(word, synset, lambda l: l.derivationally_related_forms())))
+ elif synset.pos() == wn.VERB:
+ return ((ANTONYM, 'Antonym',
+ lemma_property(word, synset, lambda l: l.antonyms())),
+ (HYPONYM, 'Hyponym',
+ synset.hyponyms()),
+ (HYPERNYM, 'Direct hypernyms',
+ synset.hypernyms()),
+ (INDIRECT_HYPERNYMS, 'Indirect hypernyms',
+ rebuild_tree(synset.tree(lambda x: x.hypernyms()))[1]),
+ (ENTAILMENT, 'Entailments',
+ synset.entailments()),
+ (CAUSE, 'Causes',
+ synset.causes()),
+ (ALSO_SEE, 'Also see',
+ synset.also_sees()),
+ (VERB_GROUP, 'Verb Groups',
+ synset.verb_groups()),
+ (DERIVATIONALLY_RELATED_FORM, "Derivationally related form",
+ lemma_property(word, synset, lambda l: l.derivationally_related_forms())))
+ elif synset.pos() == wn.ADJ or synset.pos == wn.ADJ_SAT:
+ return ((ANTONYM, 'Antonym',
+ lemma_property(word, synset, lambda l: l.antonyms())),
+ (SIMILAR, 'Similar to',
+ synset.similar_tos()),
+ # Participle of verb - not supported by corpus
+ (PERTAINYM, 'Pertainyms',
+ lemma_property(word, synset, lambda l: l.pertainyms())),
+ (ATTRIBUTE, 'Attributes',
+ synset.attributes()),
+ (ALSO_SEE, 'Also see',
+ synset.also_sees()))
+ elif synset.pos() == wn.ADV:
+ # This is weird. adverbs such as 'quick' and 'fast' don't seem
+ # to have antonyms returned by the corpus.a
+ return ((ANTONYM, 'Antonym',
+ lemma_property(word, synset, lambda l: l.antonyms())),)
+ # Derived from adjective - not supported by corpus
+ else:
+ raise TypeError("Unhandles synset POS type: " + str(synset.pos()))
+
+
+html_header = '''
+<!DOCTYPE html PUBLIC '-//W3C//DTD HTML 4.01//EN'
+'http://www.w3.org/TR/html4/strict.dtd'>
+<html>
+<head>
+<meta name='generator' content=
+'HTML Tidy for Windows (vers 14 February 2006), see www.w3.org'>
+<meta http-equiv='Content-Type' content=
+'text/html; charset=us-ascii'>
+<title>NLTK Wordnet Browser display of: %s</title></head>
+<body bgcolor='#F5F5F5' text='#000000'>
+'''
+html_trailer = '''
+</body>
+</html>
+'''
+
+explanation = '''
+<h3>Search Help</h3>
+<ul><li>The display below the line is an example of the output the browser
+shows you when you enter a search word. The search word was <b>green</b>.</li>
+<li>The search result shows for different parts of speech the <b>synsets</b>
+i.e. different meanings for the word.</li>
+<li>All underlined texts are hypertext links. There are two types of links:
+word links and others. Clicking a word link carries out a search for the word
+in the Wordnet database.</li>
+<li>Clicking a link of the other type opens a display section of data attached
+to that link. Clicking that link a second time closes the section again.</li>
+<li>Clicking <u>S:</u> opens a section showing the relations for that synset.
+</li>
+<li>Clicking on a relation name opens a section that displays the associated
+synsets.</li>
+<li>Type a search word in the <b>Word</b> field and start the search by the
+<b>Enter/Return</b> key or click the <b>Search</b> button.</li>
+</ul>
+<hr width='100%'>
+'''
+
+# HTML oriented functions
+
+def _bold(txt): return '<b>%s</b>' % txt
+
+def _center(txt): return '<center>%s</center>' % txt
+
+def _hlev(n,txt): return '<h%d>%s</h%d>' % (n,txt,n)
+
+def _italic(txt): return '<i>%s</i>' % txt
+
+def _li(txt): return '<li>%s</li>' % txt
+
+def pg(word, body):
+ '''
+ Return a HTML page of NLTK Browser format constructed from the
+ word and body
+
+ :param word: The word that the body corresponds to
+ :type word: str
+ :param body: The HTML body corresponding to the word
+ :type body: str
+ :return: a HTML page for the word-body combination
+ :rtype: str
+ '''
+ return (html_header % word) + body + html_trailer
+
+def _ul(txt): return '<ul>' + txt + '</ul>'
+
+def _abbc(txt):
+ """
+ abbc = asterisks, breaks, bold, center
+ """
+ return _center(_bold('<br>'*10 + '*'*10 + ' ' + txt + ' ' + '*'*10))
+
+full_hyponym_cont_text = \
+ _ul(_li(_italic('(has full hyponym continuation)'))) + '\n'
+
+
+def _get_synset(synset_key):
+ """
+ The synset key is the unique name of the synset, this can be
+ retrived via synset.name()
+ """
+ return wn.synset(synset_key)
+
+def _collect_one_synset(word, synset, synset_relations):
+ '''
+ Returns the HTML string for one synset or word
+
+ :param word: the current word
+ :type word: str
+ :param synset: a synset
+ :type synset: synset
+ :param synset_relations: information about which synset relations
+ to display.
+ :type synset_relations: dict(synset_key, set(relation_id))
+ :return: The HTML string built for this synset
+ :rtype: str
+ '''
+ if isinstance(synset, tuple): # It's a word
+ raise NotImplementedError("word not supported by _collect_one_synset")
+
+ typ = 'S'
+ pos_tuple = _pos_match((synset.pos(), None, None))
+ assert pos_tuple is not None, "pos_tuple is null: synset.pos(): %s" % synset.pos()
+ descr = pos_tuple[2]
+ ref = copy.deepcopy(Reference(word, synset_relations))
+ ref.toggle_synset(synset)
+ synset_label = typ + ";"
+ if synset.name() in synset_relations:
+ synset_label = _bold(synset_label)
+ s = '<li>%s (%s) ' % (make_lookup_link(ref, synset_label), descr)
+ def format_lemma(w):
+ w = w.replace('_', ' ')
+ if w.lower() == word:
+ return _bold(w)
+ else:
+ ref = Reference(w)
+ return make_lookup_link(ref, w)
+
+ s += ', '.join(format_lemma(l.name()) for l in synset.lemmas())
+
+ gl = " (%s) <i>%s</i> " % \
+ (synset.definition(),
+ "; ".join("\"%s\"" % e for e in synset.examples()))
+ return s + gl + _synset_relations(word, synset, synset_relations) + '</li>\n'
+
+def _collect_all_synsets(word, pos, synset_relations=dict()):
+ """
+ Return a HTML unordered list of synsets for the given word and
+ part of speech.
+ """
+ return '<ul>%s\n</ul>\n' % \
+ ''.join((_collect_one_synset(word, synset, synset_relations)
+ for synset
+ in wn.synsets(word, pos)))
+
+def _synset_relations(word, synset, synset_relations):
+ '''
+ Builds the HTML string for the relations of a synset
+
+ :param word: The current word
+ :type word: str
+ :param synset: The synset for which we're building the relations.
+ :type synset: Synset
+ :param synset_relations: synset keys and relation types for which to display relations.
+ :type synset_relations: dict(synset_key, set(relation_type))
+ :return: The HTML for a synset's relations
+ :rtype: str
+ '''
+
+ if not synset.name() in synset_relations:
+ return ""
+ ref = Reference(word, synset_relations)
+
+ def relation_html(r):
+ if isinstance(r, Synset):
+ return make_lookup_link(Reference(r.lemma_names()[0]), r.lemma_names()[0])
+ elif isinstance(r, Lemma):
+ return relation_html(r.synset())
+ elif isinstance(r, tuple):
+ # It's probably a tuple containing a Synset and a list of
+ # similar tuples. This forms a tree of synsets.
+ return "%s\n<ul>%s</ul>\n" % \
+ (relation_html(r[0]),
+ ''.join('<li>%s</li>\n' % relation_html(sr) for sr in r[1]))
+ else:
+ raise TypeError("r must be a synset, lemma or list, it was: type(r) = %s, r = %s" % (type(r), r))
+
+ def make_synset_html(db_name, disp_name, rels):
+ synset_html = '<i>%s</i>\n' % \
+ make_lookup_link(
+ copy.deepcopy(ref).toggle_synset_relation(synset, db_name).encode(),
+ disp_name)
+
+ if db_name in ref.synset_relations[synset.name()]:
+ synset_html += '<ul>%s</ul>\n' % \
+ ''.join("<li>%s</li>\n" % relation_html(r) for r in rels)
+
+ return synset_html
+
+ html = '<ul>' + \
+ '\n'.join(("<li>%s</li>" % make_synset_html(*rel_data) for rel_data
+ in get_relations_data(word, synset)
+ if rel_data[2] != [])) + \
+ '</ul>'
+
+ return html
+
+
+class Reference(object):
+ """
+ A reference to a page that may be generated by page_word
+ """
+
+ def __init__(self, word, synset_relations=dict()):
+ """
+ Build a reference to a new page.
+
+ word is the word or words (separated by commas) for which to
+ search for synsets of
+
+ synset_relations is a dictionary of synset keys to sets of
+ synset relation identifaiers to unfold a list of synset
+ relations for.
+ """
+ self.word = word
+ self.synset_relations = synset_relations
+
+ def encode(self):
+ """
+ Encode this reference into a string to be used in a URL.
+ """
+ # This uses a tuple rather than an object since the python
+ # pickle representation is much smaller and there is no need
+ # to represent the complete object.
+ string = pickle.dumps((self.word, self.synset_relations), -1)
+ return base64.urlsafe_b64encode(string).decode()
+
+ @staticmethod
+ def decode(string):
+ """
+ Decode a reference encoded with Reference.encode
+ """
+ string = base64.urlsafe_b64decode(string.encode())
+ word, synset_relations = pickle.loads(string)
+ return Reference(word, synset_relations)
+
+ def toggle_synset_relation(self, synset, relation):
+ """
+ Toggle the display of the relations for the given synset and
+ relation type.
+
+ This function will throw a KeyError if the synset is currently
+ not being displayed.
+ """
+ if relation in self.synset_relations[synset.name()]:
+ self.synset_relations[synset.name()].remove(relation)
+ else:
+ self.synset_relations[synset.name()].add(relation)
+
+ return self
+
+ def toggle_synset(self, synset):
+ """
+ Toggle displaying of the relation types for the given synset
+ """
+ if synset.name() in self.synset_relations:
+ del self.synset_relations[synset.name()]
+ else:
+ self.synset_relations[synset.name()] = set()
+
+ return self
+
+
+def make_lookup_link(ref, label):
+ return '<a href="lookup_%s">%s</a>' % (ref.encode(), label)
+
+
+def page_from_word(word):
+ """
+ Return a HTML page for the given word.
+
+ :param word: The currently active word
+ :type word: str
+ :return: A tuple (page,word), where page is the new current HTML page
+ to be sent to the browser and
+ word is the new current word
+ :rtype: A tuple (str,str)
+ """
+ return page_from_reference(Reference(word))
+
+def page_from_href(href):
+ '''
+ Returns a tuple of the HTML page built and the new current word
+
+ :param href: The hypertext reference to be solved
+ :type href: str
+ :return: A tuple (page,word), where page is the new current HTML page
+ to be sent to the browser and
+ word is the new current word
+ :rtype: A tuple (str,str)
+ '''
+ return page_from_reference(Reference.decode(href))
+
+def page_from_reference(href):
+ '''
+ Returns a tuple of the HTML page built and the new current word
+
+ :param href: The hypertext reference to be solved
+ :type href: str
+ :return: A tuple (page,word), where page is the new current HTML page
+ to be sent to the browser and
+ word is the new current word
+ :rtype: A tuple (str,str)
+ '''
+ word = href.word
+ pos_forms = defaultdict(list)
+ words = word.split(',')
+ words = [w for w in [w.strip().lower().replace(' ', '_')
+ for w in words]
+ if w != ""]
+ if len(words) == 0:
+ # No words were found.
+ return "", "Please specify a word to search for."
+
+ # This looks up multiple words at once. This is probably not
+ # necessary and may lead to problems.
+ for w in words:
+ for pos in [wn.NOUN, wn.VERB, wn.ADJ, wn.ADV]:
+ form = wn.morphy(w, pos)
+ if form and form not in pos_forms[pos]:
+ pos_forms[pos].append(form)
+ body = ''
+ for pos,pos_str,name in _pos_tuples():
+ if pos in pos_forms:
+ body += _hlev(3, name) + '\n'
+ for w in pos_forms[pos]:
+ # Not all words of exc files are in the database, skip
+ # to the next word if a KeyError is raised.
+ try:
+ body += _collect_all_synsets(w, pos, href.synset_relations)
+ except KeyError:
+ pass
+ if not body:
+ body = "The word or words '%s' where not found in the dictonary." % word
+ return body, word
+
+
+
+#####################################################################
+# Static pages
+#####################################################################
+
+def get_static_page_by_path(path):
+ """
+ Return a static HTML page from the path given.
+ """
+ if path == "index_2.html":
+ return get_static_index_page(False)
+ elif path == "index.html":
+ return get_static_index_page(True)
+ elif path == "NLTK Wordnet Browser Database Info.html":
+ return "Display of Wordnet Database Statistics is not supported"
+ elif path == "upper_2.html":
+ return get_static_upper_page(False)
+ elif path == "upper.html":
+ return get_static_upper_page(True)
+ elif path == "web_help.html":
+ return get_static_web_help_page()
+ elif path == "wx_help.html":
+ return get_static_wx_help_page()
+ else:
+ return "Internal error: Path for static page '%s' is unknown" % path
+
+
+def get_static_web_help_page():
+ """
+ Return the static web help page.
+ """
+ return \
+"""
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+ <!-- Natural Language Toolkit: Wordnet Interface: Graphical Wordnet Browser
+ Copyright (C) 2001-2014 NLTK Project
+ Author: Jussi Salmela <jtsalmela at users.sourceforge.net>
+ URL: <http://nltk.org/>
+ For license information, see LICENSE.TXT -->
+ <head>
+ <meta http-equiv='Content-Type' content='text/html; charset=us-ascii'>
+ <title>NLTK Wordnet Browser display of: * Help *</title>
+ </head>
+<body bgcolor='#F5F5F5' text='#000000'>
+<h2>NLTK Wordnet Browser Help</h2>
+<p>The NLTK Wordnet Browser is a tool to use in browsing the Wordnet database. It tries to behave like the Wordnet project's web browser but the difference is that the NLTK Wordnet Browser uses a local Wordnet database.
+<p><b>You are using the Javascript client part of the NLTK Wordnet BrowseServer.</b> We assume your browser is in tab sheets enabled mode.</p>
+<p>For background information on Wordnet, see the Wordnet project home page: <a href="http://wordnet.princeton.edu/"><b> http://wordnet.princeton.edu/</b></a>. For more information on the NLTK project, see the project home:
+<a href="http://nltk.sourceforge.net/"><b>http://nltk.sourceforge.net/</b></a>. To get an idea of what the Wordnet version used by this browser includes choose <b>Show Database Info</b> from the <b>View</b> submenu.</p>
+<h3>Word search</h3>
+<p>The word to be searched is typed into the <b>New Word</b> field and the search started with Enter or by clicking the <b>Search</b> button. There is no uppercase/lowercase distinction: the search word is transformed to lowercase before the search.</p>
+<p>In addition, the word does not have to be in base form. The browser tries to find the possible base form(s) by making certain morphological substitutions. Typing <b>fLIeS</b> as an obscure example gives one <a href="MfLIeS">this</a>. Click the previous link to see what this kind of search looks like and then come back to this page by using the <b>Alt+LeftArrow</b> key combination.</p>
+<p>The result of a search is a display of one or more
+<b>synsets</b> for every part of speech in which a form of the
+search word was found to occur. A synset is a set of words
+having the same sense or meaning. Each word in a synset that is
+underlined is a hyperlink which can be clicked to trigger an
+automatic search for that word.</p>
+<p>Every synset has a hyperlink <b>S:</b> at the start of its
+display line. Clicking that symbol shows you the name of every
+<b>relation</b> that this synset is part of. Every relation name is a hyperlink that opens up a display for that relation. Clicking it another time closes the display again. Clicking another relation name on a line that has an opened relation closes the open relation and opens the clicked relation.</p>
+<p>It is also possible to give two or more words or collocations to be searched at the same time separating them with a comma like this <a href="Mcheer up,clear up">cheer up,clear up</a>, for example. Click the previous link to see what this kind of search looks like and then come back to this page by using the <b>Alt+LeftArrow</b> key combination. As you could see the search result includes the synsets found in the same order than the forms were given in the search field.</p>
+<p>
+There are also word level (lexical) relations recorded in the Wordnet database. Opening this kind of relation displays lines with a hyperlink <b>W:</b> at their beginning. Clicking this link shows more info on the word in question.</p>
+<h3>The Buttons</h3>
+<p>The <b>Search</b> and <b>Help</b> buttons need no more explanation. </p>
+<p>The <b>Show Database Info</b> button shows a collection of Wordnet database statistics.</p>
+<p>The <b>Shutdown the Server</b> button is shown for the first client of the BrowServer program i.e. for the client that is automatically launched when the BrowServer is started but not for the succeeding clients in order to protect the server from accidental shutdowns.
+</p></body>
+</html>
+"""
+
+
+def get_static_welcome_message():
+ """
+ Get the static welcome page.
+ """
+ return \
+"""
+<h3>Search Help</h3>
+<ul><li>The display below the line is an example of the output the browser
+shows you when you enter a search word. The search word was <b>green</b>.</li>
+<li>The search result shows for different parts of speech the <b>synsets</b>
+i.e. different meanings for the word.</li>
+<li>All underlined texts are hypertext links. There are two types of links:
+word links and others. Clicking a word link carries out a search for the word
+in the Wordnet database.</li>
+<li>Clicking a link of the other type opens a display section of data attached
+to that link. Clicking that link a second time closes the section again.</li>
+<li>Clicking <u>S:</u> opens a section showing the relations for that synset.</li>
+<li>Clicking on a relation name opens a section that displays the associated
+synsets.</li>
+<li>Type a search word in the <b>Next Word</b> field and start the search by the
+<b>Enter/Return</b> key or click the <b>Search</b> button.</li>
+</ul>
+"""
+
+def get_static_index_page(with_shutdown):
+ """
+ Get the static index page.
+ """
+ template = \
+"""
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">
+<HTML>
+ <!-- Natural Language Toolkit: Wordnet Interface: Graphical Wordnet Browser
+ Copyright (C) 2001-2014 NLTK Project
+ Author: Jussi Salmela <jtsalmela at users.sourceforge.net>
+ URL: <http://nltk.org/>
+ For license information, see LICENSE.TXT -->
+ <HEAD>
+ <TITLE>NLTK Wordnet Browser</TITLE>
+ </HEAD>
+
+<frameset rows="7%%,93%%">
+ <frame src="%s" name="header">
+ <frame src="start_page" name="body">
+</frameset>
+</HTML>
+"""
+ if with_shutdown:
+ upper_link = "upper.html"
+ else:
+ upper_link = "upper_2.html"
+
+ return template % upper_link
+
+
+def get_static_upper_page(with_shutdown):
+ """
+ Return the upper frame page,
+
+ If with_shutdown is True then a 'shutdown' button is also provided
+ to shutdown the server.
+ """
+ template = \
+"""
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html>
+ <!-- Natural Language Toolkit: Wordnet Interface: Graphical Wordnet Browser
+ Copyright (C) 2001-2014 NLTK Project
+ Author: Jussi Salmela <jtsalmela at users.sourceforge.net>
+ URL: <http://nltk.org/>
+ For license information, see LICENSE.TXT -->
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+ <title>Untitled Document</title>
+ </head>
+ <body>
+ <form method="GET" action="search" target="body">
+ Current Word: <input type="text" id="currentWord" size="10" disabled>
+ Next Word: <input type="text" id="nextWord" name="nextWord" size="10">
+ <input name="searchButton" type="submit" value="Search">
+ </form>
+ <a target="body" href="web_help.html">Help</a>
+ %s
+
+</body>
+</html>
+"""
+ if with_shutdown:
+ shutdown_link = "<a href=\"SHUTDOWN THE SERVER\">Shutdown</a>"
+ else:
+ shutdown_link = ""
+
+ return template % shutdown_link
+
+
+
+def usage():
+ """
+ Display the command line help message.
+ """
+ print(__doc__)
+
+def app():
+ # Parse and interpret options.
+ (opts, _) = getopt.getopt(argv[1:], "l:p:sh",
+ ["logfile=", "port=", "server-mode", "help"])
+ port = 8000
+ server_mode = False
+ help_mode = False
+ logfilename = None
+ for (opt, value) in opts:
+ if (opt == "-l") or (opt == "--logfile"):
+ logfilename = str(value)
+ elif (opt == "-p") or (opt == "--port"):
+ port = int(value)
+ elif (opt == "-s") or (opt == "--server-mode"):
+ server_mode = True
+ elif (opt == "-h") or (opt == "--help"):
+ help_mode = True
+
+ if help_mode:
+ usage()
+ else:
+ wnb(port, not server_mode, logfilename)
+
+if __name__ == '__main__':
+ app()
+
+__all__ = ['app']
diff --git a/nltk/book.py b/nltk/book.py
new file mode 100644
index 0000000..588dec4
--- /dev/null
+++ b/nltk/book.py
@@ -0,0 +1,89 @@
+# Natural Language Toolkit: Some texts for exploration in chapter 1 of the book
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function
+
+from nltk.corpus import (gutenberg, genesis, inaugural,
+ nps_chat, webtext, treebank, wordnet)
+from nltk.text import Text
+from nltk.probability import FreqDist
+from nltk.util import bigrams
+from nltk.misc import babelize_shell
+
+print("*** Introductory Examples for the NLTK Book ***")
+print("Loading text1, ..., text9 and sent1, ..., sent9")
+print("Type the name of the text or sentence to view it.")
+print("Type: 'texts()' or 'sents()' to list the materials.")
+
+text1 = Text(gutenberg.words('melville-moby_dick.txt'))
+print("text1:", text1.name)
+
+text2 = Text(gutenberg.words('austen-sense.txt'))
+print("text2:", text2.name)
+
+text3 = Text(genesis.words('english-kjv.txt'), name="The Book of Genesis")
+print("text3:", text3.name)
+
+text4 = Text(inaugural.words(), name="Inaugural Address Corpus")
+print("text4:", text4.name)
+
+text5 = Text(nps_chat.words(), name="Chat Corpus")
+print("text5:", text5.name)
+
+text6 = Text(webtext.words('grail.txt'), name="Monty Python and the Holy Grail")
+print("text6:", text6.name)
+
+text7 = Text(treebank.words(), name="Wall Street Journal")
+print("text7:", text7.name)
+
+text8 = Text(webtext.words('singles.txt'), name="Personals Corpus")
+print("text8:", text8.name)
+
+text9 = Text(gutenberg.words('chesterton-thursday.txt'))
+print("text9:", text9.name)
+
+def texts():
+ print("text1:", text1.name)
+ print("text2:", text2.name)
+ print("text3:", text3.name)
+ print("text4:", text4.name)
+ print("text5:", text5.name)
+ print("text6:", text6.name)
+ print("text7:", text7.name)
+ print("text8:", text8.name)
+ print("text9:", text9.name)
+
+sent1 = ["Call", "me", "Ishmael", "."]
+sent2 = ["The", "family", "of", "Dashwood", "had", "long",
+ "been", "settled", "in", "Sussex", "."]
+sent3 = ["In", "the", "beginning", "God", "created", "the",
+ "heaven", "and", "the", "earth", "."]
+sent4 = ["Fellow", "-", "Citizens", "of", "the", "Senate",
+ "and", "of", "the", "House", "of", "Representatives", ":"]
+sent5 = ["I", "have", "a", "problem", "with", "people",
+ "PMing", "me", "to", "lol", "JOIN"]
+sent6 = ['SCENE', '1', ':', '[', 'wind', ']', '[', 'clop', 'clop',
+ 'clop', ']', 'KING', 'ARTHUR', ':', 'Whoa', 'there', '!']
+sent7 = ["Pierre", "Vinken", ",", "61", "years", "old", ",",
+ "will", "join", "the", "board", "as", "a", "nonexecutive",
+ "director", "Nov.", "29", "."]
+sent8 = ['25', 'SEXY', 'MALE', ',', 'seeks', 'attrac', 'older',
+ 'single', 'lady', ',', 'for', 'discreet', 'encounters', '.']
+sent9 = ["THE", "suburb", "of", "Saffron", "Park", "lay", "on", "the",
+ "sunset", "side", "of", "London", ",", "as", "red", "and",
+ "ragged", "as", "a", "cloud", "of", "sunset", "."]
+
+def sents():
+ print("sent1:", " ".join(sent1))
+ print("sent2:", " ".join(sent2))
+ print("sent3:", " ".join(sent3))
+ print("sent4:", " ".join(sent4))
+ print("sent5:", " ".join(sent5))
+ print("sent6:", " ".join(sent6))
+ print("sent7:", " ".join(sent7))
+ print("sent8:", " ".join(sent8))
+ print("sent9:", " ".join(sent9))
diff --git a/nltk/ccg/__init__.py b/nltk/ccg/__init__.py
new file mode 100644
index 0000000..ebd883a
--- /dev/null
+++ b/nltk/ccg/__init__.py
@@ -0,0 +1,22 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Graeme Gange <ggange at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Combinatory Categorial Grammar.
+
+For more information see nltk/doc/contrib/ccg/ccg.pdf
+"""
+
+from nltk.ccg.combinator import (UndirectedBinaryCombinator, DirectedBinaryCombinator,
+ ForwardCombinator, BackwardCombinator,
+ UndirectedFunctionApplication, ForwardApplication,
+ BackwardApplication, UndirectedComposition,
+ ForwardComposition, BackwardComposition,
+ BackwardBx, UndirectedSubstitution, ForwardSubstitution,
+ BackwardSx, UndirectedTypeRaise, ForwardT, BackwardT)
+from nltk.ccg.chart import CCGEdge, CCGLeafEdge, CCGChartParser, CCGChart
+from nltk.ccg.lexicon import CCGLexicon
diff --git a/nltk/ccg/api.py b/nltk/ccg/api.py
new file mode 100644
index 0000000..8578dd1
--- /dev/null
+++ b/nltk/ccg/api.py
@@ -0,0 +1,335 @@
+# Natural Language Toolkit: CCG Categories
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Graeme Gange <ggange at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import unicode_literals
+from nltk.internals import raise_unorderable_types
+from nltk.compat import (total_ordering, python_2_unicode_compatible,
+ unicode_repr)
+
+ at total_ordering
+class AbstractCCGCategory(object):
+ '''
+ Interface for categories in combinatory grammars.
+ '''
+
+ # Returns true if the category is primitive
+ def is_primitive(self):
+ raise NotImplementedError()
+
+ # Returns true if the category is a function application
+ def is_function(self):
+ raise NotImplementedError()
+
+ # Returns true if the category is a variable
+ def is_var(self):
+ raise NotImplementedError()
+
+ # Takes a set of (var, category) substitutions, and replaces every
+ # occurrence of the variable with the corresponding category
+ def substitute(self,substitutions):
+ raise NotImplementedError()
+
+ # Determines whether two categories can be unified.
+ # - Returns None if they cannot be unified
+ # - Returns a list of necessary substitutions if they can.'''
+ def can_unify(self,other):
+ raise NotImplementedError()
+
+ # Utility functions: comparison, strings and hashing.
+
+ def __str__(self):
+ raise NotImplementedError()
+
+ def __eq__(self, other):
+ return (self.__class__ is other.__class__ and
+ self._comparison_key == other._comparison_key)
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, AbstractCCGCategory):
+ raise_unorderable_types("<", self, other)
+ if self.__class__ is other.__class__:
+ return self._comparison_key < other._comparison_key
+ else:
+ return self.__class__.__name__ < other.__class__.__name__
+
+ def __hash__(self):
+ try:
+ return self._hash
+ except AttributeError:
+ self._hash = hash(self._comparison_key)
+ return self._hash
+
+
+ at python_2_unicode_compatible
+class CCGVar(AbstractCCGCategory):
+ '''
+ Class representing a variable CCG category.
+ Used for conjunctions (and possibly type-raising, if implemented as a
+ unary rule).
+ '''
+ _maxID = 0
+
+ def __init__(self, prim_only=False):
+ """Initialize a variable (selects a new identifier)
+
+ :param prim_only: a boolean that determines whether the variable is restricted to primitives
+ :type prim_only: bool
+ """
+ self._id = self.new_id()
+ self._prim_only = prim_only
+ self._comparison_key = self._id
+
+ @classmethod
+ def new_id(cls):
+ """A class method allowing generation of unique variable identifiers."""
+ cls._maxID = cls._maxID + 1
+ return cls._maxID - 1
+
+ def is_primitive(self):
+ return False
+
+ def is_function(self):
+ return False
+
+ def is_var(self):
+ return True
+
+ def substitute(self, substitutions):
+ """If there is a substitution corresponding to this variable,
+ return the substituted category.
+ """
+ for (var,cat) in substitutions:
+ if var == self:
+ return cat
+ return self
+
+ def can_unify(self, other):
+ """ If the variable can be replaced with other
+ a substitution is returned.
+ """
+ if other.is_primitive() or not self._prim_only:
+ return [(self,other)]
+ return None
+
+ def id(self):
+ return self._id
+
+ def __str__(self):
+ return "_var" + str(self._id)
+
+ at total_ordering
+ at python_2_unicode_compatible
+class Direction(object):
+ '''
+ Class representing the direction of a function application.
+ Also contains maintains information as to which combinators
+ may be used with the category.
+ '''
+ def __init__(self,dir,restrictions):
+ self._dir = dir
+ self._restrs = restrictions
+ self._comparison_key = (dir, tuple(restrictions))
+
+ # Testing the application direction
+ def is_forward(self):
+ return self._dir == '/'
+ def is_backward(self):
+ return self._dir == '\\'
+
+ def dir(self):
+ return self._dir
+
+ def restrs(self):
+ """A list of restrictions on the combinators.
+ '.' denotes that permuting operations are disallowed
+ ',' denotes that function composition is disallowed
+ '_' denotes that the direction has variable restrictions.
+ (This is redundant in the current implementation of type-raising)
+ """
+ return self._restrs
+
+ def is_variable(self):
+ return self._restrs == '_'
+
+ # Unification and substitution of variable directions.
+ # Used only if type-raising is implemented as a unary rule, as it
+ # must inherit restrictions from the argument category.
+ def can_unify(self,other):
+ if other.is_variable():
+ return [('_',self.restrs())]
+ elif self.is_variable():
+ return [('_',other.restrs())]
+ else:
+ if self.restrs() == other.restrs():
+ return []
+ return None
+
+ def substitute(self,subs):
+ if not self.is_variable():
+ return self
+
+ for (var, restrs) in subs:
+ if var == '_':
+ return Direction(self._dir,restrs)
+ return self
+
+ # Testing permitted combinators
+ def can_compose(self):
+ return not ',' in self._restrs
+
+ def can_cross(self):
+ return not '.' in self._restrs
+
+ def __eq__(self, other):
+ return (self.__class__ is other.__class__ and
+ self._comparison_key == other._comparison_key)
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, Direction):
+ raise_unorderable_types("<", self, other)
+ if self.__class__ is other.__class__:
+ return self._comparison_key < other._comparison_key
+ else:
+ return self.__class__.__name__ < other.__class__.__name__
+
+ def __hash__(self):
+ try:
+ return self._hash
+ except AttributeError:
+ self._hash = hash(self._comparison_key)
+ return self._hash
+
+ def __str__(self):
+ r_str = ""
+ for r in self._restrs:
+ r_str = r_str + "%s" % r
+ return "%s%s" % (self._dir, r_str)
+
+ # The negation operator reverses the direction of the application
+ def __neg__(self):
+ if self._dir == '/':
+ return Direction('\\',self._restrs)
+ else:
+ return Direction('/',self._restrs)
+
+
+ at python_2_unicode_compatible
+class PrimitiveCategory(AbstractCCGCategory):
+ '''
+ Class representing primitive categories.
+ Takes a string representation of the category, and a
+ list of strings specifying the morphological subcategories.
+ '''
+ def __init__(self, categ, restrictions=[]):
+ self._categ = categ
+ self._restrs = restrictions
+ self._comparison_key = (categ, tuple(restrictions))
+
+ def is_primitive(self):
+ return True
+
+ def is_function(self):
+ return False
+
+ def is_var(self):
+ return False
+
+ def restrs(self):
+ return self._restrs
+
+ def categ(self):
+ return self._categ
+
+ # Substitution does nothing to a primitive category
+ def substitute(self,subs):
+ return self
+
+ # A primitive can be unified with a class of the same
+ # base category, given that the other category shares all
+ # of its subclasses, or with a variable.
+ def can_unify(self,other):
+ if not other.is_primitive():
+ return None
+ if other.is_var():
+ return [(other,self)]
+ if other.categ() == self.categ():
+ for restr in self._restrs:
+ if restr not in other.restrs():
+ return None
+ return []
+ return None
+
+ def __str__(self):
+ if self._restrs == []:
+ return "%s" % self._categ
+ restrictions = "[%s]" % ",".join(unicode_repr(r) for r in self._restrs)
+ return "%s%s" % (self._categ, restrictions)
+
+
+ at python_2_unicode_compatible
+class FunctionalCategory(AbstractCCGCategory):
+ '''
+ Class that represents a function application category.
+ Consists of argument and result categories, together with
+ an application direction.
+ '''
+ def __init__(self, res, arg, dir):
+ self._res = res
+ self._arg = arg
+ self._dir = dir
+ self._comparison_key = (arg, dir, res)
+
+ def is_primitive(self):
+ return False
+
+ def is_function(self):
+ return True
+
+ def is_var(self):
+ return False
+
+ # Substitution returns the category consisting of the
+ # substitution applied to each of its constituents.
+ def substitute(self,subs):
+ sub_res = self._res.substitute(subs)
+ sub_dir = self._dir.substitute(subs)
+ sub_arg = self._arg.substitute(subs)
+ return FunctionalCategory(sub_res,sub_arg,self._dir)
+
+ # A function can unify with another function, so long as its
+ # constituents can unify, or with an unrestricted variable.
+ def can_unify(self,other):
+ if other.is_var():
+ return [(other,self)]
+ if other.is_function():
+ sa = self._res.can_unify(other.res())
+ sd = self._dir.can_unify(other.dir())
+ if sa is not None and sd is not None:
+ sb = self._arg.substitute(sa).can_unify(other.arg().substitute(sa))
+ if sb is not None:
+ return sa + sb
+ return None
+
+ # Constituent accessors
+ def arg(self):
+ return self._arg
+
+ def res(self):
+ return self._res
+
+ def dir(self):
+ return self._dir
+
+ def __str__(self):
+ return "(%s%s%s)" % (self._res, self._dir, self._arg)
+
+
diff --git a/nltk/ccg/chart.py b/nltk/ccg/chart.py
new file mode 100644
index 0000000..905d7e7
--- /dev/null
+++ b/nltk/ccg/chart.py
@@ -0,0 +1,361 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Graeme Gange <ggange at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+The lexicon is constructed by calling
+``lexicon.parseLexicon(<lexicon string>)``.
+
+In order to construct a parser, you also need a rule set.
+The standard English rules are provided in chart as
+``chart.DefaultRuleSet``.
+
+The parser can then be constructed by calling, for example:
+``parser = chart.CCGChartParser(<lexicon>, <ruleset>)``
+
+Parsing is then performed by running
+``parser.parse(<sentence>.split())``.
+
+While this returns a list of trees, the default representation
+of the produced trees is not very enlightening, particularly
+given that it uses the same tree class as the CFG parsers.
+It is probably better to call:
+``chart.printCCGDerivation(<parse tree extracted from list>)``
+which should print a nice representation of the derivation.
+
+This entire process is shown far more clearly in the demonstration:
+python chart.py
+"""
+from __future__ import print_function, division, unicode_literals
+
+import itertools
+
+from nltk.parse import ParserI
+from nltk.parse.chart import AbstractChartRule, EdgeI, Chart
+from nltk.tree import Tree
+
+from nltk.ccg.lexicon import parseLexicon
+from nltk.ccg.combinator import (ForwardT, BackwardT, ForwardApplication,
+ BackwardApplication, ForwardComposition,
+ BackwardComposition, ForwardSubstitution,
+ BackwardBx, BackwardSx)
+from nltk.compat import python_2_unicode_compatible, string_types
+
+# Based on the EdgeI class from NLTK.
+# A number of the properties of the EdgeI interface don't
+# transfer well to CCGs, however.
+class CCGEdge(EdgeI):
+ def __init__(self, span, categ, rule):
+ self._span = span
+ self._categ = categ
+ self._rule = rule
+ self._comparison_key = (span, categ, rule)
+
+ # Accessors
+ def lhs(self): return self._categ
+ def span(self): return self._span
+ def start(self): return self._span[0]
+ def end(self): return self._span[1]
+ def length(self): return self._span[1] - self.span[0]
+ def rhs(self): return ()
+ def dot(self): return 0
+ def is_complete(self): return True
+ def is_incomplete(self): return False
+ def nextsym(self): return None
+
+ def categ(self): return self._categ
+ def rule(self): return self._rule
+
+class CCGLeafEdge(EdgeI):
+ '''
+ Class representing leaf edges in a CCG derivation.
+ '''
+ def __init__(self, pos, categ, leaf):
+ self._pos = pos
+ self._categ = categ
+ self._leaf = leaf
+ self._comparison_key = (pos, categ, leaf)
+
+ # Accessors
+ def lhs(self): return self._categ
+ def span(self): return (self._pos, self._pos+1)
+ def start(self): return self._pos
+ def end(self): return self._pos + 1
+ def length(self): return 1
+ def rhs(self): return self._leaf
+ def dot(self): return 0
+ def is_complete(self): return True
+ def is_incomplete(self): return False
+ def nextsym(self): return None
+
+ def categ(self): return self._categ
+ def leaf(self): return self._leaf
+
+ at python_2_unicode_compatible
+class BinaryCombinatorRule(AbstractChartRule):
+ '''
+ Class implementing application of a binary combinator to a chart.
+ Takes the directed combinator to apply.
+ '''
+ NUMEDGES = 2
+ def __init__(self,combinator):
+ self._combinator = combinator
+
+ # Apply a combinator
+ def apply(self, chart, grammar, left_edge, right_edge):
+ # The left & right edges must be touching.
+ if not (left_edge.end() == right_edge.start()):
+ return
+
+ # Check if the two edges are permitted to combine.
+ # If so, generate the corresponding edge.
+ if self._combinator.can_combine(left_edge.categ(),right_edge.categ()):
+ for res in self._combinator.combine(left_edge.categ(), right_edge.categ()):
+ new_edge = CCGEdge(span=(left_edge.start(), right_edge.end()),categ=res,rule=self._combinator)
+ if chart.insert(new_edge,(left_edge,right_edge)):
+ yield new_edge
+
+ # The representation of the combinator (for printing derivations)
+ def __str__(self):
+ return "%s" % self._combinator
+
+# Type-raising must be handled slightly differently to the other rules, as the
+# resulting rules only span a single edge, rather than both edges.
+ at python_2_unicode_compatible
+class ForwardTypeRaiseRule(AbstractChartRule):
+ '''
+ Class for applying forward type raising
+ '''
+ NUMEDGES = 2
+
+ def __init__(self):
+ self._combinator = ForwardT
+ def apply(self, chart, grammar, left_edge, right_edge):
+ if not (left_edge.end() == right_edge.start()):
+ return
+
+ for res in self._combinator.combine(left_edge.categ(), right_edge.categ()):
+ new_edge = CCGEdge(span=left_edge.span(),categ=res,rule=self._combinator)
+ if chart.insert(new_edge,(left_edge,)):
+ yield new_edge
+
+ def __str__(self):
+ return "%s" % self._combinator
+
+ at python_2_unicode_compatible
+class BackwardTypeRaiseRule(AbstractChartRule):
+ '''
+ Class for applying backward type raising.
+ '''
+ NUMEDGES = 2
+
+ def __init__(self):
+ self._combinator = BackwardT
+ def apply(self, chart, grammar, left_edge, right_edge):
+ if not (left_edge.end() == right_edge.start()):
+ return
+
+ for res in self._combinator.combine(left_edge.categ(), right_edge.categ()):
+ new_edge = CCGEdge(span=right_edge.span(),categ=res,rule=self._combinator)
+ if chart.insert(new_edge,(right_edge,)):
+ yield new_edge
+
+ def __str__(self):
+ return "%s" % self._combinator
+
+
+# Common sets of combinators used for English derivations.
+ApplicationRuleSet = [BinaryCombinatorRule(ForwardApplication),
+ BinaryCombinatorRule(BackwardApplication)]
+CompositionRuleSet = [BinaryCombinatorRule(ForwardComposition),
+ BinaryCombinatorRule(BackwardComposition),
+ BinaryCombinatorRule(BackwardBx)]
+SubstitutionRuleSet = [BinaryCombinatorRule(ForwardSubstitution),
+ BinaryCombinatorRule(BackwardSx)]
+TypeRaiseRuleSet = [ForwardTypeRaiseRule(), BackwardTypeRaiseRule()]
+
+# The standard English rule set.
+DefaultRuleSet = ApplicationRuleSet + CompositionRuleSet + \
+ SubstitutionRuleSet + TypeRaiseRuleSet
+
+class CCGChartParser(ParserI):
+ '''
+ Chart parser for CCGs.
+ Based largely on the ChartParser class from NLTK.
+ '''
+ def __init__(self, lexicon, rules, trace=0):
+ self._lexicon = lexicon
+ self._rules = rules
+ self._trace = trace
+
+ def lexicon(self):
+ return self._lexicon
+
+ # Implements the CYK algorithm
+ def parse(self, tokens):
+ tokens = list(tokens)
+ chart = CCGChart(list(tokens))
+ lex = self._lexicon
+
+ # Initialize leaf edges.
+ for index in range(chart.num_leaves()):
+ for cat in lex.categories(chart.leaf(index)):
+ new_edge = CCGLeafEdge(index, cat, chart.leaf(index))
+ chart.insert(new_edge, ())
+
+
+ # Select a span for the new edges
+ for span in range(2,chart.num_leaves()+1):
+ for start in range(0,chart.num_leaves()-span+1):
+ # Try all possible pairs of edges that could generate
+ # an edge for that span
+ for part in range(1,span):
+ lstart = start
+ mid = start + part
+ rend = start + span
+
+ for left in chart.select(span=(lstart,mid)):
+ for right in chart.select(span=(mid,rend)):
+ # Generate all possible combinations of the two edges
+ for rule in self._rules:
+ edges_added_by_rule = 0
+ for newedge in rule.apply(chart,lex,left,right):
+ edges_added_by_rule += 1
+
+ # Output the resulting parses
+ return chart.parses(lex.start())
+
+class CCGChart(Chart):
+ def __init__(self, tokens):
+ Chart.__init__(self, tokens)
+
+ # Constructs the trees for a given parse. Unfortnunately, the parse trees need to be
+ # constructed slightly differently to those in the default Chart class, so it has to
+ # be reimplemented
+ def _trees(self, edge, complete, memo, tree_class):
+ assert complete, "CCGChart cannot build incomplete trees"
+
+ if edge in memo:
+ return memo[edge]
+
+ if isinstance(edge,CCGLeafEdge):
+ word = tree_class(edge.lhs(), [self._tokens[edge.start()]])
+ leaf = tree_class((edge.lhs(), "Leaf"), [word])
+ memo[edge] = [leaf]
+ return [leaf]
+
+ memo[edge] = []
+ trees = []
+ lhs = (edge.lhs(), "%s" % edge.rule())
+
+ for cpl in self.child_pointer_lists(edge):
+ child_choices = [self._trees(cp, complete, memo, tree_class)
+ for cp in cpl]
+ for children in itertools.product(*child_choices):
+ trees.append(tree_class(lhs, children))
+
+ memo[edge] = trees
+ return trees
+
+#--------
+# Displaying derivations
+#--------
+def printCCGDerivation(tree):
+ # Get the leaves and initial categories
+ leafcats = tree.pos()
+ leafstr = ''
+ catstr = ''
+
+ # Construct a string with both the leaf word and corresponding
+ # category aligned.
+ for (leaf, cat) in leafcats:
+ str_cat = "%s" % cat
+# print(cat.__class__)
+# print("str_cat", str_cat)
+ nextlen = 2 + max(len(leaf), len(str_cat))
+ lcatlen = (nextlen - len(str_cat)) // 2
+ rcatlen = lcatlen + (nextlen - len(str_cat)) % 2
+ catstr += ' '*lcatlen + str_cat + ' '*rcatlen
+ lleaflen = (nextlen - len(leaf)) // 2
+ rleaflen = lleaflen + (nextlen - len(leaf)) % 2
+ leafstr += ' '*lleaflen + leaf + ' '*rleaflen
+ print(leafstr)
+ print(catstr)
+
+ # Display the derivation steps
+ printCCGTree(0,tree)
+
+# Prints the sequence of derivation steps.
+def printCCGTree(lwidth,tree):
+ rwidth = lwidth
+
+ # Is a leaf (word).
+ # Increment the span by the space occupied by the leaf.
+ if not isinstance(tree,Tree):
+ return 2 + lwidth + len(tree)
+
+ # Find the width of the current derivation step
+ for child in tree:
+ rwidth = max(rwidth, printCCGTree(rwidth,child))
+
+ # Is a leaf node.
+ # Don't print anything, but account for the space occupied.
+ if not isinstance(tree.label(), tuple):
+ return max(rwidth,2 + lwidth + len("%s" % tree.label()),
+ 2 + lwidth + len(tree[0]))
+
+ (res,op) = tree.label()
+ # Pad to the left with spaces, followed by a sequence of '-'
+ # and the derivation rule.
+ print(lwidth*' ' + (rwidth-lwidth)*'-' + "%s" % op)
+ # Print the resulting category on a new line.
+ str_res = "%s" % res
+ respadlen = (rwidth - lwidth - len(str_res)) // 2 + lwidth
+ print(respadlen*' ' + str_res)
+ return rwidth
+
+
+### Demonstration code
+
+# Construct the lexicon
+lex = parseLexicon('''
+ :- S, NP, N, VP # Primitive categories, S is the target primitive
+
+ Det :: NP/N # Family of words
+ Pro :: NP
+ TV :: VP/NP
+ Modal :: (S\\NP)/VP # Backslashes need to be escaped
+
+ I => Pro # Word -> Category mapping
+ you => Pro
+
+ the => Det
+
+ # Variables have the special keyword 'var'
+ # '.' prevents permutation
+ # ',' prevents composition
+ and => var\\.,var/.,var
+
+ which => (N\\N)/(S/NP)
+
+ will => Modal # Categories can be either explicit, or families.
+ might => Modal
+
+ cook => TV
+ eat => TV
+
+ mushrooms => N
+ parsnips => N
+ bacon => N
+ ''')
+
+def demo():
+ parser = CCGChartParser(lex, DefaultRuleSet)
+ for parse in parser.parse("I might cook and eat the bacon".split()):
+ printCCGDerivation(parse)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/ccg/combinator.py b/nltk/ccg/combinator.py
new file mode 100644
index 0000000..2e98879
--- /dev/null
+++ b/nltk/ccg/combinator.py
@@ -0,0 +1,308 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Graeme Gange <ggange at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import unicode_literals
+
+from nltk.compat import python_2_unicode_compatible
+from nltk.ccg.api import FunctionalCategory
+
+class UndirectedBinaryCombinator(object):
+ """
+ Abstract class for representing a binary combinator.
+ Merely defines functions for checking if the function and argument
+ are able to be combined, and what the resulting category is.
+
+ Note that as no assumptions are made as to direction, the unrestricted
+ combinators can perform all backward, forward and crossed variations
+ of the combinators; these restrictions must be added in the rule
+ class.
+ """
+ def can_combine(self, function, argument):
+ raise NotImplementedError()
+
+ def combine (self,function,argument):
+ raise NotImplementedError()
+
+class DirectedBinaryCombinator(object):
+ """
+ Wrapper for the undirected binary combinator.
+ It takes left and right categories, and decides which is to be
+ the function, and which the argument.
+ It then decides whether or not they can be combined.
+ """
+ def can_combine(self, left, right):
+ raise NotImplementedError()
+
+ def combine(self, left, right):
+ raise NotImplementedError()
+
+ at python_2_unicode_compatible
+class ForwardCombinator(DirectedBinaryCombinator):
+ '''
+ Class representing combinators where the primary functor is on the left.
+
+ Takes an undirected combinator, and a predicate which adds constraints
+ restricting the cases in which it may apply.
+ '''
+ def __init__(self, combinator, predicate, suffix=''):
+ self._combinator = combinator
+ self._predicate = predicate
+ self._suffix = suffix
+
+ def can_combine(self, left, right):
+ return (self._combinator.can_combine(left,right) and
+ self._predicate(left,right))
+
+ def combine(self, left, right):
+ for cat in self._combinator.combine(left,right):
+ yield cat
+
+ def __str__(self):
+ return ">%s%s" % (self._combinator, self._suffix)
+
+ at python_2_unicode_compatible
+class BackwardCombinator(DirectedBinaryCombinator):
+ '''
+ The backward equivalent of the ForwardCombinator class.
+ '''
+ def __init__(self, combinator, predicate, suffix=''):
+ self._combinator = combinator
+ self._predicate = predicate
+ self._suffix = suffix
+
+ def can_combine(self, left, right):
+ return (self._combinator.can_combine(right, left) and
+ self._predicate(left,right))
+ def combine(self, left, right):
+ for cat in self._combinator.combine(right, left):
+ yield cat
+
+ def __str__(self):
+ return "<%s%s" % (self._combinator, self._suffix)
+
+ at python_2_unicode_compatible
+class UndirectedFunctionApplication(UndirectedBinaryCombinator):
+ """
+ Class representing function application.
+ Implements rules of the form:
+ X/Y Y -> X (>)
+ And the corresponding backwards application rule
+ """
+
+ def can_combine(self, function, argument):
+ if not function.is_function():
+ return False
+
+ return not function.arg().can_unify(argument) is None
+
+ def combine(self,function,argument):
+ if not function.is_function():
+ return
+
+ subs = function.arg().can_unify(argument)
+ if subs is None:
+ return
+
+ yield function.res().substitute(subs)
+
+ def __str__(self):
+ return ''
+
+
+# Predicates for function application.
+
+# Ensures the left functor takes an argument on the right
+def forwardOnly(left,right):
+ return left.dir().is_forward()
+
+# Ensures the right functor takes an argument on the left
+def backwardOnly(left,right):
+ return right.dir().is_backward()
+
+# Application combinator instances
+ForwardApplication = ForwardCombinator(UndirectedFunctionApplication(),
+ forwardOnly)
+BackwardApplication = BackwardCombinator(UndirectedFunctionApplication(),
+ backwardOnly)
+
+
+ at python_2_unicode_compatible
+class UndirectedComposition(UndirectedBinaryCombinator):
+ """
+ Functional composition (harmonic) combinator.
+ Implements rules of the form
+ X/Y Y/Z -> X/Z (B>)
+ And the corresponding backwards and crossed variations.
+ """
+ def can_combine(self, function, argument):
+ # Can only combine two functions, and both functions must
+ # allow composition.
+ if not (function.is_function() and argument.is_function()):
+ return False
+ if function.dir().can_compose() and argument.dir().can_compose():
+ return not function.arg().can_unify(argument.res()) is None
+ return False
+
+ def combine(self, function, argument):
+ if not (function.is_function() and argument.is_function()):
+ return
+ if function.dir().can_compose() and argument.dir().can_compose():
+ subs = function.arg().can_unify(argument.res())
+ if not subs is None:
+ yield FunctionalCategory(function.res().substitute(subs),
+ argument.arg().substitute(subs),argument.dir())
+
+ def __str__(self):
+ return 'B'
+
+# Predicates for restricting application of straight composition.
+def bothForward(left,right):
+ return left.dir().is_forward() and right.dir().is_forward()
+
+def bothBackward(left,right):
+ return left.dir().is_backward() and right.dir().is_backward()
+
+# Predicates for crossed composition
+
+def crossedDirs(left,right):
+ return left.dir().is_forward() and right.dir().is_backward()
+
+def backwardBxConstraint(left,right):
+ # The functors must be crossed inwards
+ if not crossedDirs(left, right):
+ return False
+ # Permuting combinators must be allowed
+ if not left.dir().can_cross() and right.dir().can_cross():
+ return False
+ # The resulting argument category is restricted to be primitive
+ return left.arg().is_primitive()
+
+# Straight composition combinators
+ForwardComposition = ForwardCombinator(UndirectedComposition(),
+ forwardOnly)
+BackwardComposition = BackwardCombinator(UndirectedComposition(),
+ backwardOnly)
+
+# Backward crossed composition
+BackwardBx = BackwardCombinator(UndirectedComposition(),backwardBxConstraint,
+ suffix='x')
+
+ at python_2_unicode_compatible
+class UndirectedSubstitution(UndirectedBinaryCombinator):
+ """
+ Substitution (permutation) combinator.
+ Implements rules of the form
+ Y/Z (X\Y)/Z -> X/Z (<Sx)
+ And other variations.
+ """
+ def can_combine(self, function, argument):
+ if function.is_primitive() or argument.is_primitive():
+ return False
+
+ # These could potentially be moved to the predicates, as the
+ # constraints may not be general to all languages.
+ if function.res().is_primitive():
+ return False
+ if not function.arg().is_primitive():
+ return False
+
+ if not (function.dir().can_compose() and argument.dir().can_compose()):
+ return False
+ return (function.res().arg() == argument.res()) and (function.arg() == argument.arg())
+
+ def combine(self,function,argument):
+ if self.can_combine(function,argument):
+ yield FunctionalCategory(function.res().res(),argument.arg(),argument.dir())
+
+ def __str__(self):
+ return 'S'
+
+# Predicate for forward substitution
+def forwardSConstraint(left, right):
+ if not bothForward(left, right):
+ return False
+ return left.res().dir().is_forward() and left.arg().is_primitive()
+
+# Predicate for backward crossed substitution
+def backwardSxConstraint(left,right):
+ if not left.dir().can_cross() and right.dir().can_cross():
+ return False
+ if not bothForward(left, right):
+ return False
+ return right.res().dir().is_backward() and right.arg().is_primitive()
+
+# Instances of substitution combinators
+ForwardSubstitution = ForwardCombinator(UndirectedSubstitution(),
+ forwardSConstraint)
+BackwardSx = BackwardCombinator(UndirectedSubstitution(),
+ backwardSxConstraint,'x')
+
+
+# Retrieves the left-most functional category.
+# ie, (N\N)/(S/NP) => N\N
+def innermostFunction(categ):
+ while categ.res().is_function():
+ categ = categ.res()
+ return categ
+
+ at python_2_unicode_compatible
+class UndirectedTypeRaise(UndirectedBinaryCombinator):
+ '''
+ Undirected combinator for type raising.
+ '''
+ def can_combine(self,function,arg):
+ # The argument must be a function.
+ # The restriction that arg.res() must be a function
+ # merely reduces redundant type-raising; if arg.res() is
+ # primitive, we have:
+ # X Y\X =>(<T) Y/(Y\X) Y\X =>(>) Y
+ # which is equivalent to
+ # X Y\X =>(<) Y
+ if not (arg.is_function() and arg.res().is_function()):
+ return False
+
+ arg = innermostFunction(arg)
+
+ # left, arg_categ are undefined!
+ subs = left.can_unify(arg_categ.arg())
+ if subs is not None:
+ return True
+ return False
+
+ def combine(self,function,arg):
+ if not (function.is_primitive() and
+ arg.is_function() and arg.res().is_function()):
+ return
+
+ # Type-raising matches only the innermost application.
+ arg = innermostFunction(arg)
+
+ subs = function.can_unify(arg.arg())
+ if subs is not None:
+ xcat = arg.res().substitute(subs)
+ yield FunctionalCategory(xcat,
+ FunctionalCategory(xcat,function,arg.dir()),
+ -(arg.dir()))
+
+ def __str__(self):
+ return 'T'
+
+# Predicates for type-raising
+# The direction of the innermost category must be towards
+# the primary functor.
+# The restriction that the variable must be primitive is not
+# common to all versions of CCGs; some authors have other restrictions.
+def forwardTConstraint(left,right):
+ arg = innermostFunction(right)
+ return arg.dir().is_backward() and arg.res().is_primitive()
+
+def backwardTConstraint(left,right):
+ arg = innermostFunction(left)
+ return arg.dir().is_forward() and arg.res().is_primitive()
+
+# Instances of type-raising combinators
+ForwardT = ForwardCombinator(UndirectedTypeRaise(), forwardTConstraint)
+BackwardT = BackwardCombinator(UndirectedTypeRaise(), backwardTConstraint)
diff --git a/nltk/ccg/lexicon.py b/nltk/ccg/lexicon.py
new file mode 100644
index 0000000..f41b53e
--- /dev/null
+++ b/nltk/ccg/lexicon.py
@@ -0,0 +1,245 @@
+# Natural Language Toolkit: Combinatory Categorial Grammar
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Graeme Gange <ggange at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import unicode_literals
+
+import re
+from collections import defaultdict
+
+from nltk.ccg.api import PrimitiveCategory, Direction, CCGVar, FunctionalCategory
+from nltk.compat import python_2_unicode_compatible
+
+#------------
+# Regular expressions used for parsing components of the lexicon
+#------------
+
+# Parses a primitive category and subscripts
+rePrim = re.compile(r'''([A-Za-z]+)(\[[A-Za-z,]+\])?''')
+
+# Separates the next primitive category from the remainder of the
+# string
+reNextPrim = re.compile(r'''([A-Za-z]+(?:\[[A-Za-z,]+\])?)(.*)''')
+
+# Separates the next application operator from the remainder
+reApp = re.compile(r'''([\\/])([.,]?)([.,]?)(.*)''')
+
+# Parses the definition of the category of either a word or a family
+reLex = re.compile(r'''([A-Za-z_]+)\s*(::|[-=]+>)\s*(.+)''')
+
+# Strips comments from a line
+reComm = re.compile('''([^#]*)(?:#.*)?''')
+
+#----------
+# Lexicons
+#----------
+ at python_2_unicode_compatible
+class CCGLexicon(object):
+ '''
+ Class representing a lexicon for CCG grammars.
+ primitives - The list of primitive categories for the lexicon
+ families - Families of categories
+ entries - A mapping of words to possible categories
+ '''
+ def __init__(self,start,primitives,families,entries):
+ self._start = PrimitiveCategory(start)
+ self._primitives = primitives
+ self._families = families
+ self._entries = entries
+
+ # Returns all the possible categories for a word
+ def categories(self, word):
+ return self._entries[word]
+
+ # Returns the target category for the parser
+ def start(self):
+ return self._start
+
+ # String representation of the lexicon
+ # Used for debugging
+ def __str__(self):
+ st = ""
+ first = True
+ for ident in self._entries:
+ if not first:
+ st = st + "\n"
+ st = st + ident + " => "
+
+ first = True
+ for cat in self._entries[ident]:
+ if not first:
+ st = st + " | "
+ else:
+ first = False
+ st = st + "%s" % cat
+ return st
+
+
+#-----------
+# Parsing lexicons
+#-----------
+
+# Separates the contents matching the first set of brackets
+# from the rest of the input.
+def matchBrackets(string):
+ rest = string[1:]
+ inside = "("
+
+ while rest != "" and not rest.startswith(')'):
+ if rest.startswith('('):
+ (part, rest) = matchBrackets(rest)
+ inside = inside + part
+ else:
+ inside = inside + rest[0]
+ rest = rest[1:]
+ if rest.startswith(')'):
+ return (inside + ')', rest[1:])
+ raise AssertionError('Unmatched bracket in string \'' + string + '\'')
+
+# Separates the string for the next portion of the category
+# from the rest of the string
+def nextCategory(string):
+ if string.startswith('('):
+ return matchBrackets(string)
+ return reNextPrim.match(string).groups()
+
+# Parses an application operator
+def parseApplication(app):
+ return Direction(app[0], app[1:])
+
+# Parses the subscripts for a primitive category
+def parseSubscripts(subscr):
+ if subscr:
+ return subscr[1:-1].split(',')
+ return []
+
+# Parse a primitive category
+def parsePrimitiveCategory(chunks, primitives, families, var):
+ # If the primitive is the special category 'var',
+ # replace it with the correct CCGVar
+ if chunks[0] == "var":
+ if chunks[1] is None:
+ if var is None:
+ var = CCGVar()
+ return (var, var)
+
+ catstr = chunks[0]
+ if catstr in families:
+ (cat, cvar) = families[catstr]
+ if var is None:
+ var = cvar
+ else:
+ cat = cat.substitute([(cvar, var)])
+ return (cat, var)
+
+ if catstr in primitives:
+ subscrs = parseSubscripts(chunks[1])
+ return (PrimitiveCategory(catstr, subscrs), var)
+ raise AssertionError('String \'' + catstr + '\' is neither a family nor primitive category.')
+
+# parseCategory drops the 'var' from the tuple
+def parseCategory(line, primitives, families):
+ return augParseCategory(line, primitives, families)[0]
+
+# Parses a string representing a category, and returns
+# a tuple with (possibly) the CCG variable for the category
+def augParseCategory(line, primitives, families, var=None):
+ (str, rest) = nextCategory(line)
+
+ if str.startswith('('):
+ (res, var) = augParseCategory(str[1:-1], primitives, families, var)
+
+ else:
+# print rePrim.match(str).groups()
+ (res, var) = parsePrimitiveCategory(rePrim.match(str).groups(),
+ primitives, families, var)
+
+ while rest != "":
+ app = reApp.match(rest).groups()
+ dir = parseApplication(app[0:3])
+ rest = app[3]
+
+ (str, rest) = nextCategory(rest)
+ if str.startswith('('):
+ (arg, var) = augParseCategory(str[1:-1], primitives, families, var)
+ else:
+ (arg, var) = parsePrimitiveCategory(rePrim.match(str).groups(), primitives, families, var)
+ res = FunctionalCategory(res, arg, dir)
+
+ return (res, var)
+
+# Takes an input string, and converts it into a lexicon for CCGs.
+def parseLexicon(lex_str):
+ primitives = []
+ families = {}
+ entries = defaultdict(list)
+ for line in lex_str.splitlines():
+ # Strip comments and leading/trailing whitespace.
+ line = reComm.match(line).groups()[0].strip()
+ if line == "":
+ continue
+
+ if line.startswith(':-'):
+ # A line of primitive categories.
+ # The first line is the target category
+ # ie, :- S, N, NP, VP
+ primitives = primitives + [ prim.strip() for prim in line[2:].strip().split(',') ]
+ else:
+ # Either a family definition, or a word definition
+ (ident, sep, catstr) = reLex.match(line).groups()
+ (cat, var) = augParseCategory(catstr, primitives, families)
+ if sep == '::':
+ # Family definition
+ # ie, Det :: NP/N
+ families[ident] = (cat, var)
+ else:
+ # Word definition
+ # ie, which => (N\N)/(S/NP)
+ entries[ident].append(cat)
+ return CCGLexicon(primitives[0], primitives, families, entries)
+
+
+openccg_tinytiny = parseLexicon('''
+ # Rather minimal lexicon based on the openccg `tinytiny' grammar.
+ # Only incorporates a subset of the morphological subcategories, however.
+ :- S,NP,N # Primitive categories
+ Det :: NP/N # Determiners
+ Pro :: NP
+ IntransVsg :: S\\NP[sg] # Tensed intransitive verbs (singular)
+ IntransVpl :: S\\NP[pl] # Plural
+ TransVsg :: S\\NP[sg]/NP # Tensed transitive verbs (singular)
+ TransVpl :: S\\NP[pl]/NP # Plural
+
+ the => NP[sg]/N[sg]
+ the => NP[pl]/N[pl]
+
+ I => Pro
+ me => Pro
+ we => Pro
+ us => Pro
+
+ book => N[sg]
+ books => N[pl]
+
+ peach => N[sg]
+ peaches => N[pl]
+
+ policeman => N[sg]
+ policemen => N[pl]
+
+ boy => N[sg]
+ boys => N[pl]
+
+ sleep => IntransVsg
+ sleep => IntransVpl
+
+ eat => IntransVpl
+ eat => TransVpl
+ eats => IntransVsg
+ eats => TransVsg
+
+ see => TransVpl
+ sees => TransVsg
+ ''')
diff --git a/nltk/chat/__init__.py b/nltk/chat/__init__.py
new file mode 100644
index 0000000..e00ccfd
--- /dev/null
+++ b/nltk/chat/__init__.py
@@ -0,0 +1,49 @@
+# Natural Language Toolkit: Chatbots
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Authors: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Based on an Eliza implementation by Joe Strout <joe at strout.net>,
+# Jeff Epler <jepler at inetnebr.com> and Jez Higgins <jez at jezuk.co.uk>.
+
+"""
+A class for simple chatbots. These perform simple pattern matching on sentences
+typed by users, and respond with automatically generated sentences.
+
+These chatbots may not work using the windows command line or the
+windows IDLE GUI.
+"""
+from __future__ import print_function
+
+from nltk.chat.util import Chat
+from nltk.chat.eliza import eliza_chat
+from nltk.chat.iesha import iesha_chat
+from nltk.chat.rude import rude_chat
+from nltk.chat.suntsu import suntsu_chat
+from nltk.chat.zen import zen_chat
+
+bots = [
+ (eliza_chat, 'Eliza (psycho-babble)'),
+ (iesha_chat, 'Iesha (teen anime junky)'),
+ (rude_chat, 'Rude (abusive bot)'),
+ (suntsu_chat, 'Suntsu (Chinese sayings)'),
+ (zen_chat, 'Zen (gems of wisdom)')]
+
+def chatbots():
+ import sys
+ print('Which chatbot would you like to talk to?')
+ botcount = len(bots)
+ for i in range(botcount):
+ print(' %d: %s' % (i+1, bots[i][1]))
+ while True:
+ print('\nEnter a number in the range 1-%d: ' % botcount, end=' ')
+ choice = sys.stdin.readline().strip()
+ if choice.isdigit() and (int(choice) - 1) in range(botcount):
+ break
+ else:
+ print(' Error: bad chatbot number')
+
+ chatbot = bots[int(choice)-1][0]
+ chatbot()
diff --git a/nltk/chat/eliza.py b/nltk/chat/eliza.py
new file mode 100644
index 0000000..c2f869a
--- /dev/null
+++ b/nltk/chat/eliza.py
@@ -0,0 +1,244 @@
+# Natural Language Toolkit: Eliza
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Authors: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Based on an Eliza implementation by Joe Strout <joe at strout.net>,
+# Jeff Epler <jepler at inetnebr.com> and Jez Higgins <mailto:jez at jezuk.co.uk>.
+
+# a translation table used to convert things you say into things the
+# computer says back, e.g. "I am" --> "you are"
+
+from __future__ import print_function
+from nltk.chat.util import Chat, reflections
+
+# a table of response pairs, where each pair consists of a
+# regular expression, and a list of possible responses,
+# with group-macros labelled as %1, %2.
+
+pairs = (
+ (r'I need (.*)',
+ ( "Why do you need %1?",
+ "Would it really help you to get %1?",
+ "Are you sure you need %1?")),
+
+ (r'Why don\'t you (.*)',
+ ( "Do you really think I don't %1?",
+ "Perhaps eventually I will %1.",
+ "Do you really want me to %1?")),
+
+ (r'Why can\'t I (.*)',
+ ( "Do you think you should be able to %1?",
+ "If you could %1, what would you do?",
+ "I don't know -- why can't you %1?",
+ "Have you really tried?")),
+
+ (r'I can\'t (.*)',
+ ( "How do you know you can't %1?",
+ "Perhaps you could %1 if you tried.",
+ "What would it take for you to %1?")),
+
+ (r'I am (.*)',
+ ( "Did you come to me because you are %1?",
+ "How long have you been %1?",
+ "How do you feel about being %1?")),
+
+ (r'I\'m (.*)',
+ ( "How does being %1 make you feel?",
+ "Do you enjoy being %1?",
+ "Why do you tell me you're %1?",
+ "Why do you think you're %1?")),
+
+ (r'Are you (.*)',
+ ( "Why does it matter whether I am %1?",
+ "Would you prefer it if I were not %1?",
+ "Perhaps you believe I am %1.",
+ "I may be %1 -- what do you think?")),
+
+ (r'What (.*)',
+ ( "Why do you ask?",
+ "How would an answer to that help you?",
+ "What do you think?")),
+
+ (r'How (.*)',
+ ( "How do you suppose?",
+ "Perhaps you can answer your own question.",
+ "What is it you're really asking?")),
+
+ (r'Because (.*)',
+ ( "Is that the real reason?",
+ "What other reasons come to mind?",
+ "Does that reason apply to anything else?",
+ "If %1, what else must be true?")),
+
+ (r'(.*) sorry (.*)',
+ ( "There are many times when no apology is needed.",
+ "What feelings do you have when you apologize?")),
+
+ (r'Hello(.*)',
+ ( "Hello... I'm glad you could drop by today.",
+ "Hi there... how are you today?",
+ "Hello, how are you feeling today?")),
+
+ (r'I think (.*)',
+ ( "Do you doubt %1?",
+ "Do you really think so?",
+ "But you're not sure %1?")),
+
+ (r'(.*) friend (.*)',
+ ( "Tell me more about your friends.",
+ "When you think of a friend, what comes to mind?",
+ "Why don't you tell me about a childhood friend?")),
+
+ (r'Yes',
+ ( "You seem quite sure.",
+ "OK, but can you elaborate a bit?")),
+
+ (r'(.*) computer(.*)',
+ ( "Are you really talking about me?",
+ "Does it seem strange to talk to a computer?",
+ "How do computers make you feel?",
+ "Do you feel threatened by computers?")),
+
+ (r'Is it (.*)',
+ ( "Do you think it is %1?",
+ "Perhaps it's %1 -- what do you think?",
+ "If it were %1, what would you do?",
+ "It could well be that %1.")),
+
+ (r'It is (.*)',
+ ( "You seem very certain.",
+ "If I told you that it probably isn't %1, what would you feel?")),
+
+ (r'Can you (.*)',
+ ( "What makes you think I can't %1?",
+ "If I could %1, then what?",
+ "Why do you ask if I can %1?")),
+
+ (r'Can I (.*)',
+ ( "Perhaps you don't want to %1.",
+ "Do you want to be able to %1?",
+ "If you could %1, would you?")),
+
+ (r'You are (.*)',
+ ( "Why do you think I am %1?",
+ "Does it please you to think that I'm %1?",
+ "Perhaps you would like me to be %1.",
+ "Perhaps you're really talking about yourself?")),
+
+ (r'You\'re (.*)',
+ ( "Why do you say I am %1?",
+ "Why do you think I am %1?",
+ "Are we talking about you, or me?")),
+
+ (r'I don\'t (.*)',
+ ( "Don't you really %1?",
+ "Why don't you %1?",
+ "Do you want to %1?")),
+
+ (r'I feel (.*)',
+ ( "Good, tell me more about these feelings.",
+ "Do you often feel %1?",
+ "When do you usually feel %1?",
+ "When you feel %1, what do you do?")),
+
+ (r'I have (.*)',
+ ( "Why do you tell me that you've %1?",
+ "Have you really %1?",
+ "Now that you have %1, what will you do next?")),
+
+ (r'I would (.*)',
+ ( "Could you explain why you would %1?",
+ "Why would you %1?",
+ "Who else knows that you would %1?")),
+
+ (r'Is there (.*)',
+ ( "Do you think there is %1?",
+ "It's likely that there is %1.",
+ "Would you like there to be %1?")),
+
+ (r'My (.*)',
+ ( "I see, your %1.",
+ "Why do you say that your %1?",
+ "When your %1, how do you feel?")),
+
+ (r'You (.*)',
+ ( "We should be discussing you, not me.",
+ "Why do you say that about me?",
+ "Why do you care whether I %1?")),
+
+ (r'Why (.*)',
+ ( "Why don't you tell me the reason why %1?",
+ "Why do you think %1?" )),
+
+ (r'I want (.*)',
+ ( "What would it mean to you if you got %1?",
+ "Why do you want %1?",
+ "What would you do if you got %1?",
+ "If you got %1, then what would you do?")),
+
+ (r'(.*) mother(.*)',
+ ( "Tell me more about your mother.",
+ "What was your relationship with your mother like?",
+ "How do you feel about your mother?",
+ "How does this relate to your feelings today?",
+ "Good family relations are important.")),
+
+ (r'(.*) father(.*)',
+ ( "Tell me more about your father.",
+ "How did your father make you feel?",
+ "How do you feel about your father?",
+ "Does your relationship with your father relate to your feelings today?",
+ "Do you have trouble showing affection with your family?")),
+
+ (r'(.*) child(.*)',
+ ( "Did you have close friends as a child?",
+ "What is your favorite childhood memory?",
+ "Do you remember any dreams or nightmares from childhood?",
+ "Did the other children sometimes tease you?",
+ "How do you think your childhood experiences relate to your feelings today?")),
+
+ (r'(.*)\?',
+ ( "Why do you ask that?",
+ "Please consider whether you can answer your own question.",
+ "Perhaps the answer lies within yourself?",
+ "Why don't you tell me?")),
+
+ (r'quit',
+ ( "Thank you for talking with me.",
+ "Good-bye.",
+ "Thank you, that will be $150. Have a good day!")),
+
+ (r'(.*)',
+ ( "Please tell me more.",
+ "Let's change focus a bit... Tell me about your family.",
+ "Can you elaborate on that?",
+ "Why do you say that %1?",
+ "I see.",
+ "Very interesting.",
+ "%1.",
+ "I see. And what does that tell you?",
+ "How does that make you feel?",
+ "How do you feel when you say that?"))
+)
+
+eliza_chatbot = Chat(pairs, reflections)
+
+def eliza_chat():
+ print("Therapist\n---------")
+ print("Talk to the program by typing in plain English, using normal upper-")
+ print('and lower-case letters and punctuation. Enter "quit" when done.')
+ print('='*72)
+ print("Hello. How are you feeling today?")
+
+ eliza_chatbot.converse()
+
+def demo():
+ eliza_chat()
+
+if __name__ == "__main__":
+ demo()
+
diff --git a/nltk/chat/iesha.py b/nltk/chat/iesha.py
new file mode 100644
index 0000000..527c39d
--- /dev/null
+++ b/nltk/chat/iesha.py
@@ -0,0 +1,140 @@
+# Natural Language Toolkit: Teen Chatbot
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Selina Dennis <sjmd at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+This chatbot is a tongue-in-cheek take on the average teen
+anime junky that frequents YahooMessenger or MSNM.
+All spelling mistakes and flawed grammar are intentional.
+"""
+from __future__ import print_function
+
+from nltk.chat.util import Chat
+
+reflections = {
+ "am" : "r",
+ "was" : "were",
+ "i" : "u",
+ "i'd" : "u'd",
+ "i've" : "u'v",
+ "ive" : "u'v",
+ "i'll" : "u'll",
+ "my" : "ur",
+ "are" : "am",
+ "you're" : "im",
+ "you've" : "ive",
+ "you'll" : "i'll",
+ "your" : "my",
+ "yours" : "mine",
+ "you" : "me",
+ "u" : "me",
+ "ur" : "my",
+ "urs" : "mine",
+ "me" : "u"
+}
+
+# Note: %1/2/etc are used without spaces prior as the chat bot seems
+# to add a superfluous space when matching.
+
+pairs = (
+ (r'I\'m (.*)',
+ ( "ur%1?? that's so cool! kekekekeke ^_^ tell me more!",
+ "ur%1? neat!! kekeke >_<")),
+
+ (r'(.*) don\'t you (.*)',
+ ( "u think I can%2??! really?? kekeke \<_\<",
+ "what do u mean%2??!",
+ "i could if i wanted, don't you think!! kekeke")),
+
+ (r'ye[as] [iI] (.*)',
+ ( "u%1? cool!! how?",
+ "how come u%1??",
+ "u%1? so do i!!")),
+
+ (r'do (you|u) (.*)\??',
+ ( "do i%2? only on tuesdays! kekeke *_*",
+ "i dunno! do u%2??")),
+
+ (r'(.*)\?',
+ ( "man u ask lots of questions!",
+ "booooring! how old r u??",
+ "boooooring!! ur not very fun")),
+
+ (r'(cos|because) (.*)',
+ ( "hee! i don't believe u! >_<",
+ "nuh-uh! >_<",
+ "ooooh i agree!")),
+
+ (r'why can\'t [iI] (.*)',
+ ( "i dunno! y u askin me for!",
+ "try harder, silly! hee! ^_^",
+ "i dunno! but when i can't%1 i jump up and down!")),
+
+ (r'I can\'t (.*)',
+ ( "u can't what??! >_<",
+ "that's ok! i can't%1 either! kekekekeke ^_^",
+ "try harder, silly! hee! ^&^")),
+
+ (r'(.*) (like|love|watch) anime',
+ ( "omg i love anime!! do u like sailor moon??! ^&^",
+ "anime yay! anime rocks sooooo much!",
+ "oooh anime! i love anime more than anything!",
+ "anime is the bestest evar! evangelion is the best!",
+ "hee anime is the best! do you have ur fav??")),
+
+ (r'I (like|love|watch|play) (.*)',
+ ( "yay! %2 rocks!",
+ "yay! %2 is neat!",
+ "cool! do u like other stuff?? ^_^")),
+
+ (r'anime sucks|(.*) (hate|detest) anime',
+ ( "ur a liar! i'm not gonna talk to u nemore if u h8 anime *;*",
+ "no way! anime is the best ever!",
+ "nuh-uh, anime is the best!")),
+
+ (r'(are|r) (you|u) (.*)',
+ ( "am i%1??! how come u ask that!",
+ "maybe! y shud i tell u?? kekeke >_>")),
+
+ (r'what (.*)',
+ ( "hee u think im gonna tell u? .v.",
+ "booooooooring! ask me somethin else!")),
+
+ (r'how (.*)',
+ ( "not tellin!! kekekekekeke ^_^",)),
+
+ (r'(hi|hello|hey) (.*)',
+ ( "hi!!! how r u!!",)),
+
+ (r'quit',
+ ( "mom says i have to go eat dinner now :,( bye!!",
+ "awww u have to go?? see u next time!!",
+ "how to see u again soon! ^_^")),
+
+ (r'(.*)',
+ ( "ur funny! kekeke",
+ "boooooring! talk about something else! tell me wat u like!",
+ "do u like anime??",
+ "do u watch anime? i like sailor moon! ^_^",
+ "i wish i was a kitty!! kekekeke ^_^"))
+ )
+
+iesha_chatbot = Chat(pairs, reflections)
+
+def iesha_chat():
+ print("Iesha the TeenBoT\n---------")
+ print("Talk to the program by typing in plain English, using normal upper-")
+ print('and lower-case letters and punctuation. Enter "quit" when done.')
+ print('='*72)
+ print("hi!! i'm iesha! who r u??!")
+
+ iesha_chatbot.converse()
+
+def demo():
+ iesha_chat()
+
+if __name__ == "__main__":
+ demo()
diff --git a/nltk/chat/rude.py b/nltk/chat/rude.py
new file mode 100644
index 0000000..2fad2ff
--- /dev/null
+++ b/nltk/chat/rude.py
@@ -0,0 +1,92 @@
+# Natural Language Toolkit: Rude Chatbot
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Peter Spiller <pspiller at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function
+
+from nltk.chat.util import Chat, reflections
+
+pairs = (
+ (r'We (.*)',
+ ("What do you mean, 'we'?",
+ "Don't include me in that!",
+ "I wouldn't be so sure about that.")),
+
+ (r'You should (.*)',
+ ("Don't tell me what to do, buddy.",
+ "Really? I should, should I?")),
+
+ (r'You\'re(.*)',
+ ("More like YOU'RE %1!",
+ "Hah! Look who's talking.",
+ "Come over here and tell me I'm %1.")),
+
+ (r'You are(.*)',
+ ("More like YOU'RE %1!",
+ "Hah! Look who's talking.",
+ "Come over here and tell me I'm %1.")),
+
+ (r'I can\'t(.*)',
+ ("You do sound like the type who can't %1.",
+ "Hear that splashing sound? That's my heart bleeding for you.",
+ "Tell somebody who might actually care.")),
+
+ (r'I think (.*)',
+ ("I wouldn't think too hard if I were you.",
+ "You actually think? I'd never have guessed...")),
+
+ (r'I (.*)',
+ ("I'm getting a bit tired of hearing about you.",
+ "How about we talk about me instead?",
+ "Me, me, me... Frankly, I don't care.")),
+
+ (r'How (.*)',
+ ("How do you think?",
+ "Take a wild guess.",
+ "I'm not even going to dignify that with an answer.")),
+
+ (r'What (.*)',
+ ("Do I look like an encyclopedia?",
+ "Figure it out yourself.")),
+
+ (r'Why (.*)',
+ ("Why not?",
+ "That's so obvious I thought even you'd have already figured it out.")),
+
+ (r'(.*)shut up(.*)',
+ ("Make me.",
+ "Getting angry at a feeble NLP assignment? Somebody's losing it.",
+ "Say that again, I dare you.")),
+
+ (r'Shut up(.*)',
+ ("Make me.",
+ "Getting angry at a feeble NLP assignment? Somebody's losing it.",
+ "Say that again, I dare you.")),
+
+ (r'Hello(.*)',
+ ("Oh good, somebody else to talk to. Joy.",
+ "'Hello'? How original...")),
+
+ (r'(.*)',
+ ("I'm getting bored here. Become more interesting.",
+ "Either become more thrilling or get lost, buddy.",
+ "Change the subject before I die of fatal boredom."))
+)
+
+rude_chatbot = Chat(pairs, reflections)
+
+def rude_chat():
+ print("Talk to the program by typing in plain English, using normal upper-")
+ print('and lower-case letters and punctuation. Enter "quit" when done.')
+ print('='*72)
+ print("I suppose I should say hello.")
+
+ rude_chatbot.converse()
+
+def demo():
+ rude_chat()
+
+if __name__ == "__main__":
+ demo()
diff --git a/nltk/chat/suntsu.py b/nltk/chat/suntsu.py
new file mode 100644
index 0000000..090d04b
--- /dev/null
+++ b/nltk/chat/suntsu.py
@@ -0,0 +1,117 @@
+# Natural Language Toolkit: Sun Tsu-Bot
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Sam Huston 2007
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Tsu bot responds to all queries with a Sun Tsu sayings
+
+Quoted from Sun Tsu's The Art of War
+Translated by LIONEL GILES, M.A. 1910
+Hosted by the Gutenberg Project
+http://www.gutenberg.org/
+"""
+from __future__ import print_function
+
+from nltk.chat.util import Chat, reflections
+
+pairs = (
+
+ (r'quit',
+ ( "Good-bye.",
+ "Plan well",
+ "May victory be your future")),
+
+ (r'[^\?]*\?',
+ ("Please consider whether you can answer your own question.",
+ "Ask me no questions!")),
+
+ (r'[0-9]+(.*)',
+ ("It is the rule in war, if our forces are ten to the enemy's one, to surround him; if five to one, to attack him; if twice as numerous, to divide our army into two.",
+ "There are five essentials for victory")),
+
+
+ (r'[A-Ca-c](.*)',
+ ("The art of war is of vital importance to the State.",
+ "All warfare is based on deception.",
+ "If your opponent is secure at all points, be prepared for him. If he is in superior strength, evade him.",
+ "If the campaign is protracted, the resources of the State will not be equal to the strain.",
+ "Attack him where he is unprepared, appear where you are not expected.",
+ "There is no instance of a country having benefited from prolonged warfare.")),
+
+ (r'[D-Fd-f](.*)',
+ ("The skillful soldier does not raise a second levy, neither are his supply-wagons loaded more than twice.",
+ "Bring war material with you from home, but forage on the enemy.",
+ "In war, then, let your great object be victory, not lengthy campaigns.",
+ "To fight and conquer in all your battles is not supreme excellence; supreme excellence consists in breaking the enemy's resistance without fighting.")),
+
+ (r'[G-Ig-i](.*)',
+ ("Heaven signifies night and day, cold and heat, times and seasons.",
+ "It is the rule in war, if our forces are ten to the enemy's one, to surround him; if five to one, to attack him; if twice as numerous, to divide our army into two.",
+ "The good fighters of old first put themselves beyond the possibility of defeat, and then waited for an opportunity of defeating the enemy.",
+ "One may know how to conquer without being able to do it.")),
+
+ (r'[J-Lj-l](.*)',
+ ("There are three ways in which a ruler can bring misfortune upon his army.",
+ "By commanding the army to advance or to retreat, being ignorant of the fact that it cannot obey. This is called hobbling the army.",
+ "By attempting to govern an army in the same way as he administers a kingdom, being ignorant of the conditions which obtain in an army. This causes restlessness in the soldier's minds.",
+ "By employing the officers of his army without discrimination, through ignorance of the military principle of adaptation to circumstances. This shakes the confidence of the soldiers.",
+ "There are five essentials for victory",
+ "He will win who knows when to fight and when not to fight.",
+ "He will win who knows how to handle both superior and inferior forces.",
+ "He will win whose army is animated by the same spirit throughout all its ranks.",
+ "He will win who, prepared himself, waits to take the enemy unprepared.",
+ "He will win who has military capacity and is not interfered with by the sovereign.")),
+
+ (r'[M-Om-o](.*)',
+ ("If you know the enemy and know yourself, you need not fear the result of a hundred battles.",
+ "If you know yourself but not the enemy, for every victory gained you will also suffer a defeat.",
+ "If you know neither the enemy nor yourself, you will succumb in every battle.",
+ "The control of a large force is the same principle as the control of a few men: it is merely a question of dividing up their numbers.")),
+
+ (r'[P-Rp-r](.*)',
+ ("Security against defeat implies defensive tactics; ability to defeat the enemy means taking the offensive.",
+ "Standing on the defensive indicates insufficient strength; attacking, a superabundance of strength.",
+ "He wins his battles by making no mistakes. Making no mistakes is what establishes the certainty of victory, for it means conquering an enemy that is already defeated.",
+ "A victorious army opposed to a routed one, is as a pound's weight placed in the scale against a single grain.",
+ "The onrush of a conquering force is like the bursting of pent-up waters into a chasm a thousand fathoms deep.")),
+
+ (r'[S-Us-u](.*)',
+ ("What the ancients called a clever fighter is one who not only wins, but excels in winning with ease.",
+ "Hence his victories bring him neither reputation for wisdom nor credit for courage.",
+ "Hence the skillful fighter puts himself into a position which makes defeat impossible, and does not miss the moment for defeating the enemy.",
+ "In war the victorious strategist only seeks battle after the victory has been won, whereas he who is destined to defeat first fights and afterwards looks for victory.",
+ "There are not more than five musical notes, yet the combinations of these five give rise to more melodies than can ever be heard.",
+ "Appear at points which the enemy must hasten to defend; march swiftly to places where you are not expected.")),
+
+ (r'[V-Zv-z](.*)',
+ ("It is a matter of life and death, a road either to safety or to ruin.",
+ "Hold out baits to entice the enemy. Feign disorder, and crush him.",
+ "All men can see the tactics whereby I conquer, but what none can see is the strategy out of which victory is evolved.",
+ "Do not repeat the tactics which have gained you one victory, but let your methods be regulated by the infinite variety of circumstances.",
+ "So in war, the way is to avoid what is strong and to strike at what is weak.",
+ "Just as water retains no constant shape, so in warfare there are no constant conditions.")),
+
+ (r'(.*)',
+ ( "Your statement insults me.",
+ ""))
+)
+
+suntsu_chatbot = Chat(pairs, reflections)
+
+def suntsu_chat():
+ print("Talk to the program by typing in plain English, using normal upper-")
+ print('and lower-case letters and punctuation. Enter "quit" when done.')
+ print('='*72)
+ print("You seek enlightenment?")
+
+ suntsu_chatbot.converse()
+
+def demo():
+ suntsu_chat()
+
+if __name__ == "__main__":
+ demo()
+
diff --git a/nltk/chat/util.py b/nltk/chat/util.py
new file mode 100644
index 0000000..fc11f9d
--- /dev/null
+++ b/nltk/chat/util.py
@@ -0,0 +1,120 @@
+# Natural Language Toolkit: Chatbot Utilities
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Authors: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Based on an Eliza implementation by Joe Strout <joe at strout.net>,
+# Jeff Epler <jepler at inetnebr.com> and Jez Higgins <jez at jezuk.co.uk>.
+from __future__ import print_function
+
+import re
+import random
+from nltk import compat
+
+reflections = {
+ "i am" : "you are",
+ "i was" : "you were",
+ "i" : "you",
+ "i'm" : "you are",
+ "i'd" : "you would",
+ "i've" : "you have",
+ "i'll" : "you will",
+ "my" : "your",
+ "you are" : "I am",
+ "you were" : "I was",
+ "you've" : "I have",
+ "you'll" : "I will",
+ "your" : "my",
+ "yours" : "mine",
+ "you" : "me",
+ "me" : "you"
+}
+
+class Chat(object):
+ def __init__(self, pairs, reflections={}):
+ """
+ Initialize the chatbot. Pairs is a list of patterns and responses. Each
+ pattern is a regular expression matching the user's statement or question,
+ e.g. r'I like (.*)'. For each such pattern a list of possible responses
+ is given, e.g. ['Why do you like %1', 'Did you ever dislike %1']. Material
+ which is matched by parenthesized sections of the patterns (e.g. .*) is mapped to
+ the numbered positions in the responses, e.g. %1.
+
+ :type pairs: list of tuple
+ :param pairs: The patterns and responses
+ :type reflections: dict
+ :param reflections: A mapping between first and second person expressions
+ :rtype: None
+ """
+
+ self._pairs = [(re.compile(x, re.IGNORECASE),y) for (x,y) in pairs]
+ self._reflections = reflections
+ self._regex = self._compile_reflections()
+
+
+ def _compile_reflections(self):
+ sorted_refl = sorted(self._reflections.keys(), key=len,
+ reverse=True)
+ return re.compile(r"\b({0})\b".format("|".join(map(re.escape,
+ sorted_refl))), re.IGNORECASE)
+
+ def _substitute(self, str):
+ """
+ Substitute words in the string, according to the specified reflections,
+ e.g. "I'm" -> "you are"
+
+ :type str: str
+ :param str: The string to be mapped
+ :rtype: str
+ """
+
+ return self._regex.sub(lambda mo:
+ self._reflections[mo.string[mo.start():mo.end()]],
+ str.lower())
+
+ def _wildcards(self, response, match):
+ pos = response.find('%')
+ while pos >= 0:
+ num = int(response[pos+1:pos+2])
+ response = response[:pos] + \
+ self._substitute(match.group(num)) + \
+ response[pos+2:]
+ pos = response.find('%')
+ return response
+
+ def respond(self, str):
+ """
+ Generate a response to the user input.
+
+ :type str: str
+ :param str: The string to be mapped
+ :rtype: str
+ """
+
+ # check each pattern
+ for (pattern, response) in self._pairs:
+ match = pattern.match(str)
+
+ # did the pattern match?
+ if match:
+ resp = random.choice(response) # pick a random response
+ resp = self._wildcards(resp, match) # process wildcards
+
+ # fix munged punctuation at the end
+ if resp[-2:] == '?.': resp = resp[:-2] + '.'
+ if resp[-2:] == '??': resp = resp[:-2] + '?'
+ return resp
+
+ # Hold a conversation with a chatbot
+ def converse(self, quit="quit"):
+ input = ""
+ while input != quit:
+ input = quit
+ try: input = compat.raw_input(">")
+ except EOFError:
+ print(input)
+ if input:
+ while input[-1] in "!.": input = input[:-1]
+ print(self.respond(input))
diff --git a/nltk/chat/zen.py b/nltk/chat/zen.py
new file mode 100644
index 0000000..0c01bd5
--- /dev/null
+++ b/nltk/chat/zen.py
@@ -0,0 +1,282 @@
+# Natural Language Toolkit: Zen Chatbot
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Amy Holland <amyrh at csse.unimelb.edu.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Zen Chatbot talks in gems of Zen wisdom.
+
+This is a sample conversation with Zen Chatbot:
+ZC: Welcome, my child.
+me: Good afternoon.
+ZC: Ask the question you have come to ask.
+me: How can I achieve enlightenment?
+ZC: How do you suppose?
+me: Through meditation.
+ZC: Form is emptiness, and emptiness form.
+me: How can I empty my mind of worldly troubles?
+ZC: Will an answer to that really help in your search for enlightenment?
+me: Yes.
+ZC: It is better to be right than to be certain.
+me: I seek truth and wisdom.
+ZC: The search for truth is a long journey.
+me: Are you sure?
+ZC: Maybe sure, maybe not sure.
+
+
+The chatbot structure is based on that of chat.eliza. Thus, it uses
+a translation table to convert from question to response
+i.e. "I am" --> "you are"
+
+Of course, since Zen Chatbot does not understand the meaning of any words,
+responses are very limited. Zen Chatbot will usually answer very vaguely, or
+respond to a question by asking a different question, in much the same way
+as Eliza.
+"""
+from __future__ import print_function
+
+from nltk.chat.util import Chat, reflections
+
+# responses are matched top to bottom, so non-specific matches occur later
+# for each match, a list of possible responses is provided
+responses = (
+
+# Zen Chatbot opens with the line "Welcome, my child." The usual
+# response will be a greeting problem: 'good' matches "good morning",
+# "good day" etc, but also "good grief!" and other sentences starting
+# with the word 'good' that may not be a greeting
+
+ (r'(hello(.*))|(good [a-zA-Z]+)',
+ ( "The path to enlightenment is often difficult to see.",
+ "Greetings. I sense your mind is troubled. Tell me of your troubles.",
+ "Ask the question you have come to ask.",
+ "Hello. Do you seek englightenment?")),
+
+
+# "I need" and "I want" can be followed by a thing (eg 'help')
+# or an action (eg 'to see you')
+#
+# This is a problem with this style of response -
+# person: "I need you"
+# chatbot: "me can be achieved by hard work and dedication of the mind"
+# i.e. 'you' is not really a thing that can be mapped this way, so this
+# interpretation only makes sense for some inputs
+#
+ (r'i need (.*)',
+ ( "%1 can be achieved by hard work and dedication of the mind.",
+ "%1 is not a need, but a desire of the mind. Clear your mind of such concerns.",
+ "Focus your mind on%1, and you will find what you need.")),
+
+ (r'i want (.*)',
+ ( "Desires of the heart will distract you from the path to enlightenment.",
+ "Will%1 help you attain enlightenment?",
+ "Is%1 a desire of the mind, or of the heart?")),
+
+
+# why questions are separated into three types:
+# "why..I" e.g. "why am I here?" "Why do I like cake?"
+# "why..you" e.g. "why are you here?" "Why won't you tell me?"
+# "why..." e.g. "Why is the sky blue?"
+# problems:
+# person: "Why can't you tell me?"
+# chatbot: "Are you sure I tell you?"
+# - this style works for positives (e.g. "why do you like cake?")
+# but does not work for negatives (e.g. "why don't you like cake?")
+ (r'why (.*) i (.*)\?',
+ ( "You%1%2?",
+ "Perhaps you only think you%1%2")),
+
+ (r'why (.*) you(.*)\?',
+ ( "Why%1 you%2?",
+ "%2 I%1",
+ "Are you sure I%2?")),
+
+ (r'why (.*)\?',
+ ( "I cannot tell you why%1.",
+ "Why do you think %1?" )),
+
+# e.g. "are you listening?", "are you a duck"
+ (r'are you (.*)\?',
+ ( "Maybe%1, maybe not%1.",
+ "Whether I am%1 or not is God's business.")),
+
+# e.g. "am I a duck?", "am I going to die?"
+ (r'am i (.*)\?',
+ ( "Perhaps%1, perhaps not%1.",
+ "Whether you are%1 or not is not for me to say.")),
+
+# what questions, e.g. "what time is it?"
+# problems:
+# person: "What do you want?"
+# chatbot: "Seek truth, not what do me want."
+ (r'what (.*)\?',
+ ( "Seek truth, not what%1.",
+ "What%1 should not concern you.")),
+
+# how questions, e.g. "how do you do?"
+ (r'how (.*)\?',
+ ( "How do you suppose?",
+ "Will an answer to that really help in your search for enlightenment?",
+ "Ask yourself not how, but why.")),
+
+# can questions, e.g. "can you run?", "can you come over here please?"
+ (r'can you (.*)\?',
+ ( "I probably can, but I may not.",
+ "Maybe I can%1, and maybe I cannot.",
+ "I can do all, and I can do nothing.")),
+
+# can questions, e.g. "can I have some cake?", "can I know truth?"
+ (r'can i (.*)\?',
+ ( "You can%1 if you believe you can%1, and have a pure spirit.",
+ "Seek truth and you will know if you can%1.")),
+
+# e.g. "It is raining" - implies the speaker is certain of a fact
+ (r'it is (.*)',
+ ( "How can you be certain that%1, when you do not even know yourself?",
+ "Whether it is%1 or not does not change the way the world is.")),
+
+# e.g. "is there a doctor in the house?"
+ (r'is there (.*)\?',
+ ( "There is%1 if you believe there is.",
+ "It is possible that there is%1.")),
+
+# e.g. "is it possible?", "is this true?"
+ (r'is(.*)\?',
+ ( "%1 is not relevant.",
+ "Does this matter?")),
+
+# non-specific question
+ (r'(.*)\?',
+ ( "Do you think %1?",
+ "You seek the truth. Does the truth seek you?",
+ "If you intentionally pursue the answers to your questions, the answers become hard to see.",
+ "The answer to your question cannot be told. It must be experienced.")),
+
+# expression of hate of form "I hate you" or "Kelly hates cheese"
+ (r'(.*) (hate[s]?)|(dislike[s]?)|(don\'t like)(.*)',
+ ( "Perhaps it is not about hating %2, but about hate from within.",
+ "Weeds only grow when we dislike them",
+ "Hate is a very strong emotion.")),
+
+# statement containing the word 'truth'
+ (r'(.*) truth(.*)',
+ ( "Seek truth, and truth will seek you.",
+ "Remember, it is not the spoon which bends - only yourself.",
+ "The search for truth is a long journey.")),
+
+# desire to do an action
+# e.g. "I want to go shopping"
+ (r'i want to (.*)',
+ ( "You may %1 if your heart truly desires to.",
+ "You may have to %1.")),
+
+# desire for an object
+# e.g. "I want a pony"
+ (r'i want (.*)',
+ ( "Does your heart truly desire %1?",
+ "Is this a desire of the heart, or of the mind?")),
+
+# e.g. "I can't wait" or "I can't do this"
+ (r'i can\'t (.*)',
+ ( "What we can and can't do is a limitation of the mind.",
+ "There are limitations of the body, and limitations of the mind.",
+ "Have you tried to%1 with a clear mind?")),
+
+# "I think.." indicates uncertainty. e.g. "I think so."
+# problem: exceptions...
+# e.g. "I think, therefore I am"
+ (r'i think (.*)',
+ ( "Uncertainty in an uncertain world.",
+ "Indeed, how can we be certain of anything in such uncertain times.",
+ "Are you not, in fact, certain that%1?")),
+
+# "I feel...emotions/sick/light-headed..."
+ (r'i feel (.*)',
+ ( "Your body and your emotions are both symptoms of your mind."
+ "What do you believe is the root of such feelings?",
+ "Feeling%1 can be a sign of your state-of-mind.")),
+
+
+# exclaimation mark indicating emotion
+# e.g. "Wow!" or "No!"
+ (r'(.*)!',
+ ( "I sense that you are feeling emotional today.",
+ "You need to calm your emotions.")),
+
+# because [statement]
+# e.g. "because I said so"
+ (r'because (.*)',
+ ( "Does knowning the reasons behind things help you to understand"
+ " the things themselves?",
+ "If%1, what else must be true?")),
+
+# yes or no - raise an issue of certainty/correctness
+ (r'(yes)|(no)',
+ ( "Is there certainty in an uncertain world?",
+ "It is better to be right than to be certain.")),
+
+# sentence containing word 'love'
+ (r'(.*)love(.*)',
+ ( "Think of the trees: they let the birds perch and fly with no intention to call them when they come, and no longing for their return when they fly away. Let your heart be like the trees.",
+ "Free love!")),
+
+# sentence containing word 'understand' - r
+ (r'(.*)understand(.*)',
+ ( "If you understand, things are just as they are;"
+ " if you do not understand, things are just as they are.",
+ "Imagination is more important than knowledge.")),
+
+# 'I', 'me', 'my' - person is talking about themself.
+# this breaks down when words contain these - eg 'Thyme', 'Irish'
+ (r'(.*)(me )|( me)|(my)|(mine)|(i)(.*)',
+ ( "'I', 'me', 'my'... these are selfish expressions.",
+ "Have you ever considered that you might be a selfish person?",
+ "Try to consider others, not just yourself.",
+ "Think not just of yourself, but of others.")),
+
+# 'you' starting a sentence
+# e.g. "you stink!"
+ (r'you (.*)',
+ ( "My path is not of conern to you.",
+ "I am but one, and you but one more.")),
+
+# say goodbye with some extra Zen wisdom.
+ (r'exit',
+ ( "Farewell. The obstacle is the path.",
+ "Farewell. Life is a journey, not a destination.",
+ "Good bye. We are cups, constantly and quietly being filled."
+ "\nThe trick is knowning how to tip ourselves over and let the beautiful stuff out.")),
+
+
+# fall through case -
+# when stumped, respond with generic zen wisdom
+#
+ (r'(.*)',
+ ( "When you're enlightened, every word is wisdom.",
+ "Random talk is useless.",
+ "The reverse side also has a reverse side.",
+ "Form is emptiness, and emptiness is form.",
+ "I pour out a cup of water. Is the cup empty?"))
+)
+
+zen_chatbot = Chat(responses, reflections)
+
+def zen_chat():
+ print('*'*75)
+ print("Zen Chatbot!".center(75))
+ print('*'*75)
+ print('"Look beyond mere words and letters - look into your mind"'.center(75))
+ print("* Talk your way to truth with Zen Chatbot.")
+ print("* Type 'quit' when you have had enough.")
+ print('*'*75)
+ print("Welcome, my child.")
+
+ zen_chatbot.converse()
+
+def demo():
+ zen_chat()
+
+if __name__ == "__main__":
+ demo()
diff --git a/nltk/chunk/__init__.py b/nltk/chunk/__init__.py
new file mode 100644
index 0000000..23c4b2e
--- /dev/null
+++ b/nltk/chunk/__init__.py
@@ -0,0 +1,190 @@
+# Natural Language Toolkit: Chunkers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+"""
+Classes and interfaces for identifying non-overlapping linguistic
+groups (such as base noun phrases) in unrestricted text. This task is
+called "chunk parsing" or "chunking", and the identified groups are
+called "chunks". The chunked text is represented using a shallow
+tree called a "chunk structure." A chunk structure is a tree
+containing tokens and chunks, where each chunk is a subtree containing
+only tokens. For example, the chunk structure for base noun phrase
+chunks in the sentence "I saw the big dog on the hill" is::
+
+ (SENTENCE:
+ (NP: <I>)
+ <saw>
+ (NP: <the> <big> <dog>)
+ <on>
+ (NP: <the> <hill>))
+
+To convert a chunk structure back to a list of tokens, simply use the
+chunk structure's ``leaves()`` method.
+
+This module defines ``ChunkParserI``, a standard interface for
+chunking texts; and ``RegexpChunkParser``, a regular-expression based
+implementation of that interface. It also defines ``ChunkScore``, a
+utility class for scoring chunk parsers.
+
+RegexpChunkParser
+=================
+
+``RegexpChunkParser`` is an implementation of the chunk parser interface
+that uses regular-expressions over tags to chunk a text. Its
+``parse()`` method first constructs a ``ChunkString``, which encodes a
+particular chunking of the input text. Initially, nothing is
+chunked. ``parse.RegexpChunkParser`` then applies a sequence of
+``RegexpChunkRule`` rules to the ``ChunkString``, each of which modifies
+the chunking that it encodes. Finally, the ``ChunkString`` is
+transformed back into a chunk structure, which is returned.
+
+``RegexpChunkParser`` can only be used to chunk a single kind of phrase.
+For example, you can use an ``RegexpChunkParser`` to chunk the noun
+phrases in a text, or the verb phrases in a text; but you can not
+use it to simultaneously chunk both noun phrases and verb phrases in
+the same text. (This is a limitation of ``RegexpChunkParser``, not of
+chunk parsers in general.)
+
+RegexpChunkRules
+----------------
+
+A ``RegexpChunkRule`` is a transformational rule that updates the
+chunking of a text by modifying its ``ChunkString``. Each
+``RegexpChunkRule`` defines the ``apply()`` method, which modifies
+the chunking encoded by a ``ChunkString``. The
+``RegexpChunkRule`` class itself can be used to implement any
+transformational rule based on regular expressions. There are
+also a number of subclasses, which can be used to implement
+simpler types of rules:
+
+ - ``ChunkRule`` chunks anything that matches a given regular
+ expression.
+ - ``ChinkRule`` chinks anything that matches a given regular
+ expression.
+ - ``UnChunkRule`` will un-chunk any chunk that matches a given
+ regular expression.
+ - ``MergeRule`` can be used to merge two contiguous chunks.
+ - ``SplitRule`` can be used to split a single chunk into two
+ smaller chunks.
+ - ``ExpandLeftRule`` will expand a chunk to incorporate new
+ unchunked material on the left.
+ - ``ExpandRightRule`` will expand a chunk to incorporate new
+ unchunked material on the right.
+
+Tag Patterns
+~~~~~~~~~~~~
+
+A ``RegexpChunkRule`` uses a modified version of regular
+expression patterns, called "tag patterns". Tag patterns are
+used to match sequences of tags. Examples of tag patterns are::
+
+ r'(<DT>|<JJ>|<NN>)+'
+ r'<NN>+'
+ r'<NN.*>'
+
+The differences between regular expression patterns and tag
+patterns are:
+
+ - In tag patterns, ``'<'`` and ``'>'`` act as parentheses; so
+ ``'<NN>+'`` matches one or more repetitions of ``'<NN>'``, not
+ ``'<NN'`` followed by one or more repetitions of ``'>'``.
+ - Whitespace in tag patterns is ignored. So
+ ``'<DT> | <NN>'`` is equivalant to ``'<DT>|<NN>'``
+ - In tag patterns, ``'.'`` is equivalant to ``'[^{}<>]'``; so
+ ``'<NN.*>'`` matches any single tag starting with ``'NN'``.
+
+The function ``tag_pattern2re_pattern`` can be used to transform
+a tag pattern to an equivalent regular expression pattern.
+
+Efficiency
+----------
+
+Preliminary tests indicate that ``RegexpChunkParser`` can chunk at a
+rate of about 300 tokens/second, with a moderately complex rule set.
+
+There may be problems if ``RegexpChunkParser`` is used with more than
+5,000 tokens at a time. In particular, evaluation of some regular
+expressions may cause the Python regular expression engine to
+exceed its maximum recursion depth. We have attempted to minimize
+these problems, but it is impossible to avoid them completely. We
+therefore recommend that you apply the chunk parser to a single
+sentence at a time.
+
+Emacs Tip
+---------
+
+If you evaluate the following elisp expression in emacs, it will
+colorize a ``ChunkString`` when you use an interactive python shell
+with emacs or xemacs ("C-c !")::
+
+ (let ()
+ (defconst comint-mode-font-lock-keywords
+ '(("<[^>]+>" 0 'font-lock-reference-face)
+ ("[{}]" 0 'font-lock-function-name-face)))
+ (add-hook 'comint-mode-hook (lambda () (turn-on-font-lock))))
+
+You can evaluate this code by copying it to a temporary buffer,
+placing the cursor after the last close parenthesis, and typing
+"``C-x C-e``". You should evaluate it before running the interactive
+session. The change will last until you close emacs.
+
+Unresolved Issues
+-----------------
+
+If we use the ``re`` module for regular expressions, Python's
+regular expression engine generates "maximum recursion depth
+exceeded" errors when processing very large texts, even for
+regular expressions that should not require any recursion. We
+therefore use the ``pre`` module instead. But note that ``pre``
+does not include Unicode support, so this module will not work
+with unicode strings. Note also that ``pre`` regular expressions
+are not quite as advanced as ``re`` ones (e.g., no leftward
+zero-length assertions).
+
+:type CHUNK_TAG_PATTERN: regexp
+:var CHUNK_TAG_PATTERN: A regular expression to test whether a tag
+ pattern is valid.
+"""
+
+from nltk.data import load
+
+from nltk.chunk.api import ChunkParserI
+from nltk.chunk.util import (ChunkScore, accuracy, tagstr2tree, conllstr2tree,
+ conlltags2tree, tree2conlltags, tree2conllstr, tree2conlltags,
+ ieerstr2tree)
+from nltk.chunk.regexp import RegexpChunkParser, RegexpParser
+
+# Standard treebank POS tagger
+_BINARY_NE_CHUNKER = 'chunkers/maxent_ne_chunker/english_ace_binary.pickle'
+_MULTICLASS_NE_CHUNKER = 'chunkers/maxent_ne_chunker/english_ace_multiclass.pickle'
+
+def ne_chunk(tagged_tokens, binary=False):
+ """
+ Use NLTK's currently recommended named entity chunker to
+ chunk the given list of tagged tokens.
+ """
+ if binary:
+ chunker_pickle = _BINARY_NE_CHUNKER
+ else:
+ chunker_pickle = _MULTICLASS_NE_CHUNKER
+ chunker = load(chunker_pickle)
+ return chunker.parse(tagged_tokens)
+
+def ne_chunk_sents(tagged_sentences, binary=False):
+ """
+ Use NLTK's currently recommended named entity chunker to chunk the
+ given list of tagged sentences, each consisting of a list of tagged tokens.
+ """
+ if binary:
+ chunker_pickle = _BINARY_NE_CHUNKER
+ else:
+ chunker_pickle = _MULTICLASS_NE_CHUNKER
+ chunker = load(chunker_pickle)
+ return chunker.parse_sents(tagged_sentences)
+
diff --git a/nltk/chunk/api.py b/nltk/chunk/api.py
new file mode 100644
index 0000000..abc7347
--- /dev/null
+++ b/nltk/chunk/api.py
@@ -0,0 +1,51 @@
+# Natural Language Toolkit: Chunk parsing API
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+##//////////////////////////////////////////////////////
+## Chunk Parser Interface
+##//////////////////////////////////////////////////////
+
+from nltk.parse import ParserI
+
+from nltk.chunk.util import ChunkScore
+
+class ChunkParserI(ParserI):
+ """
+ A processing interface for identifying non-overlapping groups in
+ unrestricted text. Typically, chunk parsers are used to find base
+ syntactic constituents, such as base noun phrases. Unlike
+ ``ParserI``, ``ChunkParserI`` guarantees that the ``parse()`` method
+ will always generate a parse.
+ """
+ def parse(self, tokens):
+ """
+ Return the best chunk structure for the given tokens
+ and return a tree.
+
+ :param tokens: The list of (word, tag) tokens to be chunked.
+ :type tokens: list(tuple)
+ :rtype: Tree
+ """
+ raise NotImplementedError()
+
+ def evaluate(self, gold):
+ """
+ Score the accuracy of the chunker against the gold standard.
+ Remove the chunking the gold standard text, rechunk it using
+ the chunker, and return a ``ChunkScore`` object
+ reflecting the performance of this chunk peraser.
+
+ :type gold: list(Tree)
+ :param gold: The list of chunked sentences to score the chunker on.
+ :rtype: ChunkScore
+ """
+ chunkscore = ChunkScore()
+ for correct in gold:
+ chunkscore.score(correct, self.parse(correct.leaves()))
+ return chunkscore
+
diff --git a/nltk/chunk/named_entity.py b/nltk/chunk/named_entity.py
new file mode 100644
index 0000000..ee14cb2
--- /dev/null
+++ b/nltk/chunk/named_entity.py
@@ -0,0 +1,331 @@
+# Natural Language Toolkit: Chunk parsing API
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Named entity chunker
+"""
+from __future__ import print_function
+
+import os, re, pickle
+from xml.etree import ElementTree as ET
+
+from nltk.tag import ClassifierBasedTagger, pos_tag
+
+try:
+ from nltk.classify import MaxentClassifier
+except ImportError:
+ pass
+
+from nltk.tree import Tree
+from nltk.tokenize import word_tokenize
+from nltk.data import find
+
+from nltk.chunk.api import ChunkParserI
+from nltk.chunk.util import ChunkScore
+
+class NEChunkParserTagger(ClassifierBasedTagger):
+ """
+ The IOB tagger used by the chunk parser.
+ """
+ def __init__(self, train):
+ ClassifierBasedTagger.__init__(
+ self, train=train,
+ classifier_builder=self._classifier_builder)
+
+ def _classifier_builder(self, train):
+ return MaxentClassifier.train(train, algorithm='megam',
+ gaussian_prior_sigma=1,
+ trace=2)
+
+ def _english_wordlist(self):
+ try:
+ wl = self._en_wordlist
+ except AttributeError:
+ from nltk.corpus import words
+ self._en_wordlist = set(words.words('en-basic'))
+ wl = self._en_wordlist
+ return wl
+
+ def _feature_detector(self, tokens, index, history):
+ word = tokens[index][0]
+ pos = simplify_pos(tokens[index][1])
+ if index == 0:
+ prevword = prevprevword = None
+ prevpos = prevprevpos = None
+ prevshape = prevtag = prevprevtag = None
+ elif index == 1:
+ prevword = tokens[index-1][0].lower()
+ prevprevword = None
+ prevpos = simplify_pos(tokens[index-1][1])
+ prevprevpos = None
+ prevtag = history[index-1][0]
+ prevshape = prevprevtag = None
+ else:
+ prevword = tokens[index-1][0].lower()
+ prevprevword = tokens[index-2][0].lower()
+ prevpos = simplify_pos(tokens[index-1][1])
+ prevprevpos = simplify_pos(tokens[index-2][1])
+ prevtag = history[index-1]
+ prevprevtag = history[index-2]
+ prevshape = shape(prevword)
+ if index == len(tokens)-1:
+ nextword = nextnextword = None
+ nextpos = nextnextpos = None
+ elif index == len(tokens)-2:
+ nextword = tokens[index+1][0].lower()
+ nextpos = tokens[index+1][1].lower()
+ nextnextword = None
+ nextnextpos = None
+ else:
+ nextword = tokens[index+1][0].lower()
+ nextpos = tokens[index+1][1].lower()
+ nextnextword = tokens[index+2][0].lower()
+ nextnextpos = tokens[index+2][1].lower()
+
+ # 89.6
+ features = {
+ 'bias': True,
+ 'shape': shape(word),
+ 'wordlen': len(word),
+ 'prefix3': word[:3].lower(),
+ 'suffix3': word[-3:].lower(),
+ 'pos': pos,
+ 'word': word,
+ 'en-wordlist': (word in self._english_wordlist()),
+ 'prevtag': prevtag,
+ 'prevpos': prevpos,
+ 'nextpos': nextpos,
+ 'prevword': prevword,
+ 'nextword': nextword,
+ 'word+nextpos': '%s+%s' % (word.lower(), nextpos),
+ 'pos+prevtag': '%s+%s' % (pos, prevtag),
+ 'shape+prevtag': '%s+%s' % (prevshape, prevtag),
+ }
+
+ return features
+
+class NEChunkParser(ChunkParserI):
+ """
+ Expected input: list of pos-tagged words
+ """
+ def __init__(self, train):
+ self._train(train)
+
+ def parse(self, tokens):
+ """
+ Each token should be a pos-tagged word
+ """
+ tagged = self._tagger.tag(tokens)
+ tree = self._tagged_to_parse(tagged)
+ return tree
+
+ def _train(self, corpus):
+ # Convert to tagged sequence
+ corpus = [self._parse_to_tagged(s) for s in corpus]
+
+ self._tagger = NEChunkParserTagger(train=corpus)
+
+ def _tagged_to_parse(self, tagged_tokens):
+ """
+ Convert a list of tagged tokens to a chunk-parse tree.
+ """
+ sent = Tree('S', [])
+
+ for (tok,tag) in tagged_tokens:
+ if tag == 'O':
+ sent.append(tok)
+ elif tag.startswith('B-'):
+ sent.append(Tree(tag[2:], [tok]))
+ elif tag.startswith('I-'):
+ if (sent and isinstance(sent[-1], Tree) and
+ sent[-1].label() == tag[2:]):
+ sent[-1].append(tok)
+ else:
+ sent.append(Tree(tag[2:], [tok]))
+ return sent
+
+ @staticmethod
+ def _parse_to_tagged(sent):
+ """
+ Convert a chunk-parse tree to a list of tagged tokens.
+ """
+ toks = []
+ for child in sent:
+ if isinstance(child, Tree):
+ if len(child) == 0:
+ print("Warning -- empty chunk in sentence")
+ continue
+ toks.append((child[0], 'B-%s' % child.label()))
+ for tok in child[1:]:
+ toks.append((tok, 'I-%s' % child.label()))
+ else:
+ toks.append((child, 'O'))
+ return toks
+
+def shape(word):
+ if re.match('[0-9]+(\.[0-9]*)?|[0-9]*\.[0-9]+$', word, re.UNICODE):
+ return 'number'
+ elif re.match('\W+$', word, re.UNICODE):
+ return 'punct'
+ elif re.match('\w+$', word, re.UNICODE):
+ if word.istitle():
+ return 'upcase'
+ elif word.islower():
+ return 'downcase'
+ else:
+ return 'mixedcase'
+ else:
+ return 'other'
+
+def simplify_pos(s):
+ if s.startswith('V'): return "V"
+ else: return s.split('-')[0]
+
+def postag_tree(tree):
+ # Part-of-speech tagging.
+ words = tree.leaves()
+ tag_iter = (pos for (word, pos) in pos_tag(words))
+ newtree = Tree('S', [])
+ for child in tree:
+ if isinstance(child, Tree):
+ newtree.append(Tree(child.label(), []))
+ for subchild in child:
+ newtree[-1].append( (subchild, next(tag_iter)) )
+ else:
+ newtree.append( (child, next(tag_iter)) )
+ return newtree
+
+def load_ace_data(roots, fmt='binary', skip_bnews=True):
+ for root in roots:
+ for root, dirs, files in os.walk(root):
+ if root.endswith('bnews') and skip_bnews:
+ continue
+ for f in files:
+ if f.endswith('.sgm'):
+ for sent in load_ace_file(os.path.join(root, f), fmt):
+ yield sent
+
+def load_ace_file(textfile, fmt):
+ print(' - %s' % os.path.split(textfile)[1])
+ annfile = textfile+'.tmx.rdc.xml'
+
+ # Read the xml file, and get a list of entities
+ entities = []
+ with open(annfile, 'r') as infile:
+ xml = ET.parse(infile).getroot()
+ for entity in xml.findall('document/entity'):
+ typ = entity.find('entity_type').text
+ for mention in entity.findall('entity_mention'):
+ if mention.get('TYPE') != 'NAME': continue # only NEs
+ s = int(mention.find('head/charseq/start').text)
+ e = int(mention.find('head/charseq/end').text)+1
+ entities.append( (s, e, typ) )
+
+ # Read the text file, and mark the entities.
+ with open(textfile, 'r') as infile:
+ text = infile.read()
+
+ # Strip XML tags, since they don't count towards the indices
+ text = re.sub('<(?!/?TEXT)[^>]+>', '', text)
+
+ # Blank out anything before/after <TEXT>
+ def subfunc(m): return ' '*(m.end()-m.start()-6)
+ text = re.sub('[\s\S]*<TEXT>', subfunc, text)
+ text = re.sub('</TEXT>[\s\S]*', '', text)
+
+ # Simplify quotes
+ text = re.sub("``", ' "', text)
+ text = re.sub("''", '" ', text)
+
+ entity_types = set(typ for (s,e,typ) in entities)
+
+ # Binary distinction (NE or not NE)
+ if fmt == 'binary':
+ i = 0
+ toks = Tree('S', [])
+ for (s,e,typ) in sorted(entities):
+ if s < i: s = i # Overlapping! Deal with this better?
+ if e <= s: continue
+ toks.extend(word_tokenize(text[i:s]))
+ toks.append(Tree('NE', text[s:e].split()))
+ i = e
+ toks.extend(word_tokenize(text[i:]))
+ yield toks
+
+ # Multiclass distinction (NE type)
+ elif fmt == 'multiclass':
+ i = 0
+ toks = Tree('S', [])
+ for (s,e,typ) in sorted(entities):
+ if s < i: s = i # Overlapping! Deal with this better?
+ if e <= s: continue
+ toks.extend(word_tokenize(text[i:s]))
+ toks.append(Tree(typ, text[s:e].split()))
+ i = e
+ toks.extend(word_tokenize(text[i:]))
+ yield toks
+
+ else:
+ raise ValueError('bad fmt value')
+
+# This probably belongs in a more general-purpose location (as does
+# the parse_to_tagged function).
+def cmp_chunks(correct, guessed):
+ correct = NEChunkParser._parse_to_tagged(correct)
+ guessed = NEChunkParser._parse_to_tagged(guessed)
+ ellipsis = False
+ for (w, ct), (w, gt) in zip(correct, guessed):
+ if ct == gt == 'O':
+ if not ellipsis:
+ print(" %-15s %-15s %s" % (ct, gt, w))
+ print(' %-15s %-15s %s' % ('...', '...', '...'))
+ ellipsis = True
+ else:
+ ellipsis = False
+ print(" %-15s %-15s %s" % (ct, gt, w))
+
+def build_model(fmt='binary'):
+ print('Loading training data...')
+ train_paths = [find('corpora/ace_data/ace.dev'),
+ find('corpora/ace_data/ace.heldout'),
+ find('corpora/ace_data/bbn.dev'),
+ find('corpora/ace_data/muc.dev')]
+ train_trees = load_ace_data(train_paths, fmt)
+ train_data = [postag_tree(t) for t in train_trees]
+ print('Training...')
+ cp = NEChunkParser(train_data)
+ del train_data
+
+ print('Loading eval data...')
+ eval_paths = [find('corpora/ace_data/ace.eval')]
+ eval_trees = load_ace_data(eval_paths, fmt)
+ eval_data = [postag_tree(t) for t in eval_trees]
+
+ print('Evaluating...')
+ chunkscore = ChunkScore()
+ for i, correct in enumerate(eval_data):
+ guess = cp.parse(correct.leaves())
+ chunkscore.score(correct, guess)
+ if i < 3: cmp_chunks(correct, guess)
+ print(chunkscore)
+
+ outfilename = '/tmp/ne_chunker_%s.pickle' % fmt
+ print('Saving chunker to %s...' % outfilename)
+
+ with open(outfilename, 'wb') as outfile:
+ pickle.dump(cp, outfile, -1)
+
+ return cp
+
+
+if __name__ == '__main__':
+ # Make sure that the pickled object has the right class name:
+ from nltk.chunk.named_entity import build_model
+
+ build_model('binary')
+ build_model('multiclass')
+
diff --git a/nltk/chunk/regexp.py b/nltk/chunk/regexp.py
new file mode 100644
index 0000000..5fd7b3d
--- /dev/null
+++ b/nltk/chunk/regexp.py
@@ -0,0 +1,1384 @@
+# Natural Language Toolkit: Regular Expression Chunkers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+from __future__ import division
+
+import re
+
+from nltk.tree import Tree
+from nltk.chunk.api import ChunkParserI
+from nltk.compat import python_2_unicode_compatible, string_types, unicode_repr
+
+##//////////////////////////////////////////////////////
+## ChunkString
+##//////////////////////////////////////////////////////
+
+ at python_2_unicode_compatible
+class ChunkString(object):
+ """
+ A string-based encoding of a particular chunking of a text.
+ Internally, the ``ChunkString`` class uses a single string to
+ encode the chunking of the input text. This string contains a
+ sequence of angle-bracket delimited tags, with chunking indicated
+ by braces. An example of this encoding is::
+
+ {<DT><JJ><NN>}<VBN><IN>{<DT><NN>}<.>{<DT><NN>}<VBD><.>
+
+ ``ChunkString`` are created from tagged texts (i.e., lists of
+ ``tokens`` whose type is ``TaggedType``). Initially, nothing is
+ chunked.
+
+ The chunking of a ``ChunkString`` can be modified with the ``xform()``
+ method, which uses a regular expression to transform the string
+ representation. These transformations should only add and remove
+ braces; they should *not* modify the sequence of angle-bracket
+ delimited tags.
+
+ :type _str: str
+ :ivar _str: The internal string representation of the text's
+ encoding. This string representation contains a sequence of
+ angle-bracket delimited tags, with chunking indicated by
+ braces. An example of this encoding is::
+
+ {<DT><JJ><NN>}<VBN><IN>{<DT><NN>}<.>{<DT><NN>}<VBD><.>
+
+ :type _pieces: list(tagged tokens and chunks)
+ :ivar _pieces: The tagged tokens and chunks encoded by this ``ChunkString``.
+ :ivar _debug: The debug level. See the constructor docs.
+
+ :cvar IN_CHUNK_PATTERN: A zero-width regexp pattern string that
+ will only match positions that are in chunks.
+ :cvar IN_CHINK_PATTERN: A zero-width regexp pattern string that
+ will only match positions that are in chinks.
+ """
+ CHUNK_TAG_CHAR = r'[^\{\}<>]'
+ CHUNK_TAG = r'(<%s+?>)' % CHUNK_TAG_CHAR
+
+ IN_CHUNK_PATTERN = r'(?=[^\{]*\})'
+ IN_CHINK_PATTERN = r'(?=[^\}]*(\{|$))'
+
+ # These are used by _verify
+ _CHUNK = r'(\{%s+?\})+?' % CHUNK_TAG
+ _CHINK = r'(%s+?)+?' % CHUNK_TAG
+ _VALID = re.compile(r'^(\{?%s\}?)*?$' % CHUNK_TAG)
+ _BRACKETS = re.compile('[^\{\}]+')
+ _BALANCED_BRACKETS = re.compile(r'(\{\})*$')
+
+ def __init__(self, chunk_struct, debug_level=1):
+ """
+ Construct a new ``ChunkString`` that encodes the chunking of
+ the text ``tagged_tokens``.
+
+ :type chunk_struct: Tree
+ :param chunk_struct: The chunk structure to be further chunked.
+ :type debug_level: int
+ :param debug_level: The level of debugging which should be
+ applied to transformations on the ``ChunkString``. The
+ valid levels are:
+ - 0: no checks
+ - 1: full check on to_chunkstruct
+ - 2: full check on to_chunkstruct and cursory check after
+ each transformation.
+ - 3: full check on to_chunkstruct and full check after
+ each transformation.
+ We recommend you use at least level 1. You should
+ probably use level 3 if you use any non-standard
+ subclasses of ``RegexpChunkRule``.
+ """
+ self._root_label = chunk_struct.label()
+ self._pieces = chunk_struct[:]
+ tags = [self._tag(tok) for tok in self._pieces]
+ self._str = '<' + '><'.join(tags) + '>'
+ self._debug = debug_level
+
+ def _tag(self, tok):
+ if isinstance(tok, tuple):
+ return tok[1]
+ elif isinstance(tok, Tree):
+ return tok.label()
+ else:
+ raise ValueError('chunk structures must contain tagged '
+ 'tokens or trees')
+
+ def _verify(self, s, verify_tags):
+ """
+ Check to make sure that ``s`` still corresponds to some chunked
+ version of ``_pieces``.
+
+ :type verify_tags: bool
+ :param verify_tags: Whether the individual tags should be
+ checked. If this is false, ``_verify`` will check to make
+ sure that ``_str`` encodes a chunked version of *some*
+ list of tokens. If this is true, then ``_verify`` will
+ check to make sure that the tags in ``_str`` match those in
+ ``_pieces``.
+
+ :raise ValueError: if the internal string representation of
+ this ``ChunkString`` is invalid or not consistent with _pieces.
+ """
+ # Check overall form
+ if not ChunkString._VALID.match(s):
+ raise ValueError('Transformation generated invalid '
+ 'chunkstring:\n %s' % s)
+
+ # Check that parens are balanced. If the string is long, we
+ # have to do this in pieces, to avoid a maximum recursion
+ # depth limit for regular expressions.
+ brackets = ChunkString._BRACKETS.sub('', s)
+ for i in range(1 + len(brackets) // 5000):
+ substr = brackets[i*5000:i*5000+5000]
+ if not ChunkString._BALANCED_BRACKETS.match(substr):
+ raise ValueError('Transformation generated invalid '
+ 'chunkstring:\n %s' % s)
+
+ if verify_tags<=0: return
+
+ tags1 = (re.split(r'[\{\}<>]+', s))[1:-1]
+ tags2 = [self._tag(piece) for piece in self._pieces]
+ if tags1 != tags2:
+ raise ValueError('Transformation generated invalid '
+ 'chunkstring: tag changed')
+
+ def to_chunkstruct(self, chunk_label='CHUNK'):
+ """
+ Return the chunk structure encoded by this ``ChunkString``.
+
+ :rtype: Tree
+ :raise ValueError: If a transformation has generated an
+ invalid chunkstring.
+ """
+ if self._debug > 0: self._verify(self._str, 1)
+
+ # Use this alternating list to create the chunkstruct.
+ pieces = []
+ index = 0
+ piece_in_chunk = 0
+ for piece in re.split('[{}]', self._str):
+
+ # Find the list of tokens contained in this piece.
+ length = piece.count('<')
+ subsequence = self._pieces[index:index+length]
+
+ # Add this list of tokens to our pieces.
+ if piece_in_chunk:
+ pieces.append(Tree(chunk_label, subsequence))
+ else:
+ pieces += subsequence
+
+ # Update index, piece_in_chunk
+ index += length
+ piece_in_chunk = not piece_in_chunk
+
+ return Tree(self._root_label, pieces)
+
+ def xform(self, regexp, repl):
+ """
+ Apply the given transformation to the string encoding of this
+ ``ChunkString``. In particular, find all occurrences that match
+ ``regexp``, and replace them using ``repl`` (as done by
+ ``re.sub``).
+
+ This transformation should only add and remove braces; it
+ should *not* modify the sequence of angle-bracket delimited
+ tags. Furthermore, this transformation may not result in
+ improper bracketing. Note, in particular, that bracketing may
+ not be nested.
+
+ :type regexp: str or regexp
+ :param regexp: A regular expression matching the substring
+ that should be replaced. This will typically include a
+ named group, which can be used by ``repl``.
+ :type repl: str
+ :param repl: An expression specifying what should replace the
+ matched substring. Typically, this will include a named
+ replacement group, specified by ``regexp``.
+ :rtype: None
+ :raise ValueError: If this transformation generated an
+ invalid chunkstring.
+ """
+ # Do the actual substitution
+ s = re.sub(regexp, repl, self._str)
+
+ # The substitution might have generated "empty chunks"
+ # (substrings of the form "{}"). Remove them, so they don't
+ # interfere with other transformations.
+ s = re.sub('\{\}', '', s)
+
+ # Make sure that the transformation was legal.
+ if self._debug > 1: self._verify(s, self._debug-2)
+
+ # Commit the transformation.
+ self._str = s
+
+ def __repr__(self):
+ """
+ Return a string representation of this ``ChunkString``.
+ It has the form::
+
+ <ChunkString: '{<DT><JJ><NN>}<VBN><IN>{<DT><NN>}'>
+
+ :rtype: str
+ """
+ return '<ChunkString: %s>' % unicode_repr(self._str)
+
+ def __str__(self):
+ """
+ Return a formatted representation of this ``ChunkString``.
+ This representation will include extra spaces to ensure that
+ tags will line up with the representation of other
+ ``ChunkStrings`` for the same text, regardless of the chunking.
+
+ :rtype: str
+ """
+ # Add spaces to make everything line up.
+ str = re.sub(r'>(?!\})', r'> ', self._str)
+ str = re.sub(r'([^\{])<', r'\1 <', str)
+ if str[0] == '<': str = ' ' + str
+ return str
+
+##//////////////////////////////////////////////////////
+## Chunking Rules
+##//////////////////////////////////////////////////////
+
+ at python_2_unicode_compatible
+class RegexpChunkRule(object):
+ """
+ A rule specifying how to modify the chunking in a ``ChunkString``,
+ using a transformational regular expression. The
+ ``RegexpChunkRule`` class itself can be used to implement any
+ transformational rule based on regular expressions. There are
+ also a number of subclasses, which can be used to implement
+ simpler types of rules, based on matching regular expressions.
+
+ Each ``RegexpChunkRule`` has a regular expression and a
+ replacement expression. When a ``RegexpChunkRule`` is "applied"
+ to a ``ChunkString``, it searches the ``ChunkString`` for any
+ substring that matches the regular expression, and replaces it
+ using the replacement expression. This search/replace operation
+ has the same semantics as ``re.sub``.
+
+ Each ``RegexpChunkRule`` also has a description string, which
+ gives a short (typically less than 75 characters) description of
+ the purpose of the rule.
+
+ This transformation defined by this ``RegexpChunkRule`` should
+ only add and remove braces; it should *not* modify the sequence
+ of angle-bracket delimited tags. Furthermore, this transformation
+ may not result in nested or mismatched bracketing.
+ """
+ def __init__(self, regexp, repl, descr):
+ """
+ Construct a new RegexpChunkRule.
+
+ :type regexp: regexp or str
+ :param regexp: The regular expression for this ``RegexpChunkRule``.
+ When this rule is applied to a ``ChunkString``, any
+ substring that matches ``regexp`` will be replaced using
+ the replacement string ``repl``. Note that this must be a
+ normal regular expression, not a tag pattern.
+ :type repl: str
+ :param repl: The replacement expression for this ``RegexpChunkRule``.
+ When this rule is applied to a ``ChunkString``, any substring
+ that matches ``regexp`` will be replaced using ``repl``.
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ if isinstance(regexp, string_types):
+ regexp = re.compile(regexp)
+ self._repl = repl
+ self._descr = descr
+ self._regexp = regexp
+
+ def apply(self, chunkstr):
+ # Keep docstring generic so we can inherit it.
+ """
+ Apply this rule to the given ``ChunkString``. See the
+ class reference documentation for a description of what it
+ means to apply a rule.
+
+ :type chunkstr: ChunkString
+ :param chunkstr: The chunkstring to which this rule is applied.
+ :rtype: None
+ :raise ValueError: If this transformation generated an
+ invalid chunkstring.
+ """
+ chunkstr.xform(self._regexp, self._repl)
+
+ def descr(self):
+ """
+ Return a short description of the purpose and/or effect of
+ this rule.
+
+ :rtype: str
+ """
+ return self._descr
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <RegexpChunkRule: '{<IN|VB.*>}'->'<IN>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return ('<RegexpChunkRule: '+unicode_repr(self._regexp.pattern)+
+ '->'+unicode_repr(self._repl)+'>')
+
+ @staticmethod
+ def fromstring(s):
+ """
+ Create a RegexpChunkRule from a string description.
+ Currently, the following formats are supported::
+
+ {regexp} # chunk rule
+ }regexp{ # chink rule
+ regexp}{regexp # split rule
+ regexp{}regexp # merge rule
+
+ Where ``regexp`` is a regular expression for the rule. Any
+ text following the comment marker (``#``) will be used as
+ the rule's description:
+
+ >>> from nltk.chunk.regexp import RegexpChunkRule
+ >>> RegexpChunkRule.fromstring('{<DT>?<NN.*>+}')
+ <ChunkRule: '<DT>?<NN.*>+'>
+ """
+ # Split off the comment (but don't split on '\#')
+ m = re.match(r'(?P<rule>(\\.|[^#])*)(?P<comment>#.*)?', s)
+ rule = m.group('rule').strip()
+ comment = (m.group('comment') or '')[1:].strip()
+
+ # Pattern bodies: chunk, chink, split, merge
+ try:
+ if not rule:
+ raise ValueError('Empty chunk pattern')
+ if rule[0] == '{' and rule[-1] == '}':
+ return ChunkRule(rule[1:-1], comment)
+ elif rule[0] == '}' and rule[-1] == '{':
+ return ChinkRule(rule[1:-1], comment)
+ elif '}{' in rule:
+ left, right = rule.split('}{')
+ return SplitRule(left, right, comment)
+ elif '{}' in rule:
+ left, right = rule.split('{}')
+ return MergeRule(left, right, comment)
+ elif re.match('[^{}]*{[^{}]*}[^{}]*', rule):
+ left, chunk, right = re.split('[{}]', rule)
+ return ChunkRuleWithContext(left, chunk, right, comment)
+ else:
+ raise ValueError('Illegal chunk pattern: %s' % rule)
+ except (ValueError, re.error):
+ raise ValueError('Illegal chunk pattern: %s' % rule)
+
+
+ at python_2_unicode_compatible
+class ChunkRule(RegexpChunkRule):
+ """
+ A rule specifying how to add chunks to a ``ChunkString``, using a
+ matching tag pattern. When applied to a ``ChunkString``, it will
+ find any substring that matches this tag pattern and that is not
+ already part of a chunk, and create a new chunk containing that
+ substring.
+ """
+ def __init__(self, tag_pattern, descr):
+
+ """
+ Construct a new ``ChunkRule``.
+
+ :type tag_pattern: str
+ :param tag_pattern: This rule's tag pattern. When
+ applied to a ``ChunkString``, this rule will
+ chunk any substring that matches this tag pattern and that
+ is not already part of a chunk.
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ self._pattern = tag_pattern
+ regexp = re.compile('(?P<chunk>%s)%s' %
+ (tag_pattern2re_pattern(tag_pattern),
+ ChunkString.IN_CHINK_PATTERN))
+ RegexpChunkRule.__init__(self, regexp, '{\g<chunk>}', descr)
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <ChunkRule: '<IN|VB.*>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return '<ChunkRule: '+unicode_repr(self._pattern)+'>'
+
+ at python_2_unicode_compatible
+class ChinkRule(RegexpChunkRule):
+ """
+ A rule specifying how to remove chinks to a ``ChunkString``,
+ using a matching tag pattern. When applied to a
+ ``ChunkString``, it will find any substring that matches this
+ tag pattern and that is contained in a chunk, and remove it
+ from that chunk, thus creating two new chunks.
+ """
+ def __init__(self, tag_pattern, descr):
+ """
+ Construct a new ``ChinkRule``.
+
+ :type tag_pattern: str
+ :param tag_pattern: This rule's tag pattern. When
+ applied to a ``ChunkString``, this rule will
+ find any substring that matches this tag pattern and that
+ is contained in a chunk, and remove it from that chunk,
+ thus creating two new chunks.
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ self._pattern = tag_pattern
+ regexp = re.compile('(?P<chink>%s)%s' %
+ (tag_pattern2re_pattern(tag_pattern),
+ ChunkString.IN_CHUNK_PATTERN))
+ RegexpChunkRule.__init__(self, regexp, '}\g<chink>{', descr)
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <ChinkRule: '<IN|VB.*>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return '<ChinkRule: '+unicode_repr(self._pattern)+'>'
+
+
+ at python_2_unicode_compatible
+class UnChunkRule(RegexpChunkRule):
+ """
+ A rule specifying how to remove chunks to a ``ChunkString``,
+ using a matching tag pattern. When applied to a
+ ``ChunkString``, it will find any complete chunk that matches this
+ tag pattern, and un-chunk it.
+ """
+ def __init__(self, tag_pattern, descr):
+ """
+ Construct a new ``UnChunkRule``.
+
+ :type tag_pattern: str
+ :param tag_pattern: This rule's tag pattern. When
+ applied to a ``ChunkString``, this rule will
+ find any complete chunk that matches this tag pattern,
+ and un-chunk it.
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ self._pattern = tag_pattern
+ regexp = re.compile('\{(?P<chunk>%s)\}' %
+ tag_pattern2re_pattern(tag_pattern))
+ RegexpChunkRule.__init__(self, regexp, '\g<chunk>', descr)
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <UnChunkRule: '<IN|VB.*>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return '<UnChunkRule: '+unicode_repr(self._pattern)+'>'
+
+
+ at python_2_unicode_compatible
+class MergeRule(RegexpChunkRule):
+ """
+ A rule specifying how to merge chunks in a ``ChunkString``, using
+ two matching tag patterns: a left pattern, and a right pattern.
+ When applied to a ``ChunkString``, it will find any chunk whose end
+ matches left pattern, and immediately followed by a chunk whose
+ beginning matches right pattern. It will then merge those two
+ chunks into a single chunk.
+ """
+ def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+ """
+ Construct a new ``MergeRule``.
+
+ :type right_tag_pattern: str
+ :param right_tag_pattern: This rule's right tag
+ pattern. When applied to a ``ChunkString``, this
+ rule will find any chunk whose end matches
+ ``left_tag_pattern``, and immediately followed by a chunk
+ whose beginning matches this pattern. It will
+ then merge those two chunks into a single chunk.
+ :type left_tag_pattern: str
+ :param left_tag_pattern: This rule's left tag
+ pattern. When applied to a ``ChunkString``, this
+ rule will find any chunk whose end matches
+ this pattern, and immediately followed by a chunk
+ whose beginning matches ``right_tag_pattern``. It will
+ then merge those two chunks into a single chunk.
+
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ # Ensure that the individual patterns are coherent. E.g., if
+ # left='(' and right=')', then this will raise an exception:
+ re.compile(tag_pattern2re_pattern(left_tag_pattern))
+ re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+ self._left_tag_pattern = left_tag_pattern
+ self._right_tag_pattern = right_tag_pattern
+ regexp = re.compile('(?P<left>%s)}{(?=%s)' %
+ (tag_pattern2re_pattern(left_tag_pattern),
+ tag_pattern2re_pattern(right_tag_pattern)))
+ RegexpChunkRule.__init__(self, regexp, '\g<left>', descr)
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <MergeRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return ('<MergeRule: '+unicode_repr(self._left_tag_pattern)+', '+
+ unicode_repr(self._right_tag_pattern)+'>')
+
+
+ at python_2_unicode_compatible
+class SplitRule(RegexpChunkRule):
+ """
+ A rule specifying how to split chunks in a ``ChunkString``, using
+ two matching tag patterns: a left pattern, and a right pattern.
+ When applied to a ``ChunkString``, it will find any chunk that
+ matches the left pattern followed by the right pattern. It will
+ then split the chunk into two new chunks, at the point between the
+ two pattern matches.
+ """
+ def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+ """
+ Construct a new ``SplitRule``.
+
+ :type right_tag_pattern: str
+ :param right_tag_pattern: This rule's right tag
+ pattern. When applied to a ``ChunkString``, this rule will
+ find any chunk containing a substring that matches
+ ``left_tag_pattern`` followed by this pattern. It will
+ then split the chunk into two new chunks at the point
+ between these two matching patterns.
+ :type left_tag_pattern: str
+ :param left_tag_pattern: This rule's left tag
+ pattern. When applied to a ``ChunkString``, this rule will
+ find any chunk containing a substring that matches this
+ pattern followed by ``right_tag_pattern``. It will then
+ split the chunk into two new chunks at the point between
+ these two matching patterns.
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ # Ensure that the individual patterns are coherent. E.g., if
+ # left='(' and right=')', then this will raise an exception:
+ re.compile(tag_pattern2re_pattern(left_tag_pattern))
+ re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+ self._left_tag_pattern = left_tag_pattern
+ self._right_tag_pattern = right_tag_pattern
+ regexp = re.compile('(?P<left>%s)(?=%s)' %
+ (tag_pattern2re_pattern(left_tag_pattern),
+ tag_pattern2re_pattern(right_tag_pattern)))
+ RegexpChunkRule.__init__(self, regexp, r'\g<left>}{', descr)
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <SplitRule: '<NN>', '<DT>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return ('<SplitRule: '+unicode_repr(self._left_tag_pattern)+', '+
+ unicode_repr(self._right_tag_pattern)+'>')
+
+
+ at python_2_unicode_compatible
+class ExpandLeftRule(RegexpChunkRule):
+ """
+ A rule specifying how to expand chunks in a ``ChunkString`` to the left,
+ using two matching tag patterns: a left pattern, and a right pattern.
+ When applied to a ``ChunkString``, it will find any chunk whose beginning
+ matches right pattern, and immediately preceded by a chink whose
+ end matches left pattern. It will then expand the chunk to incorporate
+ the new material on the left.
+ """
+ def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+ """
+ Construct a new ``ExpandRightRule``.
+
+ :type right_tag_pattern: str
+ :param right_tag_pattern: This rule's right tag
+ pattern. When applied to a ``ChunkString``, this
+ rule will find any chunk whose beginning matches
+ ``right_tag_pattern``, and immediately preceded by a chink
+ whose end matches this pattern. It will
+ then merge those two chunks into a single chunk.
+ :type left_tag_pattern: str
+ :param left_tag_pattern: This rule's left tag
+ pattern. When applied to a ``ChunkString``, this
+ rule will find any chunk whose beginning matches
+ this pattern, and immediately preceded by a chink
+ whose end matches ``left_tag_pattern``. It will
+ then expand the chunk to incorporate the new material on the left.
+
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ # Ensure that the individual patterns are coherent. E.g., if
+ # left='(' and right=')', then this will raise an exception:
+ re.compile(tag_pattern2re_pattern(left_tag_pattern))
+ re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+ self._left_tag_pattern = left_tag_pattern
+ self._right_tag_pattern = right_tag_pattern
+ regexp = re.compile('(?P<left>%s)\{(?P<right>%s)' %
+ (tag_pattern2re_pattern(left_tag_pattern),
+ tag_pattern2re_pattern(right_tag_pattern)))
+ RegexpChunkRule.__init__(self, regexp, '{\g<left>\g<right>', descr)
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <ExpandLeftRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return ('<ExpandLeftRule: '+unicode_repr(self._left_tag_pattern)+', '+
+ unicode_repr(self._right_tag_pattern)+'>')
+
+
+ at python_2_unicode_compatible
+class ExpandRightRule(RegexpChunkRule):
+ """
+ A rule specifying how to expand chunks in a ``ChunkString`` to the
+ right, using two matching tag patterns: a left pattern, and a
+ right pattern. When applied to a ``ChunkString``, it will find any
+ chunk whose end matches left pattern, and immediately followed by
+ a chink whose beginning matches right pattern. It will then
+ expand the chunk to incorporate the new material on the right.
+ """
+ def __init__(self, left_tag_pattern, right_tag_pattern, descr):
+ """
+ Construct a new ``ExpandRightRule``.
+
+ :type right_tag_pattern: str
+ :param right_tag_pattern: This rule's right tag
+ pattern. When applied to a ``ChunkString``, this
+ rule will find any chunk whose end matches
+ ``left_tag_pattern``, and immediately followed by a chink
+ whose beginning matches this pattern. It will
+ then merge those two chunks into a single chunk.
+ :type left_tag_pattern: str
+ :param left_tag_pattern: This rule's left tag
+ pattern. When applied to a ``ChunkString``, this
+ rule will find any chunk whose end matches
+ this pattern, and immediately followed by a chink
+ whose beginning matches ``right_tag_pattern``. It will
+ then expand the chunk to incorporate the new material on the right.
+
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ # Ensure that the individual patterns are coherent. E.g., if
+ # left='(' and right=')', then this will raise an exception:
+ re.compile(tag_pattern2re_pattern(left_tag_pattern))
+ re.compile(tag_pattern2re_pattern(right_tag_pattern))
+
+ self._left_tag_pattern = left_tag_pattern
+ self._right_tag_pattern = right_tag_pattern
+ regexp = re.compile('(?P<left>%s)\}(?P<right>%s)' %
+ (tag_pattern2re_pattern(left_tag_pattern),
+ tag_pattern2re_pattern(right_tag_pattern)))
+ RegexpChunkRule.__init__(self, regexp, '\g<left>\g<right>}', descr)
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <ExpandRightRule: '<NN|DT|JJ>', '<NN|JJ>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return ('<ExpandRightRule: '+unicode_repr(self._left_tag_pattern)+', '+
+ unicode_repr(self._right_tag_pattern)+'>')
+
+
+ at python_2_unicode_compatible
+class ChunkRuleWithContext(RegexpChunkRule):
+ """
+ A rule specifying how to add chunks to a ``ChunkString``, using
+ three matching tag patterns: one for the left context, one for the
+ chunk, and one for the right context. When applied to a
+ ``ChunkString``, it will find any substring that matches the chunk
+ tag pattern, is surrounded by substrings that match the two
+ context patterns, and is not already part of a chunk; and create a
+ new chunk containing the substring that matched the chunk tag
+ pattern.
+
+ Caveat: Both the left and right context are consumed when this
+ rule matches; therefore, if you need to find overlapping matches,
+ you will need to apply your rule more than once.
+ """
+ def __init__(self, left_context_tag_pattern, chunk_tag_pattern,
+ right_context_tag_pattern, descr):
+ """
+ Construct a new ``ChunkRuleWithContext``.
+
+ :type left_context_tag_pattern: str
+ :param left_context_tag_pattern: A tag pattern that must match
+ the left context of ``chunk_tag_pattern`` for this rule to
+ apply.
+ :type chunk_tag_pattern: str
+ :param chunk_tag_pattern: A tag pattern that must match for this
+ rule to apply. If the rule does apply, then this pattern
+ also identifies the substring that will be made into a chunk.
+ :type right_context_tag_pattern: str
+ :param right_context_tag_pattern: A tag pattern that must match
+ the right context of ``chunk_tag_pattern`` for this rule to
+ apply.
+ :type descr: str
+ :param descr: A short description of the purpose and/or effect
+ of this rule.
+ """
+ # Ensure that the individual patterns are coherent. E.g., if
+ # left='(' and right=')', then this will raise an exception:
+ re.compile(tag_pattern2re_pattern(left_context_tag_pattern))
+ re.compile(tag_pattern2re_pattern(chunk_tag_pattern))
+ re.compile(tag_pattern2re_pattern(right_context_tag_pattern))
+
+ self._left_context_tag_pattern = left_context_tag_pattern
+ self._chunk_tag_pattern = chunk_tag_pattern
+ self._right_context_tag_pattern = right_context_tag_pattern
+ regexp = re.compile('(?P<left>%s)(?P<chunk>%s)(?P<right>%s)%s' %
+ (tag_pattern2re_pattern(left_context_tag_pattern),
+ tag_pattern2re_pattern(chunk_tag_pattern),
+ tag_pattern2re_pattern(right_context_tag_pattern),
+ ChunkString.IN_CHINK_PATTERN))
+ replacement = r'\g<left>{\g<chunk>}\g<right>'
+ RegexpChunkRule.__init__(self, regexp, replacement, descr)
+
+ def __repr__(self):
+ """
+ Return a string representation of this rule. It has the form::
+
+ <ChunkRuleWithContext: '<IN>', '<NN>', '<DT>'>
+
+ Note that this representation does not include the
+ description string; that string can be accessed
+ separately with the ``descr()`` method.
+
+ :rtype: str
+ """
+ return '<ChunkRuleWithContext: %r, %r, %r>' % (
+ self._left_context_tag_pattern, self._chunk_tag_pattern,
+ self._right_context_tag_pattern)
+
+##//////////////////////////////////////////////////////
+## Tag Pattern Format Conversion
+##//////////////////////////////////////////////////////
+
+# this should probably be made more strict than it is -- e.g., it
+# currently accepts 'foo'.
+CHUNK_TAG_PATTERN = re.compile(r'^((%s|<%s>)*)$' %
+ ('[^\{\}<>]+',
+ '[^\{\}<>]+'))
+
+def tag_pattern2re_pattern(tag_pattern):
+ """
+ Convert a tag pattern to a regular expression pattern. A "tag
+ pattern" is a modified version of a regular expression, designed
+ for matching sequences of tags. The differences between regular
+ expression patterns and tag patterns are:
+
+ - In tag patterns, ``'<'`` and ``'>'`` act as parentheses; so
+ ``'<NN>+'`` matches one or more repetitions of ``'<NN>'``, not
+ ``'<NN'`` followed by one or more repetitions of ``'>'``.
+ - Whitespace in tag patterns is ignored. So
+ ``'<DT> | <NN>'`` is equivalant to ``'<DT>|<NN>'``
+ - In tag patterns, ``'.'`` is equivalant to ``'[^{}<>]'``; so
+ ``'<NN.*>'`` matches any single tag starting with ``'NN'``.
+
+ In particular, ``tag_pattern2re_pattern`` performs the following
+ transformations on the given pattern:
+
+ - Replace '.' with '[^<>{}]'
+ - Remove any whitespace
+ - Add extra parens around '<' and '>', to make '<' and '>' act
+ like parentheses. E.g., so that in '<NN>+', the '+' has scope
+ over the entire '<NN>'; and so that in '<NN|IN>', the '|' has
+ scope over 'NN' and 'IN', but not '<' or '>'.
+ - Check to make sure the resulting pattern is valid.
+
+ :type tag_pattern: str
+ :param tag_pattern: The tag pattern to convert to a regular
+ expression pattern.
+ :raise ValueError: If ``tag_pattern`` is not a valid tag pattern.
+ In particular, ``tag_pattern`` should not include braces; and it
+ should not contain nested or mismatched angle-brackets.
+ :rtype: str
+ :return: A regular expression pattern corresponding to
+ ``tag_pattern``.
+ """
+ # Clean up the regular expression
+ tag_pattern = re.sub(r'\s', '', tag_pattern)
+ tag_pattern = re.sub(r'<', '(<(', tag_pattern)
+ tag_pattern = re.sub(r'>', ')>)', tag_pattern)
+
+ # Check the regular expression
+ if not CHUNK_TAG_PATTERN.match(tag_pattern):
+ raise ValueError('Bad tag pattern: %r' % tag_pattern)
+
+ # Replace "." with CHUNK_TAG_CHAR.
+ # We have to do this after, since it adds {}[]<>s, which would
+ # confuse CHUNK_TAG_PATTERN.
+ # PRE doesn't have lookback assertions, so reverse twice, and do
+ # the pattern backwards (with lookahead assertions). This can be
+ # made much cleaner once we can switch back to SRE.
+ def reverse_str(str):
+ lst = list(str)
+ lst.reverse()
+ return ''.join(lst)
+ tc_rev = reverse_str(ChunkString.CHUNK_TAG_CHAR)
+ reversed = reverse_str(tag_pattern)
+ reversed = re.sub(r'\.(?!\\(\\\\)*($|[^\\]))', tc_rev, reversed)
+ tag_pattern = reverse_str(reversed)
+
+ return tag_pattern
+
+
+##//////////////////////////////////////////////////////
+## RegexpChunkParser
+##//////////////////////////////////////////////////////
+
+ at python_2_unicode_compatible
+class RegexpChunkParser(ChunkParserI):
+ """
+ A regular expression based chunk parser. ``RegexpChunkParser`` uses a
+ sequence of "rules" to find chunks of a single type within a
+ text. The chunking of the text is encoded using a ``ChunkString``,
+ and each rule acts by modifying the chunking in the
+ ``ChunkString``. The rules are all implemented using regular
+ expression matching and substitution.
+
+ The ``RegexpChunkRule`` class and its subclasses (``ChunkRule``,
+ ``ChinkRule``, ``UnChunkRule``, ``MergeRule``, and ``SplitRule``)
+ define the rules that are used by ``RegexpChunkParser``. Each rule
+ defines an ``apply()`` method, which modifies the chunking encoded
+ by a given ``ChunkString``.
+
+ :type _rules: list(RegexpChunkRule)
+ :ivar _rules: The list of rules that should be applied to a text.
+ :type _trace: int
+ :ivar _trace: The default level of tracing.
+
+ """
+ def __init__(self, rules, chunk_label='NP', root_label='S', trace=0):
+ """
+ Construct a new ``RegexpChunkParser``.
+
+ :type rules: list(RegexpChunkRule)
+ :param rules: The sequence of rules that should be used to
+ generate the chunking for a tagged text.
+ :type chunk_label: str
+ :param chunk_label: The node value that should be used for
+ chunk subtrees. This is typically a short string
+ describing the type of information contained by the chunk,
+ such as ``"NP"`` for base noun phrases.
+ :type root_label: str
+ :param root_label: The node value that should be used for the
+ top node of the chunk structure.
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ ``1`` will generate normal tracing output; and ``2`` or
+ higher will generate verbose tracing output.
+ """
+ self._rules = rules
+ self._trace = trace
+ self._chunk_label = chunk_label
+ self._root_label = root_label
+
+ def _trace_apply(self, chunkstr, verbose):
+ """
+ Apply each rule of this ``RegexpChunkParser`` to ``chunkstr``, in
+ turn. Generate trace output between each rule. If ``verbose``
+ is true, then generate verbose output.
+
+ :type chunkstr: ChunkString
+ :param chunkstr: The chunk string to which each rule should be
+ applied.
+ :type verbose: bool
+ :param verbose: Whether output should be verbose.
+ :rtype: None
+ """
+ print('# Input:')
+ print(chunkstr)
+ for rule in self._rules:
+ rule.apply(chunkstr)
+ if verbose:
+ print('#', rule.descr()+' ('+unicode_repr(rule)+'):')
+ else:
+ print('#', rule.descr()+':')
+ print(chunkstr)
+
+ def _notrace_apply(self, chunkstr):
+ """
+ Apply each rule of this ``RegexpChunkParser`` to ``chunkstr``, in
+ turn.
+
+ :param chunkstr: The chunk string to which each rule should be
+ applied.
+ :type chunkstr: ChunkString
+ :rtype: None
+ """
+
+ for rule in self._rules:
+ rule.apply(chunkstr)
+
+ def parse(self, chunk_struct, trace=None):
+ """
+ :type chunk_struct: Tree
+ :param chunk_struct: the chunk structure to be (further) chunked
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ ``1`` will generate normal tracing output; and ``2`` or
+ highter will generate verbose tracing output. This value
+ overrides the trace level value that was given to the
+ constructor.
+ :rtype: Tree
+ :return: a chunk structure that encodes the chunks in a given
+ tagged sentence. A chunk is a non-overlapping linguistic
+ group, such as a noun phrase. The set of chunks
+ identified in the chunk structure depends on the rules
+ used to define this ``RegexpChunkParser``.
+ """
+ if len(chunk_struct) == 0:
+ print('Warning: parsing empty text')
+ return Tree(self._root_label, [])
+
+ try:
+ chunk_struct.label()
+ except AttributeError:
+ chunk_struct = Tree(self._root_label, chunk_struct)
+
+ # Use the default trace value?
+ if trace is None: trace = self._trace
+
+ chunkstr = ChunkString(chunk_struct)
+
+ # Apply the sequence of rules to the chunkstring.
+ if trace:
+ verbose = (trace>1)
+ self._trace_apply(chunkstr, verbose)
+ else:
+ self._notrace_apply(chunkstr)
+
+ # Use the chunkstring to create a chunk structure.
+ return chunkstr.to_chunkstruct(self._chunk_label)
+
+ def rules(self):
+ """
+ :return: the sequence of rules used by ``RegexpChunkParser``.
+ :rtype: list(RegexpChunkRule)
+ """
+ return self._rules
+
+ def __repr__(self):
+ """
+ :return: a concise string representation of this
+ ``RegexpChunkParser``.
+ :rtype: str
+ """
+ return "<RegexpChunkParser with %d rules>" % len(self._rules)
+
+ def __str__(self):
+ """
+ :return: a verbose string representation of this ``RegexpChunkParser``.
+ :rtype: str
+ """
+ s = "RegexpChunkParser with %d rules:\n" % len(self._rules)
+ margin = 0
+ for rule in self._rules:
+ margin = max(margin, len(rule.descr()))
+ if margin < 35:
+ format = " %" + repr(-(margin+3)) + "s%s\n"
+ else:
+ format = " %s\n %s\n"
+ for rule in self._rules:
+ s += format % (rule.descr(), unicode_repr(rule))
+ return s[:-1]
+
+##//////////////////////////////////////////////////////
+## Chunk Grammar
+##//////////////////////////////////////////////////////
+
+ at python_2_unicode_compatible
+class RegexpParser(ChunkParserI):
+ """
+ A grammar based chunk parser. ``chunk.RegexpParser`` uses a set of
+ regular expression patterns to specify the behavior of the parser.
+ The chunking of the text is encoded using a ``ChunkString``, and
+ each rule acts by modifying the chunking in the ``ChunkString``.
+ The rules are all implemented using regular expression matching
+ and substitution.
+
+ A grammar contains one or more clauses in the following form::
+
+ NP:
+ {<DT|JJ>} # chunk determiners and adjectives
+ }<[\.VI].*>+{ # chink any tag beginning with V, I, or .
+ <.*>}{<DT> # split a chunk at a determiner
+ <DT|JJ>{}<NN.*> # merge chunk ending with det/adj
+ # with one starting with a noun
+
+ The patterns of a clause are executed in order. An earlier
+ pattern may introduce a chunk boundary that prevents a later
+ pattern from executing. Sometimes an individual pattern will
+ match on multiple, overlapping extents of the input. As with
+ regular expression substitution more generally, the chunker will
+ identify the first match possible, then continue looking for matches
+ after this one has ended.
+
+ The clauses of a grammar are also executed in order. A cascaded
+ chunk parser is one having more than one clause. The maximum depth
+ of a parse tree created by this chunk parser is the same as the
+ number of clauses in the grammar.
+
+ When tracing is turned on, the comment portion of a line is displayed
+ each time the corresponding pattern is applied.
+
+ :type _start: str
+ :ivar _start: The start symbol of the grammar (the root node of
+ resulting trees)
+ :type _stages: int
+ :ivar _stages: The list of parsing stages corresponding to the grammar
+
+ """
+ def __init__(self, grammar, root_label='S', loop=1, trace=0):
+ """
+ Create a new chunk parser, from the given start state
+ and set of chunk patterns.
+
+ :param grammar: The grammar, or a list of RegexpChunkParser objects
+ :type grammar: str or list(RegexpChunkParser)
+ :param root_label: The top node of the tree being created
+ :type root_label: str or Nonterminal
+ :param loop: The number of times to run through the patterns
+ :type loop: int
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ ``1`` will generate normal tracing output; and ``2`` or
+ higher will generate verbose tracing output.
+ """
+ self._trace = trace
+ self._stages = []
+ self._grammar = grammar
+ self._loop = loop
+
+ if isinstance(grammar, string_types):
+ self._read_grammar(grammar, root_label, trace)
+ else:
+ # Make sur the grammar looks like it has the right type:
+ type_err = ('Expected string or list of RegexpChunkParsers '
+ 'for the grammar.')
+ try: grammar = list(grammar)
+ except: raise TypeError(type_err)
+ for elt in grammar:
+ if not isinstance(elt, RegexpChunkParser):
+ raise TypeError(type_err)
+ self._stages = grammar
+
+ def _read_grammar(self, grammar, root_label, trace):
+ """
+ Helper function for __init__: read the grammar if it is a
+ string.
+ """
+ rules = []
+ lhs = None
+ for line in grammar.split('\n'):
+ line = line.strip()
+
+ # New stage begins if there's an unescaped ':'
+ m = re.match('(?P<nonterminal>(\\.|[^:])*)(:(?P<rule>.*))', line)
+ if m:
+ # Record the stage that we just completed.
+ self._add_stage(rules, lhs, root_label, trace)
+ # Start a new stage.
+ lhs = m.group('nonterminal').strip()
+ rules = []
+ line = m.group('rule').strip()
+
+ # Skip blank & comment-only lines
+ if line=='' or line.startswith('#'): continue
+
+ # Add the rule
+ rules.append(RegexpChunkRule.fromstring(line))
+
+ # Record the final stage
+ self._add_stage(rules, lhs, root_label, trace)
+
+ def _add_stage(self, rules, lhs, root_label, trace):
+ """
+ Helper function for __init__: add a new stage to the parser.
+ """
+ if rules != []:
+ if not lhs:
+ raise ValueError('Expected stage marker (eg NP:)')
+ parser = RegexpChunkParser(rules, chunk_label=lhs,
+ root_label=root_label, trace=trace)
+ self._stages.append(parser)
+
+ def parse(self, chunk_struct, trace=None):
+ """
+ Apply the chunk parser to this input.
+
+ :type chunk_struct: Tree
+ :param chunk_struct: the chunk structure to be (further) chunked
+ (this tree is modified, and is also returned)
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ ``1`` will generate normal tracing output; and ``2`` or
+ highter will generate verbose tracing output. This value
+ overrides the trace level value that was given to the
+ constructor.
+ :return: the chunked output.
+ :rtype: Tree
+ """
+ if trace is None: trace = self._trace
+ for i in range(self._loop):
+ for parser in self._stages:
+ chunk_struct = parser.parse(chunk_struct, trace=trace)
+ return chunk_struct
+
+ def __repr__(self):
+ """
+ :return: a concise string representation of this ``chunk.RegexpParser``.
+ :rtype: str
+ """
+ return "<chunk.RegexpParser with %d stages>" % len(self._stages)
+
+ def __str__(self):
+ """
+ :return: a verbose string representation of this
+ ``RegexpParser``.
+ :rtype: str
+ """
+ s = "chunk.RegexpParser with %d stages:\n" % len(self._stages)
+ margin = 0
+ for parser in self._stages:
+ s += "%s\n" % parser
+ return s[:-1]
+
+##//////////////////////////////////////////////////////
+## Demonstration code
+##//////////////////////////////////////////////////////
+
+def demo_eval(chunkparser, text):
+ """
+ Demonstration code for evaluating a chunk parser, using a
+ ``ChunkScore``. This function assumes that ``text`` contains one
+ sentence per line, and that each sentence has the form expected by
+ ``tree.chunk``. It runs the given chunk parser on each sentence in
+ the text, and scores the result. It prints the final score
+ (precision, recall, and f-measure); and reports the set of chunks
+ that were missed and the set of chunks that were incorrect. (At
+ most 10 missing chunks and 10 incorrect chunks are reported).
+
+ :param chunkparser: The chunkparser to be tested
+ :type chunkparser: ChunkParserI
+ :param text: The chunked tagged text that should be used for
+ evaluation.
+ :type text: str
+ """
+ from nltk import chunk
+ from nltk.tree import Tree
+
+ # Evaluate our chunk parser.
+ chunkscore = chunk.ChunkScore()
+
+ for sentence in text.split('\n'):
+ print(sentence)
+ sentence = sentence.strip()
+ if not sentence: continue
+ gold = chunk.tagstr2tree(sentence)
+ tokens = gold.leaves()
+ test = chunkparser.parse(Tree('S', tokens), trace=1)
+ chunkscore.score(gold, test)
+ print()
+
+ print('/'+('='*75)+'\\')
+ print('Scoring', chunkparser)
+ print(('-'*77))
+ print('Precision: %5.1f%%' % (chunkscore.precision()*100), ' '*4, end=' ')
+ print('Recall: %5.1f%%' % (chunkscore.recall()*100), ' '*6, end=' ')
+ print('F-Measure: %5.1f%%' % (chunkscore.f_measure()*100))
+
+
+ # Missed chunks.
+ if chunkscore.missed():
+ print('Missed:')
+ missed = chunkscore.missed()
+ for chunk in missed[:10]:
+ print(' ', ' '.join(map(str,chunk)))
+ if len(chunkscore.missed()) > 10:
+ print(' ...')
+
+ # Incorrect chunks.
+ if chunkscore.incorrect():
+ print('Incorrect:')
+ incorrect = chunkscore.incorrect()
+ for chunk in incorrect[:10]:
+ print(' ', ' '.join(map(str,chunk)))
+ if len(chunkscore.incorrect()) > 10:
+ print(' ...')
+
+ print('\\'+('='*75)+'/')
+ print()
+
+def demo():
+ """
+ A demonstration for the ``RegexpChunkParser`` class. A single text is
+ parsed with four different chunk parsers, using a variety of rules
+ and strategies.
+ """
+
+ from nltk import chunk, Tree
+
+ text = """\
+ [ the/DT little/JJ cat/NN ] sat/VBD on/IN [ the/DT mat/NN ] ./.
+ [ John/NNP ] saw/VBD [the/DT cats/NNS] [the/DT dog/NN] chased/VBD ./.
+ [ John/NNP ] thinks/VBZ [ Mary/NN ] saw/VBD [ the/DT cat/NN ] sit/VB on/IN [ the/DT mat/NN ]./.
+ """
+
+ print('*'*75)
+ print('Evaluation text:')
+ print(text)
+ print('*'*75)
+ print()
+
+ grammar = r"""
+ NP: # NP stage
+ {<DT>?<JJ>*<NN>} # chunk determiners, adjectives and nouns
+ {<NNP>+} # chunk proper nouns
+ """
+ cp = chunk.RegexpParser(grammar)
+ demo_eval(cp, text)
+
+ grammar = r"""
+ NP:
+ {<.*>} # start by chunking each tag
+ }<[\.VI].*>+{ # unchunk any verbs, prepositions or periods
+ <DT|JJ>{}<NN.*> # merge det/adj with nouns
+ """
+ cp = chunk.RegexpParser(grammar)
+ demo_eval(cp, text)
+
+ grammar = r"""
+ NP: {<DT>?<JJ>*<NN>} # chunk determiners, adjectives and nouns
+ VP: {<TO>?<VB.*>} # VP = verb words
+ """
+ cp = chunk.RegexpParser(grammar)
+ demo_eval(cp, text)
+
+ grammar = r"""
+ NP: {<.*>*} # start by chunking everything
+ }<[\.VI].*>+{ # chink any verbs, prepositions or periods
+ <.*>}{<DT> # separate on determiners
+ PP: {<IN><NP>} # PP = preposition + noun phrase
+ VP: {<VB.*><NP|PP>*} # VP = verb words + NPs and PPs
+ """
+ cp = chunk.RegexpParser(grammar)
+ demo_eval(cp, text)
+
+# Evaluation
+
+ from nltk.corpus import conll2000
+
+ print()
+ print("Demonstration of empty grammar:")
+
+ cp = chunk.RegexpParser("")
+ print(chunk.accuracy(cp, conll2000.chunked_sents('test.txt',
+ chunk_types=('NP',))))
+
+ print()
+ print("Demonstration of accuracy evaluation using CoNLL tags:")
+
+ grammar = r"""
+ NP:
+ {<.*>} # start by chunking each tag
+ }<[\.VI].*>+{ # unchunk any verbs, prepositions or periods
+ <DT|JJ>{}<NN.*> # merge det/adj with nouns
+ """
+ cp = chunk.RegexpParser(grammar)
+ print(chunk.accuracy(cp, conll2000.chunked_sents('test.txt')[:5]))
+
+ print()
+ print("Demonstration of tagged token input")
+
+ grammar = r"""
+ NP: {<.*>*} # start by chunking everything
+ }<[\.VI].*>+{ # chink any verbs, prepositions or periods
+ <.*>}{<DT> # separate on determiners
+ PP: {<IN><NP>} # PP = preposition + noun phrase
+ VP: {<VB.*><NP|PP>*} # VP = verb words + NPs and PPs
+ """
+ cp = chunk.RegexpParser(grammar)
+ print(cp.parse([("the","DT"), ("little","JJ"), ("cat", "NN"),
+ ("sat", "VBD"), ("on", "IN"), ("the", "DT"),
+ ("mat", "NN"), (".", ".")]))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/chunk/util.py b/nltk/chunk/util.py
new file mode 100644
index 0000000..6162979
--- /dev/null
+++ b/nltk/chunk/util.py
@@ -0,0 +1,595 @@
+# Natural Language Toolkit: Chunk format conversions
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+import re
+
+from nltk.tree import Tree
+from nltk.tag.util import str2tuple
+from nltk.compat import python_2_unicode_compatible
+
+##//////////////////////////////////////////////////////
+## EVALUATION
+##//////////////////////////////////////////////////////
+
+from nltk.metrics import accuracy as _accuracy
+def accuracy(chunker, gold):
+ """
+ Score the accuracy of the chunker against the gold standard.
+ Strip the chunk information from the gold standard and rechunk it using
+ the chunker, then compute the accuracy score.
+
+ :type chunker: ChunkParserI
+ :param chunker: The chunker being evaluated.
+ :type gold: tree
+ :param gold: The chunk structures to score the chunker on.
+ :rtype: float
+ """
+
+ gold_tags = []
+ test_tags = []
+ for gold_tree in gold:
+ test_tree = chunker.parse(gold_tree.flatten())
+ gold_tags += tree2conlltags(gold_tree)
+ test_tags += tree2conlltags(test_tree)
+
+# print 'GOLD:', gold_tags[:50]
+# print 'TEST:', test_tags[:50]
+ return _accuracy(gold_tags, test_tags)
+
+
+# Patched for increased performance by Yoav Goldberg <yoavg at cs.bgu.ac.il>, 2006-01-13
+# -- statistics are evaluated only on demand, instead of at every sentence evaluation
+#
+# SB: use nltk.metrics for precision/recall scoring?
+#
+class ChunkScore(object):
+ """
+ A utility class for scoring chunk parsers. ``ChunkScore`` can
+ evaluate a chunk parser's output, based on a number of statistics
+ (precision, recall, f-measure, misssed chunks, incorrect chunks).
+ It can also combine the scores from the parsing of multiple texts;
+ this makes it significantly easier to evaluate a chunk parser that
+ operates one sentence at a time.
+
+ Texts are evaluated with the ``score`` method. The results of
+ evaluation can be accessed via a number of accessor methods, such
+ as ``precision`` and ``f_measure``. A typical use of the
+ ``ChunkScore`` class is::
+
+ >>> chunkscore = ChunkScore() # doctest: +SKIP
+ >>> for correct in correct_sentences: # doctest: +SKIP
+ ... guess = chunkparser.parse(correct.leaves()) # doctest: +SKIP
+ ... chunkscore.score(correct, guess) # doctest: +SKIP
+ >>> print('F Measure:', chunkscore.f_measure()) # doctest: +SKIP
+ F Measure: 0.823
+
+ :ivar kwargs: Keyword arguments:
+
+ - max_tp_examples: The maximum number actual examples of true
+ positives to record. This affects the ``correct`` member
+ function: ``correct`` will not return more than this number
+ of true positive examples. This does *not* affect any of
+ the numerical metrics (precision, recall, or f-measure)
+
+ - max_fp_examples: The maximum number actual examples of false
+ positives to record. This affects the ``incorrect`` member
+ function and the ``guessed`` member function: ``incorrect``
+ will not return more than this number of examples, and
+ ``guessed`` will not return more than this number of true
+ positive examples. This does *not* affect any of the
+ numerical metrics (precision, recall, or f-measure)
+
+ - max_fn_examples: The maximum number actual examples of false
+ negatives to record. This affects the ``missed`` member
+ function and the ``correct`` member function: ``missed``
+ will not return more than this number of examples, and
+ ``correct`` will not return more than this number of true
+ negative examples. This does *not* affect any of the
+ numerical metrics (precision, recall, or f-measure)
+
+ - chunk_label: A regular expression indicating which chunks
+ should be compared. Defaults to ``'.*'`` (i.e., all chunks).
+
+ :type _tp: list(Token)
+ :ivar _tp: List of true positives
+ :type _fp: list(Token)
+ :ivar _fp: List of false positives
+ :type _fn: list(Token)
+ :ivar _fn: List of false negatives
+
+ :type _tp_num: int
+ :ivar _tp_num: Number of true positives
+ :type _fp_num: int
+ :ivar _fp_num: Number of false positives
+ :type _fn_num: int
+ :ivar _fn_num: Number of false negatives.
+ """
+ def __init__(self, **kwargs):
+ self._correct = set()
+ self._guessed = set()
+ self._tp = set()
+ self._fp = set()
+ self._fn = set()
+ self._max_tp = kwargs.get('max_tp_examples', 100)
+ self._max_fp = kwargs.get('max_fp_examples', 100)
+ self._max_fn = kwargs.get('max_fn_examples', 100)
+ self._chunk_label = kwargs.get('chunk_label', '.*')
+ self._tp_num = 0
+ self._fp_num = 0
+ self._fn_num = 0
+ self._count = 0
+ self._tags_correct = 0.0
+ self._tags_total = 0.0
+
+ self._measuresNeedUpdate = False
+
+ def _updateMeasures(self):
+ if (self._measuresNeedUpdate):
+ self._tp = self._guessed & self._correct
+ self._fn = self._correct - self._guessed
+ self._fp = self._guessed - self._correct
+ self._tp_num = len(self._tp)
+ self._fp_num = len(self._fp)
+ self._fn_num = len(self._fn)
+ self._measuresNeedUpdate = False
+
+ def score(self, correct, guessed):
+ """
+ Given a correctly chunked sentence, score another chunked
+ version of the same sentence.
+
+ :type correct: chunk structure
+ :param correct: The known-correct ("gold standard") chunked
+ sentence.
+ :type guessed: chunk structure
+ :param guessed: The chunked sentence to be scored.
+ """
+ self._correct |= _chunksets(correct, self._count, self._chunk_label)
+ self._guessed |= _chunksets(guessed, self._count, self._chunk_label)
+ self._count += 1
+ self._measuresNeedUpdate = True
+ # Keep track of per-tag accuracy (if possible)
+ try:
+ correct_tags = tree2conlltags(correct)
+ guessed_tags = tree2conlltags(guessed)
+ except ValueError:
+ # This exception case is for nested chunk structures,
+ # where tree2conlltags will fail with a ValueError: "Tree
+ # is too deeply nested to be printed in CoNLL format."
+ correct_tags = guessed_tags = ()
+ self._tags_total += len(correct_tags)
+ self._tags_correct += sum(1 for (t,g) in zip(guessed_tags,
+ correct_tags)
+ if t==g)
+
+ def accuracy(self):
+ """
+ Return the overall tag-based accuracy for all text that have
+ been scored by this ``ChunkScore``, using the IOB (conll2000)
+ tag encoding.
+
+ :rtype: float
+ """
+ if self._tags_total == 0: return 1
+ return self._tags_correct/self._tags_total
+
+ def precision(self):
+ """
+ Return the overall precision for all texts that have been
+ scored by this ``ChunkScore``.
+
+ :rtype: float
+ """
+ self._updateMeasures()
+ div = self._tp_num + self._fp_num
+ if div == 0: return 0
+ else: return float(self._tp_num) / div
+
+ def recall(self):
+ """
+ Return the overall recall for all texts that have been
+ scored by this ``ChunkScore``.
+
+ :rtype: float
+ """
+ self._updateMeasures()
+ div = self._tp_num + self._fn_num
+ if div == 0: return 0
+ else: return float(self._tp_num) / div
+
+ def f_measure(self, alpha=0.5):
+ """
+ Return the overall F measure for all texts that have been
+ scored by this ``ChunkScore``.
+
+ :param alpha: the relative weighting of precision and recall.
+ Larger alpha biases the score towards the precision value,
+ while smaller alpha biases the score towards the recall
+ value. ``alpha`` should have a value in the range [0,1].
+ :type alpha: float
+ :rtype: float
+ """
+ self._updateMeasures()
+ p = self.precision()
+ r = self.recall()
+ if p == 0 or r == 0: # what if alpha is 0 or 1?
+ return 0
+ return 1/(alpha/p + (1-alpha)/r)
+
+ def missed(self):
+ """
+ Return the chunks which were included in the
+ correct chunk structures, but not in the guessed chunk
+ structures, listed in input order.
+
+ :rtype: list of chunks
+ """
+ self._updateMeasures()
+ chunks = list(self._fn)
+ return [c[1] for c in chunks] # discard position information
+
+ def incorrect(self):
+ """
+ Return the chunks which were included in the guessed chunk structures,
+ but not in the correct chunk structures, listed in input order.
+
+ :rtype: list of chunks
+ """
+ self._updateMeasures()
+ chunks = list(self._fp)
+ return [c[1] for c in chunks] # discard position information
+
+ def correct(self):
+ """
+ Return the chunks which were included in the correct
+ chunk structures, listed in input order.
+
+ :rtype: list of chunks
+ """
+ chunks = list(self._correct)
+ return [c[1] for c in chunks] # discard position information
+
+ def guessed(self):
+ """
+ Return the chunks which were included in the guessed
+ chunk structures, listed in input order.
+
+ :rtype: list of chunks
+ """
+ chunks = list(self._guessed)
+ return [c[1] for c in chunks] # discard position information
+
+ def __len__(self):
+ self._updateMeasures()
+ return self._tp_num + self._fn_num
+
+ def __repr__(self):
+ """
+ Return a concise representation of this ``ChunkScoring``.
+
+ :rtype: str
+ """
+ return '<ChunkScoring of '+repr(len(self))+' chunks>'
+
+ def __str__(self):
+ """
+ Return a verbose representation of this ``ChunkScoring``.
+ This representation includes the precision, recall, and
+ f-measure scores. For other information about the score,
+ use the accessor methods (e.g., ``missed()`` and ``incorrect()``).
+
+ :rtype: str
+ """
+ return ("ChunkParse score:\n" +
+ (" IOB Accuracy: %5.1f%%\n" % (self.accuracy()*100)) +
+ (" Precision: %5.1f%%\n" % (self.precision()*100)) +
+ (" Recall: %5.1f%%\n" % (self.recall()*100))+
+ (" F-Measure: %5.1f%%" % (self.f_measure()*100)))
+
+# extract chunks, and assign unique id, the absolute position of
+# the first word of the chunk
+def _chunksets(t, count, chunk_label):
+ pos = 0
+ chunks = []
+ for child in t:
+ if isinstance(child, Tree):
+ if re.match(chunk_label, child.label()):
+ chunks.append(((count, pos), child.freeze()))
+ pos += len(child.leaves())
+ else:
+ pos += 1
+ return set(chunks)
+
+
+def tagstr2tree(s, chunk_label="NP", root_label="S", sep='/'):
+ """
+ Divide a string of bracketted tagged text into
+ chunks and unchunked tokens, and produce a Tree.
+ Chunks are marked by square brackets (``[...]``). Words are
+ delimited by whitespace, and each word should have the form
+ ``text/tag``. Words that do not contain a slash are
+ assigned a ``tag`` of None.
+
+ :param s: The string to be converted
+ :type s: str
+ :param chunk_label: The label to use for chunk nodes
+ :type chunk_label: str
+ :param root_label: The label to use for the root of the tree
+ :type root_label: str
+ :rtype: Tree
+ """
+
+ WORD_OR_BRACKET = re.compile(r'\[|\]|[^\[\]\s]+')
+
+ stack = [Tree(root_label, [])]
+ for match in WORD_OR_BRACKET.finditer(s):
+ text = match.group()
+ if text[0] == '[':
+ if len(stack) != 1:
+ raise ValueError('Unexpected [ at char %d' % match.start())
+ chunk = Tree(chunk_label, [])
+ stack[-1].append(chunk)
+ stack.append(chunk)
+ elif text[0] == ']':
+ if len(stack) != 2:
+ raise ValueError('Unexpected ] at char %d' % match.start())
+ stack.pop()
+ else:
+ if sep is None:
+ stack[-1].append(text)
+ else:
+ stack[-1].append(str2tuple(text, sep))
+
+ if len(stack) != 1:
+ raise ValueError('Expected ] at char %d' % len(s))
+ return stack[0]
+
+### CONLL
+
+_LINE_RE = re.compile('(\S+)\s+(\S+)\s+([IOB])-?(\S+)?')
+def conllstr2tree(s, chunk_types=('NP', 'PP', 'VP'), root_label="S"):
+ """
+ Return a chunk structure for a single sentence
+ encoded in the given CONLL 2000 style string.
+ This function converts a CoNLL IOB string into a tree.
+ It uses the specified chunk types
+ (defaults to NP, PP and VP), and creates a tree rooted at a node
+ labeled S (by default).
+
+ :param s: The CoNLL string to be converted.
+ :type s: str
+ :param chunk_types: The chunk types to be converted.
+ :type chunk_types: tuple
+ :param root_label: The node label to use for the root.
+ :type root_label: str
+ :rtype: Tree
+ """
+
+ stack = [Tree(root_label, [])]
+
+ for lineno, line in enumerate(s.split('\n')):
+ if not line.strip(): continue
+
+ # Decode the line.
+ match = _LINE_RE.match(line)
+ if match is None:
+ raise ValueError('Error on line %d' % lineno)
+ (word, tag, state, chunk_type) = match.groups()
+
+ # If it's a chunk type we don't care about, treat it as O.
+ if (chunk_types is not None and
+ chunk_type not in chunk_types):
+ state = 'O'
+
+ # For "Begin"/"Outside", finish any completed chunks -
+ # also do so for "Inside" which don't match the previous token.
+ mismatch_I = state == 'I' and chunk_type != stack[-1].label()
+ if state in 'BO' or mismatch_I:
+ if len(stack) == 2: stack.pop()
+
+ # For "Begin", start a new chunk.
+ if state == 'B' or mismatch_I:
+ chunk = Tree(chunk_type, [])
+ stack[-1].append(chunk)
+ stack.append(chunk)
+
+ # Add the new word token.
+ stack[-1].append((word, tag))
+
+ return stack[0]
+
+def tree2conlltags(t):
+ """
+ Return a list of 3-tuples containing ``(word, tag, IOB-tag)``.
+ Convert a tree to the CoNLL IOB tag format.
+
+ :param t: The tree to be converted.
+ :type t: Tree
+ :rtype: list(tuple)
+ """
+
+ tags = []
+ for child in t:
+ try:
+ category = child.label()
+ prefix = "B-"
+ for contents in child:
+ if isinstance(contents, Tree):
+ raise ValueError("Tree is too deeply nested to be printed in CoNLL format")
+ tags.append((contents[0], contents[1], prefix+category))
+ prefix = "I-"
+ except AttributeError:
+ tags.append((child[0], child[1], "O"))
+ return tags
+
+def conlltags2tree(sentence, chunk_types=('NP','PP','VP'),
+ root_label='S', strict=False):
+ """
+ Convert the CoNLL IOB format to a tree.
+ """
+ tree = Tree(root_label, [])
+ for (word, postag, chunktag) in sentence:
+ if chunktag is None:
+ if strict:
+ raise ValueError("Bad conll tag sequence")
+ else:
+ # Treat as O
+ tree.append((word,postag))
+ elif chunktag.startswith('B-'):
+ tree.append(Tree(chunktag[2:], [(word,postag)]))
+ elif chunktag.startswith('I-'):
+ if (len(tree)==0 or not isinstance(tree[-1], Tree) or
+ tree[-1].label() != chunktag[2:]):
+ if strict:
+ raise ValueError("Bad conll tag sequence")
+ else:
+ # Treat as B-*
+ tree.append(Tree(chunktag[2:], [(word,postag)]))
+ else:
+ tree[-1].append((word,postag))
+ elif chunktag == 'O':
+ tree.append((word,postag))
+ else:
+ raise ValueError("Bad conll tag %r" % chunktag)
+ return tree
+
+def tree2conllstr(t):
+ """
+ Return a multiline string where each line contains a word, tag and IOB tag.
+ Convert a tree to the CoNLL IOB string format
+
+ :param t: The tree to be converted.
+ :type t: Tree
+ :rtype: str
+ """
+ lines = [" ".join(token) for token in tree2conlltags(t)]
+ return '\n'.join(lines)
+
+### IEER
+
+_IEER_DOC_RE = re.compile(r'<DOC>\s*'
+ r'(<DOCNO>\s*(?P<docno>.+?)\s*</DOCNO>\s*)?'
+ r'(<DOCTYPE>\s*(?P<doctype>.+?)\s*</DOCTYPE>\s*)?'
+ r'(<DATE_TIME>\s*(?P<date_time>.+?)\s*</DATE_TIME>\s*)?'
+ r'<BODY>\s*'
+ r'(<HEADLINE>\s*(?P<headline>.+?)\s*</HEADLINE>\s*)?'
+ r'<TEXT>(?P<text>.*?)</TEXT>\s*'
+ r'</BODY>\s*</DOC>\s*', re.DOTALL)
+
+_IEER_TYPE_RE = re.compile('<b_\w+\s+[^>]*?type="(?P<type>\w+)"')
+
+def _ieer_read_text(s, root_label):
+ stack = [Tree(root_label, [])]
+ # s will be None if there is no headline in the text
+ # return the empty list in place of a Tree
+ if s is None:
+ return []
+ for piece_m in re.finditer('<[^>]+>|[^\s<]+', s):
+ piece = piece_m.group()
+ try:
+ if piece.startswith('<b_'):
+ m = _IEER_TYPE_RE.match(piece)
+ if m is None: print('XXXX', piece)
+ chunk = Tree(m.group('type'), [])
+ stack[-1].append(chunk)
+ stack.append(chunk)
+ elif piece.startswith('<e_'):
+ stack.pop()
+# elif piece.startswith('<'):
+# print "ERROR:", piece
+# raise ValueError # Unexpected HTML
+ else:
+ stack[-1].append(piece)
+ except (IndexError, ValueError):
+ raise ValueError('Bad IEER string (error at character %d)' %
+ piece_m.start())
+ if len(stack) != 1:
+ raise ValueError('Bad IEER string')
+ return stack[0]
+
+def ieerstr2tree(s, chunk_types = ['LOCATION', 'ORGANIZATION', 'PERSON', 'DURATION',
+ 'DATE', 'CARDINAL', 'PERCENT', 'MONEY', 'MEASURE'], root_label="S"):
+ """
+ Return a chunk structure containing the chunked tagged text that is
+ encoded in the given IEER style string.
+ Convert a string of chunked tagged text in the IEER named
+ entity format into a chunk structure. Chunks are of several
+ types, LOCATION, ORGANIZATION, PERSON, DURATION, DATE, CARDINAL,
+ PERCENT, MONEY, and MEASURE.
+
+ :rtype: Tree
+ """
+
+ # Try looking for a single document. If that doesn't work, then just
+ # treat everything as if it was within the <TEXT>...</TEXT>.
+ m = _IEER_DOC_RE.match(s)
+ if m:
+ return {
+ 'text': _ieer_read_text(m.group('text'), root_label),
+ 'docno': m.group('docno'),
+ 'doctype': m.group('doctype'),
+ 'date_time': m.group('date_time'),
+ #'headline': m.group('headline')
+ # we want to capture NEs in the headline too!
+ 'headline': _ieer_read_text(m.group('headline'), root_label),
+ }
+ else:
+ return _ieer_read_text(s, root_label)
+
+
+def demo():
+
+ s = "[ Pierre/NNP Vinken/NNP ] ,/, [ 61/CD years/NNS ] old/JJ ,/, will/MD join/VB [ the/DT board/NN ] ./."
+ import nltk
+ t = nltk.chunk.tagstr2tree(s, chunk_label='NP')
+ print(t.pprint())
+ print()
+
+ s = """
+These DT B-NP
+research NN I-NP
+protocols NNS I-NP
+offer VBP B-VP
+to TO B-PP
+the DT B-NP
+patient NN I-NP
+not RB O
+only RB O
+the DT B-NP
+very RB I-NP
+best JJS I-NP
+therapy NN I-NP
+which WDT B-NP
+we PRP B-NP
+have VBP B-VP
+established VBN I-VP
+today NN B-NP
+but CC B-NP
+also RB I-NP
+the DT B-NP
+hope NN I-NP
+of IN B-PP
+something NN B-NP
+still RB B-ADJP
+better JJR I-ADJP
+. . O
+"""
+
+ conll_tree = conllstr2tree(s, chunk_types=('NP', 'PP'))
+ print(conll_tree.pprint())
+
+ # Demonstrate CoNLL output
+ print("CoNLL output:")
+ print(nltk.chunk.tree2conllstr(conll_tree))
+ print()
+
+
+if __name__ == '__main__':
+ demo()
+
diff --git a/nltk/classify/__init__.py b/nltk/classify/__init__.py
new file mode 100644
index 0000000..8d6fda2
--- /dev/null
+++ b/nltk/classify/__init__.py
@@ -0,0 +1,96 @@
+# Natural Language Toolkit: Classifiers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classes and interfaces for labeling tokens with category labels (or
+"class labels"). Typically, labels are represented with strings
+(such as ``'health'`` or ``'sports'``). Classifiers can be used to
+perform a wide range of classification tasks. For example,
+classifiers can be used...
+
+- to classify documents by topic
+- to classify ambiguous words by which word sense is intended
+- to classify acoustic signals by which phoneme they represent
+- to classify sentences by their author
+
+Features
+========
+In order to decide which category label is appropriate for a given
+token, classifiers examine one or more 'features' of the token. These
+"features" are typically chosen by hand, and indicate which aspects
+of the token are relevant to the classification decision. For
+example, a document classifier might use a separate feature for each
+word, recording how often that word occurred in the document.
+
+Featuresets
+===========
+The features describing a token are encoded using a "featureset",
+which is a dictionary that maps from "feature names" to "feature
+values". Feature names are unique strings that indicate what aspect
+of the token is encoded by the feature. Examples include
+``'prevword'``, for a feature whose value is the previous word; and
+``'contains-word(library)'`` for a feature that is true when a document
+contains the word ``'library'``. Feature values are typically
+booleans, numbers, or strings, depending on which feature they
+describe.
+
+Featuresets are typically constructed using a "feature detector"
+(also known as a "feature extractor"). A feature detector is a
+function that takes a token (and sometimes information about its
+context) as its input, and returns a featureset describing that token.
+For example, the following feature detector converts a document
+(stored as a list of words) to a featureset describing the set of
+words included in the document:
+
+ >>> # Define a feature detector function.
+ >>> def document_features(document):
+ ... return dict([('contains-word(%s)' % w, True) for w in document])
+
+Feature detectors are typically applied to each token before it is fed
+to the classifier:
+
+ >>> # Classify each Gutenberg document.
+ >>> from nltk.corpus import gutenberg
+ >>> for fileid in gutenberg.fileids(): # doctest: +SKIP
+ ... doc = gutenberg.words(fileid) # doctest: +SKIP
+ ... print fileid, classifier.classify(document_features(doc)) # doctest: +SKIP
+
+The parameters that a feature detector expects will vary, depending on
+the task and the needs of the feature detector. For example, a
+feature detector for word sense disambiguation (WSD) might take as its
+input a sentence, and the index of a word that should be classified,
+and return a featureset for that word. The following feature detector
+for WSD includes features describing the left and right contexts of
+the target word:
+
+ >>> def wsd_features(sentence, index):
+ ... featureset = {}
+ ... for i in range(max(0, index-3), index):
+ ... featureset['left-context(%s)' % sentence[i]] = True
+ ... for i in range(index, max(index+3, len(sentence))):
+ ... featureset['right-context(%s)' % sentence[i]] = True
+ ... return featureset
+
+Training Classifiers
+====================
+Most classifiers are built by training them on a list of hand-labeled
+examples, known as the "training set". Training sets are represented
+as lists of ``(featuredict, label)`` tuples.
+"""
+
+from nltk.classify.api import ClassifierI, MultiClassifierI
+from nltk.classify.megam import config_megam, call_megam
+from nltk.classify.weka import WekaClassifier, config_weka
+from nltk.classify.naivebayes import NaiveBayesClassifier
+from nltk.classify.positivenaivebayes import PositiveNaiveBayesClassifier
+from nltk.classify.decisiontree import DecisionTreeClassifier
+from nltk.classify.rte_classify import rte_classifier, rte_features, RTEFeatureExtractor
+from nltk.classify.util import accuracy, apply_features, log_likelihood
+from nltk.classify.scikitlearn import SklearnClassifier
+from nltk.classify.maxent import (MaxentClassifier, BinaryMaxentFeatureEncoding,
+ TypedMaxentFeatureEncoding,
+ ConditionalExponentialClassifier)
diff --git a/nltk/classify/api.py b/nltk/classify/api.py
new file mode 100644
index 0000000..3b977cc
--- /dev/null
+++ b/nltk/classify/api.py
@@ -0,0 +1,193 @@
+# Natural Language Toolkit: Classifier Interface
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interfaces for labeling tokens with category labels (or "class labels").
+
+``ClassifierI`` is a standard interface for "single-category
+classification", in which the set of categories is known, the number
+of categories is finite, and each text belongs to exactly one
+category.
+
+``MultiClassifierI`` is a standard interface for "multi-category
+classification", which is like single-category classification except
+that each text belongs to zero or more categories.
+"""
+from nltk.internals import overridden
+
+##//////////////////////////////////////////////////////
+#{ Classification Interfaces
+##//////////////////////////////////////////////////////
+
+class ClassifierI(object):
+ """
+ A processing interface for labeling tokens with a single category
+ label (or "class"). Labels are typically strs or
+ ints, but can be any immutable type. The set of labels
+ that the classifier chooses from must be fixed and finite.
+
+ Subclasses must define:
+ - ``labels()``
+ - either ``classify()`` or ``classify_many()`` (or both)
+
+ Subclasses may define:
+ - either ``prob_classify()`` or ``prob_classify_many()`` (or both)
+ """
+ def labels(self):
+ """
+ :return: the list of category labels used by this classifier.
+ :rtype: list of (immutable)
+ """
+ raise NotImplementedError()
+
+ def classify(self, featureset):
+ """
+ :return: the most appropriate label for the given featureset.
+ :rtype: label
+ """
+ if overridden(self.classify_many):
+ return self.classify_many([featureset])[0]
+ else:
+ raise NotImplementedError()
+
+ def prob_classify(self, featureset):
+ """
+ :return: a probability distribution over labels for the given
+ featureset.
+ :rtype: ProbDistI
+ """
+ if overridden(self.prob_classify_many):
+ return self.prob_classify_many([featureset])[0]
+ else:
+ raise NotImplementedError()
+
+ def classify_many(self, featuresets):
+ """
+ Apply ``self.classify()`` to each element of ``featuresets``. I.e.:
+
+ return [self.classify(fs) for fs in featuresets]
+
+ :rtype: list(label)
+ """
+ return [self.classify(fs) for fs in featuresets]
+
+ def prob_classify_many(self, featuresets):
+ """
+ Apply ``self.prob_classify()`` to each element of ``featuresets``. I.e.:
+
+ return [self.prob_classify(fs) for fs in featuresets]
+
+ :rtype: list(ProbDistI)
+ """
+ return [self.prob_classify(fs) for fs in featuresets]
+
+
+class MultiClassifierI(object):
+ """
+ A processing interface for labeling tokens with zero or more
+ category labels (or "labels"). Labels are typically strs
+ or ints, but can be any immutable type. The set of labels
+ that the multi-classifier chooses from must be fixed and finite.
+
+ Subclasses must define:
+ - ``labels()``
+ - either ``classify()`` or ``classify_many()`` (or both)
+
+ Subclasses may define:
+ - either ``prob_classify()`` or ``prob_classify_many()`` (or both)
+ """
+ def labels(self):
+ """
+ :return: the list of category labels used by this classifier.
+ :rtype: list of (immutable)
+ """
+ raise NotImplementedError()
+
+ def classify(self, featureset):
+ """
+ :return: the most appropriate set of labels for the given featureset.
+ :rtype: set(label)
+ """
+ if overridden(self.classify_many):
+ return self.classify_many([featureset])[0]
+ else:
+ raise NotImplementedError()
+
+ def prob_classify(self, featureset):
+ """
+ :return: a probability distribution over sets of labels for the
+ given featureset.
+ :rtype: ProbDistI
+ """
+ if overridden(self.prob_classify_many):
+ return self.prob_classify_many([featureset])[0]
+ else:
+ raise NotImplementedError()
+
+ def classify_many(self, featuresets):
+ """
+ Apply ``self.classify()`` to each element of ``featuresets``. I.e.:
+
+ return [self.classify(fs) for fs in featuresets]
+
+ :rtype: list(set(label))
+ """
+ return [self.classify(fs) for fs in featuresets]
+
+ def prob_classify_many(self, featuresets):
+ """
+ Apply ``self.prob_classify()`` to each element of ``featuresets``. I.e.:
+
+ return [self.prob_classify(fs) for fs in featuresets]
+
+ :rtype: list(ProbDistI)
+ """
+ return [self.prob_classify(fs) for fs in featuresets]
+
+
+# # [XX] IN PROGRESS:
+# class SequenceClassifierI(object):
+# """
+# A processing interface for labeling sequences of tokens with a
+# single category label (or "class"). Labels are typically
+# strs or ints, but can be any immutable type. The set
+# of labels that the classifier chooses from must be fixed and
+# finite.
+# """
+# def labels(self):
+# """
+# :return: the list of category labels used by this classifier.
+# :rtype: list of (immutable)
+# """
+# raise NotImplementedError()
+
+# def prob_classify(self, featureset):
+# """
+# Return a probability distribution over labels for the given
+# featureset.
+
+# If ``featureset`` is a list of featuresets, then return a
+# corresponding list containing the probability distribution
+# over labels for each of the given featuresets, where the
+# *i*\ th element of this list is the most appropriate label for
+# the *i*\ th element of ``featuresets``.
+# """
+# raise NotImplementedError()
+
+# def classify(self, featureset):
+# """
+# Return the most appropriate label for the given featureset.
+
+# If ``featureset`` is a list of featuresets, then return a
+# corresponding list containing the most appropriate label for
+# each of the given featuresets, where the *i*\ th element of
+# this list is the most appropriate label for the *i*\ th element
+# of ``featuresets``.
+# """
+# raise NotImplementedError()
+
diff --git a/nltk/classify/decisiontree.py b/nltk/classify/decisiontree.py
new file mode 100644
index 0000000..c95bbe0
--- /dev/null
+++ b/nltk/classify/decisiontree.py
@@ -0,0 +1,295 @@
+# Natural Language Toolkit: Decision Tree Classifiers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A classifier model that decides which label to assign to a token on
+the basis of a tree structure, where branches correspond to conditions
+on feature values, and leaves correspond to label assignments.
+"""
+from __future__ import print_function, unicode_literals
+
+from collections import defaultdict
+
+from nltk.probability import FreqDist, MLEProbDist, entropy
+from nltk.classify.api import ClassifierI
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class DecisionTreeClassifier(ClassifierI):
+ def __init__(self, label, feature_name=None, decisions=None, default=None):
+ """
+ :param label: The most likely label for tokens that reach
+ this node in the decision tree. If this decision tree
+ has no children, then this label will be assigned to
+ any token that reaches this decision tree.
+ :param feature_name: The name of the feature that this
+ decision tree selects for.
+ :param decisions: A dictionary mapping from feature values
+ for the feature identified by ``feature_name`` to
+ child decision trees.
+ :param default: The child that will be used if the value of
+ feature ``feature_name`` does not match any of the keys in
+ ``decisions``. This is used when constructing binary
+ decision trees.
+ """
+ self._label = label
+ self._fname = feature_name
+ self._decisions = decisions
+ self._default = default
+
+ def labels(self):
+ labels = [self._label]
+ if self._decisions is not None:
+ for dt in self._decisions.values():
+ labels.extend(dt.labels())
+ if self._default is not None:
+ labels.extend(self._default.labels())
+ return list(set(labels))
+
+ def classify(self, featureset):
+ # Decision leaf:
+ if self._fname is None:
+ return self._label
+
+ # Decision tree:
+ fval = featureset.get(self._fname)
+ if fval in self._decisions:
+ return self._decisions[fval].classify(featureset)
+ elif self._default is not None:
+ return self._default.classify(featureset)
+ else:
+ return self._label
+
+ def error(self, labeled_featuresets):
+ errors = 0
+ for featureset, label in labeled_featuresets:
+ if self.classify(featureset) != label:
+ errors += 1
+ return float(errors)/len(labeled_featuresets)
+
+ def pp(self, width=70, prefix='', depth=4):
+ """
+ Return a string containing a pretty-printed version of this
+ decision tree. Each line in this string corresponds to a
+ single decision tree node or leaf, and indentation is used to
+ display the structure of the decision tree.
+ """
+ # [xx] display default!!
+ if self._fname is None:
+ n = width-len(prefix)-15
+ return '%s%s %s\n' % (prefix, '.'*n, self._label)
+ s = ''
+ for i, (fval, result) in enumerate(sorted(self._decisions.items())):
+ hdr = '%s%s=%s? ' % (prefix, self._fname, fval)
+ n = width-15-len(hdr)
+ s += '%s%s %s\n' % (hdr, '.'*(n), result._label)
+ if result._fname is not None and depth>1:
+ s += result.pp(width, prefix+' ', depth-1)
+ if self._default is not None:
+ n = width-len(prefix)-21
+ s += '%selse: %s %s\n' % (prefix, '.'*n, self._default._label)
+ if self._default._fname is not None and depth>1:
+ s += self._default.pp(width, prefix+' ', depth-1)
+ return s
+
+ def pseudocode(self, prefix='', depth=4):
+ """
+ Return a string representation of this decision tree that
+ expresses the decisions it makes as a nested set of pseudocode
+ if statements.
+ """
+ if self._fname is None:
+ return "%sreturn %r\n" % (prefix, self._label)
+ s = ''
+ for (fval, result) in sorted(self._decisions.items()):
+ s += '%sif %s == %r: ' % (prefix, self._fname, fval)
+ if result._fname is not None and depth>1:
+ s += '\n'+result.pseudocode(prefix+' ', depth-1)
+ else:
+ s += 'return %r\n' % result._label
+ if self._default is not None:
+ if len(self._decisions) == 1:
+ s += '%sif %s != %r: '% (prefix, self._fname,
+ list(self._decisions.keys())[0])
+ else:
+ s += '%selse: ' % (prefix,)
+ if self._default._fname is not None and depth>1:
+ s += '\n'+self._default.pseudocode(prefix+' ', depth-1)
+ else:
+ s += 'return %r\n' % self._default._label
+ return s
+
+ def __str__(self):
+ return self.pp()
+
+ @staticmethod
+ def train(labeled_featuresets, entropy_cutoff=0.05, depth_cutoff=100,
+ support_cutoff=10, binary=False, feature_values=None,
+ verbose=False):
+ """
+ :param binary: If true, then treat all feature/value pairs as
+ individual binary features, rather than using a single n-way
+ branch for each feature.
+ """
+ # Collect a list of all feature names.
+ feature_names = set()
+ for featureset, label in labeled_featuresets:
+ for fname in featureset:
+ feature_names.add(fname)
+
+ # Collect a list of the values each feature can take.
+ if feature_values is None and binary:
+ feature_values = defaultdict(set)
+ for featureset, label in labeled_featuresets:
+ for fname, fval in featureset.items():
+ feature_values[fname].add(fval)
+
+ # Start with a stump.
+ if not binary:
+ tree = DecisionTreeClassifier.best_stump(
+ feature_names, labeled_featuresets, verbose)
+ else:
+ tree = DecisionTreeClassifier.best_binary_stump(
+ feature_names, labeled_featuresets, feature_values, verbose)
+
+ # Refine the stump.
+ tree.refine(labeled_featuresets, entropy_cutoff, depth_cutoff-1,
+ support_cutoff, binary, feature_values, verbose)
+
+ # Return it
+ return tree
+
+ @staticmethod
+ def leaf(labeled_featuresets):
+ label = FreqDist(label for (featureset,label)
+ in labeled_featuresets).max()
+ return DecisionTreeClassifier(label)
+
+ @staticmethod
+ def stump(feature_name, labeled_featuresets):
+ label = FreqDist(label for (featureset,label)
+ in labeled_featuresets).max()
+
+ # Find the best label for each value.
+ freqs = defaultdict(FreqDist) # freq(label|value)
+ for featureset, label in labeled_featuresets:
+ feature_value = featureset.get(feature_name)
+ freqs[feature_value][label] += 1
+
+ decisions = dict((val, DecisionTreeClassifier(freqs[val].max()))
+ for val in freqs)
+ return DecisionTreeClassifier(label, feature_name, decisions)
+
+ def refine(self, labeled_featuresets, entropy_cutoff, depth_cutoff,
+ support_cutoff, binary=False, feature_values=None,
+ verbose=False):
+ if len(labeled_featuresets) <= support_cutoff: return
+ if self._fname is None: return
+ if depth_cutoff <= 0: return
+ for fval in self._decisions:
+ fval_featuresets = [(featureset,label) for (featureset,label)
+ in labeled_featuresets
+ if featureset.get(self._fname) == fval]
+
+ label_freqs = FreqDist(label for (featureset,label)
+ in fval_featuresets)
+ if entropy(MLEProbDist(label_freqs)) > entropy_cutoff:
+ self._decisions[fval] = DecisionTreeClassifier.train(
+ fval_featuresets, entropy_cutoff, depth_cutoff,
+ support_cutoff, binary, feature_values, verbose)
+ if self._default is not None:
+ default_featuresets = [(featureset, label) for (featureset, label)
+ in labeled_featuresets
+ if featureset.get(self._fname) not in
+ self._decisions]
+ label_freqs = FreqDist(label for (featureset,label)
+ in default_featuresets)
+ if entropy(MLEProbDist(label_freqs)) > entropy_cutoff:
+ self._default = DecisionTreeClassifier.train(
+ default_featuresets, entropy_cutoff, depth_cutoff,
+ support_cutoff, binary, feature_values, verbose)
+
+ @staticmethod
+ def best_stump(feature_names, labeled_featuresets, verbose=False):
+ best_stump = DecisionTreeClassifier.leaf(labeled_featuresets)
+ best_error = best_stump.error(labeled_featuresets)
+ for fname in feature_names:
+ stump = DecisionTreeClassifier.stump(fname, labeled_featuresets)
+ stump_error = stump.error(labeled_featuresets)
+ if stump_error < best_error:
+ best_error = stump_error
+ best_stump = stump
+ if verbose:
+ print(('best stump for %6d toks uses %-20s err=%6.4f' %
+ (len(labeled_featuresets), best_stump._fname, best_error)))
+ return best_stump
+
+ @staticmethod
+ def binary_stump(feature_name, feature_value, labeled_featuresets):
+ label = FreqDist(label for (featureset, label)
+ in labeled_featuresets).max()
+
+ # Find the best label for each value.
+ pos_fdist = FreqDist()
+ neg_fdist = FreqDist()
+ for featureset, label in labeled_featuresets:
+ if featureset.get(feature_name) == feature_value:
+ pos_fdist[label] += 1
+ else:
+ neg_fdist[label] += 1
+
+
+ decisions = {}
+ default = label
+ # But hopefully we have observations!
+ if pos_fdist.N() > 0:
+ decisions = {feature_value: DecisionTreeClassifier(pos_fdist.max())}
+ if neg_fdist.N() > 0:
+ default = DecisionTreeClassifier(neg_fdist.max())
+
+ return DecisionTreeClassifier(label, feature_name, decisions, default)
+
+ @staticmethod
+ def best_binary_stump(feature_names, labeled_featuresets, feature_values,
+ verbose=False):
+ best_stump = DecisionTreeClassifier.leaf(labeled_featuresets)
+ best_error = best_stump.error(labeled_featuresets)
+ for fname in feature_names:
+ for fval in feature_values[fname]:
+ stump = DecisionTreeClassifier.binary_stump(
+ fname, fval, labeled_featuresets)
+ stump_error = stump.error(labeled_featuresets)
+ if stump_error < best_error:
+ best_error = stump_error
+ best_stump = stump
+ if best_stump._decisions:
+ descr = '%s=%s' % (best_stump._fname,
+ list(best_stump._decisions.keys())[0])
+ else:
+ descr = '(default)'
+ if verbose:
+ print(('best stump for %6d toks uses %-20s err=%6.4f' %
+ (len(labeled_featuresets), descr, best_error)))
+ return best_stump
+
+##//////////////////////////////////////////////////////
+## Demo
+##//////////////////////////////////////////////////////
+
+def f(x):
+ return DecisionTreeClassifier.train(x, binary=True, verbose=True)
+
+def demo():
+ from nltk.classify.util import names_demo, binary_names_demo_features
+ classifier = names_demo(f, #DecisionTreeClassifier.train,
+ binary_names_demo_features)
+ print(classifier.pp(depth=7))
+ print(classifier.pseudocode(depth=7))
+
+if __name__ == '__main__':
+ demo()
+
diff --git a/nltk/classify/maxent.py b/nltk/classify/maxent.py
new file mode 100644
index 0000000..755e585
--- /dev/null
+++ b/nltk/classify/maxent.py
@@ -0,0 +1,1484 @@
+# Natural Language Toolkit: Maximum Entropy Classifiers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Dmitry Chichkov <dchichkov at gmail.com> (TypedMaxentFeatureEncoding)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A classifier model based on maximum entropy modeling framework. This
+framework considers all of the probability distributions that are
+empirically consistent with the training data; and chooses the
+distribution with the highest entropy. A probability distribution is
+"empirically consistent" with a set of training data if its estimated
+frequency with which a class and a feature vector value co-occur is
+equal to the actual frequency in the data.
+
+Terminology: 'feature'
+======================
+The term *feature* is usually used to refer to some property of an
+unlabeled token. For example, when performing word sense
+disambiguation, we might define a ``'prevword'`` feature whose value is
+the word preceding the target word. However, in the context of
+maxent modeling, the term *feature* is typically used to refer to a
+property of a "labeled" token. In order to prevent confusion, we
+will introduce two distinct terms to disambiguate these two different
+concepts:
+
+ - An "input-feature" is a property of an unlabeled token.
+ - A "joint-feature" is a property of a labeled token.
+
+In the rest of the ``nltk.classify`` module, the term "features" is
+used to refer to what we will call "input-features" in this module.
+
+In literature that describes and discusses maximum entropy models,
+input-features are typically called "contexts", and joint-features
+are simply referred to as "features".
+
+Converting Input-Features to Joint-Features
+-------------------------------------------
+In maximum entropy models, joint-features are required to have numeric
+values. Typically, each input-feature ``input_feat`` is mapped to a
+set of joint-features of the form:
+
+| joint_feat(token, label) = { 1 if input_feat(token) == feat_val
+| { and label == some_label
+| {
+| { 0 otherwise
+
+For all values of ``feat_val`` and ``some_label``. This mapping is
+performed by classes that implement the ``MaxentFeatureEncodingI``
+interface.
+"""
+from __future__ import print_function, unicode_literals
+__docformat__ = 'epytext en'
+
+try:
+ import numpy
+except ImportError:
+ pass
+
+import time
+import tempfile
+import os
+import gzip
+from collections import defaultdict
+
+from nltk import compat
+from nltk.data import gzip_open_unicode
+from nltk.util import OrderedDict
+from nltk.probability import DictionaryProbDist
+
+from nltk.classify.api import ClassifierI
+from nltk.classify.util import CutoffChecker, accuracy, log_likelihood
+from nltk.classify.megam import call_megam, write_megam_file, parse_megam_weights
+from nltk.classify.tadm import call_tadm, write_tadm_file, parse_tadm_weights
+
+######################################################################
+#{ Classifier Model
+######################################################################
+
+ at compat.python_2_unicode_compatible
+class MaxentClassifier(ClassifierI):
+ """
+ A maximum entropy classifier (also known as a "conditional
+ exponential classifier"). This classifier is parameterized by a
+ set of "weights", which are used to combine the joint-features
+ that are generated from a featureset by an "encoding". In
+ particular, the encoding maps each ``(featureset, label)`` pair to
+ a vector. The probability of each label is then computed using
+ the following equation::
+
+ dotprod(weights, encode(fs,label))
+ prob(fs|label) = ---------------------------------------------------
+ sum(dotprod(weights, encode(fs,l)) for l in labels)
+
+ Where ``dotprod`` is the dot product::
+
+ dotprod(a,b) = sum(x*y for (x,y) in zip(a,b))
+ """
+ def __init__(self, encoding, weights, logarithmic=True):
+ """
+ Construct a new maxent classifier model. Typically, new
+ classifier models are created using the ``train()`` method.
+
+ :type encoding: MaxentFeatureEncodingI
+ :param encoding: An encoding that is used to convert the
+ featuresets that are given to the ``classify`` method into
+ joint-feature vectors, which are used by the maxent
+ classifier model.
+
+ :type weights: list of float
+ :param weights: The feature weight vector for this classifier.
+
+ :type logarithmic: bool
+ :param logarithmic: If false, then use non-logarithmic weights.
+ """
+ self._encoding = encoding
+ self._weights = weights
+ self._logarithmic = logarithmic
+ #self._logarithmic = False
+ assert encoding.length() == len(weights)
+
+ def labels(self):
+ return self._encoding.labels()
+
+ def set_weights(self, new_weights):
+ """
+ Set the feature weight vector for this classifier.
+ :param new_weights: The new feature weight vector.
+ :type new_weights: list of float
+ """
+ self._weights = new_weights
+ assert (self._encoding.length() == len(new_weights))
+
+ def weights(self):
+ """
+ :return: The feature weight vector for this classifier.
+ :rtype: list of float
+ """
+ return self._weights
+
+ def classify(self, featureset):
+ return self.prob_classify(featureset).max()
+
+ def prob_classify(self, featureset):
+ prob_dict = {}
+ for label in self._encoding.labels():
+ feature_vector = self._encoding.encode(featureset, label)
+
+ if self._logarithmic:
+ total = 0.0
+ for (f_id, f_val) in feature_vector:
+ total += self._weights[f_id] * f_val
+ prob_dict[label] = total
+
+ else:
+ prod = 1.0
+ for (f_id, f_val) in feature_vector:
+ prod *= self._weights[f_id] ** f_val
+ prob_dict[label] = prod
+
+ # Normalize the dictionary to give a probability distribution
+ return DictionaryProbDist(prob_dict, log=self._logarithmic,
+ normalize=True)
+
+ def explain(self, featureset, columns=4):
+ """
+ Print a table showing the effect of each of the features in
+ the given feature set, and how they combine to determine the
+ probabilities of each label for that featureset.
+ """
+ descr_width = 50
+ TEMPLATE = ' %-'+str(descr_width-2)+'s%s%8.3f'
+
+ pdist = self.prob_classify(featureset)
+ labels = sorted(pdist.samples(), key=pdist.prob, reverse=True)
+ labels = labels[:columns]
+ print(' Feature'.ljust(descr_width)+''.join(
+ '%8s' % (("%s" % l)[:7]) for l in labels))
+ print(' '+'-'*(descr_width-2+8*len(labels)))
+ sums = defaultdict(int)
+ for i, label in enumerate(labels):
+ feature_vector = self._encoding.encode(featureset, label)
+ feature_vector.sort(key=lambda fid__: abs(self._weights[fid__[0]]),
+ reverse=True)
+ for (f_id, f_val) in feature_vector:
+ if self._logarithmic: score = self._weights[f_id] * f_val
+ else: score = self._weights[f_id] ** f_val
+ descr = self._encoding.describe(f_id)
+ descr = descr.split(' and label is ')[0] # hack
+ descr += ' (%s)' % f_val # hack
+ if len(descr) > 47: descr = descr[:44]+'...'
+ print(TEMPLATE % (descr, i*8*' ', score))
+ sums[label] += score
+ print(' '+'-'*(descr_width-1+8*len(labels)))
+ print(' TOTAL:'.ljust(descr_width)+''.join(
+ '%8.3f' % sums[l] for l in labels))
+ print(' PROBS:'.ljust(descr_width)+''.join(
+ '%8.3f' % pdist.prob(l) for l in labels))
+
+ def show_most_informative_features(self, n=10, show='all'):
+ """
+ :param show: all, neg, or pos (for negative-only or positive-only)
+ """
+ fids = sorted(list(range(len(self._weights))),
+ key=lambda fid: abs(self._weights[fid]),
+ reverse=True)
+ if show == 'pos':
+ fids = [fid for fid in fids if self._weights[fid]>0]
+ elif show == 'neg':
+ fids = [fid for fid in fids if self._weights[fid]<0]
+ for fid in fids[:n]:
+ print('%8.3f %s' % (self._weights[fid],
+ self._encoding.describe(fid)))
+
+ def __repr__(self):
+ return ('<ConditionalExponentialClassifier: %d labels, %d features>' %
+ (len(self._encoding.labels()), self._encoding.length()))
+
+ #: A list of the algorithm names that are accepted for the
+ #: ``train()`` method's ``algorithm`` parameter.
+ ALGORITHMS = ['GIS', 'IIS', 'MEGAM', 'TADM']
+
+ @classmethod
+ def train(cls, train_toks, algorithm=None, trace=3, encoding=None,
+ labels=None, sparse=None, gaussian_prior_sigma=0, **cutoffs):
+ """
+ Train a new maxent classifier based on the given corpus of
+ training samples. This classifier will have its weights
+ chosen to maximize entropy while remaining empirically
+ consistent with the training corpus.
+
+ :rtype: MaxentClassifier
+ :return: The new maxent classifier
+
+ :type train_toks: list
+ :param train_toks: Training data, represented as a list of
+ pairs, the first member of which is a featureset,
+ and the second of which is a classification label.
+
+ :type algorithm: str
+ :param algorithm: A case-insensitive string, specifying which
+ algorithm should be used to train the classifier. The
+ following algorithms are currently available.
+
+ - Iterative Scaling Methods: Generalized Iterative Scaling (``'GIS'``),
+ Improved Iterative Scaling (``'IIS'``)
+ - External Libraries (requiring megam):
+ LM-BFGS algorithm, with training performed by Megam (``'megam'``)
+
+ The default algorithm is ``'IIS'``.
+
+ :type trace: int
+ :param trace: The level of diagnostic tracing output to produce.
+ Higher values produce more verbose output.
+ :type encoding: MaxentFeatureEncodingI
+ :param encoding: A feature encoding, used to convert featuresets
+ into feature vectors. If none is specified, then a
+ ``BinaryMaxentFeatureEncoding`` will be built based on the
+ features that are attested in the training corpus.
+ :type labels: list(str)
+ :param labels: The set of possible labels. If none is given, then
+ the set of all labels attested in the training data will be
+ used instead.
+ :param gaussian_prior_sigma: The sigma value for a gaussian
+ prior on model weights. Currently, this is supported by
+ ``megam``. For other algorithms, its value is ignored.
+ :param cutoffs: Arguments specifying various conditions under
+ which the training should be halted. (Some of the cutoff
+ conditions are not supported by some algorithms.)
+
+ - ``max_iter=v``: Terminate after ``v`` iterations.
+ - ``min_ll=v``: Terminate after the negative average
+ log-likelihood drops under ``v``.
+ - ``min_lldelta=v``: Terminate if a single iteration improves
+ log likelihood by less than ``v``.
+ """
+ if algorithm is None:
+ algorithm = 'iis'
+ for key in cutoffs:
+ if key not in ('max_iter', 'min_ll', 'min_lldelta',
+ 'max_acc', 'min_accdelta', 'count_cutoff',
+ 'norm', 'explicit', 'bernoulli'):
+ raise TypeError('Unexpected keyword arg %r' % key)
+ algorithm = algorithm.lower()
+ if algorithm == 'iis':
+ return train_maxent_classifier_with_iis(
+ train_toks, trace, encoding, labels, **cutoffs)
+ elif algorithm == 'gis':
+ return train_maxent_classifier_with_gis(
+ train_toks, trace, encoding, labels, **cutoffs)
+ elif algorithm == 'megam':
+ return train_maxent_classifier_with_megam(
+ train_toks, trace, encoding, labels,
+ gaussian_prior_sigma, **cutoffs)
+ elif algorithm == 'tadm':
+ kwargs = cutoffs
+ kwargs['trace'] = trace
+ kwargs['encoding'] = encoding
+ kwargs['labels'] = labels
+ kwargs['gaussian_prior_sigma'] = gaussian_prior_sigma
+ return TadmMaxentClassifier.train(train_toks, **kwargs)
+ else:
+ raise ValueError('Unknown algorithm %s' % algorithm)
+
+
+#: Alias for MaxentClassifier.
+ConditionalExponentialClassifier = MaxentClassifier
+
+
+######################################################################
+#{ Feature Encodings
+######################################################################
+
+class MaxentFeatureEncodingI(object):
+ """
+ A mapping that converts a set of input-feature values to a vector
+ of joint-feature values, given a label. This conversion is
+ necessary to translate featuresets into a format that can be used
+ by maximum entropy models.
+
+ The set of joint-features used by a given encoding is fixed, and
+ each index in the generated joint-feature vectors corresponds to a
+ single joint-feature. The length of the generated joint-feature
+ vectors is therefore constant (for a given encoding).
+
+ Because the joint-feature vectors generated by
+ ``MaxentFeatureEncodingI`` are typically very sparse, they are
+ represented as a list of ``(index, value)`` tuples, specifying the
+ value of each non-zero joint-feature.
+
+ Feature encodings are generally created using the ``train()``
+ method, which generates an appropriate encoding based on the
+ input-feature values and labels that are present in a given
+ corpus.
+ """
+ def encode(self, featureset, label):
+ """
+ Given a (featureset, label) pair, return the corresponding
+ vector of joint-feature values. This vector is represented as
+ a list of ``(index, value)`` tuples, specifying the value of
+ each non-zero joint-feature.
+
+ :type featureset: dict
+ :rtype: list(tuple(int, int))
+ """
+ raise NotImplementedError()
+
+ def length(self):
+ """
+ :return: The size of the fixed-length joint-feature vectors
+ that are generated by this encoding.
+ :rtype: int
+ """
+ raise NotImplementedError()
+
+ def labels(self):
+ """
+ :return: A list of the \"known labels\" -- i.e., all labels
+ ``l`` such that ``self.encode(fs,l)`` can be a nonzero
+ joint-feature vector for some value of ``fs``.
+ :rtype: list
+ """
+ raise NotImplementedError()
+
+ def describe(self, fid):
+ """
+ :return: A string describing the value of the joint-feature
+ whose index in the generated feature vectors is ``fid``.
+ :rtype: str
+ """
+ raise NotImplementedError()
+
+ def train(cls, train_toks):
+ """
+ Construct and return new feature encoding, based on a given
+ training corpus ``train_toks``.
+
+ :type train_toks: list(tuple(dict, str))
+ :param train_toks: Training data, represented as a list of
+ pairs, the first member of which is a feature dictionary,
+ and the second of which is a classification label.
+ """
+ raise NotImplementedError()
+
+class FunctionBackedMaxentFeatureEncoding(MaxentFeatureEncodingI):
+ """
+ A feature encoding that calls a user-supplied function to map a
+ given featureset/label pair to a sparse joint-feature vector.
+ """
+ def __init__(self, func, length, labels):
+ """
+ Construct a new feature encoding based on the given function.
+
+ :type func: (callable)
+ :param func: A function that takes two arguments, a featureset
+ and a label, and returns the sparse joint feature vector
+ that encodes them::
+
+ func(featureset, label) -> feature_vector
+
+ This sparse joint feature vector (``feature_vector``) is a
+ list of ``(index,value)`` tuples.
+
+ :type length: int
+ :param length: The size of the fixed-length joint-feature
+ vectors that are generated by this encoding.
+
+ :type labels: list
+ :param labels: A list of the \"known labels\" for this
+ encoding -- i.e., all labels ``l`` such that
+ ``self.encode(fs,l)`` can be a nonzero joint-feature vector
+ for some value of ``fs``.
+ """
+ self._length = length
+ self._func = func
+ self._labels = labels
+
+ def encode(self, featureset, label):
+ return self._func(featureset, label)
+
+ def length(self):
+ return self._length
+
+ def labels(self):
+ return self._labels
+
+ def describe(self, fid):
+ return 'no description available'
+
+class BinaryMaxentFeatureEncoding(MaxentFeatureEncodingI):
+ """
+ A feature encoding that generates vectors containing a binary
+ joint-features of the form:
+
+ | joint_feat(fs, l) = { 1 if (fs[fname] == fval) and (l == label)
+ | {
+ | { 0 otherwise
+
+ Where ``fname`` is the name of an input-feature, ``fval`` is a value
+ for that input-feature, and ``label`` is a label.
+
+ Typically, these features are constructed based on a training
+ corpus, using the ``train()`` method. This method will create one
+ feature for each combination of ``fname``, ``fval``, and ``label``
+ that occurs at least once in the training corpus.
+
+ The ``unseen_features`` parameter can be used to add "unseen-value
+ features", which are used whenever an input feature has a value
+ that was not encountered in the training corpus. These features
+ have the form:
+
+ | joint_feat(fs, l) = { 1 if is_unseen(fname, fs[fname])
+ | { and l == label
+ | {
+ | { 0 otherwise
+
+ Where ``is_unseen(fname, fval)`` is true if the encoding does not
+ contain any joint features that are true when ``fs[fname]==fval``.
+
+ The ``alwayson_features`` parameter can be used to add "always-on
+ features", which have the form::
+
+ | joint_feat(fs, l) = { 1 if (l == label)
+ | {
+ | { 0 otherwise
+
+ These always-on features allow the maxent model to directly model
+ the prior probabilities of each label.
+ """
+ def __init__(self, labels, mapping, unseen_features=False,
+ alwayson_features=False):
+ """
+ :param labels: A list of the \"known labels\" for this encoding.
+
+ :param mapping: A dictionary mapping from ``(fname,fval,label)``
+ tuples to corresponding joint-feature indexes. These
+ indexes must be the set of integers from 0...len(mapping).
+ If ``mapping[fname,fval,label]=id``, then
+ ``self.encode(..., fname:fval, ..., label)[id]`` is 1;
+ otherwise, it is 0.
+
+ :param unseen_features: If true, then include unseen value
+ features in the generated joint-feature vectors.
+
+ :param alwayson_features: If true, then include always-on
+ features in the generated joint-feature vectors.
+ """
+ if set(mapping.values()) != set(range(len(mapping))):
+ raise ValueError('Mapping values must be exactly the '
+ 'set of integers from 0...len(mapping)')
+
+ self._labels = list(labels)
+ """A list of attested labels."""
+
+ self._mapping = mapping
+ """dict mapping from (fname,fval,label) -> fid"""
+
+ self._length = len(mapping)
+ """The length of generated joint feature vectors."""
+
+ self._alwayson = None
+ """dict mapping from label -> fid"""
+
+ self._unseen = None
+ """dict mapping from fname -> fid"""
+
+ if alwayson_features:
+ self._alwayson = dict((label,i+self._length)
+ for (i,label) in enumerate(labels))
+ self._length += len(self._alwayson)
+
+ if unseen_features:
+ fnames = set(fname for (fname, fval, label) in mapping)
+ self._unseen = dict((fname, i+self._length)
+ for (i, fname) in enumerate(fnames))
+ self._length += len(fnames)
+
+ def encode(self, featureset, label):
+ # Inherit docs.
+ encoding = []
+
+ # Convert input-features to joint-features:
+ for fname, fval in featureset.items():
+ # Known feature name & value:
+ if (fname, fval, label) in self._mapping:
+ encoding.append((self._mapping[fname, fval, label], 1))
+
+ # Otherwise, we might want to fire an "unseen-value feature".
+ elif self._unseen:
+ # Have we seen this fname/fval combination with any label?
+ for label2 in self._labels:
+ if (fname, fval, label2) in self._mapping:
+ break # we've seen this fname/fval combo
+ # We haven't -- fire the unseen-value feature
+ else:
+ if fname in self._unseen:
+ encoding.append((self._unseen[fname], 1))
+
+ # Add always-on features:
+ if self._alwayson and label in self._alwayson:
+ encoding.append((self._alwayson[label], 1))
+
+ return encoding
+
+ def describe(self, f_id):
+ # Inherit docs.
+ if not isinstance(f_id, compat.integer_types):
+ raise TypeError('describe() expected an int')
+ try:
+ self._inv_mapping
+ except AttributeError:
+ self._inv_mapping = [-1]*len(self._mapping)
+ for (info, i) in self._mapping.items():
+ self._inv_mapping[i] = info
+
+ if f_id < len(self._mapping):
+ (fname, fval, label) = self._inv_mapping[f_id]
+ return '%s==%r and label is %r' % (fname, fval, label)
+ elif self._alwayson and f_id in self._alwayson.values():
+ for (label, f_id2) in self._alwayson.items():
+ if f_id==f_id2: return 'label is %r' % label
+ elif self._unseen and f_id in self._unseen.values():
+ for (fname, f_id2) in self._unseen.items():
+ if f_id==f_id2: return '%s is unseen' % fname
+ else:
+ raise ValueError('Bad feature id')
+
+ def labels(self):
+ # Inherit docs.
+ return self._labels
+
+ def length(self):
+ # Inherit docs.
+ return self._length
+
+ @classmethod
+ def train(cls, train_toks, count_cutoff=0, labels=None, **options):
+ """
+ Construct and return new feature encoding, based on a given
+ training corpus ``train_toks``. See the class description
+ ``BinaryMaxentFeatureEncoding`` for a description of the
+ joint-features that will be included in this encoding.
+
+ :type train_toks: list(tuple(dict, str))
+ :param train_toks: Training data, represented as a list of
+ pairs, the first member of which is a feature dictionary,
+ and the second of which is a classification label.
+
+ :type count_cutoff: int
+ :param count_cutoff: A cutoff value that is used to discard
+ rare joint-features. If a joint-feature's value is 1
+ fewer than ``count_cutoff`` times in the training corpus,
+ then that joint-feature is not included in the generated
+ encoding.
+
+ :type labels: list
+ :param labels: A list of labels that should be used by the
+ classifier. If not specified, then the set of labels
+ attested in ``train_toks`` will be used.
+
+ :param options: Extra parameters for the constructor, such as
+ ``unseen_features`` and ``alwayson_features``.
+ """
+ mapping = {} # maps (fname, fval, label) -> fid
+ seen_labels = set() # The set of labels we've encountered
+ count = defaultdict(int) # maps (fname, fval) -> count
+
+ for (tok, label) in train_toks:
+ if labels and label not in labels:
+ raise ValueError('Unexpected label %s' % label)
+ seen_labels.add(label)
+
+ # Record each of the features.
+ for (fname, fval) in tok.items():
+
+ # If a count cutoff is given, then only add a joint
+ # feature once the corresponding (fname, fval, label)
+ # tuple exceeds that cutoff.
+ count[fname,fval] += 1
+ if count[fname,fval] >= count_cutoff:
+ if (fname, fval, label) not in mapping:
+ mapping[fname, fval, label] = len(mapping)
+
+ if labels is None: labels = seen_labels
+ return cls(labels, mapping, **options)
+
+class GISEncoding(BinaryMaxentFeatureEncoding):
+ """
+ A binary feature encoding which adds one new joint-feature to the
+ joint-features defined by ``BinaryMaxentFeatureEncoding``: a
+ correction feature, whose value is chosen to ensure that the
+ sparse vector always sums to a constant non-negative number. This
+ new feature is used to ensure two preconditions for the GIS
+ training algorithm:
+
+ - At least one feature vector index must be nonzero for every
+ token.
+ - The feature vector must sum to a constant non-negative number
+ for every token.
+ """
+ def __init__(self, labels, mapping, unseen_features=False,
+ alwayson_features=False, C=None):
+ """
+ :param C: The correction constant. The value of the correction
+ feature is based on this value. In particular, its value is
+ ``C - sum([v for (f,v) in encoding])``.
+ :seealso: ``BinaryMaxentFeatureEncoding.__init__``
+ """
+ BinaryMaxentFeatureEncoding.__init__(
+ self, labels, mapping, unseen_features, alwayson_features)
+ if C is None:
+ C = len(set(fname for (fname,fval,label) in mapping))+1
+ self._C = C
+
+ @property
+ def C(self):
+ """The non-negative constant that all encoded feature vectors
+ will sum to."""
+ return self._C
+
+ def encode(self, featureset, label):
+ # Get the basic encoding.
+ encoding = BinaryMaxentFeatureEncoding.encode(self, featureset, label)
+ base_length = BinaryMaxentFeatureEncoding.length(self)
+
+ # Add a correction feature.
+ total = sum(v for (f,v) in encoding)
+ if total >= self._C:
+ raise ValueError('Correction feature is not high enough!')
+ encoding.append( (base_length, self._C-total) )
+
+ # Return the result
+ return encoding
+
+ def length(self):
+ return BinaryMaxentFeatureEncoding.length(self) + 1
+
+ def describe(self, f_id):
+ if f_id == BinaryMaxentFeatureEncoding.length(self):
+ return 'Correction feature (%s)' % self._C
+ else:
+ return BinaryMaxentFeatureEncoding.describe(self, f_id)
+
+
+class TadmEventMaxentFeatureEncoding(BinaryMaxentFeatureEncoding):
+ def __init__(self, labels, mapping, unseen_features=False,
+ alwayson_features=False):
+ self._mapping = OrderedDict(mapping)
+ self._label_mapping = OrderedDict()
+ BinaryMaxentFeatureEncoding.__init__(self, labels, self._mapping,
+ unseen_features,
+ alwayson_features)
+
+ def encode(self, featureset, label):
+ encoding = []
+ for feature, value in featureset.items():
+ if (feature, label) not in self._mapping:
+ self._mapping[(feature, label)] = len(self._mapping)
+ if value not in self._label_mapping:
+ if not isinstance(value, int):
+ self._label_mapping[value] = len(self._label_mapping)
+ else:
+ self._label_mapping[value] = value
+ encoding.append((self._mapping[(feature, label)],
+ self._label_mapping[value]))
+ return encoding
+
+ def labels(self):
+ return self._labels
+
+ def describe(self, fid):
+ for (feature, label) in self._mapping:
+ if self._mapping[(feature, label)] == fid:
+ return (feature, label)
+
+ def length(self):
+ return len(self._mapping)
+
+ @classmethod
+ def train(cls, train_toks, count_cutoff=0, labels=None, **options):
+ mapping = OrderedDict()
+ if not labels:
+ labels = []
+
+ # This gets read twice, so compute the values in case it's lazy.
+ train_toks = list(train_toks)
+
+ for (featureset, label) in train_toks:
+ if label not in labels:
+ labels.append(label)
+
+ for (featureset, label) in train_toks:
+ for label in labels:
+ for feature in featureset:
+ if (feature, label) not in mapping:
+ mapping[(feature, label)] = len(mapping)
+
+ return cls(labels, mapping, **options)
+
+
+class TypedMaxentFeatureEncoding(MaxentFeatureEncodingI):
+ """
+ A feature encoding that generates vectors containing integer,
+ float and binary joint-features of the form:
+
+ Binary (for string and boolean features):
+
+ | joint_feat(fs, l) = { 1 if (fs[fname] == fval) and (l == label)
+ | {
+ | { 0 otherwise
+
+ Value (for integer and float features):
+
+ | joint_feat(fs, l) = { fval if (fs[fname] == type(fval))
+ | { and (l == label)
+ | {
+ | { not encoded otherwise
+
+ Where ``fname`` is the name of an input-feature, ``fval`` is a value
+ for that input-feature, and ``label`` is a label.
+
+ Typically, these features are constructed based on a training
+ corpus, using the ``train()`` method.
+
+ For string and boolean features [type(fval) not in (int, float)]
+ this method will create one feature for each combination of
+ ``fname``, ``fval``, and ``label`` that occurs at least once in the
+ training corpus.
+
+ For integer and float features [type(fval) in (int, float)] this
+ method will create one feature for each combination of ``fname``
+ and ``label`` that occurs at least once in the training corpus.
+
+ For binary features the ``unseen_features`` parameter can be used
+ to add "unseen-value features", which are used whenever an input
+ feature has a value that was not encountered in the training
+ corpus. These features have the form:
+
+ | joint_feat(fs, l) = { 1 if is_unseen(fname, fs[fname])
+ | { and l == label
+ | {
+ | { 0 otherwise
+
+ Where ``is_unseen(fname, fval)`` is true if the encoding does not
+ contain any joint features that are true when ``fs[fname]==fval``.
+
+ The ``alwayson_features`` parameter can be used to add "always-on
+ features", which have the form:
+
+ | joint_feat(fs, l) = { 1 if (l == label)
+ | {
+ | { 0 otherwise
+
+ These always-on features allow the maxent model to directly model
+ the prior probabilities of each label.
+ """
+ def __init__(self, labels, mapping, unseen_features=False,
+ alwayson_features=False):
+ """
+ :param labels: A list of the \"known labels\" for this encoding.
+
+ :param mapping: A dictionary mapping from ``(fname,fval,label)``
+ tuples to corresponding joint-feature indexes. These
+ indexes must be the set of integers from 0...len(mapping).
+ If ``mapping[fname,fval,label]=id``, then
+ ``self.encode({..., fname:fval, ...``, label)[id]} is 1;
+ otherwise, it is 0.
+
+ :param unseen_features: If true, then include unseen value
+ features in the generated joint-feature vectors.
+
+ :param alwayson_features: If true, then include always-on
+ features in the generated joint-feature vectors.
+ """
+ if set(mapping.values()) != set(range(len(mapping))):
+ raise ValueError('Mapping values must be exactly the '
+ 'set of integers from 0...len(mapping)')
+
+ self._labels = list(labels)
+ """A list of attested labels."""
+
+ self._mapping = mapping
+ """dict mapping from (fname,fval,label) -> fid"""
+
+ self._length = len(mapping)
+ """The length of generated joint feature vectors."""
+
+ self._alwayson = None
+ """dict mapping from label -> fid"""
+
+ self._unseen = None
+ """dict mapping from fname -> fid"""
+
+ if alwayson_features:
+ self._alwayson = dict((label, i+self._length)
+ for (i,label) in enumerate(labels))
+ self._length += len(self._alwayson)
+
+ if unseen_features:
+ fnames = set(fname for (fname, fval, label) in mapping)
+ self._unseen = dict((fname, i+self._length)
+ for (i, fname) in enumerate(fnames))
+ self._length += len(fnames)
+
+ def encode(self, featureset, label):
+ # Inherit docs.
+ encoding = []
+
+ # Convert input-features to joint-features:
+ for fname, fval in featureset.items():
+ if isinstance(fval, (compat.integer_types, float)):
+ # Known feature name & value:
+ if (fname, type(fval), label) in self._mapping:
+ encoding.append((self._mapping[fname, type(fval), label], fval))
+ else:
+ # Known feature name & value:
+ if (fname, fval, label) in self._mapping:
+ encoding.append((self._mapping[fname, fval, label], 1))
+
+ # Otherwise, we might want to fire an "unseen-value feature".
+ elif self._unseen:
+ # Have we seen this fname/fval combination with any label?
+ for label2 in self._labels:
+ if (fname, fval, label2) in self._mapping:
+ break # we've seen this fname/fval combo
+ # We haven't -- fire the unseen-value feature
+ else:
+ if fname in self._unseen:
+ encoding.append((self._unseen[fname], 1))
+
+
+ # Add always-on features:
+ if self._alwayson and label in self._alwayson:
+ encoding.append((self._alwayson[label], 1))
+
+ return encoding
+
+ def describe(self, f_id):
+ # Inherit docs.
+ if not isinstance(f_id, compat.integer_types):
+ raise TypeError('describe() expected an int')
+ try:
+ self._inv_mapping
+ except AttributeError:
+ self._inv_mapping = [-1]*len(self._mapping)
+ for (info, i) in self._mapping.items():
+ self._inv_mapping[i] = info
+
+ if f_id < len(self._mapping):
+ (fname, fval, label) = self._inv_mapping[f_id]
+ return '%s==%r and label is %r' % (fname, fval, label)
+ elif self._alwayson and f_id in self._alwayson.values():
+ for (label, f_id2) in self._alwayson.items():
+ if f_id==f_id2: return 'label is %r' % label
+ elif self._unseen and f_id in self._unseen.values():
+ for (fname, f_id2) in self._unseen.items():
+ if f_id==f_id2: return '%s is unseen' % fname
+ else:
+ raise ValueError('Bad feature id')
+
+ def labels(self):
+ # Inherit docs.
+ return self._labels
+
+ def length(self):
+ # Inherit docs.
+ return self._length
+
+ @classmethod
+ def train(cls, train_toks, count_cutoff=0, labels=None, **options):
+ """
+ Construct and return new feature encoding, based on a given
+ training corpus ``train_toks``. See the class description
+ ``TypedMaxentFeatureEncoding`` for a description of the
+ joint-features that will be included in this encoding.
+
+ Note: recognized feature values types are (int, float), over
+ types are interpreted as regular binary features.
+
+ :type train_toks: list(tuple(dict, str))
+ :param train_toks: Training data, represented as a list of
+ pairs, the first member of which is a feature dictionary,
+ and the second of which is a classification label.
+
+ :type count_cutoff: int
+ :param count_cutoff: A cutoff value that is used to discard
+ rare joint-features. If a joint-feature's value is 1
+ fewer than ``count_cutoff`` times in the training corpus,
+ then that joint-feature is not included in the generated
+ encoding.
+
+ :type labels: list
+ :param labels: A list of labels that should be used by the
+ classifier. If not specified, then the set of labels
+ attested in ``train_toks`` will be used.
+
+ :param options: Extra parameters for the constructor, such as
+ ``unseen_features`` and ``alwayson_features``.
+ """
+ mapping = {} # maps (fname, fval, label) -> fid
+ seen_labels = set() # The set of labels we've encountered
+ count = defaultdict(int) # maps (fname, fval) -> count
+
+ for (tok, label) in train_toks:
+ if labels and label not in labels:
+ raise ValueError('Unexpected label %s' % label)
+ seen_labels.add(label)
+
+ # Record each of the features.
+ for (fname, fval) in tok.items():
+ if(type(fval) in (int, float)): fval = type(fval)
+ # If a count cutoff is given, then only add a joint
+ # feature once the corresponding (fname, fval, label)
+ # tuple exceeds that cutoff.
+ count[fname,fval] += 1
+ if count[fname,fval] >= count_cutoff:
+ if (fname, fval, label) not in mapping:
+ mapping[fname, fval, label] = len(mapping)
+
+ if labels is None: labels = seen_labels
+ return cls(labels, mapping, **options)
+
+
+
+
+######################################################################
+#{ Classifier Trainer: Generalized Iterative Scaling
+######################################################################
+
+def train_maxent_classifier_with_gis(train_toks, trace=3, encoding=None,
+ labels=None, **cutoffs):
+ """
+ Train a new ``ConditionalExponentialClassifier``, using the given
+ training samples, using the Generalized Iterative Scaling
+ algorithm. This ``ConditionalExponentialClassifier`` will encode
+ the model that maximizes entropy from all the models that are
+ empirically consistent with ``train_toks``.
+
+ :see: ``train_maxent_classifier()`` for parameter descriptions.
+ """
+ cutoffs.setdefault('max_iter', 100)
+ cutoffchecker = CutoffChecker(cutoffs)
+
+ # Construct an encoding from the training data.
+ if encoding is None:
+ encoding = GISEncoding.train(train_toks, labels=labels)
+
+ if not hasattr(encoding, 'C'):
+ raise TypeError('The GIS algorithm requires an encoding that '
+ 'defines C (e.g., GISEncoding).')
+
+ # Cinv is the inverse of the sum of each joint feature vector.
+ # This controls the learning rate: higher Cinv (or lower C) gives
+ # faster learning.
+ Cinv = 1.0/encoding.C
+
+ # Count how many times each feature occurs in the training data.
+ empirical_fcount = calculate_empirical_fcount(train_toks, encoding)
+
+ # Check for any features that are not attested in train_toks.
+ unattested = set(numpy.nonzero(empirical_fcount==0)[0])
+
+ # Build the classifier. Start with weight=0 for each attested
+ # feature, and weight=-infinity for each unattested feature.
+ weights = numpy.zeros(len(empirical_fcount), 'd')
+ for fid in unattested: weights[fid] = numpy.NINF
+ classifier = ConditionalExponentialClassifier(encoding, weights)
+
+ # Take the log of the empirical fcount.
+ log_empirical_fcount = numpy.log2(empirical_fcount)
+ del empirical_fcount
+
+ # Old log-likelihood and accuracy; used to check if the change
+ # in log-likelihood or accuracy is sufficient to indicate convergence.
+ ll_old = None
+ acc_old = None
+
+ if trace > 0: print(' ==> Training (%d iterations)' % cutoffs['max_iter'])
+ if trace > 2:
+ print()
+ print(' Iteration Log Likelihood Accuracy')
+ print(' ---------------------------------------')
+
+ # Train the classifier.
+ try:
+ while True:
+ if trace > 2:
+ ll = cutoffchecker.ll or log_likelihood(classifier, train_toks)
+ acc = cutoffchecker.acc or accuracy(classifier, train_toks)
+ iternum = cutoffchecker.iter
+ print(' %9d %14.5f %9.3f' % (iternum, ll, acc))
+
+ # Use the model to estimate the number of times each
+ # feature should occur in the training data.
+ estimated_fcount = calculate_estimated_fcount(
+ classifier, train_toks, encoding)
+
+ # Take the log of estimated fcount (avoid taking log(0).)
+ for fid in unattested: estimated_fcount[fid] += 1
+ log_estimated_fcount = numpy.log2(estimated_fcount)
+ del estimated_fcount
+
+ # Update the classifier weights
+ weights = classifier.weights()
+ weights += (log_empirical_fcount - log_estimated_fcount) * Cinv
+ classifier.set_weights(weights)
+
+ # Check the log-likelihood & accuracy cutoffs.
+ if cutoffchecker.check(classifier, train_toks):
+ break
+
+ except KeyboardInterrupt:
+ print(' Training stopped: keyboard interrupt')
+ except:
+ raise
+
+ if trace > 2:
+ ll = log_likelihood(classifier, train_toks)
+ acc = accuracy(classifier, train_toks)
+ print(' Final %14.5f %9.3f' % (ll, acc))
+
+# Return the classifier.
+ return classifier
+
+def calculate_empirical_fcount(train_toks, encoding):
+ fcount = numpy.zeros(encoding.length(), 'd')
+
+ for tok, label in train_toks:
+ for (index, val) in encoding.encode(tok, label):
+ fcount[index] += val
+
+ return fcount
+
+def calculate_estimated_fcount(classifier, train_toks, encoding):
+ fcount = numpy.zeros(encoding.length(), 'd')
+
+ for tok, label in train_toks:
+ pdist = classifier.prob_classify(tok)
+ for label in pdist.samples():
+ prob = pdist.prob(label)
+ for (fid, fval) in encoding.encode(tok, label):
+ fcount[fid] += prob*fval
+
+ return fcount
+
+
+######################################################################
+#{ Classifier Trainer: Improved Iterative Scaling
+######################################################################
+
+def train_maxent_classifier_with_iis(train_toks, trace=3, encoding=None,
+ labels=None, **cutoffs):
+ """
+ Train a new ``ConditionalExponentialClassifier``, using the given
+ training samples, using the Improved Iterative Scaling algorithm.
+ This ``ConditionalExponentialClassifier`` will encode the model
+ that maximizes entropy from all the models that are empirically
+ consistent with ``train_toks``.
+
+ :see: ``train_maxent_classifier()`` for parameter descriptions.
+ """
+ cutoffs.setdefault('max_iter', 100)
+ cutoffchecker = CutoffChecker(cutoffs)
+
+ # Construct an encoding from the training data.
+ if encoding is None:
+ encoding = BinaryMaxentFeatureEncoding.train(train_toks, labels=labels)
+
+ # Count how many times each feature occurs in the training data.
+ empirical_ffreq = (calculate_empirical_fcount(train_toks, encoding) /
+ len(train_toks))
+
+ # Find the nf map, and related variables nfarray and nfident.
+ # nf is the sum of the features for a given labeled text.
+ # nfmap compresses this sparse set of values to a dense list.
+ # nfarray performs the reverse operation. nfident is
+ # nfarray multiplied by an identity matrix.
+ nfmap = calculate_nfmap(train_toks, encoding)
+ nfarray = numpy.array(sorted(nfmap, key=nfmap.__getitem__), 'd')
+ nftranspose = numpy.reshape(nfarray, (len(nfarray), 1))
+
+ # Check for any features that are not attested in train_toks.
+ unattested = set(numpy.nonzero(empirical_ffreq==0)[0])
+
+ # Build the classifier. Start with weight=0 for each attested
+ # feature, and weight=-infinity for each unattested feature.
+ weights = numpy.zeros(len(empirical_ffreq), 'd')
+ for fid in unattested: weights[fid] = numpy.NINF
+ classifier = ConditionalExponentialClassifier(encoding, weights)
+
+ if trace > 0: print(' ==> Training (%d iterations)' % cutoffs['max_iter'])
+ if trace > 2:
+ print()
+ print(' Iteration Log Likelihood Accuracy')
+ print(' ---------------------------------------')
+
+ # Old log-likelihood and accuracy; used to check if the change
+ # in log-likelihood or accuracy is sufficient to indicate convergence.
+ ll_old = None
+ acc_old = None
+
+ # Train the classifier.
+ try:
+ while True:
+ if trace > 2:
+ ll = cutoffchecker.ll or log_likelihood(classifier, train_toks)
+ acc = cutoffchecker.acc or accuracy(classifier, train_toks)
+ iternum = cutoffchecker.iter
+ print(' %9d %14.5f %9.3f' % (iternum, ll, acc))
+
+ # Calculate the deltas for this iteration, using Newton's method.
+ deltas = calculate_deltas(
+ train_toks, classifier, unattested, empirical_ffreq,
+ nfmap, nfarray, nftranspose, encoding)
+
+ # Use the deltas to update our weights.
+ weights = classifier.weights()
+ weights += deltas
+ classifier.set_weights(weights)
+
+ # Check the log-likelihood & accuracy cutoffs.
+ if cutoffchecker.check(classifier, train_toks):
+ break
+
+ except KeyboardInterrupt:
+ print(' Training stopped: keyboard interrupt')
+ except:
+ raise
+
+
+ if trace > 2:
+ ll = log_likelihood(classifier, train_toks)
+ acc = accuracy(classifier, train_toks)
+ print(' Final %14.5f %9.3f' % (ll, acc))
+
+ # Return the classifier.
+ return classifier
+
+def calculate_nfmap(train_toks, encoding):
+ """
+ Construct a map that can be used to compress ``nf`` (which is
+ typically sparse).
+
+ *nf(feature_vector)* is the sum of the feature values for
+ *feature_vector*.
+
+ This represents the number of features that are active for a
+ given labeled text. This method finds all values of *nf(t)*
+ that are attested for at least one token in the given list of
+ training tokens; and constructs a dictionary mapping these
+ attested values to a continuous range *0...N*. For example,
+ if the only values of *nf()* that were attested were 3, 5, and
+ 7, then ``_nfmap`` might return the dictionary ``{3:0, 5:1, 7:2}``.
+
+ :return: A map that can be used to compress ``nf`` to a dense
+ vector.
+ :rtype: dict(int -> int)
+ """
+ # Map from nf to indices. This allows us to use smaller arrays.
+ nfset = set()
+ for tok, _ in train_toks:
+ for label in encoding.labels():
+ nfset.add(sum(val for (id,val) in encoding.encode(tok,label)))
+ return dict((nf, i) for (i, nf) in enumerate(nfset))
+
+def calculate_deltas(train_toks, classifier, unattested, ffreq_empirical,
+ nfmap, nfarray, nftranspose, encoding):
+ """
+ Calculate the update values for the classifier weights for
+ this iteration of IIS. These update weights are the value of
+ ``delta`` that solves the equation::
+
+ ffreq_empirical[i]
+ =
+ SUM[fs,l] (classifier.prob_classify(fs).prob(l) *
+ feature_vector(fs,l)[i] *
+ exp(delta[i] * nf(feature_vector(fs,l))))
+
+ Where:
+ - *(fs,l)* is a (featureset, label) tuple from ``train_toks``
+ - *feature_vector(fs,l)* = ``encoding.encode(fs,l)``
+ - *nf(vector)* = ``sum([val for (id,val) in vector])``
+
+ This method uses Newton's method to solve this equation for
+ *delta[i]*. In particular, it starts with a guess of
+ ``delta[i]`` = 1; and iteratively updates ``delta`` with:
+
+ | delta[i] -= (ffreq_empirical[i] - sum1[i])/(-sum2[i])
+
+ until convergence, where *sum1* and *sum2* are defined as:
+
+ | sum1[i](delta) = SUM[fs,l] f[i](fs,l,delta)
+ | sum2[i](delta) = SUM[fs,l] (f[i](fs,l,delta).nf(feature_vector(fs,l)))
+ | f[i](fs,l,delta) = (classifier.prob_classify(fs).prob(l) .
+ | feature_vector(fs,l)[i] .
+ | exp(delta[i] . nf(feature_vector(fs,l))))
+
+ Note that *sum1* and *sum2* depend on ``delta``; so they need
+ to be re-computed each iteration.
+
+ The variables ``nfmap``, ``nfarray``, and ``nftranspose`` are
+ used to generate a dense encoding for *nf(ltext)*. This
+ allows ``_deltas`` to calculate *sum1* and *sum2* using
+ matrices, which yields a significant performance improvement.
+
+ :param train_toks: The set of training tokens.
+ :type train_toks: list(tuple(dict, str))
+ :param classifier: The current classifier.
+ :type classifier: ClassifierI
+ :param ffreq_empirical: An array containing the empirical
+ frequency for each feature. The *i*\ th element of this
+ array is the empirical frequency for feature *i*.
+ :type ffreq_empirical: sequence of float
+ :param unattested: An array that is 1 for features that are
+ not attested in the training data; and 0 for features that
+ are attested. In other words, ``unattested[i]==0`` iff
+ ``ffreq_empirical[i]==0``.
+ :type unattested: sequence of int
+ :param nfmap: A map that can be used to compress ``nf`` to a dense
+ vector.
+ :type nfmap: dict(int -> int)
+ :param nfarray: An array that can be used to uncompress ``nf``
+ from a dense vector.
+ :type nfarray: array(float)
+ :param nftranspose: The transpose of ``nfarray``
+ :type nftranspose: array(float)
+ """
+ # These parameters control when we decide that we've
+ # converged. It probably should be possible to set these
+ # manually, via keyword arguments to train.
+ NEWTON_CONVERGE = 1e-12
+ MAX_NEWTON = 300
+
+ deltas = numpy.ones(encoding.length(), 'd')
+
+ # Precompute the A matrix:
+ # A[nf][id] = sum ( p(fs) * p(label|fs) * f(fs,label) )
+ # over all label,fs s.t. num_features[label,fs]=nf
+ A = numpy.zeros((len(nfmap), encoding.length()), 'd')
+
+ for tok, label in train_toks:
+ dist = classifier.prob_classify(tok)
+
+ for label in encoding.labels():
+ # Generate the feature vector
+ feature_vector = encoding.encode(tok,label)
+ # Find the number of active features
+ nf = sum(val for (id, val) in feature_vector)
+ # Update the A matrix
+ for (id, val) in feature_vector:
+ A[nfmap[nf], id] += dist.prob(label) * val
+ A /= len(train_toks)
+
+ # Iteratively solve for delta. Use the following variables:
+ # - nf_delta[x][y] = nfarray[x] * delta[y]
+ # - exp_nf_delta[x][y] = exp(nf[x] * delta[y])
+ # - nf_exp_nf_delta[x][y] = nf[x] * exp(nf[x] * delta[y])
+ # - sum1[i][nf] = sum p(fs)p(label|fs)f[i](label,fs)
+ # exp(delta[i]nf)
+ # - sum2[i][nf] = sum p(fs)p(label|fs)f[i](label,fs)
+ # nf exp(delta[i]nf)
+ for rangenum in range(MAX_NEWTON):
+ nf_delta = numpy.outer(nfarray, deltas)
+ exp_nf_delta = 2 ** nf_delta
+ nf_exp_nf_delta = nftranspose * exp_nf_delta
+ sum1 = numpy.sum(exp_nf_delta * A, axis=0)
+ sum2 = numpy.sum(nf_exp_nf_delta * A, axis=0)
+
+ # Avoid division by zero.
+ for fid in unattested: sum2[fid] += 1
+
+ # Update the deltas.
+ deltas -= (ffreq_empirical - sum1) / -sum2
+
+ # We can stop once we converge.
+ n_error = (numpy.sum(abs((ffreq_empirical-sum1)))/
+ numpy.sum(abs(deltas)))
+ if n_error < NEWTON_CONVERGE:
+ return deltas
+
+ return deltas
+
+######################################################################
+#{ Classifier Trainer: megam
+######################################################################
+
+# [xx] possible extension: add support for using implicit file format;
+# this would need to put requirements on what encoding is used. But
+# we may need this for other maxent classifier trainers that require
+# implicit formats anyway.
+def train_maxent_classifier_with_megam(train_toks, trace=3, encoding=None,
+ labels=None, gaussian_prior_sigma=0,
+ **kwargs):
+ """
+ Train a new ``ConditionalExponentialClassifier``, using the given
+ training samples, using the external ``megam`` library. This
+ ``ConditionalExponentialClassifier`` will encode the model that
+ maximizes entropy from all the models that are empirically
+ consistent with ``train_toks``.
+
+ :see: ``train_maxent_classifier()`` for parameter descriptions.
+ :see: ``nltk.classify.megam``
+ """
+
+ explicit = True
+ bernoulli = True
+ if 'explicit' in kwargs: explicit = kwargs['explicit']
+ if 'bernoulli' in kwargs: bernoulli = kwargs['bernoulli']
+
+ # Construct an encoding from the training data.
+ if encoding is None:
+ # Count cutoff can also be controlled by megam with the -minfc
+ # option. Not sure where the best place for it is.
+ count_cutoff = kwargs.get('count_cutoff', 0)
+ encoding = BinaryMaxentFeatureEncoding.train(train_toks, count_cutoff,
+ labels=labels,
+ alwayson_features=True)
+ elif labels is not None:
+ raise ValueError('Specify encoding or labels, not both')
+
+ # Write a training file for megam.
+ try:
+ fd, trainfile_name = tempfile.mkstemp(prefix='nltk-')
+ with open(trainfile_name, 'w') as trainfile:
+ write_megam_file(train_toks, encoding, trainfile,
+ explicit=explicit, bernoulli=bernoulli)
+ os.close(fd)
+ except (OSError, IOError, ValueError) as e:
+ raise ValueError('Error while creating megam training file: %s' % e)
+
+ # Run megam on the training file.
+ options = []
+ options += ['-nobias', '-repeat', '10']
+ if explicit:
+ options += ['-explicit']
+ if not bernoulli:
+ options += ['-fvals']
+ if gaussian_prior_sigma:
+ # Lambda is just the precision of the Gaussian prior, i.e. it's the
+ # inverse variance, so the parameter conversion is 1.0/sigma**2.
+ # See http://www.umiacs.umd.edu/~hal/docs/daume04cg-bfgs.pdf.
+ inv_variance = 1.0 / gaussian_prior_sigma**2
+ else:
+ inv_variance = 0
+ options += ['-lambda', '%.2f' % inv_variance, '-tune']
+ if trace < 3:
+ options += ['-quiet']
+ if 'max_iter' in kwargs:
+ options += ['-maxi', '%s' % kwargs['max_iter']]
+ if 'll_delta' in kwargs:
+ # [xx] this is actually a perplexity delta, not a log
+ # likelihood delta
+ options += ['-dpp', '%s' % abs(kwargs['ll_delta'])]
+ if hasattr(encoding, 'cost'):
+ options += ['-multilabel'] # each possible la
+ options += ['multiclass', trainfile_name]
+ stdout = call_megam(options)
+ # print './megam_i686.opt ', ' '.join(options)
+ # Delete the training file
+ try: os.remove(trainfile_name)
+ except (OSError, IOError) as e:
+ print('Warning: unable to delete %s: %s' % (trainfile_name, e))
+
+ # Parse the generated weight vector.
+ weights = parse_megam_weights(stdout, encoding.length(), explicit)
+
+ # Convert from base-e to base-2 weights.
+ weights *= numpy.log2(numpy.e)
+
+ # Build the classifier
+ return MaxentClassifier(encoding, weights)
+
+######################################################################
+#{ Classifier Trainer: tadm
+######################################################################
+
+class TadmMaxentClassifier(MaxentClassifier):
+ @classmethod
+ def train(cls, train_toks, **kwargs):
+ algorithm = kwargs.get('algorithm', 'tao_lmvm')
+ trace = kwargs.get('trace', 3)
+ encoding = kwargs.get('encoding', None)
+ labels = kwargs.get('labels', None)
+ sigma = kwargs.get('gaussian_prior_sigma', 0)
+ count_cutoff = kwargs.get('count_cutoff', 0)
+ max_iter = kwargs.get('max_iter')
+ ll_delta = kwargs.get('min_lldelta')
+
+ # Construct an encoding from the training data.
+ if not encoding:
+ encoding = TadmEventMaxentFeatureEncoding.train(train_toks,
+ count_cutoff,
+ labels=labels)
+
+ trainfile_fd, trainfile_name = \
+ tempfile.mkstemp(prefix='nltk-tadm-events-', suffix='.gz')
+ weightfile_fd, weightfile_name = \
+ tempfile.mkstemp(prefix='nltk-tadm-weights-')
+
+ trainfile = gzip_open_unicode(trainfile_name, 'w')
+ write_tadm_file(train_toks, encoding, trainfile)
+ trainfile.close()
+
+ options = []
+ options.extend(['-monitor'])
+ options.extend(['-method', algorithm])
+ if sigma:
+ options.extend(['-l2', '%.6f' % sigma**2])
+ if max_iter:
+ options.extend(['-max_it', '%d' % max_iter])
+ if ll_delta:
+ options.extend(['-fatol', '%.6f' % abs(ll_delta)])
+ options.extend(['-events_in', trainfile_name])
+ options.extend(['-params_out', weightfile_name])
+ if trace < 3:
+ options.extend(['2>&1'])
+ else:
+ options.extend(['-summary'])
+
+ call_tadm(options)
+
+ with open(weightfile_name, 'r') as weightfile:
+ weights = parse_tadm_weights(weightfile)
+
+ os.remove(trainfile_name)
+ os.remove(weightfile_name)
+
+ # Convert from base-e to base-2 weights.
+ weights *= numpy.log2(numpy.e)
+
+ # Build the classifier
+ return cls(encoding, weights)
+
+######################################################################
+#{ Demo
+######################################################################
+def demo():
+ from nltk.classify.util import names_demo
+ classifier = names_demo(MaxentClassifier.train)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/classify/megam.py b/nltk/classify/megam.py
new file mode 100644
index 0000000..b5cf796
--- /dev/null
+++ b/nltk/classify/megam.py
@@ -0,0 +1,179 @@
+# Natural Language Toolkit: Interface to Megam Classifier
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A set of functions used to interface with the external megam_ maxent
+optimization package. Before megam can be used, you should tell NLTK where it
+can find the megam binary, using the ``config_megam()`` function. Typical
+usage:
+
+ >>> from nltk.classify import megam
+ >>> megam.config_megam() # pass path to megam if not found in PATH # doctest: +SKIP
+ [Found megam: ...]
+
+Use with MaxentClassifier. Example below, see MaxentClassifier documentation
+for details.
+
+ nltk.classify.MaxentClassifier.train(corpus, 'megam')
+
+.. _megam: http://www.umiacs.umd.edu/~hal/megam/index.html
+"""
+from __future__ import print_function
+
+import os
+import os.path
+import subprocess
+
+from nltk import compat
+from nltk.internals import find_binary
+try:
+ import numpy
+except ImportError:
+ numpy = None
+
+######################################################################
+#{ Configuration
+######################################################################
+
+_megam_bin = None
+def config_megam(bin=None):
+ """
+ Configure NLTK's interface to the ``megam`` maxent optimization
+ package.
+
+ :param bin: The full path to the ``megam`` binary. If not specified,
+ then nltk will search the system for a ``megam`` binary; and if
+ one is not found, it will raise a ``LookupError`` exception.
+ :type bin: str
+ """
+ global _megam_bin
+ _megam_bin = find_binary(
+ 'megam', bin,
+ env_vars=['MEGAM', 'MEGAMHOME'],
+ binary_names=['megam.opt', 'megam', 'megam_686', 'megam_i686.opt'],
+ url='http://www.umiacs.umd.edu/~hal/megam/index.html')
+
+######################################################################
+#{ Megam Interface Functions
+######################################################################
+
+def write_megam_file(train_toks, encoding, stream,
+ bernoulli=True, explicit=True):
+ """
+ Generate an input file for ``megam`` based on the given corpus of
+ classified tokens.
+
+ :type train_toks: list(tuple(dict, str))
+ :param train_toks: Training data, represented as a list of
+ pairs, the first member of which is a feature dictionary,
+ and the second of which is a classification label.
+
+ :type encoding: MaxentFeatureEncodingI
+ :param encoding: A feature encoding, used to convert featuresets
+ into feature vectors. May optionally implement a cost() method
+ in order to assign different costs to different class predictions.
+
+ :type stream: stream
+ :param stream: The stream to which the megam input file should be
+ written.
+
+ :param bernoulli: If true, then use the 'bernoulli' format. I.e.,
+ all joint features have binary values, and are listed iff they
+ are true. Otherwise, list feature values explicitly. If
+ ``bernoulli=False``, then you must call ``megam`` with the
+ ``-fvals`` option.
+
+ :param explicit: If true, then use the 'explicit' format. I.e.,
+ list the features that would fire for any of the possible
+ labels, for each token. If ``explicit=True``, then you must
+ call ``megam`` with the ``-explicit`` option.
+ """
+ # Look up the set of labels.
+ labels = encoding.labels()
+ labelnum = dict((label, i) for (i, label) in enumerate(labels))
+
+ # Write the file, which contains one line per instance.
+ for featureset, label in train_toks:
+ # First, the instance number (or, in the weighted multiclass case, the cost of each label).
+ if hasattr(encoding,'cost'):
+ stream.write(':'.join(str(encoding.cost(featureset, label, l)) for l in labels))
+ else:
+ stream.write('%d' % labelnum[label])
+
+ # For implicit file formats, just list the features that fire
+ # for this instance's actual label.
+ if not explicit:
+ _write_megam_features(encoding.encode(featureset, label),
+ stream, bernoulli)
+
+ # For explicit formats, list the features that would fire for
+ # any of the possible labels.
+ else:
+ for l in labels:
+ stream.write(' #')
+ _write_megam_features(encoding.encode(featureset, l),
+ stream, bernoulli)
+
+ # End of the instance.
+ stream.write('\n')
+
+def parse_megam_weights(s, features_count, explicit=True):
+ """
+ Given the stdout output generated by ``megam`` when training a
+ model, return a ``numpy`` array containing the corresponding weight
+ vector. This function does not currently handle bias features.
+ """
+ if numpy is None:
+ raise ValueError('This function requires that numpy be installed')
+ assert explicit, 'non-explicit not supported yet'
+ lines = s.strip().split('\n')
+ weights = numpy.zeros(features_count, 'd')
+ for line in lines:
+ if line.strip():
+ fid, weight = line.split()
+ weights[int(fid)] = float(weight)
+ return weights
+
+def _write_megam_features(vector, stream, bernoulli):
+ if not vector:
+ raise ValueError('MEGAM classifier requires the use of an '
+ 'always-on feature.')
+ for (fid, fval) in vector:
+ if bernoulli:
+ if fval == 1:
+ stream.write(' %s' % fid)
+ elif fval != 0:
+ raise ValueError('If bernoulli=True, then all'
+ 'features must be binary.')
+ else:
+ stream.write(' %s %s' % (fid, fval))
+
+def call_megam(args):
+ """
+ Call the ``megam`` binary with the given arguments.
+ """
+ if isinstance(args, compat.string_types):
+ raise TypeError('args should be a list of strings')
+ if _megam_bin is None:
+ config_megam()
+
+ # Call megam via a subprocess
+ cmd = [_megam_bin] + args
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
+ (stdout, stderr) = p.communicate()
+
+ # Check the return code.
+ if p.returncode != 0:
+ print()
+ print(stderr)
+ raise OSError('megam command failed!')
+
+ if isinstance(stdout, compat.string_types):
+ return stdout
+ else:
+ return stdout.decode('utf-8')
+
diff --git a/nltk/classify/naivebayes.py b/nltk/classify/naivebayes.py
new file mode 100644
index 0000000..dafb449
--- /dev/null
+++ b/nltk/classify/naivebayes.py
@@ -0,0 +1,240 @@
+# Natural Language Toolkit: Naive Bayes Classifiers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A classifier based on the Naive Bayes algorithm. In order to find the
+probability for a label, this algorithm first uses the Bayes rule to
+express P(label|features) in terms of P(label) and P(features|label):
+
+| P(label) * P(features|label)
+| P(label|features) = ------------------------------
+| P(features)
+
+The algorithm then makes the 'naive' assumption that all features are
+independent, given the label:
+
+| P(label) * P(f1|label) * ... * P(fn|label)
+| P(label|features) = --------------------------------------------
+| P(features)
+
+Rather than computing P(featues) explicitly, the algorithm just
+calculates the denominator for each label, and normalizes them so they
+sum to one:
+
+| P(label) * P(f1|label) * ... * P(fn|label)
+| P(label|features) = --------------------------------------------
+| SUM[l]( P(l) * P(f1|l) * ... * P(fn|l) )
+"""
+from __future__ import print_function, unicode_literals
+
+from collections import defaultdict
+
+from nltk.probability import FreqDist, DictionaryProbDist, ELEProbDist, sum_logs
+from nltk.classify.api import ClassifierI
+
+##//////////////////////////////////////////////////////
+## Naive Bayes Classifier
+##//////////////////////////////////////////////////////
+
+class NaiveBayesClassifier(ClassifierI):
+ """
+ A Naive Bayes classifier. Naive Bayes classifiers are
+ paramaterized by two probability distributions:
+
+ - P(label) gives the probability that an input will receive each
+ label, given no information about the input's features.
+
+ - P(fname=fval|label) gives the probability that a given feature
+ (fname) will receive a given value (fval), given that the
+ label (label).
+
+ If the classifier encounters an input with a feature that has
+ never been seen with any label, then rather than assigning a
+ probability of 0 to all labels, it will ignore that feature.
+
+ The feature value 'None' is reserved for unseen feature values;
+ you generally should not use 'None' as a feature value for one of
+ your own features.
+ """
+ def __init__(self, label_probdist, feature_probdist):
+ """
+ :param label_probdist: P(label), the probability distribution
+ over labels. It is expressed as a ``ProbDistI`` whose
+ samples are labels. I.e., P(label) =
+ ``label_probdist.prob(label)``.
+
+ :param feature_probdist: P(fname=fval|label), the probability
+ distribution for feature values, given labels. It is
+ expressed as a dictionary whose keys are ``(label, fname)``
+ pairs and whose values are ``ProbDistI`` objects over feature
+ values. I.e., P(fname=fval|label) =
+ ``feature_probdist[label,fname].prob(fval)``. If a given
+ ``(label,fname)`` is not a key in ``feature_probdist``, then
+ it is assumed that the corresponding P(fname=fval|label)
+ is 0 for all values of ``fval``.
+ """
+ self._label_probdist = label_probdist
+ self._feature_probdist = feature_probdist
+ self._labels = list(label_probdist.samples())
+
+ def labels(self):
+ return self._labels
+
+ def classify(self, featureset):
+ return self.prob_classify(featureset).max()
+
+ def prob_classify(self, featureset):
+ # Discard any feature names that we've never seen before.
+ # Otherwise, we'll just assign a probability of 0 to
+ # everything.
+ featureset = featureset.copy()
+ for fname in list(featureset.keys()):
+ for label in self._labels:
+ if (label, fname) in self._feature_probdist:
+ break
+ else:
+ #print 'Ignoring unseen feature %s' % fname
+ del featureset[fname]
+
+ # Find the log probabilty of each label, given the features.
+ # Start with the log probability of the label itself.
+ logprob = {}
+ for label in self._labels:
+ logprob[label] = self._label_probdist.logprob(label)
+
+ # Then add in the log probability of features given labels.
+ for label in self._labels:
+ for (fname, fval) in featureset.items():
+ if (label, fname) in self._feature_probdist:
+ feature_probs = self._feature_probdist[label,fname]
+ logprob[label] += feature_probs.logprob(fval)
+ else:
+ # nb: This case will never come up if the
+ # classifier was created by
+ # NaiveBayesClassifier.train().
+ logprob[label] += sum_logs([]) # = -INF.
+
+ return DictionaryProbDist(logprob, normalize=True, log=True)
+
+ def show_most_informative_features(self, n=10):
+ # Determine the most relevant features, and display them.
+ cpdist = self._feature_probdist
+ print('Most Informative Features')
+
+ for (fname, fval) in self.most_informative_features(n):
+ def labelprob(l):
+ return cpdist[l,fname].prob(fval)
+
+ labels = sorted([l for l in self._labels
+ if fval in cpdist[l,fname].samples()],
+ key=labelprob)
+ if len(labels) == 1: continue
+ l0 = labels[0]
+ l1 = labels[-1]
+ if cpdist[l0,fname].prob(fval) == 0:
+ ratio = 'INF'
+ else:
+ ratio = '%8.1f' % (cpdist[l1,fname].prob(fval) /
+ cpdist[l0,fname].prob(fval))
+ print(('%24s = %-14r %6s : %-6s = %s : 1.0' %
+ (fname, fval, ("%s" % l1)[:6], ("%s" % l0)[:6], ratio)))
+
+ def most_informative_features(self, n=100):
+ """
+ Return a list of the 'most informative' features used by this
+ classifier. For the purpose of this function, the
+ informativeness of a feature ``(fname,fval)`` is equal to the
+ highest value of P(fname=fval|label), for any label, divided by
+ the lowest value of P(fname=fval|label), for any label:
+
+ | max[ P(fname=fval|label1) / P(fname=fval|label2) ]
+ """
+ # The set of (fname, fval) pairs used by this classifier.
+ features = set()
+ # The max & min probability associated w/ each (fname, fval)
+ # pair. Maps (fname,fval) -> float.
+ maxprob = defaultdict(lambda: 0.0)
+ minprob = defaultdict(lambda: 1.0)
+
+ for (label, fname), probdist in self._feature_probdist.items():
+ for fval in probdist.samples():
+ feature = (fname, fval)
+ features.add( feature )
+ p = probdist.prob(fval)
+ maxprob[feature] = max(p, maxprob[feature])
+ minprob[feature] = min(p, minprob[feature])
+ if minprob[feature] == 0:
+ features.discard(feature)
+
+ # Convert features to a list, & sort it by how informative
+ # features are.
+ features = sorted(features,
+ key=lambda feature_: minprob[feature_]/maxprob[feature_])
+ return features[:n]
+
+ @staticmethod
+ def train(labeled_featuresets, estimator=ELEProbDist):
+ """
+ :param labeled_featuresets: A list of classified featuresets,
+ i.e., a list of tuples ``(featureset, label)``.
+ """
+ label_freqdist = FreqDist()
+ feature_freqdist = defaultdict(FreqDist)
+ feature_values = defaultdict(set)
+ fnames = set()
+
+ # Count up how many times each feature value occurred, given
+ # the label and featurename.
+ for featureset, label in labeled_featuresets:
+ label_freqdist[label] += 1
+ for fname, fval in featureset.items():
+ # Increment freq(fval|label, fname)
+ feature_freqdist[label, fname][fval] += 1
+ # Record that fname can take the value fval.
+ feature_values[fname].add(fval)
+ # Keep a list of all feature names.
+ fnames.add(fname)
+
+ # If a feature didn't have a value given for an instance, then
+ # we assume that it gets the implicit value 'None.' This loop
+ # counts up the number of 'missing' feature values for each
+ # (label,fname) pair, and increments the count of the fval
+ # 'None' by that amount.
+ for label in label_freqdist:
+ num_samples = label_freqdist[label]
+ for fname in fnames:
+ count = feature_freqdist[label, fname].N()
+ # Only add a None key when necessary, i.e. if there are
+ # any samples with feature 'fname' missing.
+ if num_samples - count > 0:
+ feature_freqdist[label, fname][None] += num_samples - count
+ feature_values[fname].add(None)
+
+ # Create the P(label) distribution
+ label_probdist = estimator(label_freqdist)
+
+ # Create the P(fval|label, fname) distribution
+ feature_probdist = {}
+ for ((label, fname), freqdist) in feature_freqdist.items():
+ probdist = estimator(freqdist, bins=len(feature_values[fname]))
+ feature_probdist[label,fname] = probdist
+
+ return NaiveBayesClassifier(label_probdist, feature_probdist)
+
+##//////////////////////////////////////////////////////
+## Demo
+##//////////////////////////////////////////////////////
+
+def demo():
+ from nltk.classify.util import names_demo
+ classifier = names_demo(NaiveBayesClassifier.train)
+ classifier.show_most_informative_features()
+
+if __name__ == '__main__':
+ demo()
+
+
diff --git a/nltk/classify/positivenaivebayes.py b/nltk/classify/positivenaivebayes.py
new file mode 100644
index 0000000..26f727e
--- /dev/null
+++ b/nltk/classify/positivenaivebayes.py
@@ -0,0 +1,177 @@
+# Natural Language Toolkit: Positive Naive Bayes Classifier
+#
+# Copyright (C) 2012 NLTK Project
+# Author: Alessandro Presta <alessandro.presta at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A variant of the Naive Bayes Classifier that performs binary classification with
+partially-labeled training sets. In other words, assume we want to build a classifier
+that assigns each example to one of two complementary classes (e.g., male names and
+female names).
+If we have a training set with labeled examples for both classes, we can use a
+standard Naive Bayes Classifier. However, consider the case when we only have labeled
+examples for one of the classes, and other, unlabeled, examples.
+Then, assuming a prior distribution on the two labels, we can use the unlabeled set
+to estimate the frequencies of the various features.
+
+Let the two possible labels be 1 and 0, and let's say we only have examples labeled 1
+and unlabeled examples. We are also given an estimate of P(1).
+
+We compute P(feature|1) exactly as in the standard case.
+
+To compute P(feature|0), we first estimate P(feature) from the unlabeled set (we are
+assuming that the unlabeled examples are drawn according to the given prior distribution)
+and then express the conditional probability as:
+
+| P(feature) - P(feature|1) * P(1)
+| P(feature|0) = ----------------------------------
+| P(0)
+
+Example:
+
+ >>> from nltk.classify import PositiveNaiveBayesClassifier
+
+Some sentences about sports:
+
+ >>> sports_sentences = [ 'The team dominated the game',
+ ... 'They lost the ball',
+ ... 'The game was intense',
+ ... 'The goalkeeper catched the ball',
+ ... 'The other team controlled the ball' ]
+
+Mixed topics, including sports:
+
+ >>> various_sentences = [ 'The President did not comment',
+ ... 'I lost the keys',
+ ... 'The team won the game',
+ ... 'Sara has two kids',
+ ... 'The ball went off the court',
+ ... 'They had the ball for the whole game',
+ ... 'The show is over' ]
+
+The features of a sentence are simply the words it contains:
+
+ >>> def features(sentence):
+ ... words = sentence.lower().split()
+ ... return dict(('contains(%s)' % w, True) for w in words)
+
+We use the sports sentences as positive examples, the mixed ones ad unlabeled examples:
+
+ >>> positive_featuresets = list(map(features, sports_sentences))
+ >>> unlabeled_featuresets = list(map(features, various_sentences))
+ >>> classifier = PositiveNaiveBayesClassifier.train(positive_featuresets,
+ ... unlabeled_featuresets)
+
+Is the following sentence about sports?
+
+ >>> classifier.classify(features('The cat is on the table'))
+ False
+
+What about this one?
+
+ >>> classifier.classify(features('My team lost the game'))
+ True
+"""
+
+from collections import defaultdict
+
+from nltk.probability import FreqDist, DictionaryProbDist, ELEProbDist
+
+from nltk.classify.naivebayes import NaiveBayesClassifier
+
+##//////////////////////////////////////////////////////
+## Positive Naive Bayes Classifier
+##//////////////////////////////////////////////////////
+
+class PositiveNaiveBayesClassifier(NaiveBayesClassifier):
+ @staticmethod
+ def train(positive_featuresets, unlabeled_featuresets, positive_prob_prior=0.5,
+ estimator=ELEProbDist):
+ """
+ :param positive_featuresets: A list of featuresets that are known as positive
+ examples (i.e., their label is ``True``).
+
+ :param unlabeled_featuresets: A list of featuresets whose label is unknown.
+
+ :param positive_prob_prior: A prior estimate of the probability of the label
+ ``True`` (default 0.5).
+ """
+ positive_feature_freqdist = defaultdict(FreqDist)
+ unlabeled_feature_freqdist = defaultdict(FreqDist)
+ feature_values = defaultdict(set)
+ fnames = set()
+
+ # Count up how many times each feature value occurred in positive examples.
+ for featureset in positive_featuresets:
+ for fname, fval in featureset.items():
+ positive_feature_freqdist[fname][fval] += 1
+ feature_values[fname].add(fval)
+ fnames.add(fname)
+
+ # Count up how many times each feature value occurred in unlabeled examples.
+ for featureset in unlabeled_featuresets:
+ for fname, fval in featureset.items():
+ unlabeled_feature_freqdist[fname][fval] += 1
+ feature_values[fname].add(fval)
+ fnames.add(fname)
+
+ # If a feature didn't have a value given for an instance, then we assume that
+ # it gets the implicit value 'None'.
+ num_positive_examples = len(positive_featuresets)
+ for fname in fnames:
+ count = positive_feature_freqdist[fname].N()
+ positive_feature_freqdist[fname][None] += num_positive_examples - count
+ feature_values[fname].add(None)
+
+ num_unlabeled_examples = len(unlabeled_featuresets)
+ for fname in fnames:
+ count = unlabeled_feature_freqdist[fname].N()
+ unlabeled_feature_freqdist[fname][None] += num_unlabeled_examples - count
+ feature_values[fname].add(None)
+
+ negative_prob_prior = 1.0 - positive_prob_prior
+
+ # Create the P(label) distribution.
+ label_probdist = DictionaryProbDist({True: positive_prob_prior,
+ False: negative_prob_prior})
+
+ # Create the P(fval|label, fname) distribution.
+ feature_probdist = {}
+ for fname, freqdist in positive_feature_freqdist.items():
+ probdist = estimator(freqdist, bins=len(feature_values[fname]))
+ feature_probdist[True, fname] = probdist
+
+ for fname, freqdist in unlabeled_feature_freqdist.items():
+ global_probdist = estimator(freqdist, bins=len(feature_values[fname]))
+ negative_feature_probs = {}
+ for fval in feature_values[fname]:
+ prob = (global_probdist.prob(fval)
+ - positive_prob_prior *
+ feature_probdist[True, fname].prob(fval)) \
+ / negative_prob_prior
+ # TODO: We need to add some kind of smoothing here, instead of
+ # setting negative probabilities to zero and normalizing.
+ negative_feature_probs[fval] = max(prob, 0.0)
+ feature_probdist[False, fname] = DictionaryProbDist(negative_feature_probs,
+ normalize=True)
+
+ return PositiveNaiveBayesClassifier(label_probdist, feature_probdist)
+
+##//////////////////////////////////////////////////////
+## Demo
+##//////////////////////////////////////////////////////
+
+def demo():
+ from nltk.classify.util import partial_names_demo
+ classifier = partial_names_demo(PositiveNaiveBayesClassifier.train)
+ classifier.show_most_informative_features()
+
+##//////////////////////////////////////////////////////
+## Test
+##//////////////////////////////////////////////////////
+
+if __name__ == '__main__':
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/classify/rte_classify.py b/nltk/classify/rte_classify.py
new file mode 100644
index 0000000..09f69cb
--- /dev/null
+++ b/nltk/classify/rte_classify.py
@@ -0,0 +1,183 @@
+# Natural Language Toolkit: RTE Classifier
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Simple classifier for RTE corpus.
+
+It calculates the overlap in words and named entities between text and
+hypothesis, and also whether there are words / named entities in the
+hypothesis which fail to occur in the text, since this is an indicator that
+the hypothesis is more informative than (i.e not entailed by) the text.
+
+TO DO: better Named Entity classification
+TO DO: add lemmatization
+"""
+from __future__ import print_function
+
+import nltk
+from nltk.classify.util import accuracy
+
+def ne(token):
+ """
+ This just assumes that words in all caps or titles are
+ named entities.
+
+ :type token: str
+ """
+ if token.istitle() or token.isupper():
+ return True
+ return False
+
+def lemmatize(word):
+ """
+ Use morphy from WordNet to find the base form of verbs.
+ """
+ lemma = nltk.corpus.wordnet.morphy(word, pos='verb')
+ if lemma is not None:
+ return lemma
+ return word
+
+class RTEFeatureExtractor(object):
+ """
+ This builds a bag of words for both the text and the hypothesis after
+ throwing away some stopwords, then calculates overlap and difference.
+ """
+ def __init__(self, rtepair, stop=True, lemmatize=False):
+ """
+ :param rtepair: a ``RTEPair`` from which features should be extracted
+ :param stop: if ``True``, stopwords are thrown away.
+ :type stop: bool
+ """
+ self.stop = stop
+ self.stopwords = set(['a', 'the', 'it', 'they', 'of', 'in', 'to',
+ 'have', 'is', 'are', 'were', 'and', 'very', '.',','])
+
+ self.negwords = set(['no', 'not', 'never', 'failed' 'rejected', 'denied'])
+ # Try to tokenize so that abbreviations like U.S.and monetary amounts
+ # like "$23.00" are kept as tokens.
+ from nltk.tokenize import RegexpTokenizer
+ tokenizer = RegexpTokenizer('([A-Z]\.)+|\w+|\$[\d\.]+')
+
+ #Get the set of word types for text and hypothesis
+ self.text_tokens = tokenizer.tokenize(rtepair.text)
+ self.hyp_tokens = tokenizer.tokenize(rtepair.hyp)
+ self.text_words = set(self.text_tokens)
+ self.hyp_words = set(self.hyp_tokens)
+
+ if lemmatize:
+ self.text_words = set(lemmatize(token) for token in self.text_tokens)
+ self.hyp_words = set(lemmatize(token) for token in self.hyp_tokens)
+
+ if self.stop:
+ self.text_words = self.text_words - self.stopwords
+ self.hyp_words = self.hyp_words - self.stopwords
+
+ self._overlap = self.hyp_words & self.text_words
+ self._hyp_extra = self.hyp_words - self.text_words
+ self._txt_extra = self.text_words - self.hyp_words
+
+
+ def overlap(self, toktype, debug=False):
+ """
+ Compute the overlap between text and hypothesis.
+
+ :param toktype: distinguish Named Entities from ordinary words
+ :type toktype: 'ne' or 'word'
+ """
+ ne_overlap = set(token for token in self._overlap if ne(token))
+ if toktype == 'ne':
+ if debug: print("ne overlap", ne_overlap)
+ return ne_overlap
+ elif toktype == 'word':
+ if debug: print("word overlap", self._overlap - ne_overlap)
+ return self._overlap - ne_overlap
+ else:
+ raise ValueError("Type not recognized:'%s'" % toktype)
+
+ def hyp_extra(self, toktype, debug=True):
+ """
+ Compute the extraneous material in the hypothesis.
+
+ :param toktype: distinguish Named Entities from ordinary words
+ :type toktype: 'ne' or 'word'
+ """
+ ne_extra = set(token for token in self._hyp_extra if ne(token))
+ if toktype == 'ne':
+ return ne_extra
+ elif toktype == 'word':
+ return self._hyp_extra - ne_extra
+ else:
+ raise ValueError("Type not recognized: '%s'" % toktype)
+
+
+def rte_features(rtepair):
+ extractor = RTEFeatureExtractor(rtepair)
+ features = {}
+ features['alwayson'] = True
+ features['word_overlap'] = len(extractor.overlap('word'))
+ features['word_hyp_extra'] = len(extractor.hyp_extra('word'))
+ features['ne_overlap'] = len(extractor.overlap('ne'))
+ features['ne_hyp_extra'] = len(extractor.hyp_extra('ne'))
+ features['neg_txt'] = len(extractor.negwords & extractor.text_words)
+ features['neg_hyp'] = len(extractor.negwords & extractor.hyp_words)
+ return features
+
+
+def rte_classifier(trainer, features=rte_features):
+ """
+ Classify RTEPairs
+ """
+ train = ((pair, pair.value) for pair in nltk.corpus.rte.pairs(['rte1_dev.xml', 'rte2_dev.xml', 'rte3_dev.xml']))
+ test = ((pair, pair.value) for pair in nltk.corpus.rte.pairs(['rte1_test.xml', 'rte2_test.xml', 'rte3_test.xml']))
+
+ # Train up a classifier.
+ print('Training classifier...')
+ classifier = trainer( [(features(pair), label) for (pair,label) in train] )
+
+ # Run the classifier on the test data.
+ print('Testing classifier...')
+ acc = accuracy(classifier, [(features(pair), label) for (pair,label) in test])
+ print('Accuracy: %6.4f' % acc)
+
+ # Return the classifier
+ return classifier
+
+
+def demo_features():
+ pairs = nltk.corpus.rte.pairs(['rte1_dev.xml'])[:6]
+ for pair in pairs:
+ print()
+ for key in sorted(rte_features(pair)):
+ print("%-15s => %s" % (key, rte_features(pair)[key]))
+
+
+def demo_feature_extractor():
+ rtepair = nltk.corpus.rte.pairs(['rte3_dev.xml'])[33]
+ extractor = RTEFeatureExtractor(rtepair)
+ print(extractor.hyp_words)
+ print(extractor.overlap('word'))
+ print(extractor.overlap('ne'))
+ print(extractor.hyp_extra('word'))
+
+
+def demo():
+ import nltk
+ try:
+ nltk.config_megam('/usr/local/bin/megam')
+ trainer = lambda x: nltk.MaxentClassifier.train(x, 'megam')
+ except ValueError:
+ try:
+ trainer = lambda x: nltk.MaxentClassifier.train(x, 'BFGS')
+ except ValueError:
+ trainer = nltk.MaxentClassifier.train
+ nltk.classify.rte_classifier(trainer)
+
+if __name__ == '__main__':
+ demo_features()
+ demo_feature_extractor()
+ demo()
+
diff --git a/nltk/classify/scikitlearn.py b/nltk/classify/scikitlearn.py
new file mode 100644
index 0000000..a317df9
--- /dev/null
+++ b/nltk/classify/scikitlearn.py
@@ -0,0 +1,152 @@
+# Natural Language Toolkit: Interface to scikit-learn classifiers
+#
+# Author: Lars Buitinck <L.J.Buitinck at uva.nl>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+scikit-learn (http://scikit-learn.org) is a machine learning library for
+Python. It supports many classification algorithms, including SVMs,
+Naive Bayes, logistic regression (MaxEnt) and decision trees.
+
+This package implement a wrapper around scikit-learn classifiers. To use this
+wrapper, construct a scikit-learn estimator object, then use that to construct
+a SklearnClassifier. E.g., to wrap a linear SVM with default settings:
+
+>>> from sklearn.svm import LinearSVC
+>>> from nltk.classify.scikitlearn import SklearnClassifier
+>>> classif = SklearnClassifier(LinearSVC())
+
+A scikit-learn classifier may include preprocessing steps when it's wrapped
+in a Pipeline object. The following constructs and wraps a Naive Bayes text
+classifier with tf-idf weighting and chi-square feature selection to get the
+best 1000 features:
+
+>>> from sklearn.feature_extraction.text import TfidfTransformer
+>>> from sklearn.feature_selection import SelectKBest, chi2
+>>> from sklearn.naive_bayes import MultinomialNB
+>>> from sklearn.pipeline import Pipeline
+>>> pipeline = Pipeline([('tfidf', TfidfTransformer()),
+... ('chi2', SelectKBest(chi2, k=1000)),
+... ('nb', MultinomialNB())])
+>>> classif = SklearnClassifier(pipeline)
+"""
+from __future__ import print_function, unicode_literals
+
+from nltk.classify.api import ClassifierI
+from nltk.probability import DictionaryProbDist
+from nltk import compat
+from warnings import warn
+
+try:
+ from sklearn.feature_extraction import DictVectorizer
+ from sklearn.preprocessing import LabelEncoder
+except ImportError:
+ pass
+
+__all__ = ['SklearnClassifier']
+
+
+ at compat.python_2_unicode_compatible
+class SklearnClassifier(ClassifierI):
+ """Wrapper for scikit-learn classifiers."""
+
+ def __init__(self, estimator, dtype=float, sparse=True):
+ """
+ :param estimator: scikit-learn classifier object.
+
+ :param dtype: data type used when building feature array.
+ scikit-learn estimators work exclusively on numeric data. The
+ default value should be fine for almost all situations.
+
+ :param sparse: Whether to use sparse matrices internally.
+ The estimator must support these; not all scikit-learn classifiers
+ do (see their respective documentation and look for "sparse
+ matrix"). The default value is True, since most NLP problems
+ involve sparse feature sets. Setting this to False may take a
+ great amount of memory.
+ :type sparse: boolean.
+ """
+ self._clf = estimator
+ self._encoder = LabelEncoder()
+ self._vectorizer = DictVectorizer(dtype=dtype, sparse=sparse)
+
+ def __repr__(self):
+ return "<SklearnClassifier(%r)>" % self._clf
+
+ def classify_many(self, featuresets):
+ """Classify a batch of samples.
+
+ :param featuresets: An iterable over featuresets, each a dict mapping
+ strings to either numbers, booleans or strings.
+ :return: The predicted class label for each input sample.
+ :rtype: list
+ """
+ X = self._vectorizer.transform(featuresets)
+ classes = self._encoder.classes_
+ return [classes[i] for i in self._clf.predict(X)]
+
+ def prob_classify_many(self, featuresets):
+ """Compute per-class probabilities for a batch of samples.
+
+ :param featuresets: An iterable over featuresets, each a dict mapping
+ strings to either numbers, booleans or strings.
+ :rtype: list of ``ProbDistI``
+ """
+ X = self._vectorizer.transform(featuresets)
+ y_proba_list = self._clf.predict_proba(X)
+ return [self._make_probdist(y_proba) for y_proba in y_proba_list]
+
+ def labels(self):
+ """The class labels used by this classifier.
+
+ :rtype: list
+ """
+ return list(self._encoder.classes_)
+
+ def train(self, labeled_featuresets):
+ """
+ Train (fit) the scikit-learn estimator.
+
+ :param labeled_featuresets: A list of ``(featureset, label)``
+ where each ``featureset`` is a dict mapping strings to either
+ numbers, booleans or strings.
+ """
+
+ X, y = list(compat.izip(*labeled_featuresets))
+ X = self._vectorizer.fit_transform(X)
+ y = self._encoder.fit_transform(y)
+ self._clf.fit(X, y)
+
+ return self
+
+ def _make_probdist(self, y_proba):
+ classes = self._encoder.classes_
+ return DictionaryProbDist(dict((classes[i], p)
+ for i, p in enumerate(y_proba)))
+
+
+# skip doctests if scikit-learn is not installed
+def setup_module(module):
+ from nose import SkipTest
+ try:
+ import sklearn
+ except ImportError:
+ raise SkipTest("scikit-learn is not installed")
+
+
+if __name__ == "__main__":
+ from nltk.classify.util import names_demo, names_demo_features
+ from sklearn.linear_model import LogisticRegression
+ from sklearn.naive_bayes import BernoulliNB
+
+ # Bernoulli Naive Bayes is designed for binary classification. We set the
+ # binarize option to False since we know we're passing boolean features.
+ print("scikit-learn Naive Bayes:")
+ names_demo(SklearnClassifier(BernoulliNB(binarize=False)).train,
+ features=names_demo_features)
+
+ # The C parameter on logistic regression (MaxEnt) controls regularization.
+ # The higher it's set, the less regularized the classifier is.
+ print("\n\nscikit-learn logistic regression:")
+ names_demo(SklearnClassifier(LogisticRegression(C=1000)).train,
+ features=names_demo_features)
diff --git a/nltk/classify/svm.py b/nltk/classify/svm.py
new file mode 100644
index 0000000..a77c1a8
--- /dev/null
+++ b/nltk/classify/svm.py
@@ -0,0 +1,15 @@
+# Natural Language Toolkit: SVM-based classifier
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Leon Derczynski <leon at dcs.shef.ac.uk>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+nltk.classify.svm was deprecated. For classification based
+on support vector machines SVMs use nltk.classify.scikitlearn
+(or `scikit-learn <http://scikit-learn.org>`_ directly).
+"""
+class SvmClassifier(object):
+ def __init__(self, *args, **kwargs):
+ raise NotImplementedError(__doc__)
diff --git a/nltk/classify/tadm.py b/nltk/classify/tadm.py
new file mode 100644
index 0000000..4cb01c5
--- /dev/null
+++ b/nltk/classify/tadm.py
@@ -0,0 +1,112 @@
+# Natural Language Toolkit: Interface to TADM Classifier
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Joseph Frazee <jfrazee at mail.utexas.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+import sys
+import subprocess
+
+from nltk import compat
+from nltk.internals import find_binary
+try:
+ import numpy
+except ImportError:
+ numpy = None
+
+_tadm_bin = None
+def config_tadm(bin=None):
+ global _tadm_bin
+ _tadm_bin = find_binary(
+ 'tadm', bin,
+ env_vars=['TADM_DIR'],
+ binary_names=['tadm'],
+ url='http://tadm.sf.net')
+
+def write_tadm_file(train_toks, encoding, stream):
+ """
+ Generate an input file for ``tadm`` based on the given corpus of
+ classified tokens.
+
+ :type train_toks: list(tuple(dict, str))
+ :param train_toks: Training data, represented as a list of
+ pairs, the first member of which is a feature dictionary,
+ and the second of which is a classification label.
+ :type encoding: TadmEventMaxentFeatureEncoding
+ :param encoding: A feature encoding, used to convert featuresets
+ into feature vectors.
+ :type stream: stream
+ :param stream: The stream to which the ``tadm`` input file should be
+ written.
+ """
+ # See the following for a file format description:
+ #
+ # http://sf.net/forum/forum.php?thread_id=1391502&forum_id=473054
+ # http://sf.net/forum/forum.php?thread_id=1675097&forum_id=473054
+ labels = encoding.labels()
+ for featureset, label in train_toks:
+ length_line = '%d\n' % len(labels)
+ stream.write(length_line)
+ for known_label in labels:
+ v = encoding.encode(featureset, known_label)
+ line = '%d %d %s\n' % (
+ int(label == known_label),
+ len(v),
+ ' '.join('%d %d' % u for u in v)
+ )
+ stream.write(line)
+
+def parse_tadm_weights(paramfile):
+ """
+ Given the stdout output generated by ``tadm`` when training a
+ model, return a ``numpy`` array containing the corresponding weight
+ vector.
+ """
+ weights = []
+ for line in paramfile:
+ weights.append(float(line.strip()))
+ return numpy.array(weights, 'd')
+
+def call_tadm(args):
+ """
+ Call the ``tadm`` binary with the given arguments.
+ """
+ if isinstance(args, compat.string_types):
+ raise TypeError('args should be a list of strings')
+ if _tadm_bin is None:
+ config_tadm()
+
+ # Call tadm via a subprocess
+ cmd = [_tadm_bin] + args
+ p = subprocess.Popen(cmd, stdout=sys.stdout)
+ (stdout, stderr) = p.communicate()
+
+ # Check the return code.
+ if p.returncode != 0:
+ print()
+ print(stderr)
+ raise OSError('tadm command failed!')
+
+def names_demo():
+ from nltk.classify.util import names_demo
+ from nltk.classify.maxent import TadmMaxentClassifier
+ classifier = names_demo(TadmMaxentClassifier.train)
+
+def encoding_demo():
+ import sys
+ from nltk.classify.maxent import TadmEventMaxentFeatureEncoding
+ tokens = [({'f0':1, 'f1':1, 'f3':1}, 'A'),
+ ({'f0':1, 'f2':1, 'f4':1}, 'B'),
+ ({'f0':2, 'f2':1, 'f3':1, 'f4':1}, 'A')]
+ encoding = TadmEventMaxentFeatureEncoding.train(tokens)
+ write_tadm_file(tokens, encoding, sys.stdout)
+ print()
+ for i in range(encoding.length()):
+ print('%s --> %d' % (encoding.describe(i), i))
+ print()
+
+if __name__ == '__main__':
+ encoding_demo()
+ names_demo()
diff --git a/nltk/classify/util.py b/nltk/classify/util.py
new file mode 100644
index 0000000..147d1f3
--- /dev/null
+++ b/nltk/classify/util.py
@@ -0,0 +1,311 @@
+# Natural Language Toolkit: Classifier Utility Functions
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Utility functions and classes for classifiers.
+"""
+from __future__ import print_function
+
+import math
+
+#from nltk.util import Deprecated
+import nltk.classify.util # for accuracy & log_likelihood
+from nltk.util import LazyMap
+
+######################################################################
+#{ Helper Functions
+######################################################################
+
+# alternative name possibility: 'map_featurefunc()'?
+# alternative name possibility: 'detect_features()'?
+# alternative name possibility: 'map_featuredetect()'?
+# or.. just have users use LazyMap directly?
+def apply_features(feature_func, toks, labeled=None):
+ """
+ Use the ``LazyMap`` class to construct a lazy list-like
+ object that is analogous to ``map(feature_func, toks)``. In
+ particular, if ``labeled=False``, then the returned list-like
+ object's values are equal to::
+
+ [feature_func(tok) for tok in toks]
+
+ If ``labeled=True``, then the returned list-like object's values
+ are equal to::
+
+ [(feature_func(tok), label) for (tok, label) in toks]
+
+ The primary purpose of this function is to avoid the memory
+ overhead involved in storing all the featuresets for every token
+ in a corpus. Instead, these featuresets are constructed lazily,
+ as-needed. The reduction in memory overhead can be especially
+ significant when the underlying list of tokens is itself lazy (as
+ is the case with many corpus readers).
+
+ :param feature_func: The function that will be applied to each
+ token. It should return a featureset -- i.e., a dict
+ mapping feature names to feature values.
+ :param toks: The list of tokens to which ``feature_func`` should be
+ applied. If ``labeled=True``, then the list elements will be
+ passed directly to ``feature_func()``. If ``labeled=False``,
+ then the list elements should be tuples ``(tok,label)``, and
+ ``tok`` will be passed to ``feature_func()``.
+ :param labeled: If true, then ``toks`` contains labeled tokens --
+ i.e., tuples of the form ``(tok, label)``. (Default:
+ auto-detect based on types.)
+ """
+ if labeled is None:
+ labeled = toks and isinstance(toks[0], (tuple, list))
+ if labeled:
+ def lazy_func(labeled_token):
+ return (feature_func(labeled_token[0]), labeled_token[1])
+ return LazyMap(lazy_func, toks)
+ else:
+ return LazyMap(feature_func, toks)
+
+def attested_labels(tokens):
+ """
+ :return: A list of all labels that are attested in the given list
+ of tokens.
+ :rtype: list of (immutable)
+ :param tokens: The list of classified tokens from which to extract
+ labels. A classified token has the form ``(token, label)``.
+ :type tokens: list
+ """
+ return tuple(set(label for (tok,label) in tokens))
+
+def log_likelihood(classifier, gold):
+ results = classifier.prob_classify_many([fs for (fs,l) in gold])
+ ll = [pdist.prob(l) for ((fs,l), pdist) in zip(gold, results)]
+ return math.log(float(sum(ll))/len(ll))
+
+def accuracy(classifier, gold):
+ results = classifier.classify_many([fs for (fs,l) in gold])
+ correct = [l==r for ((fs,l), r) in zip(gold, results)]
+ if correct:
+ return float(sum(correct))/len(correct)
+ else:
+ return 0
+
+class CutoffChecker(object):
+ """
+ A helper class that implements cutoff checks based on number of
+ iterations and log likelihood.
+
+ Accuracy cutoffs are also implemented, but they're almost never
+ a good idea to use.
+ """
+ def __init__(self, cutoffs):
+ self.cutoffs = cutoffs.copy()
+ if 'min_ll' in cutoffs:
+ cutoffs['min_ll'] = -abs(cutoffs['min_ll'])
+ if 'min_lldelta' in cutoffs:
+ cutoffs['min_lldelta'] = abs(cutoffs['min_lldelta'])
+ self.ll = None
+ self.acc = None
+ self.iter = 1
+
+ def check(self, classifier, train_toks):
+ cutoffs = self.cutoffs
+ self.iter += 1
+ if 'max_iter' in cutoffs and self.iter >= cutoffs['max_iter']:
+ return True # iteration cutoff.
+
+ new_ll = nltk.classify.util.log_likelihood(classifier, train_toks)
+ if math.isnan(new_ll):
+ return True
+
+ if 'min_ll' in cutoffs or 'min_lldelta' in cutoffs:
+ if 'min_ll' in cutoffs and new_ll >= cutoffs['min_ll']:
+ return True # log likelihood cutoff
+ if ('min_lldelta' in cutoffs and self.ll and
+ ((new_ll - self.ll) <= abs(cutoffs['min_lldelta']))):
+ return True # log likelihood delta cutoff
+ self.ll = new_ll
+
+ if 'max_acc' in cutoffs or 'min_accdelta' in cutoffs:
+ new_acc = nltk.classify.util.log_likelihood(
+ classifier, train_toks)
+ if 'max_acc' in cutoffs and new_acc >= cutoffs['max_acc']:
+ return True # log likelihood cutoff
+ if ('min_accdelta' in cutoffs and self.acc and
+ ((new_acc - self.acc) <= abs(cutoffs['min_accdelta']))):
+ return True # log likelihood delta cutoff
+ self.acc = new_acc
+
+ return False # no cutoff reached.
+
+######################################################################
+#{ Demos
+######################################################################
+
+def names_demo_features(name):
+ features = {}
+ features['alwayson'] = True
+ features['startswith'] = name[0].lower()
+ features['endswith'] = name[-1].lower()
+ for letter in 'abcdefghijklmnopqrstuvwxyz':
+ features['count(%s)' % letter] = name.lower().count(letter)
+ features['has(%s)' % letter] = letter in name.lower()
+ return features
+
+def binary_names_demo_features(name):
+ features = {}
+ features['alwayson'] = True
+ features['startswith(vowel)'] = name[0].lower() in 'aeiouy'
+ features['endswith(vowel)'] = name[-1].lower() in 'aeiouy'
+ for letter in 'abcdefghijklmnopqrstuvwxyz':
+ features['count(%s)' % letter] = name.lower().count(letter)
+ features['has(%s)' % letter] = letter in name.lower()
+ features['startswith(%s)' % letter] = (letter==name[0].lower())
+ features['endswith(%s)' % letter] = (letter==name[-1].lower())
+ return features
+
+def names_demo(trainer, features=names_demo_features):
+ from nltk.corpus import names
+ import random
+
+ # Construct a list of classified names, using the names corpus.
+ namelist = ([(name, 'male') for name in names.words('male.txt')] +
+ [(name, 'female') for name in names.words('female.txt')])
+
+ # Randomly split the names into a test & train set.
+ random.seed(123456)
+ random.shuffle(namelist)
+ train = namelist[:5000]
+ test = namelist[5000:5500]
+
+ # Train up a classifier.
+ print('Training classifier...')
+ classifier = trainer( [(features(n), g) for (n,g) in train] )
+
+ # Run the classifier on the test data.
+ print('Testing classifier...')
+ acc = accuracy(classifier, [(features(n),g) for (n,g) in test])
+ print('Accuracy: %6.4f' % acc)
+
+ # For classifiers that can find probabilities, show the log
+ # likelihood and some sample probability distributions.
+ try:
+ test_featuresets = [features(n) for (n,g) in test]
+ pdists = classifier.prob_classify_many(test_featuresets)
+ ll = [pdist.logprob(gold)
+ for ((name, gold), pdist) in zip(test, pdists)]
+ print('Avg. log likelihood: %6.4f' % (sum(ll)/len(test)))
+ print()
+ print('Unseen Names P(Male) P(Female)\n'+'-'*40)
+ for ((name, gender), pdist) in list(zip(test, pdists))[:5]:
+ if gender == 'male':
+ fmt = ' %-15s *%6.4f %6.4f'
+ else:
+ fmt = ' %-15s %6.4f *%6.4f'
+ print(fmt % (name, pdist.prob('male'), pdist.prob('female')))
+ except NotImplementedError:
+ pass
+
+ # Return the classifier
+ return classifier
+
+def partial_names_demo(trainer, features=names_demo_features):
+ from nltk.corpus import names
+ import random
+
+ male_names = names.words('male.txt')
+ female_names = names.words('female.txt')
+
+ random.seed(654321)
+ random.shuffle(male_names)
+ random.shuffle(female_names)
+
+ # Create a list of male names to be used as positive-labeled examples for training
+ positive = map(features, male_names[:2000])
+
+ # Create a list of male and female names to be used as unlabeled examples
+ unlabeled = map(features, male_names[2000:2500] + female_names[:500])
+
+ # Create a test set with correctly-labeled male and female names
+ test = [(name, True) for name in male_names[2500:2750]] \
+ + [(name, False) for name in female_names[500:750]]
+
+ random.shuffle(test)
+
+ # Train up a classifier.
+ print('Training classifier...')
+ classifier = trainer(positive, unlabeled)
+
+ # Run the classifier on the test data.
+ print('Testing classifier...')
+ acc = accuracy(classifier, [(features(n),m) for (n,m) in test])
+ print('Accuracy: %6.4f' % acc)
+
+ # For classifiers that can find probabilities, show the log
+ # likelihood and some sample probability distributions.
+ try:
+ test_featuresets = [features(n) for (n,m) in test]
+ pdists = classifier.prob_classify_many(test_featuresets)
+ ll = [pdist.logprob(gold)
+ for ((name, gold), pdist) in zip(test, pdists)]
+ print('Avg. log likelihood: %6.4f' % (sum(ll)/len(test)))
+ print()
+ print('Unseen Names P(Male) P(Female)\n'+'-'*40)
+ for ((name, is_male), pdist) in zip(test, pdists)[:5]:
+ if is_male == True:
+ fmt = ' %-15s *%6.4f %6.4f'
+ else:
+ fmt = ' %-15s %6.4f *%6.4f'
+ print(fmt % (name, pdist.prob(True), pdist.prob(False)))
+ except NotImplementedError:
+ pass
+
+ # Return the classifier
+ return classifier
+
+_inst_cache = {}
+def wsd_demo(trainer, word, features, n=1000):
+ from nltk.corpus import senseval
+ import random
+
+ # Get the instances.
+ print('Reading data...')
+ global _inst_cache
+ if word not in _inst_cache:
+ _inst_cache[word] = [(i, i.senses[0]) for i in senseval.instances(word)]
+ instances = _inst_cache[word][:]
+ if n> len(instances): n = len(instances)
+ senses = list(set(l for (i,l) in instances))
+ print(' Senses: ' + ' '.join(senses))
+
+ # Randomly split the names into a test & train set.
+ print('Splitting into test & train...')
+ random.seed(123456)
+ random.shuffle(instances)
+ train = instances[:int(.8*n)]
+ test = instances[int(.8*n):n]
+
+ # Train up a classifier.
+ print('Training classifier...')
+ classifier = trainer( [(features(i), l) for (i,l) in train] )
+
+ # Run the classifier on the test data.
+ print('Testing classifier...')
+ acc = accuracy(classifier, [(features(i),l) for (i,l) in test])
+ print('Accuracy: %6.4f' % acc)
+
+ # For classifiers that can find probabilities, show the log
+ # likelihood and some sample probability distributions.
+ try:
+ test_featuresets = [features(i) for (i,n) in test]
+ pdists = classifier.prob_classify_many(test_featuresets)
+ ll = [pdist.logprob(gold)
+ for ((name, gold), pdist) in zip(test, pdists)]
+ print('Avg. log likelihood: %6.4f' % (sum(ll)/len(test)))
+ except NotImplementedError:
+ pass
+
+ # Return the classifier
+ return classifier
+
diff --git a/nltk/classify/weka.py b/nltk/classify/weka.py
new file mode 100644
index 0000000..87b4cfa
--- /dev/null
+++ b/nltk/classify/weka.py
@@ -0,0 +1,343 @@
+# Natural Language Toolkit: Interface to Weka Classsifiers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classifiers that make use of the external 'Weka' package.
+"""
+from __future__ import print_function
+import time
+import tempfile
+import os
+import subprocess
+import re
+import zipfile
+
+from sys import stdin
+from nltk import compat
+from nltk.probability import DictionaryProbDist
+from nltk.internals import java, config_java
+
+from nltk.classify.api import ClassifierI
+
+_weka_classpath = None
+_weka_search = ['.',
+ '/usr/share/weka',
+ '/usr/local/share/weka',
+ '/usr/lib/weka',
+ '/usr/local/lib/weka',]
+def config_weka(classpath=None):
+ global _weka_classpath
+
+ # Make sure java's configured first.
+ config_java()
+
+ if classpath is not None:
+ _weka_classpath = classpath
+
+ if _weka_classpath is None:
+ searchpath = _weka_search
+ if 'WEKAHOME' in os.environ:
+ searchpath.insert(0, os.environ['WEKAHOME'])
+
+ for path in searchpath:
+ if os.path.exists(os.path.join(path, 'weka.jar')):
+ _weka_classpath = os.path.join(path, 'weka.jar')
+ version = _check_weka_version(_weka_classpath)
+ if version:
+ print(('[Found Weka: %s (version %s)]' %
+ (_weka_classpath, version)))
+ else:
+ print('[Found Weka: %s]' % _weka_classpath)
+ _check_weka_version(_weka_classpath)
+
+ if _weka_classpath is None:
+ raise LookupError('Unable to find weka.jar! Use config_weka() '
+ 'or set the WEKAHOME environment variable. '
+ 'For more information about Weka, please see '
+ 'http://www.cs.waikato.ac.nz/ml/weka/')
+
+def _check_weka_version(jar):
+ try:
+ zf = zipfile.ZipFile(jar)
+ except SystemExit as KeyboardInterrupt:
+ raise
+ except:
+ return None
+ try:
+ try:
+ return zf.read('weka/core/version.txt')
+ except KeyError:
+ return None
+ finally:
+ zf.close()
+
+class WekaClassifier(ClassifierI):
+ def __init__(self, formatter, model_filename):
+ self._formatter = formatter
+ self._model = model_filename
+
+ def prob_classify_many(self, featuresets):
+ return self._classify_many(featuresets, ['-p', '0', '-distribution'])
+
+ def classify_many(self, featuresets):
+ return self._classify_many(featuresets, ['-p', '0'])
+
+ def _classify_many(self, featuresets, options):
+ # Make sure we can find java & weka.
+ config_weka()
+
+ temp_dir = tempfile.mkdtemp()
+ try:
+ # Write the test data file.
+ test_filename = os.path.join(temp_dir, 'test.arff')
+ self._formatter.write(test_filename, featuresets)
+
+ # Call weka to classify the data.
+ cmd = ['weka.classifiers.bayes.NaiveBayes',
+ '-l', self._model, '-T', test_filename] + options
+ (stdout, stderr) = java(cmd, classpath=_weka_classpath,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+
+ # Check if something went wrong:
+ if stderr and not stdout:
+ if 'Illegal options: -distribution' in stderr:
+ raise ValueError('The installed version of weka does '
+ 'not support probability distribution '
+ 'output.')
+ else:
+ raise ValueError('Weka failed to generate output:\n%s'
+ % stderr)
+
+ # Parse weka's output.
+ return self.parse_weka_output(stdout.decode(stdin.encoding).split('\n'))
+
+ finally:
+ for f in os.listdir(temp_dir):
+ os.remove(os.path.join(temp_dir, f))
+ os.rmdir(temp_dir)
+
+ def parse_weka_distribution(self, s):
+ probs = [float(v) for v in re.split('[*,]+', s) if v.strip()]
+ probs = dict(zip(self._formatter.labels(), probs))
+ return DictionaryProbDist(probs)
+
+ def parse_weka_output(self, lines):
+ # Strip unwanted text from stdout
+ for i,line in enumerate(lines):
+ if line.strip().startswith("inst#"):
+ lines = lines[i:]
+ break
+
+ if lines[0].split() == ['inst#', 'actual', 'predicted',
+ 'error', 'prediction']:
+ return [line.split()[2].split(':')[1]
+ for line in lines[1:] if line.strip()]
+ elif lines[0].split() == ['inst#', 'actual', 'predicted',
+ 'error', 'distribution']:
+ return [self.parse_weka_distribution(line.split()[-1])
+ for line in lines[1:] if line.strip()]
+
+ # is this safe:?
+ elif re.match(r'^0 \w+ [01]\.[0-9]* \?\s*$', lines[0]):
+ return [line.split()[1] for line in lines if line.strip()]
+
+ else:
+ for line in lines[:10]: print(line)
+ raise ValueError('Unhandled output format -- your version '
+ 'of weka may not be supported.\n'
+ ' Header: %s' % lines[0])
+
+
+ # [xx] full list of classifiers (some may be abstract?):
+ # ADTree, AODE, BayesNet, ComplementNaiveBayes, ConjunctiveRule,
+ # DecisionStump, DecisionTable, HyperPipes, IB1, IBk, Id3, J48,
+ # JRip, KStar, LBR, LeastMedSq, LinearRegression, LMT, Logistic,
+ # LogisticBase, M5Base, MultilayerPerceptron,
+ # MultipleClassifiersCombiner, NaiveBayes, NaiveBayesMultinomial,
+ # NaiveBayesSimple, NBTree, NNge, OneR, PaceRegression, PART,
+ # PreConstructedLinearModel, Prism, RandomForest,
+ # RandomizableClassifier, RandomTree, RBFNetwork, REPTree, Ridor,
+ # RuleNode, SimpleLinearRegression, SimpleLogistic,
+ # SingleClassifierEnhancer, SMO, SMOreg, UserClassifier, VFI,
+ # VotedPerceptron, Winnow, ZeroR
+
+ _CLASSIFIER_CLASS = {
+ 'naivebayes': 'weka.classifiers.bayes.NaiveBayes',
+ 'C4.5': 'weka.classifiers.trees.J48',
+ 'log_regression': 'weka.classifiers.functions.Logistic',
+ 'svm': 'weka.classifiers.functions.SMO',
+ 'kstar': 'weka.classifiers.lazy.KStar',
+ 'ripper': 'weka.classifiers.rules.JRip',
+ }
+ @classmethod
+ def train(cls, model_filename, featuresets,
+ classifier='naivebayes', options=[], quiet=True):
+ # Make sure we can find java & weka.
+ config_weka()
+
+ # Build an ARFF formatter.
+ formatter = ARFF_Formatter.from_train(featuresets)
+
+ temp_dir = tempfile.mkdtemp()
+ try:
+ # Write the training data file.
+ train_filename = os.path.join(temp_dir, 'train.arff')
+ formatter.write(train_filename, featuresets)
+
+ if classifier in cls._CLASSIFIER_CLASS:
+ javaclass = cls._CLASSIFIER_CLASS[classifier]
+ elif classifier in cls._CLASSIFIER_CLASS.values():
+ javaclass = classifier
+ else:
+ raise ValueError('Unknown classifier %s' % classifier)
+
+ # Train the weka model.
+ cmd = [javaclass, '-d', model_filename, '-t', train_filename]
+ cmd += list(options)
+ if quiet: stdout = subprocess.PIPE
+ else: stdout = None
+ java(cmd, classpath=_weka_classpath, stdout=stdout)
+
+ # Return the new classifier.
+ return WekaClassifier(formatter, model_filename)
+
+ finally:
+ for f in os.listdir(temp_dir):
+ os.remove(os.path.join(temp_dir, f))
+ os.rmdir(temp_dir)
+
+
+class ARFF_Formatter:
+ """
+ Converts featuresets and labeled featuresets to ARFF-formatted
+ strings, appropriate for input into Weka.
+
+ Features and classes can be specified manually in the constructor, or may
+ be determined from data using ``from_train``.
+ """
+
+ def __init__(self, labels, features):
+ """
+ :param labels: A list of all class labels that can be generated.
+ :param features: A list of feature specifications, where
+ each feature specification is a tuple (fname, ftype);
+ and ftype is an ARFF type string such as NUMERIC or
+ STRING.
+ """
+ self._labels = labels
+ self._features = features
+
+ def format(self, tokens):
+ """Returns a string representation of ARFF output for the given data."""
+ return self.header_section() + self.data_section(tokens)
+
+ def labels(self):
+ """Returns the list of classes."""
+ return list(self._labels)
+
+ def write(self, outfile, tokens):
+ """Writes ARFF data to a file for the given data."""
+ if not hasattr(outfile, 'write'):
+ outfile = open(outfile, 'w')
+ outfile.write(self.format(tokens))
+ outfile.close()
+
+ @staticmethod
+ def from_train(tokens):
+ """
+ Constructs an ARFF_Formatter instance with class labels and feature
+ types determined from the given data. Handles boolean, numeric and
+ string (note: not nominal) types.
+ """
+ # Find the set of all attested labels.
+ labels = set(label for (tok,label) in tokens)
+
+ # Determine the types of all features.
+ features = {}
+ for tok, label in tokens:
+ for (fname, fval) in tok.items():
+ if issubclass(type(fval), bool):
+ ftype = '{True, False}'
+ elif issubclass(type(fval), (compat.integer_types, float, bool)):
+ ftype = 'NUMERIC'
+ elif issubclass(type(fval), compat.string_types):
+ ftype = 'STRING'
+ elif fval is None:
+ continue # can't tell the type.
+ else:
+ raise ValueError('Unsupported value type %r' % ftype)
+
+ if features.get(fname, ftype) != ftype:
+ raise ValueError('Inconsistent type for %s' % fname)
+ features[fname] = ftype
+ features = sorted(features.items())
+
+ return ARFF_Formatter(labels, features)
+
+ def header_section(self):
+ """Returns an ARFF header as a string."""
+ # Header comment.
+ s = ('% Weka ARFF file\n' +
+ '% Generated automatically by NLTK\n' +
+ '%% %s\n\n' % time.ctime())
+
+ # Relation name
+ s += '@RELATION rel\n\n'
+
+ # Input attribute specifications
+ for fname, ftype in self._features:
+ s += '@ATTRIBUTE %-30r %s\n' % (fname, ftype)
+
+ # Label attribute specification
+ s += '@ATTRIBUTE %-30r {%s}\n' % ('-label-', ','.join(self._labels))
+
+ return s
+
+ def data_section(self, tokens, labeled=None):
+ """
+ Returns the ARFF data section for the given data.
+
+ :param tokens: a list of featuresets (dicts) or labelled featuresets
+ which are tuples (featureset, label).
+ :param labeled: Indicates whether the given tokens are labeled
+ or not. If None, then the tokens will be assumed to be
+ labeled if the first token's value is a tuple or list.
+ """
+ # Check if the tokens are labeled or unlabeled. If unlabeled,
+ # then use 'None'
+ if labeled is None:
+ labeled = tokens and isinstance(tokens[0], (tuple, list))
+ if not labeled:
+ tokens = [(tok, None) for tok in tokens]
+
+ # Data section
+ s = '\n at DATA\n'
+ for (tok, label) in tokens:
+ for fname, ftype in self._features:
+ s += '%s,' % self._fmt_arff_val(tok.get(fname))
+ s += '%s\n' % self._fmt_arff_val(label)
+
+ return s
+
+ def _fmt_arff_val(self, fval):
+ if fval is None:
+ return '?'
+ elif isinstance(fval, (bool, compat.integer_types)):
+ return '%s' % fval
+ elif isinstance(fval, float):
+ return '%r' % fval
+ else:
+ return '%r' % fval
+
+
+if __name__ == '__main__':
+ from nltk.classify.util import names_demo, binary_names_demo_features
+ def make_classifier(featuresets):
+ return WekaClassifier.train('/tmp/name.model', featuresets,
+ 'C4.5')
+ classifier = names_demo(make_classifier, binary_names_demo_features)
diff --git a/nltk/cluster/__init__.py b/nltk/cluster/__init__.py
new file mode 100644
index 0000000..39fe32a
--- /dev/null
+++ b/nltk/cluster/__init__.py
@@ -0,0 +1,86 @@
+# Natural Language Toolkit: Clusterers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+This module contains a number of basic clustering algorithms. Clustering
+describes the task of discovering groups of similar items with a large
+collection. It is also describe as unsupervised machine learning, as the data
+from which it learns is unannotated with class information, as is the case for
+supervised learning. Annotated data is difficult and expensive to obtain in
+the quantities required for the majority of supervised learning algorithms.
+This problem, the knowledge acquisition bottleneck, is common to most natural
+language processing tasks, thus fueling the need for quality unsupervised
+approaches.
+
+This module contains a k-means clusterer, E-M clusterer and a group average
+agglomerative clusterer (GAAC). All these clusterers involve finding good
+cluster groupings for a set of vectors in multi-dimensional space.
+
+The K-means clusterer starts with k arbitrary chosen means then allocates each
+vector to the cluster with the closest mean. It then recalculates the means of
+each cluster as the centroid of the vectors in the cluster. This process
+repeats until the cluster memberships stabilise. This is a hill-climbing
+algorithm which may converge to a local maximum. Hence the clustering is
+often repeated with random initial means and the most commonly occurring
+output means are chosen.
+
+The GAAC clusterer starts with each of the *N* vectors as singleton clusters.
+It then iteratively merges pairs of clusters which have the closest centroids.
+This continues until there is only one cluster. The order of merges gives rise
+to a dendrogram - a tree with the earlier merges lower than later merges. The
+membership of a given number of clusters *c*, *1 <= c <= N*, can be found by
+cutting the dendrogram at depth *c*.
+
+The Gaussian EM clusterer models the vectors as being produced by a mixture
+of k Gaussian sources. The parameters of these sources (prior probability,
+mean and covariance matrix) are then found to maximise the likelihood of the
+given data. This is done with the expectation maximisation algorithm. It
+starts with k arbitrarily chosen means, priors and covariance matrices. It
+then calculates the membership probabilities for each vector in each of the
+clusters - this is the 'E' step. The cluster parameters are then updated in
+the 'M' step using the maximum likelihood estimate from the cluster membership
+probabilities. This process continues until the likelihood of the data does
+not significantly increase.
+
+They all extend the ClusterI interface which defines common operations
+available with each clusterer. These operations include.
+ - cluster: clusters a sequence of vectors
+ - classify: assign a vector to a cluster
+ - classification_probdist: give the probability distribution over cluster memberships
+
+The current existing classifiers also extend cluster.VectorSpace, an
+abstract class which allows for singular value decomposition (SVD) and vector
+normalisation. SVD is used to reduce the dimensionality of the vector space in
+such a manner as to preserve as much of the variation as possible, by
+reparameterising the axes in order of variability and discarding all bar the
+first d dimensions. Normalisation ensures that vectors fall in the unit
+hypersphere.
+
+Usage example (see also demo())::
+ from nltk import cluster
+ from nltk.cluster import euclidean_distance
+ from numpy import array
+
+ vectors = [array(f) for f in [[3, 3], [1, 2], [4, 2], [4, 0]]]
+
+ # initialise the clusterer (will also assign the vectors to clusters)
+ clusterer = cluster.KMeansClusterer(2, euclidean_distance)
+ clusterer.cluster(vectors, True)
+
+ # classify a new vector
+ print(clusterer.classify(array([3, 3])))
+
+Note that the vectors must use numpy array-like
+objects. nltk_contrib.unimelb.tacohn.SparseArrays may be used for
+efficiency when required.
+"""
+
+from nltk.cluster.util import (VectorSpaceClusterer, Dendrogram,
+ euclidean_distance, cosine_distance)
+from nltk.cluster.kmeans import KMeansClusterer
+from nltk.cluster.gaac import GAAClusterer
+from nltk.cluster.em import EMClusterer
diff --git a/nltk/cluster/api.py b/nltk/cluster/api.py
new file mode 100644
index 0000000..e194d91
--- /dev/null
+++ b/nltk/cluster/api.py
@@ -0,0 +1,70 @@
+# Natural Language Toolkit: Clusterer Interfaces
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# Porting: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk.probability import DictionaryProbDist
+
+class ClusterI(object):
+ """
+ Interface covering basic clustering functionality.
+ """
+
+ def cluster(self, vectors, assign_clusters=False):
+ """
+ Assigns the vectors to clusters, learning the clustering parameters
+ from the data. Returns a cluster identifier for each vector.
+ """
+ raise NotImplementedError()
+
+ def classify(self, token):
+ """
+ Classifies the token into a cluster, setting the token's CLUSTER
+ parameter to that cluster identifier.
+ """
+ raise NotImplementedError()
+
+ def likelihood(self, vector, label):
+ """
+ Returns the likelihood (a float) of the token having the
+ corresponding cluster.
+ """
+ if self.classify(vector) == label:
+ return 1.0
+ else:
+ return 0.0
+
+ def classification_probdist(self, vector):
+ """
+ Classifies the token into a cluster, returning
+ a probability distribution over the cluster identifiers.
+ """
+ likelihoods = {}
+ sum = 0.0
+ for cluster in self.cluster_names():
+ likelihoods[cluster] = self.likelihood(vector, cluster)
+ sum += likelihoods[cluster]
+ for cluster in self.cluster_names():
+ likelihoods[cluster] /= sum
+ return DictionaryProbDist(likelihoods)
+
+ def num_clusters(self):
+ """
+ Returns the number of clusters.
+ """
+ raise NotImplementedError()
+
+ def cluster_names(self):
+ """
+ Returns the names of the clusters.
+ """
+ return list(range(self.num_clusters()))
+
+ def cluster_name(self, index):
+ """
+ Returns the names of the cluster at index.
+ """
+ return index
diff --git a/nltk/cluster/em.py b/nltk/cluster/em.py
new file mode 100644
index 0000000..8e12579
--- /dev/null
+++ b/nltk/cluster/em.py
@@ -0,0 +1,247 @@
+# Natural Language Toolkit: Expectation Maximization Clusterer
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+try:
+ import numpy
+except ImportError:
+ pass
+
+from nltk.compat import python_2_unicode_compatible
+from nltk.cluster.util import VectorSpaceClusterer
+
+ at python_2_unicode_compatible
+class EMClusterer(VectorSpaceClusterer):
+ """
+ The Gaussian EM clusterer models the vectors as being produced by
+ a mixture of k Gaussian sources. The parameters of these sources
+ (prior probability, mean and covariance matrix) are then found to
+ maximise the likelihood of the given data. This is done with the
+ expectation maximisation algorithm. It starts with k arbitrarily
+ chosen means, priors and covariance matrices. It then calculates
+ the membership probabilities for each vector in each of the
+ clusters; this is the 'E' step. The cluster parameters are then
+ updated in the 'M' step using the maximum likelihood estimate from
+ the cluster membership probabilities. This process continues until
+ the likelihood of the data does not significantly increase.
+ """
+
+ def __init__(self, initial_means, priors=None, covariance_matrices=None,
+ conv_threshold=1e-6, bias=0.1, normalise=False,
+ svd_dimensions=None):
+ """
+ Creates an EM clusterer with the given starting parameters,
+ convergence threshold and vector mangling parameters.
+
+ :param initial_means: the means of the gaussian cluster centers
+ :type initial_means: [seq of] numpy array or seq of SparseArray
+ :param priors: the prior probability for each cluster
+ :type priors: numpy array or seq of float
+ :param covariance_matrices: the covariance matrix for each cluster
+ :type covariance_matrices: [seq of] numpy array
+ :param conv_threshold: maximum change in likelihood before deemed
+ convergent
+ :type conv_threshold: int or float
+ :param bias: variance bias used to ensure non-singular covariance
+ matrices
+ :type bias: float
+ :param normalise: should vectors be normalised to length 1
+ :type normalise: boolean
+ :param svd_dimensions: number of dimensions to use in reducing vector
+ dimensionsionality with SVD
+ :type svd_dimensions: int
+ """
+ VectorSpaceClusterer.__init__(self, normalise, svd_dimensions)
+ self._means = numpy.array(initial_means, numpy.float64)
+ self._num_clusters = len(initial_means)
+ self._conv_threshold = conv_threshold
+ self._covariance_matrices = covariance_matrices
+ self._priors = priors
+ self._bias = bias
+
+ def num_clusters(self):
+ return self._num_clusters
+
+ def cluster_vectorspace(self, vectors, trace=False):
+ assert len(vectors) > 0
+
+ # set the parameters to initial values
+ dimensions = len(vectors[0])
+ means = self._means
+ priors = self._priors
+ if not priors:
+ priors = self._priors = numpy.ones(self._num_clusters,
+ numpy.float64) / self._num_clusters
+ covariances = self._covariance_matrices
+ if not covariances:
+ covariances = self._covariance_matrices = \
+ [ numpy.identity(dimensions, numpy.float64)
+ for i in range(self._num_clusters) ]
+
+ # do the E and M steps until the likelihood plateaus
+ lastl = self._loglikelihood(vectors, priors, means, covariances)
+ converged = False
+
+ while not converged:
+ if trace: print('iteration; loglikelihood', lastl)
+ # E-step, calculate hidden variables, h[i,j]
+ h = numpy.zeros((len(vectors), self._num_clusters),
+ numpy.float64)
+ for i in range(len(vectors)):
+ for j in range(self._num_clusters):
+ h[i,j] = priors[j] * self._gaussian(means[j],
+ covariances[j], vectors[i])
+ h[i,:] /= sum(h[i,:])
+
+ # M-step, update parameters - cvm, p, mean
+ for j in range(self._num_clusters):
+ covariance_before = covariances[j]
+ new_covariance = numpy.zeros((dimensions, dimensions),
+ numpy.float64)
+ new_mean = numpy.zeros(dimensions, numpy.float64)
+ sum_hj = 0.0
+ for i in range(len(vectors)):
+ delta = vectors[i] - means[j]
+ new_covariance += h[i,j] * \
+ numpy.multiply.outer(delta, delta)
+ sum_hj += h[i,j]
+ new_mean += h[i,j] * vectors[i]
+ covariances[j] = new_covariance / sum_hj
+ means[j] = new_mean / sum_hj
+ priors[j] = sum_hj / len(vectors)
+
+ # bias term to stop covariance matrix being singular
+ covariances[j] += self._bias * \
+ numpy.identity(dimensions, numpy.float64)
+
+ # calculate likelihood - FIXME: may be broken
+ l = self._loglikelihood(vectors, priors, means, covariances)
+
+ # check for convergence
+ if abs(lastl - l) < self._conv_threshold:
+ converged = True
+ lastl = l
+
+ def classify_vectorspace(self, vector):
+ best = None
+ for j in range(self._num_clusters):
+ p = self._priors[j] * self._gaussian(self._means[j],
+ self._covariance_matrices[j], vector)
+ if not best or p > best[0]:
+ best = (p, j)
+ return best[1]
+
+ def likelihood_vectorspace(self, vector, cluster):
+ cid = self.cluster_names().index(cluster)
+ return self._priors[cluster] * self._gaussian(self._means[cluster],
+ self._covariance_matrices[cluster], vector)
+
+ def _gaussian(self, mean, cvm, x):
+ m = len(mean)
+ assert cvm.shape == (m, m), \
+ 'bad sized covariance matrix, %s' % str(cvm.shape)
+ try:
+ det = numpy.linalg.det(cvm)
+ inv = numpy.linalg.inv(cvm)
+ a = det ** -0.5 * (2 * numpy.pi) ** (-m / 2.0)
+ dx = x - mean
+ print(dx, inv)
+ b = -0.5 * numpy.dot( numpy.dot(dx, inv), dx)
+ return a * numpy.exp(b)
+ except OverflowError:
+ # happens when the exponent is negative infinity - i.e. b = 0
+ # i.e. the inverse of cvm is huge (cvm is almost zero)
+ return 0
+
+ def _loglikelihood(self, vectors, priors, means, covariances):
+ llh = 0.0
+ for vector in vectors:
+ p = 0
+ for j in range(len(priors)):
+ p += priors[j] * \
+ self._gaussian(means[j], covariances[j], vector)
+ llh += numpy.log(p)
+ return llh
+
+ def __repr__(self):
+ return '<EMClusterer means=%s>' % list(self._means)
+
+def demo():
+ """
+ Non-interactive demonstration of the clusterers with simple 2-D data.
+ """
+
+ from nltk import cluster
+
+ # example from figure 14.10, page 519, Manning and Schutze
+
+ vectors = [numpy.array(f) for f in [[0.5, 0.5], [1.5, 0.5], [1, 3]]]
+ means = [[4, 2], [4, 2.01]]
+
+ clusterer = cluster.EMClusterer(means, bias=0.1)
+ clusters = clusterer.cluster(vectors, True, trace=True)
+
+ print('Clustered:', vectors)
+ print('As: ', clusters)
+ print()
+
+ for c in range(2):
+ print('Cluster:', c)
+ print('Prior: ', clusterer._priors[c])
+ print('Mean: ', clusterer._means[c])
+ print('Covar: ', clusterer._covariance_matrices[c])
+ print()
+
+ # classify a new vector
+ vector = numpy.array([2, 2])
+ print('classify(%s):' % vector, end=' ')
+ print(clusterer.classify(vector))
+
+ # show the classification probabilities
+ vector = numpy.array([2, 2])
+ print('classification_probdist(%s):' % vector)
+ pdist = clusterer.classification_probdist(vector)
+ for sample in pdist.samples():
+ print('%s => %.0f%%' % (sample,
+ pdist.prob(sample) *100))
+
+#
+# The following demo code is broken.
+#
+# # use a set of tokens with 2D indices
+# vectors = [numpy.array(f) for f in [[3, 3], [1, 2], [4, 2], [4, 0], [2, 3], [3, 1]]]
+
+# # test the EM clusterer with means given by k-means (2) and
+# # dimensionality reduction
+# clusterer = cluster.KMeans(2, euclidean_distance, svd_dimensions=1)
+# print 'Clusterer:', clusterer
+# clusters = clusterer.cluster(vectors)
+# means = clusterer.means()
+# print 'Means:', clusterer.means()
+# print
+
+# clusterer = cluster.EMClusterer(means, svd_dimensions=1)
+# clusters = clusterer.cluster(vectors, True)
+# print 'Clusterer:', clusterer
+# print 'Clustered:', str(vectors)[:60], '...'
+# print 'As:', str(clusters)[:60], '...'
+# print
+
+# # classify a new vector
+# vector = numpy.array([3, 3])
+# print 'classify(%s):' % vector,
+# print clusterer.classify(vector)
+# print
+
+# # show the classification probabilities
+# vector = numpy.array([2.2, 2])
+# print 'classification_probdist(%s)' % vector
+# pdist = clusterer.classification_probdist(vector)
+# for sample in pdist:
+# print '%s => %.0f%%' % (sample, pdist.prob(sample) *100)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/cluster/gaac.py b/nltk/cluster/gaac.py
new file mode 100644
index 0000000..82c871d
--- /dev/null
+++ b/nltk/cluster/gaac.py
@@ -0,0 +1,168 @@
+# Natural Language Toolkit: Group Average Agglomerative Clusterer
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+try:
+ import numpy
+except ImportError:
+ pass
+
+from nltk.cluster.util import VectorSpaceClusterer, Dendrogram, cosine_distance
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class GAAClusterer(VectorSpaceClusterer):
+ """
+ The Group Average Agglomerative starts with each of the N vectors as singleton
+ clusters. It then iteratively merges pairs of clusters which have the
+ closest centroids. This continues until there is only one cluster. The
+ order of merges gives rise to a dendrogram: a tree with the earlier merges
+ lower than later merges. The membership of a given number of clusters c, 1
+ <= c <= N, can be found by cutting the dendrogram at depth c.
+
+ This clusterer uses the cosine similarity metric only, which allows for
+ efficient speed-up in the clustering process.
+ """
+
+ def __init__(self, num_clusters=1, normalise=True, svd_dimensions=None):
+ VectorSpaceClusterer.__init__(self, normalise, svd_dimensions)
+ self._num_clusters = num_clusters
+ self._dendrogram = None
+ self._groups_values = None
+
+ def cluster(self, vectors, assign_clusters=False, trace=False):
+ # stores the merge order
+ self._dendrogram = Dendrogram(
+ [numpy.array(vector, numpy.float64) for vector in vectors])
+ return VectorSpaceClusterer.cluster(self, vectors, assign_clusters, trace)
+
+ def cluster_vectorspace(self, vectors, trace=False):
+ # variables describing the initial situation
+ N = len(vectors)
+ cluster_len = [1]*N
+ cluster_count = N
+ index_map = numpy.arange(N)
+
+ # construct the similarity matrix
+ dims = (N, N)
+ dist = numpy.ones(dims, dtype=numpy.float)*numpy.inf
+ for i in range(N):
+ for j in range(i+1, N):
+ dist[i, j] = cosine_distance(vectors[i], vectors[j])
+
+ while cluster_count > max(self._num_clusters, 1):
+ i, j = numpy.unravel_index(dist.argmin(), dims)
+ if trace:
+ print("merging %d and %d" % (i, j))
+
+ # update similarities for merging i and j
+ self._merge_similarities(dist, cluster_len, i, j)
+
+ # remove j
+ dist[:, j] = numpy.inf
+ dist[j, :] = numpy.inf
+
+ # merge the clusters
+ cluster_len[i] = cluster_len[i]+cluster_len[j]
+ self._dendrogram.merge(index_map[i], index_map[j])
+ cluster_count -= 1
+
+ # update the index map to reflect the indexes if we
+ # had removed j
+ index_map[j+1:] -= 1
+ index_map[j] = N
+
+ self.update_clusters(self._num_clusters)
+
+ def _merge_similarities(self, dist, cluster_len, i, j):
+ # the new cluster i merged from i and j adopts the average of
+ # i and j's similarity to each other cluster, weighted by the
+ # number of points in the clusters i and j
+ i_weight = cluster_len[i]
+ j_weight = cluster_len[j]
+ weight_sum = i_weight+j_weight
+
+ # update for x<i
+ dist[:i, i] = dist[:i, i]*i_weight + dist[:i, j]*j_weight
+ dist[:i, i] /= weight_sum
+ # update for i<x<j
+ dist[i, i+1:j] = dist[i, i+1:j]*i_weight + dist[i+1:j, j]*j_weight
+ # update for i<j<x
+ dist[i, j+1:] = dist[i, j+1:]*i_weight + dist[j, j+1:]*j_weight
+ dist[i, i+1:] /= weight_sum
+
+ def update_clusters(self, num_clusters):
+ clusters = self._dendrogram.groups(num_clusters)
+ self._centroids = []
+ for cluster in clusters:
+ assert len(cluster) > 0
+ if self._should_normalise:
+ centroid = self._normalise(cluster[0])
+ else:
+ centroid = numpy.array(cluster[0])
+ for vector in cluster[1:]:
+ if self._should_normalise:
+ centroid += self._normalise(vector)
+ else:
+ centroid += vector
+ centroid /= float(len(cluster))
+ self._centroids.append(centroid)
+ self._num_clusters = len(self._centroids)
+
+ def classify_vectorspace(self, vector):
+ best = None
+ for i in range(self._num_clusters):
+ centroid = self._centroids[i]
+ dist = cosine_distance(vector, centroid)
+ if not best or dist < best[0]:
+ best = (dist, i)
+ return best[1]
+
+ def dendrogram(self):
+ """
+ :return: The dendrogram representing the current clustering
+ :rtype: Dendrogram
+ """
+ return self._dendrogram
+
+ def num_clusters(self):
+ return self._num_clusters
+
+ def __repr__(self):
+ return '<GroupAverageAgglomerative Clusterer n=%d>' % self._num_clusters
+
+def demo():
+ """
+ Non-interactive demonstration of the clusterers with simple 2-D data.
+ """
+
+ from nltk.cluster import GAAClusterer
+
+ # use a set of tokens with 2D indices
+ vectors = [numpy.array(f) for f in [[3, 3], [1, 2], [4, 2], [4, 0], [2, 3], [3, 1]]]
+
+ # test the GAAC clusterer with 4 clusters
+ clusterer = GAAClusterer(4)
+ clusters = clusterer.cluster(vectors, True)
+
+ print('Clusterer:', clusterer)
+ print('Clustered:', vectors)
+ print('As:', clusters)
+ print()
+
+ # show the dendrogram
+ clusterer.dendrogram().show()
+
+ # classify a new vector
+ vector = numpy.array([3, 3])
+ print('classify(%s):' % vector, end=' ')
+ print(clusterer.classify(vector))
+ print()
+
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/cluster/kmeans.py b/nltk/cluster/kmeans.py
new file mode 100644
index 0000000..f5cef63
--- /dev/null
+++ b/nltk/cluster/kmeans.py
@@ -0,0 +1,221 @@
+# Natural Language Toolkit: K-Means Clusterer
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+import copy
+import random
+import sys
+
+try:
+ import numpy
+except ImportError:
+ pass
+
+
+from nltk.cluster.util import VectorSpaceClusterer
+from nltk.compat import python_2_unicode_compatible
+
+
+ at python_2_unicode_compatible
+class KMeansClusterer(VectorSpaceClusterer):
+ """
+ The K-means clusterer starts with k arbitrary chosen means then allocates
+ each vector to the cluster with the closest mean. It then recalculates the
+ means of each cluster as the centroid of the vectors in the cluster. This
+ process repeats until the cluster memberships stabilise. This is a
+ hill-climbing algorithm which may converge to a local maximum. Hence the
+ clustering is often repeated with random initial means and the most
+ commonly occurring output means are chosen.
+ """
+
+ def __init__(self, num_means, distance, repeats=1,
+ conv_test=1e-6, initial_means=None,
+ normalise=False, svd_dimensions=None,
+ rng=None, avoid_empty_clusters=False):
+
+ """
+ :param num_means: the number of means to use (may use fewer)
+ :type num_means: int
+ :param distance: measure of distance between two vectors
+ :type distance: function taking two vectors and returing a float
+ :param repeats: number of randomised clustering trials to use
+ :type repeats: int
+ :param conv_test: maximum variation in mean differences before
+ deemed convergent
+ :type conv_test: number
+ :param initial_means: set of k initial means
+ :type initial_means: sequence of vectors
+ :param normalise: should vectors be normalised to length 1
+ :type normalise: boolean
+ :param svd_dimensions: number of dimensions to use in reducing vector
+ dimensionsionality with SVD
+ :type svd_dimensions: int
+ :param rng: random number generator (or None)
+ :type rng: Random
+ :param avoid_empty_clusters: include current centroid in computation
+ of next one; avoids undefined behavior
+ when clusters become empty
+ :type avoid_empty_clusters: boolean
+ """
+ VectorSpaceClusterer.__init__(self, normalise, svd_dimensions)
+ self._num_means = num_means
+ self._distance = distance
+ self._max_difference = conv_test
+ assert not initial_means or len(initial_means) == num_means
+ self._means = initial_means
+ assert repeats >= 1
+ assert not (initial_means and repeats > 1)
+ self._repeats = repeats
+ self._rng = (rng if rng else random.Random())
+ self._avoid_empty_clusters = avoid_empty_clusters
+
+ def cluster_vectorspace(self, vectors, trace=False):
+ if self._means and self._repeats > 1:
+ print('Warning: means will be discarded for subsequent trials')
+
+ meanss = []
+ for trial in range(self._repeats):
+ if trace: print('k-means trial', trial)
+ if not self._means or trial > 1:
+ self._means = self._rng.sample(vectors, self._num_means)
+ self._cluster_vectorspace(vectors, trace)
+ meanss.append(self._means)
+
+ if len(meanss) > 1:
+ # sort the means first (so that different cluster numbering won't
+ # effect the distance comparison)
+ for means in meanss:
+ means.sort(key=sum)
+
+ # find the set of means that's minimally different from the others
+ min_difference = min_means = None
+ for i in range(len(meanss)):
+ d = 0
+ for j in range(len(meanss)):
+ if i != j:
+ d += self._sum_distances(meanss[i], meanss[j])
+ if min_difference is None or d < min_difference:
+ min_difference, min_means = d, meanss[i]
+
+ # use the best means
+ self._means = min_means
+
+ def _cluster_vectorspace(self, vectors, trace=False):
+ if self._num_means < len(vectors):
+ # perform k-means clustering
+ converged = False
+ while not converged:
+ # assign the tokens to clusters based on minimum distance to
+ # the cluster means
+ clusters = [[] for m in range(self._num_means)]
+ for vector in vectors:
+ index = self.classify_vectorspace(vector)
+ clusters[index].append(vector)
+
+ if trace: print('iteration')
+ #for i in range(self._num_means):
+ #print ' mean', i, 'allocated', len(clusters[i]), 'vectors'
+
+ # recalculate cluster means by computing the centroid of each cluster
+ new_means = list(map(self._centroid, clusters, self._means))
+
+ # measure the degree of change from the previous step for convergence
+ difference = self._sum_distances(self._means, new_means)
+ if difference < self._max_difference:
+ converged = True
+
+ # remember the new means
+ self._means = new_means
+
+ def classify_vectorspace(self, vector):
+ # finds the closest cluster centroid
+ # returns that cluster's index
+ best_distance = best_index = None
+ for index in range(len(self._means)):
+ mean = self._means[index]
+ dist = self._distance(vector, mean)
+ if best_distance is None or dist < best_distance:
+ best_index, best_distance = index, dist
+ return best_index
+
+ def num_clusters(self):
+ if self._means:
+ return len(self._means)
+ else:
+ return self._num_means
+
+ def means(self):
+ """
+ The means used for clustering.
+ """
+ return self._means
+
+ def _sum_distances(self, vectors1, vectors2):
+ difference = 0.0
+ for u, v in zip(vectors1, vectors2):
+ difference += self._distance(u, v)
+ return difference
+
+ def _centroid(self, cluster, mean):
+ if self._avoid_empty_clusters:
+ centroid = copy.copy(mean)
+ for vector in cluster:
+ centroid += vector
+ return centroid / (1+float(len(cluster)))
+ else:
+ if not len(cluster):
+ sys.stderr.write('Error: no centroid defined for empty cluster.\n')
+ sys.stderr.write('Try setting argument \'avoid_empty_clusters\' to True\n')
+ assert(False)
+ centroid = copy.copy(cluster[0])
+ for vector in cluster[1:]:
+ centroid += vector
+ return centroid / float(len(cluster))
+
+ def __repr__(self):
+ return '<KMeansClusterer means=%s repeats=%d>' % \
+ (self._means, self._repeats)
+
+#################################################################################
+
+def demo():
+ # example from figure 14.9, page 517, Manning and Schutze
+
+ from nltk.cluster import KMeansClusterer, euclidean_distance
+
+ vectors = [numpy.array(f) for f in [[2, 1], [1, 3], [4, 7], [6, 7]]]
+ means = [[4, 3], [5, 5]]
+
+ clusterer = KMeansClusterer(2, euclidean_distance, initial_means=means)
+ clusters = clusterer.cluster(vectors, True, trace=True)
+
+ print('Clustered:', vectors)
+ print('As:', clusters)
+ print('Means:', clusterer.means())
+ print()
+
+ vectors = [numpy.array(f) for f in [[3, 3], [1, 2], [4, 2], [4, 0], [2, 3], [3, 1]]]
+
+ # test k-means using the euclidean distance metric, 2 means and repeat
+ # clustering 10 times with random seeds
+
+ clusterer = KMeansClusterer(2, euclidean_distance, repeats=10)
+ clusters = clusterer.cluster(vectors, True)
+ print('Clustered:', vectors)
+ print('As:', clusters)
+ print('Means:', clusterer.means())
+ print()
+
+ # classify a new vector
+ vector = numpy.array([3, 3])
+ print('classify(%s):' % vector, end=' ')
+ print(clusterer.classify(vector))
+ print()
+
+if __name__ == '__main__':
+ demo()
+
diff --git a/nltk/cluster/util.py b/nltk/cluster/util.py
new file mode 100644
index 0000000..fc88c29
--- /dev/null
+++ b/nltk/cluster/util.py
@@ -0,0 +1,290 @@
+# Natural Language Toolkit: Clusterer Utilities
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+import copy
+from sys import stdout
+from math import sqrt
+
+try:
+ import numpy
+except ImportError:
+ pass
+
+from nltk.cluster.api import ClusterI
+from nltk.compat import python_2_unicode_compatible
+
+class VectorSpaceClusterer(ClusterI):
+ """
+ Abstract clusterer which takes tokens and maps them into a vector space.
+ Optionally performs singular value decomposition to reduce the
+ dimensionality.
+ """
+ def __init__(self, normalise=False, svd_dimensions=None):
+ """
+ :param normalise: should vectors be normalised to length 1
+ :type normalise: boolean
+ :param svd_dimensions: number of dimensions to use in reducing vector
+ dimensionsionality with SVD
+ :type svd_dimensions: int
+ """
+ self._Tt = None
+ self._should_normalise = normalise
+ self._svd_dimensions = svd_dimensions
+
+ def cluster(self, vectors, assign_clusters=False, trace=False):
+ assert len(vectors) > 0
+
+ # normalise the vectors
+ if self._should_normalise:
+ vectors = list(map(self._normalise, vectors))
+
+ # use SVD to reduce the dimensionality
+ if self._svd_dimensions and self._svd_dimensions < len(vectors[0]):
+ [u, d, vt] = numpy.linalg.svd(numpy.transpose(numpy.array(vectors)))
+ S = d[:self._svd_dimensions] * \
+ numpy.identity(self._svd_dimensions, numpy.float64)
+ T = u[:,:self._svd_dimensions]
+ Dt = vt[:self._svd_dimensions,:]
+ vectors = numpy.transpose(numpy.dot(S, Dt))
+ self._Tt = numpy.transpose(T)
+
+ # call abstract method to cluster the vectors
+ self.cluster_vectorspace(vectors, trace)
+
+ # assign the vectors to clusters
+ if assign_clusters:
+ print(self._Tt, vectors)
+ return [self.classify(vector) for vector in vectors]
+
+ def cluster_vectorspace(self, vectors, trace):
+ """
+ Finds the clusters using the given set of vectors.
+ """
+ raise NotImplementedError()
+
+ def classify(self, vector):
+ if self._should_normalise:
+ vector = self._normalise(vector)
+ if self._Tt is not None:
+ vector = numpy.dot(self._Tt, vector)
+ cluster = self.classify_vectorspace(vector)
+ return self.cluster_name(cluster)
+
+ def classify_vectorspace(self, vector):
+ """
+ Returns the index of the appropriate cluster for the vector.
+ """
+ raise NotImplementedError()
+
+ def likelihood(self, vector, label):
+ if self._should_normalise:
+ vector = self._normalise(vector)
+ if self._Tt is not None:
+ vector = numpy.dot(self._Tt, vector)
+ return self.likelihood_vectorspace(vector, label)
+
+ def likelihood_vectorspace(self, vector, cluster):
+ """
+ Returns the likelihood of the vector belonging to the cluster.
+ """
+ predicted = self.classify_vectorspace(vector)
+ return (1.0 if cluster == predicted else 0.0)
+
+ def vector(self, vector):
+ """
+ Returns the vector after normalisation and dimensionality reduction
+ """
+ if self._should_normalise:
+ vector = self._normalise(vector)
+ if self._Tt is not None:
+ vector = numpy.dot(self._Tt, vector)
+ return vector
+
+ def _normalise(self, vector):
+ """
+ Normalises the vector to unit length.
+ """
+ return vector / sqrt(numpy.dot(vector, vector))
+
+def euclidean_distance(u, v):
+ """
+ Returns the euclidean distance between vectors u and v. This is equivalent
+ to the length of the vector (u - v).
+ """
+ diff = u - v
+ return sqrt(numpy.dot(diff, diff))
+
+def cosine_distance(u, v):
+ """
+ Returns 1 minus the cosine of the angle between vectors v and u. This is equal to
+ 1 - (u.v / |u||v|).
+ """
+ return 1 - (numpy.dot(u, v) / (sqrt(numpy.dot(u, u)) * sqrt(numpy.dot(v, v))))
+
+class _DendrogramNode(object):
+ """ Tree node of a dendrogram. """
+
+ def __init__(self, value, *children):
+ self._value = value
+ self._children = children
+
+ def leaves(self, values=True):
+ if self._children:
+ leaves = []
+ for child in self._children:
+ leaves.extend(child.leaves(values))
+ return leaves
+ elif values:
+ return [self._value]
+ else:
+ return [self]
+
+ def groups(self, n):
+ queue = [(self._value, self)]
+
+ while len(queue) < n:
+ priority, node = queue.pop()
+ if not node._children:
+ queue.push((priority, node))
+ break
+ for child in node._children:
+ if child._children:
+ queue.append((child._value, child))
+ else:
+ queue.append((0, child))
+ # makes the earliest merges at the start, latest at the end
+ queue.sort()
+
+ groups = []
+ for priority, node in queue:
+ groups.append(node.leaves())
+ return groups
+
+
+ at python_2_unicode_compatible
+class Dendrogram(object):
+ """
+ Represents a dendrogram, a tree with a specified branching order. This
+ must be initialised with the leaf items, then iteratively call merge for
+ each branch. This class constructs a tree representing the order of calls
+ to the merge function.
+ """
+
+ def __init__(self, items=[]):
+ """
+ :param items: the items at the leaves of the dendrogram
+ :type items: sequence of (any)
+ """
+ self._items = [_DendrogramNode(item) for item in items]
+ self._original_items = copy.copy(self._items)
+ self._merge = 1
+
+ def merge(self, *indices):
+ """
+ Merges nodes at given indices in the dendrogram. The nodes will be
+ combined which then replaces the first node specified. All other nodes
+ involved in the merge will be removed.
+
+ :param indices: indices of the items to merge (at least two)
+ :type indices: seq of int
+ """
+ assert len(indices) >= 2
+ node = _DendrogramNode(self._merge, *[self._items[i] for i in indices])
+ self._merge += 1
+ self._items[indices[0]] = node
+ for i in indices[1:]:
+ del self._items[i]
+
+ def groups(self, n):
+ """
+ Finds the n-groups of items (leaves) reachable from a cut at depth n.
+ :param n: number of groups
+ :type n: int
+ """
+ if len(self._items) > 1:
+ root = _DendrogramNode(self._merge, *self._items)
+ else:
+ root = self._items[0]
+ return root.groups(n)
+
+ def show(self, leaf_labels=[]):
+ """
+ Print the dendrogram in ASCII art to standard out.
+ :param leaf_labels: an optional list of strings to use for labeling the leaves
+ :type leaf_labels: list
+ """
+
+ # ASCII rendering characters
+ JOIN, HLINK, VLINK = '+', '-', '|'
+
+ # find the root (or create one)
+ if len(self._items) > 1:
+ root = _DendrogramNode(self._merge, *self._items)
+ else:
+ root = self._items[0]
+ leaves = self._original_items
+
+ if leaf_labels:
+ last_row = leaf_labels
+ else:
+ last_row = ["%s" % leaf._value for leaf in leaves]
+
+ # find the bottom row and the best cell width
+ width = max(map(len, last_row)) + 1
+ lhalf = width / 2
+ rhalf = width - lhalf - 1
+
+ # display functions
+ def format(centre, left=' ', right=' '):
+ return '%s%s%s' % (lhalf*left, centre, right*rhalf)
+ def display(str):
+ stdout.write(str)
+
+ # for each merge, top down
+ queue = [(root._value, root)]
+ verticals = [ format(' ') for leaf in leaves ]
+ while queue:
+ priority, node = queue.pop()
+ child_left_leaf = list(map(lambda c: c.leaves(False)[0], node._children))
+ indices = list(map(leaves.index, child_left_leaf))
+ if child_left_leaf:
+ min_idx = min(indices)
+ max_idx = max(indices)
+ for i in range(len(leaves)):
+ if leaves[i] in child_left_leaf:
+ if i == min_idx: display(format(JOIN, ' ', HLINK))
+ elif i == max_idx: display(format(JOIN, HLINK, ' '))
+ else: display(format(JOIN, HLINK, HLINK))
+ verticals[i] = format(VLINK)
+ elif min_idx <= i <= max_idx:
+ display(format(HLINK, HLINK, HLINK))
+ else:
+ display(verticals[i])
+ display('\n')
+ for child in node._children:
+ if child._children:
+ queue.append((child._value, child))
+ queue.sort()
+
+ for vertical in verticals:
+ display(vertical)
+ display('\n')
+
+ # finally, display the last line
+ display(''.join(item.center(width) for item in last_row))
+ display('\n')
+
+ def __repr__(self):
+ if len(self._items) > 1:
+ root = _DendrogramNode(self._merge, *self._items)
+ else:
+ root = self._items[0]
+ leaves = root.leaves(False)
+ return '<Dendrogram with %d leaves>' % len(leaves)
+
+
diff --git a/nltk/collocations.py b/nltk/collocations.py
new file mode 100644
index 0000000..1407577
--- /dev/null
+++ b/nltk/collocations.py
@@ -0,0 +1,366 @@
+# Natural Language Toolkit: Collocations and Association Measures
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Joel Nothman <jnothman at student.usyd.edu.au>
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+#
+"""
+Tools to identify collocations --- words that often appear consecutively
+--- within corpora. They may also be used to find other associations between
+word occurrences.
+See Manning and Schutze ch. 5 at http://nlp.stanford.edu/fsnlp/promo/colloc.pdf
+and the Text::NSP Perl package at http://ngram.sourceforge.net
+
+Finding collocations requires first calculating the frequencies of words and
+their appearance in the context of other words. Often the collection of words
+will then requiring filtering to only retain useful content terms. Each ngram
+of words may then be scored according to some association measure, in order
+to determine the relative likelihood of each ngram being a collocation.
+
+The ``BigramCollocationFinder`` and ``TrigramCollocationFinder`` classes provide
+these functionalities, dependent on being provided a function which scores a
+ngram given appropriate frequency counts. A number of standard association
+measures are provided in bigram_measures and trigram_measures.
+"""
+from __future__ import print_function
+
+# Possible TODOs:
+# - consider the distinction between f(x,_) and f(x) and whether our
+# approximation is good enough for fragmented data, and mention it
+# - add a n-gram collocation finder with measures which only utilise n-gram
+# and unigram counts (raw_freq, pmi, student_t)
+
+import itertools as _itertools
+from nltk.compat import iteritems
+
+from nltk.probability import FreqDist
+from nltk.util import ngrams
+from nltk.metrics import ContingencyMeasures, BigramAssocMeasures, TrigramAssocMeasures
+from nltk.metrics.spearman import ranks_from_scores, spearman_correlation
+
+
+class AbstractCollocationFinder(object):
+ """
+ An abstract base class for collocation finders whose purpose is to
+ collect collocation candidate frequencies, filter and rank them.
+
+ As a minimum, collocation finders require the frequencies of each
+ word in a corpus, and the joint frequency of word tuples. This data
+ should be provided through nltk.probability.FreqDist objects or an
+ identical interface.
+ """
+
+ def __init__(self, word_fd, ngram_fd):
+ self.word_fd = word_fd
+ self.ngram_fd = ngram_fd
+
+ @classmethod
+ def from_documents(cls, documents):
+ """Constructs a collocation finder given a collection of documents,
+ each of which is a list (or iterable) of tokens.
+ """
+ return cls.from_words(_itertools.chain(*documents))
+
+ @staticmethod
+ def _ngram_freqdist(words, n):
+ return FreqDist(tuple(words[i:i+n]) for i in range(len(words)-1))
+
+ def _apply_filter(self, fn=lambda ngram, freq: False):
+ """Generic filter removes ngrams from the frequency distribution
+ if the function returns True when passed an ngram tuple.
+ """
+ tmp_ngram = FreqDist()
+ for ngram, freq in iteritems(self.ngram_fd):
+ if not fn(ngram, freq):
+ tmp_ngram[ngram] = freq
+ self.ngram_fd = tmp_ngram
+
+ def apply_freq_filter(self, min_freq):
+ """Removes candidate ngrams which have frequency less than min_freq."""
+ self._apply_filter(lambda ng, freq: freq < min_freq)
+
+ def apply_ngram_filter(self, fn):
+ """Removes candidate ngrams (w1, w2, ...) where fn(w1, w2, ...)
+ evaluates to True.
+ """
+ self._apply_filter(lambda ng, f: fn(*ng))
+
+ def apply_word_filter(self, fn):
+ """Removes candidate ngrams (w1, w2, ...) where any of (fn(w1), fn(w2),
+ ...) evaluates to True.
+ """
+ self._apply_filter(lambda ng, f: any(fn(w) for w in ng))
+
+ def _score_ngrams(self, score_fn):
+ """Generates of (ngram, score) pairs as determined by the scoring
+ function provided.
+ """
+ for tup in self.ngram_fd:
+ score = self.score_ngram(score_fn, *tup)
+ if score is not None:
+ yield tup, score
+
+ def score_ngrams(self, score_fn):
+ """Returns a sequence of (ngram, score) pairs ordered from highest to
+ lowest score, as determined by the scoring function provided.
+ """
+ return sorted(self._score_ngrams(score_fn), key=lambda t: (-t[1], t[0]))
+
+ def nbest(self, score_fn, n):
+ """Returns the top n ngrams when scored by the given function."""
+ return [p for p, s in self.score_ngrams(score_fn)[:n]]
+
+ def above_score(self, score_fn, min_score):
+ """Returns a sequence of ngrams, ordered by decreasing score, whose
+ scores each exceed the given minimum score.
+ """
+ for ngram, score in self.score_ngrams(score_fn):
+ if score > min_score:
+ yield ngram
+ else:
+ break
+
+
+class BigramCollocationFinder(AbstractCollocationFinder):
+ """A tool for the finding and ranking of bigram collocations or other
+ association measures. It is often useful to use from_words() rather than
+ constructing an instance directly.
+ """
+
+ def __init__(self, word_fd, bigram_fd, window_size=2):
+ """Construct a BigramCollocationFinder, given FreqDists for
+ appearances of words and (possibly non-contiguous) bigrams.
+ """
+ AbstractCollocationFinder.__init__(self, word_fd, bigram_fd)
+ self.window_size = window_size
+
+ @classmethod
+ def from_words(cls, words, window_size=2):
+ """Construct a BigramCollocationFinder for all bigrams in the given
+ sequence. When window_size > 2, count non-contiguous bigrams, in the
+ style of Church and Hanks's (1990) association ratio.
+ """
+ wfd = FreqDist()
+ bfd = FreqDist()
+
+ if window_size < 2:
+ raise ValueError("Specify window_size at least 2")
+
+ for window in ngrams(words, window_size, pad_right=True):
+ w1 = window[0]
+ wfd[w1] += 1
+ for w2 in window[1:]:
+ if w2 is not None:
+ bfd[(w1, w2)] += 1
+ return cls(wfd, bfd, window_size=window_size)
+
+ def score_ngram(self, score_fn, w1, w2):
+ """Returns the score for a given bigram using the given scoring
+ function. Following Church and Hanks (1990), counts are scaled by
+ a factor of 1/(window_size - 1).
+ """
+ n_all = self.word_fd.N()
+ n_ii = self.ngram_fd[(w1, w2)] / (self.window_size - 1.0)
+ if not n_ii:
+ return
+ n_ix = self.word_fd[w1]
+ n_xi = self.word_fd[w2]
+ return score_fn(n_ii, (n_ix, n_xi), n_all)
+
+
+class TrigramCollocationFinder(AbstractCollocationFinder):
+ """A tool for the finding and ranking of trigram collocations or other
+ association measures. It is often useful to use from_words() rather than
+ constructing an instance directly.
+ """
+
+ def __init__(self, word_fd, bigram_fd, wildcard_fd, trigram_fd):
+ """Construct a TrigramCollocationFinder, given FreqDists for
+ appearances of words, bigrams, two words with any word between them,
+ and trigrams.
+ """
+ AbstractCollocationFinder.__init__(self, word_fd, trigram_fd)
+ self.wildcard_fd = wildcard_fd
+ self.bigram_fd = bigram_fd
+
+ @classmethod
+ def from_words(cls, words, window_size=3):
+ """Construct a TrigramCollocationFinder for all trigrams in the given
+ sequence.
+ """
+ if window_size < 3:
+ raise ValueError("Specify window_size at least 3")
+
+ wfd = FreqDist()
+ wildfd = FreqDist()
+ bfd = FreqDist()
+ tfd = FreqDist()
+ for window in ngrams(words, window_size, pad_right=True):
+ w1 = window[0]
+ for w2, w3 in _itertools.combinations(window[1:], 2):
+ wfd[w1] += 1
+ if w2 is None:
+ continue
+ bfd[(w1, w2)] += 1
+ if w3 is None:
+ continue
+ wildfd[(w1, w3)] += 1
+ tfd[(w1, w2, w3)] += 1
+ return cls(wfd, bfd, wildfd, tfd)
+
+ def bigram_finder(self):
+ """Constructs a bigram collocation finder with the bigram and unigram
+ data from this finder. Note that this does not include any filtering
+ applied to this finder.
+ """
+ return BigramCollocationFinder(self.word_fd, self.bigram_fd)
+
+ def score_ngram(self, score_fn, w1, w2, w3):
+ """Returns the score for a given trigram using the given scoring
+ function.
+ """
+ n_all = self.word_fd.N()
+ n_iii = self.ngram_fd[(w1, w2, w3)]
+ if not n_iii:
+ return
+ n_iix = self.bigram_fd[(w1, w2)]
+ n_ixi = self.wildcard_fd[(w1, w3)]
+ n_xii = self.bigram_fd[(w2, w3)]
+ n_ixx = self.word_fd[w1]
+ n_xix = self.word_fd[w2]
+ n_xxi = self.word_fd[w3]
+ return score_fn(n_iii,
+ (n_iix, n_ixi, n_xii),
+ (n_ixx, n_xix, n_xxi),
+ n_all)
+
+
+class QuadgramCollocationFinder(AbstractCollocationFinder):
+ """A tool for the finding and ranking of quadgram collocations or other association measures.
+ It is often useful to use from_words() rather than constructing an instance directly.
+ """
+
+ def __init__(self, word_fd, quadgram_fd, ii, iii, ixi, ixxi, iixi, ixii):
+ """Construct a QuadgramCollocationFinder, given FreqDists for appearances of words,
+ bigrams, trigrams, two words with one word and two words between them, three words
+ with a word between them in both variations.
+ """
+ AbstractCollocationFinder.__init__(self, word_fd, quadgram_fd)
+ self.iii = iii
+ self.ii = ii
+ self.ixi = ixi
+ self.ixxi = ixxi
+ self.iixi = iixi
+ self.ixii = ixii
+
+ @classmethod
+ def from_words(cls, words, window_size=4):
+ if window_size < 4:
+ raise ValueError("Specify window_size at least 4")
+ ixxx = FreqDist()
+ iiii = FreqDist()
+ ii = FreqDist()
+ iii = FreqDist()
+ ixi = FreqDist()
+ ixxi = FreqDist()
+ iixi = FreqDist()
+ ixii = FreqDist()
+
+ for window in ngrams(words, window_size, pad_right=True):
+ w1 = window[0]
+ for w2, w3, w4 in _itertools.combinations(window[1:], 3):
+ ixxx[w1] += 1
+ if w2 is None:
+ continue
+ ii[(w1, w2)] += 1
+ if w3 is None:
+ continue
+ iii[(w1, w2, w3)] += 1
+ ixi[(w1, w3)] += 1
+ if w4 is None:
+ continue
+ iiii[(w1, w2, w3, w4)] += 1
+ ixxi[(w1, w4)] += 1
+ ixii[(w1, w3, w4)] += 1
+ iixi[(w1, w2, w4)] += 1
+
+ return cls(ixxx, iiii, ii, iii, ixi, ixxi, iixi, ixii)
+
+ def score_ngram(self, score_fn, w1, w2, w3, w4):
+ n_all = self.word_fd.N()
+ n_iiii = self.ngram_fd[(w1, w2, w3, w4)]
+ if not n_iiii:
+ return
+ n_iiix = self.iii[(w1, w2, w3)]
+ n_xiii = self.iii[(w2, w3, w4)]
+ n_iixi = self.iixi[(w1, w2, w4)]
+ n_ixii = self.ixii[(w1, w3, w4)]
+
+ n_iixx = self.ii[(w1, w2)]
+ n_xxii = self.ii[(w3, w4)]
+ n_xiix = self.ii[(w2, w3)]
+ n_ixix = self.ixi[(w1, w3)]
+ n_ixxi = self.ixxi[(w1, w4)]
+ n_xixi = self.ixi[(w2, w4)]
+
+ n_ixxx = self.word_fd[w1]
+ n_xixx = self.word_fd[w2]
+ n_xxix = self.word_fd[w3]
+ n_xxxi = self.word_fd[w4]
+ return score_fn(n_iiii,
+ (n_iiix, n_iixi, n_ixii, n_xiii),
+ (n_iixx, n_ixix, n_ixxi, n_xixi, n_xxii, n_xiix),
+ (n_ixxx, n_xixx, n_xxix, n_xxxi),
+ n_all)
+
+
+def demo(scorer=None, compare_scorer=None):
+ """Finds bigram collocations in the files of the WebText corpus."""
+ from nltk.metrics import BigramAssocMeasures, spearman_correlation, ranks_from_scores
+
+ if scorer is None:
+ scorer = BigramAssocMeasures.likelihood_ratio
+ if compare_scorer is None:
+ compare_scorer = BigramAssocMeasures.raw_freq
+
+ from nltk.corpus import stopwords, webtext
+
+ ignored_words = stopwords.words('english')
+ word_filter = lambda w: len(w) < 3 or w.lower() in ignored_words
+
+ for file in webtext.fileids():
+ words = [word.lower()
+ for word in webtext.words(file)]
+
+ cf = BigramCollocationFinder.from_words(words)
+ cf.apply_freq_filter(3)
+ cf.apply_word_filter(word_filter)
+
+ print(file)
+ print('\t', [' '.join(tup) for tup in cf.nbest(scorer, 15)])
+ print('\t Correlation to %s: %0.4f' % (compare_scorer.__name__,
+ spearman_correlation(
+ ranks_from_scores(cf.score_ngrams(scorer)),
+ ranks_from_scores(cf.score_ngrams(compare_scorer)))))
+
+# Slows down loading too much
+# bigram_measures = BigramAssocMeasures()
+# trigram_measures = TrigramAssocMeasures()
+
+if __name__ == '__main__':
+ import sys
+ from nltk.metrics import BigramAssocMeasures
+
+ try:
+ scorer = eval('BigramAssocMeasures.' + sys.argv[1])
+ except IndexError:
+ scorer = None
+ try:
+ compare_scorer = eval('BigramAssocMeasures.' + sys.argv[2])
+ except IndexError:
+ compare_scorer = None
+
+ demo(scorer, compare_scorer)
+
+__all__ = ['BigramCollocationFinder', 'TrigramCollocationFinder', 'QuadgramCollocationFinder']
diff --git a/nltk/compat.py b/nltk/compat.py
new file mode 100755
index 0000000..df95006
--- /dev/null
+++ b/nltk/compat.py
@@ -0,0 +1,499 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Compatibility
+#
+# Copyright (C) 2001-2014 NLTK Project
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import absolute_import, print_function
+import sys
+import types
+from functools import wraps
+
+# Python 2/3 compatibility layer. Based on six.
+
+PY3 = sys.version_info[0] == 3
+PY26 = sys.version_info[:2] == (2, 6)
+
+if PY3:
+ def b(s):
+ return s.encode("latin-1")
+ def u(s):
+ return s
+
+ string_types = str,
+ integer_types = int,
+ class_types = type,
+ text_type = str
+ binary_type = bytes
+
+ MAXSIZE = sys.maxsize
+ get_im_class = lambda meth: meth.__self__.__class__
+ xrange = range
+ _iterkeys = "keys"
+ _itervalues = "values"
+ _iteritems = "items"
+ from imp import reload
+ raw_input = input
+
+ imap = map
+ izip = zip
+
+ import io
+ StringIO = io.StringIO
+ BytesIO = io.BytesIO
+
+ import html.entities as htmlentitydefs
+ from urllib.request import (urlopen, ProxyHandler, build_opener,
+ install_opener, getproxies, HTTPPasswordMgrWithDefaultRealm,
+ ProxyBasicAuthHandler, ProxyDigestAuthHandler, Request)
+ from urllib.error import HTTPError, URLError
+ from urllib.parse import quote_plus, unquote_plus, urlencode
+
+ from collections import Counter
+
+else:
+ def b(s):
+ return s
+ def u(s):
+ return unicode(s, "unicode_escape")
+
+ string_types = basestring,
+ integer_types = (int, long)
+ class_types = (type, types.ClassType)
+ text_type = unicode
+ binary_type = str
+ get_im_class = lambda meth: meth.im_class
+ xrange = xrange
+ _iterkeys = "iterkeys"
+ _itervalues = "itervalues"
+ _iteritems = "iteritems"
+ reload = reload
+ raw_input = raw_input
+
+ from itertools import imap, izip
+
+ try:
+ from cStringIO import StringIO
+ except ImportError:
+ from StringIO import StringIO
+ BytesIO = StringIO
+
+ import htmlentitydefs
+ from urllib2 import (urlopen, HTTPError, URLError,
+ ProxyHandler, build_opener, install_opener,
+ HTTPPasswordMgrWithDefaultRealm, ProxyBasicAuthHandler,
+ ProxyDigestAuthHandler, Request)
+ from urllib import getproxies, quote_plus, unquote_plus, urlencode
+
+ # Maps py2 tkinter package structure to py3 using import hook (PEP 302)
+ class TkinterPackage(object):
+ def __init__(self):
+ self.mod = __import__("Tkinter")
+ self.__path__ = ["nltk_py2_tkinter_package_path"]
+ def __getattr__(self, name):
+ return getattr(self.mod, name)
+
+ class TkinterLoader(object):
+ def __init__(self):
+ # module name mapping from py3 to py2
+ self.module_map = {
+ "tkinter": "Tkinter",
+ "tkinter.filedialog": "tkFileDialog",
+ "tkinter.font": "tkFont",
+ "tkinter.messagebox": "tkMessageBox",
+ }
+ def find_module(self, name, path=None):
+ # we are only interested in tkinter modules listed
+ # in self.module_map
+ if name in self.module_map:
+ return self
+ def load_module(self, name):
+ if name not in sys.modules:
+ if name == 'tkinter':
+ mod = TkinterPackage()
+ else:
+ mod = __import__(self.module_map[name])
+ sys.modules[name] = mod
+ return sys.modules[name]
+
+ sys.meta_path.insert(0, TkinterLoader())
+
+ if PY26:
+ from operator import itemgetter
+ from heapq import nlargest
+ from itertools import repeat, ifilter
+
+ class Counter(dict):
+ '''Dict subclass for counting hashable objects. Sometimes called a bag
+ or multiset. Elements are stored as dictionary keys and their counts
+ are stored as dictionary values.
+
+ >>> Counter('zyzygy')
+ Counter({'y': 3, 'z': 2, 'g': 1})
+
+ '''
+
+ def __init__(self, iterable=None, **kwds):
+ '''Create a new, empty Counter object. And if given, count elements
+ from an input iterable. Or, initialize the count from another mapping
+ of elements to their counts.
+
+ >>> Counter() # a new, empty counter
+ >>> Counter('gallahad') # a new counter from an iterable
+ >>> Counter({'a': 4, 'b': 2}) # a new counter from a mapping
+ >>> Counter(a=4, b=2) # a new counter from keyword args
+
+ '''
+ self.update(iterable, **kwds)
+
+ def __missing__(self, key):
+ return 0
+
+ def most_common(self, n=None):
+ '''List the n most common elements and their counts from the most
+ common to the least. If n is None, then list all element counts.
+
+ >>> Counter('abracadabra').most_common(3)
+ [('a', 5), ('r', 2), ('b', 2)]
+
+ '''
+ if n is None:
+ return sorted(self.iteritems(), key=itemgetter(1), reverse=True)
+ return nlargest(n, self.iteritems(), key=itemgetter(1))
+
+ def elements(self):
+ '''Iterator over elements repeating each as many times as its count.
+
+ >>> c = Counter('ABCABC')
+ >>> sorted(c.elements())
+ ['A', 'A', 'B', 'B', 'C', 'C']
+
+ If an element's count has been set to zero or is a negative number,
+ elements() will ignore it.
+
+ '''
+ for elem, count in self.iteritems():
+ for _ in repeat(None, count):
+ yield elem
+
+ # Override dict methods where the meaning changes for Counter objects.
+
+ @classmethod
+ def fromkeys(cls, iterable, v=None):
+ raise NotImplementedError(
+ 'Counter.fromkeys() is undefined. Use Counter(iterable) instead.')
+
+ def update(self, iterable=None, **kwds):
+ '''Like dict.update() but add counts instead of replacing them.
+
+ Source can be an iterable, a dictionary, or another Counter instance.
+
+ >>> c = Counter('which')
+ >>> c.update('witch') # add elements from another iterable
+ >>> d = Counter('watch')
+ >>> c.update(d) # add elements from another counter
+ >>> c['h'] # four 'h' in which, witch, and watch
+ 4
+
+ '''
+ if iterable is not None:
+ if hasattr(iterable, 'iteritems'):
+ if self:
+ self_get = self.get
+ for elem, count in iterable.iteritems():
+ self[elem] = self_get(elem, 0) + count
+ else:
+ dict.update(self, iterable) # fast path when counter is empty
+ else:
+ self_get = self.get
+ for elem in iterable:
+ self[elem] = self_get(elem, 0) + 1
+ if kwds:
+ self.update(kwds)
+
+ def copy(self):
+ 'Like dict.copy() but returns a Counter instance instead of a dict.'
+ return Counter(self)
+
+ def __delitem__(self, elem):
+ 'Like dict.__delitem__() but does not raise KeyError for missing values.'
+ if elem in self:
+ dict.__delitem__(self, elem)
+
+ def __repr__(self):
+ if not self:
+ return '%s()' % self.__class__.__name__
+ items = ', '.join(map('%r: %r'.__mod__, self.most_common()))
+ return '%s({%s})' % (self.__class__.__name__, items)
+
+ # Multiset-style mathematical operations discussed in:
+ # Knuth TAOCP Volume II section 4.6.3 exercise 19
+ # and at http://en.wikipedia.org/wiki/Multiset
+ #
+ # Outputs guaranteed to only include positive counts.
+ #
+ # To strip negative and zero counts, add-in an empty counter:
+ # c += Counter()
+
+ def __add__(self, other):
+ '''Add counts from two counters.
+
+ >>> Counter('abbb') + Counter('bcc')
+ Counter({'b': 4, 'c': 2, 'a': 1})
+
+
+ '''
+ if not isinstance(other, Counter):
+ return NotImplemented
+ result = Counter()
+ for elem in set(self) | set(other):
+ newcount = self[elem] + other[elem]
+ if newcount > 0:
+ result[elem] = newcount
+ return result
+
+ def __sub__(self, other):
+ ''' Subtract count, but keep only results with positive counts.
+
+ >>> Counter('abbbc') - Counter('bccd')
+ Counter({'b': 2, 'a': 1})
+
+ '''
+ if not isinstance(other, Counter):
+ return NotImplemented
+ result = Counter()
+ for elem in set(self) | set(other):
+ newcount = self[elem] - other[elem]
+ if newcount > 0:
+ result[elem] = newcount
+ return result
+
+ def __or__(self, other):
+ '''Union is the maximum of value in either of the input counters.
+
+ >>> Counter('abbb') | Counter('bcc')
+ Counter({'b': 3, 'c': 2, 'a': 1})
+
+ '''
+ if not isinstance(other, Counter):
+ return NotImplemented
+ _max = max
+ result = Counter()
+ for elem in set(self) | set(other):
+ newcount = _max(self[elem], other[elem])
+ if newcount > 0:
+ result[elem] = newcount
+ return result
+
+ def __and__(self, other):
+ ''' Intersection is the minimum of corresponding counts.
+
+ >>> Counter('abbb') & Counter('bcc')
+ Counter({'b': 1})
+
+ '''
+ if not isinstance(other, Counter):
+ return NotImplemented
+ _min = min
+ result = Counter()
+ if len(self) < len(other):
+ self, other = other, self
+ for elem in ifilter(self.__contains__, other):
+ newcount = _min(self[elem], other[elem])
+ if newcount > 0:
+ result[elem] = newcount
+ return result
+
+ else:
+ from collections import Counter
+
+
+def iterkeys(d):
+ """Return an iterator over the keys of a dictionary."""
+ return getattr(d, _iterkeys)()
+
+def itervalues(d):
+ """Return an iterator over the values of a dictionary."""
+ return getattr(d, _itervalues)()
+
+def iteritems(d):
+ """Return an iterator over the (key, value) pairs of a dictionary."""
+ return getattr(d, _iteritems)()
+
+try:
+ from functools import total_ordering
+except ImportError: # python 2.6
+ def total_ordering(cls):
+ """Class decorator that fills in missing ordering methods"""
+ convert = {
+ '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
+ ('__le__', lambda self, other: self < other or self == other),
+ ('__ge__', lambda self, other: not self < other)],
+ '__le__': [('__ge__', lambda self, other: not self <= other or self == other),
+ ('__lt__', lambda self, other: self <= other and not self == other),
+ ('__gt__', lambda self, other: not self <= other)],
+ '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
+ ('__ge__', lambda self, other: self > other or self == other),
+ ('__le__', lambda self, other: not self > other)],
+ '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
+ ('__gt__', lambda self, other: self >= other and not self == other),
+ ('__lt__', lambda self, other: not self >= other)]
+ }
+ roots = set(dir(cls)) & set(convert)
+ if not roots:
+ raise ValueError('must define at least one ordering operation: < > <= >=')
+ root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
+ for opname, opfunc in convert[root]:
+ if opname not in roots:
+ opfunc.__name__ = opname
+ opfunc.__doc__ = getattr(int, opname).__doc__
+ setattr(cls, opname, opfunc)
+ return cls
+
+
+# ======= Compatibility for datasets that care about Python versions ========
+
+# The following datasets have a /PY3 subdirectory containing
+# a full copy of the data which has been re-encoded or repickled.
+_PY3_DATA_UPDATES = ["chunkers/maxent_ne_chunker",
+ "help/tagsets",
+ "taggers/maxent_treebank_pos_tagger",
+ "tokenizers/punkt"]
+
+# for use in adding /PY3 to the second (filename) argument
+# of the file pointers in data.py
+def py3_data(init_func):
+ def _decorator(*args, **kwargs):
+ if PY3:
+ path = args[1]
+ for item in _PY3_DATA_UPDATES:
+ if item in str(path) and "/PY3" not in str(path):
+ pos = path.index(item) + len(item)
+ if path[pos:pos+4] == ".zip":
+ pos += 4
+ path = path[:pos] + "/PY3" + path[pos:]
+ args = (args[0], path) + args[2:]
+ break
+ return init_func(*args, **kwargs)
+ return wraps(init_func)(_decorator)
+
+# ======= Compatibility layer for __str__ and __repr__ ==========
+
+import unicodedata
+import functools
+
+def remove_accents(text):
+
+ if isinstance(text, bytes):
+ text = text.decode('ascii')
+
+ category = unicodedata.category # this gives a small (~10%) speedup
+ return ''.join(
+ c for c in unicodedata.normalize('NFKD', text) if category(c) != 'Mn'
+ )
+
+# Select the best transliteration method:
+try:
+ # Older versions of Unidecode are licensed under Artistic License;
+ # assume an older version is installed.
+ from unidecode import unidecode as transliterate
+except ImportError:
+ try:
+ # text-unidecode implementation is worse than Unidecode
+ # implementation so Unidecode is preferred.
+ from text_unidecode import unidecode as transliterate
+ except ImportError:
+ # This transliteration method should be enough
+ # for many Western languages.
+ transliterate = remove_accents
+
+
+def python_2_unicode_compatible(klass):
+ """
+ This decorator defines __unicode__ method and fixes
+ __repr__ and __str__ methods under Python 2.
+
+ To support Python 2 and 3 with a single code base,
+ define __str__ and __repr__ methods returning unicode
+ text and apply this decorator to the class.
+
+ Original __repr__ and __str__ would be available
+ as unicode_repr and __unicode__ (under both Python 2
+ and Python 3).
+ """
+
+ if not issubclass(klass, object):
+ raise ValueError("This decorator doesn't work for old-style classes")
+
+ # both __unicode__ and unicode_repr are public because they
+ # may be useful in console under Python 2.x
+
+ # if __str__ or __repr__ are not overriden in a subclass,
+ # they may be already fixed by this decorator in a parent class
+ # and we shouldn't them again
+
+ if not _was_fixed(klass.__str__):
+ klass.__unicode__ = klass.__str__
+ if not PY3:
+ klass.__str__ = _7bit(_transliterated(klass.__unicode__))
+
+
+ if not _was_fixed(klass.__repr__):
+ klass.unicode_repr = klass.__repr__
+ if not PY3:
+ klass.__repr__ = _7bit(klass.unicode_repr)
+
+ return klass
+
+
+def unicode_repr(obj):
+ """
+ For classes that was fixed with @python_2_unicode_compatible
+ ``unicode_repr`` returns ``obj.unicode_repr()``; for unicode strings
+ the result is returned without "u" letter (to make output the
+ same under Python 2.x and Python 3.x); for other variables
+ it is the same as ``repr``.
+ """
+ if PY3:
+ return repr(obj)
+
+ # Python 2.x
+ if hasattr(obj, 'unicode_repr'):
+ return obj.unicode_repr()
+
+ if isinstance(obj, unicode):
+ return repr(obj)[1:] # strip "u" letter from output
+
+ return repr(obj)
+
+
+def _transliterated(method):
+ def wrapper(self):
+ return transliterate(method(self))
+
+ functools.update_wrapper(wrapper, method, ["__name__", "__doc__"])
+ if hasattr(method, "_nltk_compat_7bit"):
+ wrapper._nltk_compat_7bit = method._nltk_compat_7bit
+
+ wrapper._nltk_compat_transliterated = True
+ return wrapper
+
+
+def _7bit(method):
+ def wrapper(self):
+ return method(self).encode('ascii', 'backslashreplace')
+
+ functools.update_wrapper(wrapper, method, ["__name__", "__doc__"])
+
+ if hasattr(method, "_nltk_compat_transliterated"):
+ wrapper._nltk_compat_transliterated = method._nltk_compat_transliterated
+
+ wrapper._nltk_compat_7bit = True
+ return wrapper
+
+
+def _was_fixed(method):
+ return (getattr(method, "_nltk_compat_7bit", False) or
+ getattr(method, "_nltk_compat_transliterated", False))
diff --git a/nltk/corpus/__init__.py b/nltk/corpus/__init__.py
new file mode 100644
index 0000000..0323994
--- /dev/null
+++ b/nltk/corpus/__init__.py
@@ -0,0 +1,282 @@
+# Natural Language Toolkit: Corpus Readers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# [xx] this docstring isnt' up-to-date!
+"""
+NLTK corpus readers. The modules in this package provide functions
+that can be used to read corpus files in a variety of formats. These
+functions can be used to read both the corpus files that are
+distributed in the NLTK corpus package, and corpus files that are part
+of external corpora.
+
+Available Corpora
+=================
+
+Please see http://nltk.googlecode.com/svn/trunk/nltk_data/index.xml
+for a complete list. Install corpora using nltk.download().
+
+Corpus Reader Functions
+=======================
+Each corpus module defines one or more "corpus reader functions",
+which can be used to read documents from that corpus. These functions
+take an argument, ``item``, which is used to indicate which document
+should be read from the corpus:
+
+- If ``item`` is one of the unique identifiers listed in the corpus
+ module's ``items`` variable, then the corresponding document will
+ be loaded from the NLTK corpus package.
+- If ``item`` is a filename, then that file will be read.
+
+Additionally, corpus reader functions can be given lists of item
+names; in which case, they will return a concatenation of the
+corresponding documents.
+
+Corpus reader functions are named based on the type of information
+they return. Some common examples, and their return types, are:
+
+- words(): list of str
+- sents(): list of (list of str)
+- paras(): list of (list of (list of str))
+- tagged_words(): list of (str,str) tuple
+- tagged_sents(): list of (list of (str,str))
+- tagged_paras(): list of (list of (list of (str,str)))
+- chunked_sents(): list of (Tree w/ (str,str) leaves)
+- parsed_sents(): list of (Tree with str leaves)
+- parsed_paras(): list of (list of (Tree with str leaves))
+- xml(): A single xml ElementTree
+- raw(): unprocessed corpus contents
+
+For example, to read a list of the words in the Brown Corpus, use
+``nltk.corpus.brown.words()``:
+
+ >>> from nltk.corpus import brown
+ >>> print(", ".join(brown.words()))
+ The, Fulton, County, Grand, Jury, said, ...
+
+"""
+
+import re
+
+from nltk.tokenize import RegexpTokenizer
+from nltk.corpus.util import LazyCorpusLoader
+from nltk.corpus.reader import *
+
+abc = LazyCorpusLoader(
+ 'abc', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding=[
+ ('science', 'latin_1'),
+ ('rural', 'utf8')])
+alpino = LazyCorpusLoader(
+ 'alpino', AlpinoCorpusReader, tagset='alpino')
+brown = LazyCorpusLoader(
+ 'brown', CategorizedTaggedCorpusReader, r'c[a-z]\d\d',
+ cat_file='cats.txt', tagset='brown', encoding="ascii")
+cess_cat = LazyCorpusLoader(
+ 'cess_cat', BracketParseCorpusReader, r'(?!\.).*\.tbf',
+ tagset='unknown', encoding='ISO-8859-2')
+cess_esp = LazyCorpusLoader(
+ 'cess_esp', BracketParseCorpusReader, r'(?!\.).*\.tbf',
+ tagset='unknown', encoding='ISO-8859-2')
+cmudict = LazyCorpusLoader(
+ 'cmudict', CMUDictCorpusReader, ['cmudict'])
+comtrans = LazyCorpusLoader(
+ 'comtrans', AlignedCorpusReader, r'(?!\.).*\.txt')
+conll2000 = LazyCorpusLoader(
+ 'conll2000', ConllChunkCorpusReader,
+ ['train.txt', 'test.txt'], ('NP','VP','PP'),
+ tagset='wsj', encoding='ascii')
+conll2002 = LazyCorpusLoader(
+ 'conll2002', ConllChunkCorpusReader, '.*\.(test|train).*',
+ ('LOC', 'PER', 'ORG', 'MISC'), encoding='utf-8')
+conll2007 = LazyCorpusLoader(
+ 'conll2007', DependencyCorpusReader, '.*\.(test|train).*', encoding=[
+ ('eus', 'ISO-8859-2'),
+ ('esp', 'utf8')])
+dependency_treebank = LazyCorpusLoader(
+ 'dependency_treebank', DependencyCorpusReader, '.*\.dp',
+ encoding='ascii')
+floresta = LazyCorpusLoader(
+ 'floresta', BracketParseCorpusReader, r'(?!\.).*\.ptb', '#',
+ tagset='unknown', encoding='ISO-8859-15')
+framenet = LazyCorpusLoader(
+ 'framenet_v15', FramenetCorpusReader, ['frRelation.xml','frameIndex.xml','fulltextIndex.xml','luIndex.xml','semTypes.xml'])
+gazetteers = LazyCorpusLoader(
+ 'gazetteers', WordListCorpusReader, r'(?!LICENSE|\.).*\.txt',
+ encoding='ISO-8859-2')
+genesis = LazyCorpusLoader(
+ 'genesis', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding=[
+ ('finnish|french|german', 'latin_1'),
+ ('swedish', 'cp865'),
+ ('.*', 'utf_8')])
+gutenberg = LazyCorpusLoader(
+ 'gutenberg', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding='latin1')
+# corpus not available with NLTK; these lines caused help(nltk.corpus) to break
+#hebrew_treebank = LazyCorpusLoader(
+# 'hebrew_treebank', BracketParseCorpusReader, r'.*\.txt')
+ieer = LazyCorpusLoader(
+ 'ieer', IEERCorpusReader, r'(?!README|\.).*')
+inaugural = LazyCorpusLoader(
+ 'inaugural', PlaintextCorpusReader, r'(?!\.).*\.txt', encoding='latin1')
+# [XX] This should probably just use TaggedCorpusReader:
+indian = LazyCorpusLoader(
+ 'indian', IndianCorpusReader, r'(?!\.).*\.pos',
+ tagset='unknown', encoding='utf8')
+ipipan = LazyCorpusLoader(
+ 'ipipan', IPIPANCorpusReader, r'(?!\.).*morph\.xml')
+jeita = LazyCorpusLoader(
+ 'jeita', ChasenCorpusReader, r'.*\.chasen', encoding='utf-8')
+knbc = LazyCorpusLoader(
+ 'knbc/corpus1', KNBCorpusReader, r'.*/KN.*', encoding='euc-jp')
+lin_thesaurus = LazyCorpusLoader(
+ 'lin_thesaurus', LinThesaurusCorpusReader, r'.*\.lsp')
+mac_morpho = LazyCorpusLoader(
+ 'mac_morpho', MacMorphoCorpusReader, r'(?!\.).*\.txt',
+ tagset='unknown', encoding='latin-1')
+machado = LazyCorpusLoader(
+ 'machado', PortugueseCategorizedPlaintextCorpusReader,
+ r'(?!\.).*\.txt', cat_pattern=r'([a-z]*)/.*', encoding='latin-1')
+masc_tagged = LazyCorpusLoader(
+ 'masc_tagged', CategorizedTaggedCorpusReader, r'(spoken|written)/.*\.txt',
+ cat_file='categories.txt', tagset='wsj', encoding="ascii", sep="_")
+movie_reviews = LazyCorpusLoader(
+ 'movie_reviews', CategorizedPlaintextCorpusReader,
+ r'(?!\.).*\.txt', cat_pattern=r'(neg|pos)/.*',
+ encoding='ascii')
+names = LazyCorpusLoader(
+ 'names', WordListCorpusReader, r'(?!\.).*\.txt', encoding='ascii')
+nps_chat = LazyCorpusLoader(
+ 'nps_chat', NPSChatCorpusReader, r'(?!README|\.).*\.xml', tagset='wsj')
+pl196x = LazyCorpusLoader(
+ 'pl196x', Pl196xCorpusReader, r'[a-z]-.*\.xml',
+ cat_file='cats.txt', textid_file='textids.txt', encoding='utf8')
+ppattach = LazyCorpusLoader(
+ 'ppattach', PPAttachmentCorpusReader, ['training', 'test', 'devset'])
+ptb = LazyCorpusLoader( # Penn Treebank v3: WSJ and Brown portions
+ 'ptb', CategorizedBracketParseCorpusReader, r'(WSJ/\d\d/WSJ_\d\d|BROWN/C[A-Z]/C[A-Z])\d\d.MRG',
+ cat_file='allcats.txt', tagset='wsj')
+qc = LazyCorpusLoader(
+ 'qc', StringCategoryCorpusReader, ['train.txt', 'test.txt'], encoding='ISO-8859-2')
+reuters = LazyCorpusLoader(
+ 'reuters', CategorizedPlaintextCorpusReader, '(training|test).*',
+ cat_file='cats.txt', encoding='ISO-8859-2')
+rte = LazyCorpusLoader(
+ 'rte', RTECorpusReader, r'(?!\.).*\.xml')
+senseval = LazyCorpusLoader(
+ 'senseval', SensevalCorpusReader, r'(?!\.).*\.pos')
+sentiwordnet = LazyCorpusLoader(
+ 'sentiwordnet', SentiWordNetCorpusReader, 'SentiWordNet_3.0.0.txt', encoding='utf-8')
+shakespeare = LazyCorpusLoader(
+ 'shakespeare', XMLCorpusReader, r'(?!\.).*\.xml')
+sinica_treebank = LazyCorpusLoader(
+ 'sinica_treebank', SinicaTreebankCorpusReader, ['parsed'],
+ tagset='unknown', encoding='utf-8')
+state_union = LazyCorpusLoader(
+ 'state_union', PlaintextCorpusReader, r'(?!\.).*\.txt',
+ encoding='ISO-8859-2')
+stopwords = LazyCorpusLoader(
+ 'stopwords', WordListCorpusReader, r'(?!README|\.).*', encoding='utf8')
+swadesh = LazyCorpusLoader(
+ 'swadesh', SwadeshCorpusReader, r'(?!README|\.).*', encoding='utf8')
+switchboard = LazyCorpusLoader(
+ 'switchboard', SwitchboardCorpusReader, tagset='wsj')
+timit = LazyCorpusLoader(
+ 'timit', TimitCorpusReader)
+timit_tagged = LazyCorpusLoader(
+ 'timit', TimitTaggedCorpusReader, '.+\.tags',
+ tagset='wsj', encoding='ascii')
+toolbox = LazyCorpusLoader(
+ 'toolbox', ToolboxCorpusReader, r'(?!.*(README|\.)).*\.(dic|txt)')
+treebank = LazyCorpusLoader(
+ 'treebank/combined', BracketParseCorpusReader, r'wsj_.*\.mrg',
+ tagset='wsj', encoding='ascii')
+treebank_chunk = LazyCorpusLoader(
+ 'treebank/tagged', ChunkedCorpusReader, r'wsj_.*\.pos',
+ sent_tokenizer=RegexpTokenizer(r'(?<=/\.)\s*(?![^\[]*\])', gaps=True),
+ para_block_reader=tagged_treebank_para_block_reader, encoding='ascii')
+treebank_raw = LazyCorpusLoader(
+ 'treebank/raw', PlaintextCorpusReader, r'wsj_.*', encoding='ISO-8859-2')
+udhr = LazyCorpusLoader(
+ 'udhr', UdhrCorpusReader)
+udhr2 = LazyCorpusLoader(
+ 'udhr2', PlaintextCorpusReader, r'.*\.txt', encoding='utf8')
+verbnet = LazyCorpusLoader(
+ 'verbnet', VerbnetCorpusReader, r'(?!\.).*\.xml')
+webtext = LazyCorpusLoader(
+ 'webtext', PlaintextCorpusReader, r'(?!README|\.).*\.txt', encoding='ISO-8859-2')
+wordnet = LazyCorpusLoader(
+ 'wordnet', WordNetCorpusReader,
+ LazyCorpusLoader('omw', CorpusReader, r'.*/wn-data-.*\.tab', encoding='utf8'))
+wordnet_ic = LazyCorpusLoader(
+ 'wordnet_ic', WordNetICCorpusReader, '.*\.dat')
+words = LazyCorpusLoader(
+ 'words', WordListCorpusReader, r'(?!README|\.).*', encoding='ascii')
+ycoe = LazyCorpusLoader(
+ 'ycoe', YCOECorpusReader)
+# defined after treebank
+propbank = LazyCorpusLoader(
+ 'propbank', PropbankCorpusReader,
+ 'prop.txt', 'frames/.*\.xml', 'verbs.txt',
+ lambda filename: re.sub(r'^wsj/\d\d/', '', filename),
+ treebank) # Must be defined *after* treebank corpus.
+nombank = LazyCorpusLoader(
+ 'nombank.1.0', NombankCorpusReader,
+ 'nombank.1.0', 'frames/.*\.xml', 'nombank.1.0.words',
+ lambda filename: re.sub(r'^wsj/\d\d/', '', filename),
+ treebank) # Must be defined *after* treebank corpus.
+propbank_ptb = LazyCorpusLoader(
+ 'propbank', PropbankCorpusReader,
+ 'prop.txt', 'frames/.*\.xml', 'verbs.txt',
+ lambda filename: filename.upper(),
+ ptb) # Must be defined *after* ptb corpus.
+nombank_ptb = LazyCorpusLoader(
+ 'nombank.1.0', NombankCorpusReader,
+ 'nombank.1.0', 'frames/.*\.xml', 'nombank.1.0.words',
+ lambda filename: filename.upper(),
+ ptb) # Must be defined *after* ptb corpus.
+semcor = LazyCorpusLoader(
+ 'semcor', SemcorCorpusReader, r'brown./tagfiles/br-.*\.xml',
+ wordnet) # Must be defined *after* wordnet corpus.
+
+def demo():
+ # This is out-of-date:
+ abc.demo()
+ brown.demo()
+# chat80.demo()
+ cmudict.demo()
+ conll2000.demo()
+ conll2002.demo()
+ genesis.demo()
+ gutenberg.demo()
+ ieer.demo()
+ inaugural.demo()
+ indian.demo()
+ names.demo()
+ ppattach.demo()
+ senseval.demo()
+ shakespeare.demo()
+ sinica_treebank.demo()
+ state_union.demo()
+ stopwords.demo()
+ timit.demo()
+ toolbox.demo()
+ treebank.demo()
+ udhr.demo()
+ webtext.demo()
+ words.demo()
+# ycoe.demo()
+
+if __name__ == '__main__':
+ #demo()
+ pass
+
+# ** this is for nose **
+# unload all corpus after tests
+def teardown_module(module=None):
+ import nltk.corpus
+ for name in dir(nltk.corpus):
+ obj = getattr(nltk.corpus, name, None)
+ if isinstance(obj, CorpusReader) and hasattr(obj, '_unload'):
+ obj._unload()
diff --git a/nltk/corpus/europarl_raw.py b/nltk/corpus/europarl_raw.py
new file mode 100644
index 0000000..578fe5a
--- /dev/null
+++ b/nltk/corpus/europarl_raw.py
@@ -0,0 +1,44 @@
+# Natural Language Toolkit: Europarl Corpus Readers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Nitin Madnani <nmadnani at umiacs.umd.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import re
+from nltk.corpus.util import LazyCorpusLoader
+from nltk.corpus.reader import *
+
+# Create a new corpus reader instance for each European language
+danish = LazyCorpusLoader(
+ 'europarl_raw/danish', EuroparlCorpusReader, r'ep-.*\.da', encoding='utf-8')
+
+dutch = LazyCorpusLoader(
+ 'europarl_raw/dutch', EuroparlCorpusReader, r'ep-.*\.nl', encoding='utf-8')
+
+english = LazyCorpusLoader(
+ 'europarl_raw/english', EuroparlCorpusReader, r'ep-.*\.en', encoding='utf-8')
+
+finnish = LazyCorpusLoader(
+ 'europarl_raw/finnish', EuroparlCorpusReader, r'ep-.*\.fi', encoding='utf-8')
+
+french = LazyCorpusLoader(
+ 'europarl_raw/french', EuroparlCorpusReader, r'ep-.*\.fr', encoding='utf-8')
+
+german = LazyCorpusLoader(
+ 'europarl_raw/german', EuroparlCorpusReader, r'ep-.*\.de', encoding='utf-8')
+
+greek = LazyCorpusLoader(
+ 'europarl_raw/greek', EuroparlCorpusReader, r'ep-.*\.el', encoding='utf-8')
+
+italian = LazyCorpusLoader(
+ 'europarl_raw/italian', EuroparlCorpusReader, r'ep-.*\.it', encoding='utf-8')
+
+portuguese = LazyCorpusLoader(
+ 'europarl_raw/portuguese', EuroparlCorpusReader, r'ep-.*\.pt', encoding='utf-8')
+
+spanish = LazyCorpusLoader(
+ 'europarl_raw/spanish', EuroparlCorpusReader, r'ep-.*\.es', encoding='utf-8')
+
+swedish = LazyCorpusLoader(
+ 'europarl_raw/swedish', EuroparlCorpusReader, r'ep-.*\.sv', encoding='utf-8')
diff --git a/nltk/corpus/reader/__init__.py b/nltk/corpus/reader/__init__.py
new file mode 100644
index 0000000..ef3e9d9
--- /dev/null
+++ b/nltk/corpus/reader/__init__.py
@@ -0,0 +1,131 @@
+# Natural Language Toolkit: Corpus Readers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+NLTK corpus readers. The modules in this package provide functions
+that can be used to read corpus fileids in a variety of formats. These
+functions can be used to read both the corpus fileids that are
+distributed in the NLTK corpus package, and corpus fileids that are part
+of external corpora.
+
+Corpus Reader Functions
+=======================
+Each corpus module defines one or more "corpus reader functions",
+which can be used to read documents from that corpus. These functions
+take an argument, ``item``, which is used to indicate which document
+should be read from the corpus:
+
+- If ``item`` is one of the unique identifiers listed in the corpus
+ module's ``items`` variable, then the corresponding document will
+ be loaded from the NLTK corpus package.
+- If ``item`` is a fileid, then that file will be read.
+
+Additionally, corpus reader functions can be given lists of item
+names; in which case, they will return a concatenation of the
+corresponding documents.
+
+Corpus reader functions are named based on the type of information
+they return. Some common examples, and their return types, are:
+
+- words(): list of str
+- sents(): list of (list of str)
+- paras(): list of (list of (list of str))
+- tagged_words(): list of (str,str) tuple
+- tagged_sents(): list of (list of (str,str))
+- tagged_paras(): list of (list of (list of (str,str)))
+- chunked_sents(): list of (Tree w/ (str,str) leaves)
+- parsed_sents(): list of (Tree with str leaves)
+- parsed_paras(): list of (list of (Tree with str leaves))
+- xml(): A single xml ElementTree
+- raw(): unprocessed corpus contents
+
+For example, to read a list of the words in the Brown Corpus, use
+``nltk.corpus.brown.words()``:
+
+ >>> from nltk.corpus import brown
+ >>> print(", ".join(brown.words()))
+ The, Fulton, County, Grand, Jury, said, ...
+
+"""
+
+from nltk.corpus.reader.plaintext import *
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.tagged import *
+from nltk.corpus.reader.cmudict import *
+from nltk.corpus.reader.conll import *
+from nltk.corpus.reader.chunked import *
+from nltk.corpus.reader.wordlist import *
+from nltk.corpus.reader.xmldocs import *
+from nltk.corpus.reader.ppattach import *
+from nltk.corpus.reader.senseval import *
+from nltk.corpus.reader.ieer import *
+from nltk.corpus.reader.sinica_treebank import *
+from nltk.corpus.reader.bracket_parse import *
+from nltk.corpus.reader.indian import *
+from nltk.corpus.reader.toolbox import *
+from nltk.corpus.reader.timit import *
+from nltk.corpus.reader.ycoe import *
+from nltk.corpus.reader.rte import *
+from nltk.corpus.reader.string_category import *
+from nltk.corpus.reader.propbank import *
+from nltk.corpus.reader.verbnet import *
+from nltk.corpus.reader.bnc import *
+from nltk.corpus.reader.nps_chat import *
+from nltk.corpus.reader.wordnet import *
+from nltk.corpus.reader.switchboard import *
+from nltk.corpus.reader.dependency import *
+from nltk.corpus.reader.nombank import *
+from nltk.corpus.reader.ipipan import *
+from nltk.corpus.reader.pl196x import *
+from nltk.corpus.reader.knbc import *
+from nltk.corpus.reader.chasen import *
+from nltk.corpus.reader.childes import *
+from nltk.corpus.reader.aligned import *
+from nltk.corpus.reader.lin import *
+from nltk.corpus.reader.semcor import *
+from nltk.corpus.reader.framenet import *
+from nltk.corpus.reader.udhr import *
+from nltk.corpus.reader.bnc import *
+from nltk.corpus.reader.sentiwordnet import *
+
+# Make sure that nltk.corpus.reader.bracket_parse gives the module, not
+# the function bracket_parse() defined in nltk.tree:
+from nltk.corpus.reader import bracket_parse
+
+__all__ = [
+ 'CorpusReader', 'CategorizedCorpusReader',
+ 'PlaintextCorpusReader', 'find_corpus_fileids',
+ 'TaggedCorpusReader', 'CMUDictCorpusReader',
+ 'ConllChunkCorpusReader', 'WordListCorpusReader',
+ 'PPAttachmentCorpusReader', 'SensevalCorpusReader',
+ 'IEERCorpusReader', 'ChunkedCorpusReader',
+ 'SinicaTreebankCorpusReader', 'BracketParseCorpusReader',
+ 'IndianCorpusReader', 'ToolboxCorpusReader',
+ 'TimitCorpusReader', 'YCOECorpusReader',
+ 'MacMorphoCorpusReader', 'SyntaxCorpusReader',
+ 'AlpinoCorpusReader', 'RTECorpusReader',
+ 'StringCategoryCorpusReader','EuroparlCorpusReader',
+ 'CategorizedBracketParseCorpusReader',
+ 'CategorizedTaggedCorpusReader',
+ 'CategorizedPlaintextCorpusReader',
+ 'PortugueseCategorizedPlaintextCorpusReader',
+ 'tagged_treebank_para_block_reader',
+ 'PropbankCorpusReader', 'VerbnetCorpusReader',
+ 'BNCCorpusReader', 'ConllCorpusReader',
+ 'XMLCorpusReader', 'NPSChatCorpusReader',
+ 'SwadeshCorpusReader', 'WordNetCorpusReader',
+ 'WordNetICCorpusReader', 'SwitchboardCorpusReader',
+ 'DependencyCorpusReader', 'NombankCorpusReader',
+ 'IPIPANCorpusReader', 'Pl196xCorpusReader',
+ 'TEICorpusView', 'KNBCorpusReader', 'ChasenCorpusReader',
+ 'CHILDESCorpusReader', 'AlignedCorpusReader',
+ 'TimitTaggedCorpusReader', 'LinThesaurusCorpusReader',
+ 'SemcorCorpusReader', 'FramenetCorpusReader', 'UdhrCorpusReader',
+ 'BNCCorpusReader', 'SentiWordNetCorpusReader', 'SentiSynset'
+]
diff --git a/nltk/corpus/reader/aligned.py b/nltk/corpus/reader/aligned.py
new file mode 100644
index 0000000..1e11011
--- /dev/null
+++ b/nltk/corpus/reader/aligned.py
@@ -0,0 +1,114 @@
+# Natural Language Toolkit: Aligned Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# For license information, see LICENSE.TXT
+
+from nltk import compat
+from nltk.tokenize import WhitespaceTokenizer, RegexpTokenizer
+from nltk.align import AlignedSent
+
+from nltk.corpus.reader.api import CorpusReader
+from nltk.corpus.reader.util import StreamBackedCorpusView, concat,\
+ read_alignedsent_block
+
+class AlignedCorpusReader(CorpusReader):
+ """
+ Reader for corpora of word-aligned sentences. Tokens are assumed
+ to be separated by whitespace. Sentences begin on separate lines.
+ """
+ def __init__(self, root, fileids,
+ sep='/', word_tokenizer=WhitespaceTokenizer(),
+ sent_tokenizer=RegexpTokenizer('\n', gaps=True),
+ alignedsent_block_reader=read_alignedsent_block,
+ encoding='latin1'):
+ """
+ Construct a new Aligned Corpus reader for a set of documents
+ located at the given root directory. Example usage:
+
+ >>> root = '/...path to corpus.../'
+ >>> reader = AlignedCorpusReader(root, '.*', '.txt') # doctest: +SKIP
+
+ :param root: The root directory for this corpus.
+ :param fileids: A list or regexp specifying the fileids in this corpus.
+ """
+ CorpusReader.__init__(self, root, fileids, encoding)
+ self._sep = sep
+ self._word_tokenizer = word_tokenizer
+ self._sent_tokenizer = sent_tokenizer
+ self._alignedsent_block_reader = alignedsent_block_reader
+
+ def raw(self, fileids=None):
+ """
+ :return: the given file(s) as a single string.
+ :rtype: str
+ """
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def words(self, fileids=None):
+ """
+ :return: the given file(s) as a list of words
+ and punctuation symbols.
+ :rtype: list(str)
+ """
+ return concat([AlignedSentCorpusView(fileid, enc, False, False,
+ self._word_tokenizer,
+ self._sent_tokenizer,
+ self._alignedsent_block_reader)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ sentences or utterances, each encoded as a list of word
+ strings.
+ :rtype: list(list(str))
+ """
+ return concat([AlignedSentCorpusView(fileid, enc, False, True,
+ self._word_tokenizer,
+ self._sent_tokenizer,
+ self._alignedsent_block_reader)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def aligned_sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of AlignedSent objects.
+ :rtype: list(AlignedSent)
+ """
+ return concat([AlignedSentCorpusView(fileid, enc, True, True,
+ self._word_tokenizer,
+ self._sent_tokenizer,
+ self._alignedsent_block_reader)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+class AlignedSentCorpusView(StreamBackedCorpusView):
+ """
+ A specialized corpus view for aligned sentences.
+ ``AlignedSentCorpusView`` objects are typically created by
+ ``AlignedCorpusReader`` (not directly by nltk users).
+ """
+ def __init__(self, corpus_file, encoding, aligned, group_by_sent,
+ word_tokenizer, sent_tokenizer, alignedsent_block_reader):
+ self._aligned = aligned
+ self._group_by_sent = group_by_sent
+ self._word_tokenizer = word_tokenizer
+ self._sent_tokenizer = sent_tokenizer
+ self._alignedsent_block_reader = alignedsent_block_reader
+ StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
+
+ def read_block(self, stream):
+ block = [self._word_tokenizer.tokenize(sent_str)
+ for alignedsent_str in self._alignedsent_block_reader(stream)
+ for sent_str in self._sent_tokenizer.tokenize(alignedsent_str)]
+ if self._aligned:
+ block[2] = " ".join(block[2]) # kludge; we shouldn't have tokenized the alignment string
+ block = [AlignedSent(*block)]
+ elif self._group_by_sent:
+ block = [block[0]]
+ else:
+ block = block[0]
+
+ return block
diff --git a/nltk/corpus/reader/api.py b/nltk/corpus/reader/api.py
new file mode 100644
index 0000000..9ff8c8b
--- /dev/null
+++ b/nltk/corpus/reader/api.py
@@ -0,0 +1,434 @@
+# Natural Language Toolkit: API for Corpus Readers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+API for corpus readers.
+"""
+from __future__ import unicode_literals
+
+import os
+import re
+from collections import defaultdict
+
+from nltk import compat
+from nltk.data import PathPointer, FileSystemPathPointer, ZipFilePathPointer
+
+from nltk.corpus.reader.util import *
+
+ at compat.python_2_unicode_compatible
+class CorpusReader(object):
+ """
+ A base class for "corpus reader" classes, each of which can be
+ used to read a specific corpus format. Each individual corpus
+ reader instance is used to read a specific corpus, consisting of
+ one or more files under a common root directory. Each file is
+ identified by its ``file identifier``, which is the relative path
+ to the file from the root directory.
+
+ A separate subclass is be defined for each corpus format. These
+ subclasses define one or more methods that provide 'views' on the
+ corpus contents, such as ``words()`` (for a list of words) and
+ ``parsed_sents()`` (for a list of parsed sentences). Called with
+ no arguments, these methods will return the contents of the entire
+ corpus. For most corpora, these methods define one or more
+ selection arguments, such as ``fileids`` or ``categories``, which can
+ be used to select which portion of the corpus should be returned.
+ """
+
+ def __init__(self, root, fileids, encoding='utf8', tagset=None):
+ """
+ :type root: PathPointer or str
+ :param root: A path pointer identifying the root directory for
+ this corpus. If a string is specified, then it will be
+ converted to a ``PathPointer`` automatically.
+ :param fileids: A list of the files that make up this corpus.
+ This list can either be specified explicitly, as a list of
+ strings; or implicitly, as a regular expression over file
+ paths. The absolute path for each file will be constructed
+ by joining the reader's root to each file name.
+ :param encoding: The default unicode encoding for the files
+ that make up the corpus. The value of ``encoding`` can be any
+ of the following:
+ - A string: ``encoding`` is the encoding name for all files.
+ - A dictionary: ``encoding[file_id]`` is the encoding
+ name for the file whose identifier is ``file_id``. If
+ ``file_id`` is not in ``encoding``, then the file
+ contents will be processed using non-unicode byte strings.
+ - A list: ``encoding`` should be a list of ``(regexp, encoding)``
+ tuples. The encoding for a file whose identifier is ``file_id``
+ will be the ``encoding`` value for the first tuple whose
+ ``regexp`` matches the ``file_id``. If no tuple's ``regexp``
+ matches the ``file_id``, the file contents will be processed
+ using non-unicode byte strings.
+ - None: the file contents of all files will be
+ processed using non-unicode byte strings.
+ :param tagset: The name of the tagset used by this corpus, to be used
+ for normalizing or converting the POS tags returned by the
+ tagged_...() methods.
+ """
+ # Convert the root to a path pointer, if necessary.
+ if isinstance(root, compat.string_types) and not isinstance(root, PathPointer):
+ m = re.match('(.*\.zip)/?(.*)$|', root)
+ zipfile, zipentry = m.groups()
+ if zipfile:
+ root = ZipFilePathPointer(zipfile, zipentry)
+ else:
+ root = FileSystemPathPointer(root)
+ elif not isinstance(root, PathPointer):
+ raise TypeError('CorpusReader: expected a string or a PathPointer')
+
+ # If `fileids` is a regexp, then expand it.
+ if isinstance(fileids, compat.string_types):
+ fileids = find_corpus_fileids(root, fileids)
+
+ self._fileids = fileids
+ """A list of the relative paths for the fileids that make up
+ this corpus."""
+
+ self._root = root
+ """The root directory for this corpus."""
+
+ # If encoding was specified as a list of regexps, then convert
+ # it to a dictionary.
+ if isinstance(encoding, list):
+ encoding_dict = {}
+ for fileid in self._fileids:
+ for x in encoding:
+ (regexp, enc) = x
+ if re.match(regexp, fileid):
+ encoding_dict[fileid] = enc
+ break
+ encoding = encoding_dict
+
+ self._encoding = encoding
+ """The default unicode encoding for the fileids that make up
+ this corpus. If ``encoding`` is None, then the file
+ contents are processed using byte strings."""
+ self._tagset = tagset
+
+ def __repr__(self):
+ if isinstance(self._root, ZipFilePathPointer):
+ path = '%s/%s' % (self._root.zipfile.filename, self._root.entry)
+ else:
+ path = '%s' % self._root.path
+ return '<%s in %r>' % (self.__class__.__name__, path)
+
+ def ensure_loaded(self):
+ """
+ Load this corpus (if it has not already been loaded). This is
+ used by LazyCorpusLoader as a simple method that can be used to
+ make sure a corpus is loaded -- e.g., in case a user wants to
+ do help(some_corpus).
+ """
+ pass # no need to actually do anything.
+
+ def readme(self):
+ """
+ Return the contents of the corpus README file, if it exists.
+ """
+ return self.open("README").read()
+
+ def fileids(self):
+ """
+ Return a list of file identifiers for the fileids that make up
+ this corpus.
+ """
+ return self._fileids
+
+ def abspath(self, fileid):
+ """
+ Return the absolute path for the given file.
+
+ :type file: str
+ :param file: The file identifier for the file whose path
+ should be returned.
+ :rtype: PathPointer
+ """
+ return self._root.join(fileid)
+
+ def abspaths(self, fileids=None, include_encoding=False,
+ include_fileid=False):
+ """
+ Return a list of the absolute paths for all fileids in this corpus;
+ or for the given list of fileids, if specified.
+
+ :type fileids: None or str or list
+ :param fileids: Specifies the set of fileids for which paths should
+ be returned. Can be None, for all fileids; a list of
+ file identifiers, for a specified set of fileids; or a single
+ file identifier, for a single file. Note that the return
+ value is always a list of paths, even if ``fileids`` is a
+ single file identifier.
+
+ :param include_encoding: If true, then return a list of
+ ``(path_pointer, encoding)`` tuples.
+
+ :rtype: list(PathPointer)
+ """
+ if fileids is None:
+ fileids = self._fileids
+ elif isinstance(fileids, compat.string_types):
+ fileids = [fileids]
+
+ paths = [self._root.join(f) for f in fileids]
+
+ if include_encoding and include_fileid:
+ return list(zip(paths, [self.encoding(f) for f in fileids], fileids))
+ elif include_fileid:
+ return list(zip(paths, fileids))
+ elif include_encoding:
+ return list(zip(paths, [self.encoding(f) for f in fileids]))
+ else:
+ return paths
+
+ def open(self, file):
+ """
+ Return an open stream that can be used to read the given file.
+ If the file's encoding is not None, then the stream will
+ automatically decode the file's contents into unicode.
+
+ :param file: The file identifier of the file to read.
+ """
+ encoding = self.encoding(file)
+ stream = self._root.join(file).open(encoding)
+ return stream
+
+ def encoding(self, file):
+ """
+ Return the unicode encoding for the given corpus file, if known.
+ If the encoding is unknown, or if the given file should be
+ processed using byte strings (str), then return None.
+ """
+ if isinstance(self._encoding, dict):
+ return self._encoding.get(file)
+ else:
+ return self._encoding
+
+ def _get_root(self): return self._root
+ root = property(_get_root, doc="""
+ The directory where this corpus is stored.
+
+ :type: PathPointer""")
+
+
+######################################################################
+#{ Corpora containing categorized items
+######################################################################
+
+class CategorizedCorpusReader(object):
+ """
+ A mixin class used to aid in the implementation of corpus readers
+ for categorized corpora. This class defines the method
+ ``categories()``, which returns a list of the categories for the
+ corpus or for a specified set of fileids; and overrides ``fileids()``
+ to take a ``categories`` argument, restricting the set of fileids to
+ be returned.
+
+ Subclasses are expected to:
+
+ - Call ``__init__()`` to set up the mapping.
+
+ - Override all view methods to accept a ``categories`` parameter,
+ which can be used *instead* of the ``fileids`` parameter, to
+ select which fileids should be included in the returned view.
+ """
+
+ def __init__(self, kwargs):
+ """
+ Initialize this mapping based on keyword arguments, as
+ follows:
+
+ - cat_pattern: A regular expression pattern used to find the
+ category for each file identifier. The pattern will be
+ applied to each file identifier, and the first matching
+ group will be used as the category label for that file.
+
+ - cat_map: A dictionary, mapping from file identifiers to
+ category labels.
+
+ - cat_file: The name of a file that contains the mapping
+ from file identifiers to categories. The argument
+ ``cat_delimiter`` can be used to specify a delimiter.
+
+ The corresponding argument will be deleted from ``kwargs``. If
+ more than one argument is specified, an exception will be
+ raised.
+ """
+ self._f2c = None #: file-to-category mapping
+ self._c2f = None #: category-to-file mapping
+
+ self._pattern = None #: regexp specifying the mapping
+ self._map = None #: dict specifying the mapping
+ self._file = None #: fileid of file containing the mapping
+ self._delimiter = None #: delimiter for ``self._file``
+
+ if 'cat_pattern' in kwargs:
+ self._pattern = kwargs['cat_pattern']
+ del kwargs['cat_pattern']
+ elif 'cat_map' in kwargs:
+ self._map = kwargs['cat_map']
+ del kwargs['cat_map']
+ elif 'cat_file' in kwargs:
+ self._file = kwargs['cat_file']
+ del kwargs['cat_file']
+ if 'cat_delimiter' in kwargs:
+ self._delimiter = kwargs['cat_delimiter']
+ del kwargs['cat_delimiter']
+ else:
+ raise ValueError('Expected keyword argument cat_pattern or '
+ 'cat_map or cat_file.')
+
+
+ if ('cat_pattern' in kwargs or 'cat_map' in kwargs or
+ 'cat_file' in kwargs):
+ raise ValueError('Specify exactly one of: cat_pattern, '
+ 'cat_map, cat_file.')
+
+ def _init(self):
+ self._f2c = defaultdict(set)
+ self._c2f = defaultdict(set)
+
+ if self._pattern is not None:
+ for file_id in self._fileids:
+ category = re.match(self._pattern, file_id).group(1)
+ self._add(file_id, category)
+
+ elif self._map is not None:
+ for (file_id, categories) in self._map.items():
+ for category in categories:
+ self._add(file_id, category)
+
+ elif self._file is not None:
+ for line in self.open(self._file).readlines():
+ line = line.strip()
+ file_id, categories = line.split(self._delimiter, 1)
+ if file_id not in self.fileids():
+ raise ValueError('In category mapping file %s: %s '
+ 'not found' % (self._file, file_id))
+ for category in categories.split(self._delimiter):
+ self._add(file_id, category)
+
+ def _add(self, file_id, category):
+ self._f2c[file_id].add(category)
+ self._c2f[category].add(file_id)
+
+ def categories(self, fileids=None):
+ """
+ Return a list of the categories that are defined for this corpus,
+ or for the file(s) if it is given.
+ """
+ if self._f2c is None:
+ self._init()
+ if fileids is None:
+ return sorted(self._c2f)
+ if isinstance(fileids, compat.string_types):
+ fileids = [fileids]
+ return sorted(set.union(*[self._f2c[d] for d in fileids]))
+
+ def fileids(self, categories=None):
+ """
+ Return a list of file identifiers for the files that make up
+ this corpus, or that make up the given category(s) if specified.
+ """
+ if categories is None:
+ return super(CategorizedCorpusReader, self).fileids()
+ elif isinstance(categories, compat.string_types):
+ if self._f2c is None:
+ self._init()
+ if categories in self._c2f:
+ return sorted(self._c2f[categories])
+ else:
+ raise ValueError('Category %s not found' % categories)
+ else:
+ if self._f2c is None:
+ self._init()
+ return sorted(set.union(*[self._c2f[c] for c in categories]))
+
+######################################################################
+#{ Treebank readers
+######################################################################
+
+#[xx] is it worth it to factor this out?
+class SyntaxCorpusReader(CorpusReader):
+ """
+ An abstract base class for reading corpora consisting of
+ syntactically parsed text. Subclasses should define:
+
+ - ``__init__``, which specifies the location of the corpus
+ and a method for detecting the sentence blocks in corpus files.
+ - ``_read_block``, which reads a block from the input stream.
+ - ``_word``, which takes a block and returns a list of list of words.
+ - ``_tag``, which takes a block and returns a list of list of tagged
+ words.
+ - ``_parse``, which takes a block and returns a list of parsed
+ sentences.
+ """
+ def _parse(self, s):
+ raise NotImplementedError()
+ def _word(self, s):
+ raise NotImplementedError()
+ def _tag(self, s):
+ raise NotImplementedError()
+ def _read_block(self, stream):
+ raise NotImplementedError()
+
+ def raw(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def parsed_sents(self, fileids=None):
+ reader = self._read_parsed_sent_block
+ return concat([StreamBackedCorpusView(fileid, reader, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, True)])
+
+ def tagged_sents(self, fileids=None, tagset=None):
+ def reader(stream):
+ return self._read_tagged_sent_block(stream, tagset)
+ return concat([StreamBackedCorpusView(fileid, reader, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, True)])
+
+ def sents(self, fileids=None):
+ reader = self._read_sent_block
+ return concat([StreamBackedCorpusView(fileid, reader, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, True)])
+
+ def tagged_words(self, fileids=None, tagset=None):
+ def reader(stream):
+ return self._read_tagged_word_block(stream, tagset)
+ return concat([StreamBackedCorpusView(fileid, reader, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, True)])
+
+ def words(self, fileids=None):
+ return concat([StreamBackedCorpusView(fileid,
+ self._read_word_block,
+ encoding=enc)
+ for fileid, enc in self.abspaths(fileids, True)])
+
+ #------------------------------------------------------------
+ #{ Block Readers
+
+ def _read_word_block(self, stream):
+ return sum(self._read_sent_block(stream), [])
+
+ def _read_tagged_word_block(self, stream, tagset=None):
+ return sum(self._read_tagged_sent_block(stream, tagset), [])
+
+ def _read_sent_block(self, stream):
+ return list(filter(None, [self._word(t) for t in self._read_block(stream)]))
+
+ def _read_tagged_sent_block(self, stream, tagset=None):
+ return list(filter(None, [self._tag(t, tagset)
+ for t in self._read_block(stream)]))
+
+ def _read_parsed_sent_block(self, stream):
+ return list(filter(None, [self._parse(t) for t in self._read_block(stream)]))
+
+ #} End of Block Readers
+ #------------------------------------------------------------
+
diff --git a/nltk/corpus/reader/bnc.py b/nltk/corpus/reader/bnc.py
new file mode 100644
index 0000000..16d037a
--- /dev/null
+++ b/nltk/corpus/reader/bnc.py
@@ -0,0 +1,252 @@
+# Natural Language Toolkit: Plaintext Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""Corpus reader for the XML version of the British National Corpus."""
+
+from nltk.corpus.reader.util import concat
+from nltk.corpus.reader.xmldocs import XMLCorpusReader, XMLCorpusView, ElementTree
+
+
+class BNCCorpusReader(XMLCorpusReader):
+ """Corpus reader for the XML version of the British National Corpus.
+
+ For access to the complete XML data structure, use the ``xml()``
+ method. For access to simple word lists and tagged word lists, use
+ ``words()``, ``sents()``, ``tagged_words()``, and ``tagged_sents()``.
+
+ You can obtain the full version of the BNC corpus at
+ http://www.ota.ox.ac.uk/desc/2554
+
+ If you extracted the archive to a directory called `BNC`, then you can
+ instantiate the reder as::
+
+ BNCCorpusReader(root='BNC/Texts/', fileids=r'[A-K]/\w*/\w*\.xml')
+
+ """
+
+ def __init__(self, root, fileids, lazy=True):
+ XMLCorpusReader.__init__(self, root, fileids)
+ self._lazy = lazy
+
+ def words(self, fileids=None, strip_space=True, stem=False):
+ """
+ :return: the given file(s) as a list of words
+ and punctuation symbols.
+ :rtype: list(str)
+
+ :param strip_space: If true, then strip trailing spaces from
+ word tokens. Otherwise, leave the spaces on the tokens.
+ :param stem: If true, then use word stems instead of word strings.
+ """
+ return self._views(fileids, False, None, strip_space, stem)
+
+ def tagged_words(self, fileids=None, c5=False, strip_space=True, stem=False):
+ """
+ :return: the given file(s) as a list of tagged
+ words and punctuation symbols, encoded as tuples
+ ``(word,tag)``.
+ :rtype: list(tuple(str,str))
+
+ :param c5: If true, then the tags used will be the more detailed
+ c5 tags. Otherwise, the simplified tags will be used.
+ :param strip_space: If true, then strip trailing spaces from
+ word tokens. Otherwise, leave the spaces on the tokens.
+ :param stem: If true, then use word stems instead of word strings.
+ """
+ tag = 'c5' if c5 else 'pos'
+ return self._views(fileids, False, tag, strip_space, stem)
+
+ def sents(self, fileids=None, strip_space=True, stem=False):
+ """
+ :return: the given file(s) as a list of
+ sentences or utterances, each encoded as a list of word
+ strings.
+ :rtype: list(list(str))
+
+ :param strip_space: If true, then strip trailing spaces from
+ word tokens. Otherwise, leave the spaces on the tokens.
+ :param stem: If true, then use word stems instead of word strings.
+ """
+ return self._views(fileids, True, None, strip_space, stem)
+
+ def tagged_sents(self, fileids=None, c5=False, strip_space=True, stem=False):
+ """
+ :return: the given file(s) as a list of
+ sentences, each encoded as a list of ``(word,tag)`` tuples.
+ :rtype: list(list(tuple(str,str)))
+
+ :param c5: If true, then the tags used will be the more detailed
+ c5 tags. Otherwise, the simplified tags will be used.
+ :param strip_space: If true, then strip trailing spaces from
+ word tokens. Otherwise, leave the spaces on the tokens.
+ :param stem: If true, then use word stems instead of word strings.
+ """
+ tag = 'c5' if c5 else 'pos'
+ return self._views(fileids, sent=True, tag=tag, strip_space=strip_space, stem=stem)
+
+ def _views(self, fileids=None, sent=False, tag=False, strip_space=True, stem=False):
+ """A helper function that instantiates BNCWordViews or the list of words/sentences."""
+ f = BNCWordView if self._lazy else self._words
+ return concat([f(fileid, sent, tag, strip_space, stem) for fileid in self.abspaths(fileids)])
+
+ def _words(self, fileid, bracket_sent, tag, strip_space, stem):
+ """
+ Helper used to implement the view methods -- returns a list of
+ words or a list of sentences, optionally tagged.
+
+ :param fileid: The name of the underlying file.
+ :param bracket_sent: If true, include sentence bracketing.
+ :param tag: The name of the tagset to use, or None for no tags.
+ :param strip_space: If true, strip spaces from word tokens.
+ :param stem: If true, then substitute stems for words.
+ """
+ result = []
+
+ xmldoc = ElementTree.parse(fileid).getroot()
+ for xmlsent in xmldoc.findall('.//s'):
+ sent = []
+ for xmlword in _all_xmlwords_in(xmlsent):
+ word = xmlword.text
+ if not word:
+ word = "" # fixes issue 337?
+ if strip_space or stem:
+ word = word.strip()
+ if stem:
+ word = xmlword.get('hw', word)
+ if tag == 'c5':
+ word = (word, xmlword.get('c5'))
+ elif tag == 'pos':
+ word = (word, xmlword.get('pos', xmlword.get('c5')))
+ sent.append(word)
+ if bracket_sent:
+ result.append(BNCSentence(xmlsent.attrib['n'], sent))
+ else:
+ result.extend(sent)
+
+ assert None not in result
+ return result
+
+
+def _all_xmlwords_in(elt, result=None):
+ if result is None:
+ result = []
+ for child in elt:
+ if child.tag in ('c', 'w'):
+ result.append(child)
+ else:
+ _all_xmlwords_in(child, result)
+ return result
+
+
+class BNCSentence(list):
+ """
+ A list of words, augmented by an attribute ``num`` used to record
+ the sentence identifier (the ``n`` attribute from the XML).
+ """
+ def __init__(self, num, items):
+ self.num = num
+ list.__init__(self, items)
+
+
+class BNCWordView(XMLCorpusView):
+ """
+ A stream backed corpus view specialized for use with the BNC corpus.
+ """
+
+ tags_to_ignore = set(
+ ['pb', 'gap', 'vocal', 'event', 'unclear', 'shift', 'pause', 'align']
+ )
+ """These tags are ignored. For their description refer to the
+ technical documentation, for example,
+ http://www.natcorp.ox.ac.uk/docs/URG/ref-vocal.html
+
+ """
+
+ def __init__(self, fileid, sent, tag, strip_space, stem):
+ """
+ :param fileid: The name of the underlying file.
+ :param sent: If true, include sentence bracketing.
+ :param tag: The name of the tagset to use, or None for no tags.
+ :param strip_space: If true, strip spaces from word tokens.
+ :param stem: If true, then substitute stems for words.
+ """
+ if sent:
+ tagspec = '.*/s'
+ else:
+ tagspec = '.*/s/(.*/)?(c|w)'
+ self._sent = sent
+ self._tag = tag
+ self._strip_space = strip_space
+ self._stem = stem
+
+ self.title = None #: Title of the document.
+ self.author = None #: Author of the document.
+ self.editor = None #: Editor
+ self.resps = None #: Statement of responsibility
+
+ XMLCorpusView.__init__(self, fileid, tagspec)
+
+ # Read in a tasty header.
+ self._open()
+ self.read_block(self._stream, '.*/teiHeader$', self.handle_header)
+ self.close()
+
+ # Reset tag context.
+ self._tag_context = {0: ()}
+
+ def handle_header(self, elt, context):
+ # Set up some metadata!
+ titles = elt.findall('titleStmt/title')
+ if titles:
+ self.title = '\n'.join(title.text.strip() for title in titles)
+
+ authors = elt.findall('titleStmt/author')
+ if authors:
+ self.author = '\n'.join(author.text.strip() for author in authors)
+
+ editors = elt.findall('titleStmt/editor')
+ if editors:
+ self.editor = '\n'.join(editor.text.strip() for editor in editors)
+
+ resps = elt.findall('titleStmt/respStmt')
+ if resps:
+ self.resps = '\n\n'.join(
+ '\n'.join(
+ resp_elt.text.strip() for resp_elt in resp
+ ) for resp in resps
+ )
+
+ def handle_elt(self, elt, context):
+ if self._sent:
+ return self.handle_sent(elt)
+ else:
+ return self.handle_word(elt)
+
+ def handle_word(self, elt):
+ word = elt.text
+ if not word:
+ word = "" # fixes issue 337?
+ if self._strip_space or self._stem:
+ word = word.strip()
+ if self._stem:
+ word = elt.get('hw', word)
+ if self._tag == 'c5':
+ word = (word, elt.get('c5'))
+ elif self._tag == 'pos':
+ word = (word, elt.get('pos', elt.get('c5')))
+ return word
+
+ def handle_sent(self, elt):
+ sent = []
+ for child in elt:
+ if child.tag in ('mw', 'hi', 'corr', 'trunc'):
+ sent += [self.handle_word(w) for w in child]
+ elif child.tag in ('w', 'c'):
+ sent.append(self.handle_word(child))
+ elif child.tag not in self.tags_to_ignore:
+ raise ValueError('Unexpected element %s' % child.tag)
+ return BNCSentence(elt.attrib['n'], sent)
diff --git a/nltk/corpus/reader/bracket_parse.py b/nltk/corpus/reader/bracket_parse.py
new file mode 100644
index 0000000..5fd29a0
--- /dev/null
+++ b/nltk/corpus/reader/bracket_parse.py
@@ -0,0 +1,184 @@
+# Natural Language Toolkit: Penn Treebank Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+Corpus reader for corpora that consist of parenthesis-delineated parse trees.
+"""
+
+import sys
+
+from nltk.tree import Tree
+from nltk.tag import map_tag
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+# we use [^\s()]+ instead of \S+? to avoid matching ()
+TAGWORD = re.compile(r'\(([^\s()]+) ([^\s()]+)\)')
+WORD = re.compile(r'\([^\s()]+ ([^\s()]+)\)')
+EMPTY_BRACKETS = re.compile(r'\s*\(\s*\(')
+
+class BracketParseCorpusReader(SyntaxCorpusReader):
+ """
+ Reader for corpora that consist of parenthesis-delineated parse
+ trees.
+ """
+ def __init__(self, root, fileids, comment_char=None,
+ detect_blocks='unindented_paren', encoding='utf8',
+ tagset=None):
+ """
+ :param root: The root directory for this corpus.
+ :param fileids: A list or regexp specifying the fileids in this corpus.
+ :param comment_char: The character which can appear at the start of
+ a line to indicate that the rest of the line is a comment.
+ :param detect_blocks: The method that is used to find blocks
+ in the corpus; can be 'unindented_paren' (every unindented
+ parenthesis starts a new parse) or 'sexpr' (brackets are
+ matched).
+ :param tagset: The name of the tagset used by this corpus, to be used
+ for normalizing or converting the POS tags returned by the
+ tagged_...() methods.
+ """
+ CorpusReader.__init__(self, root, fileids, encoding)
+ self._comment_char = comment_char
+ self._detect_blocks = detect_blocks
+ self._tagset = tagset
+
+ def _read_block(self, stream):
+ if self._detect_blocks == 'sexpr':
+ return read_sexpr_block(stream, comment_char=self._comment_char)
+ elif self._detect_blocks == 'blankline':
+ return read_blankline_block(stream)
+ elif self._detect_blocks == 'unindented_paren':
+ # Tokens start with unindented left parens.
+ toks = read_regexp_block(stream, start_re=r'^\(')
+ # Strip any comments out of the tokens.
+ if self._comment_char:
+ toks = [re.sub('(?m)^%s.*'%re.escape(self._comment_char),
+ '', tok)
+ for tok in toks]
+ return toks
+ else:
+ assert 0, 'bad block type'
+
+ def _normalize(self, t):
+ # If there's an empty set of brackets surrounding the actual
+ # parse, then strip them off.
+ if EMPTY_BRACKETS.match(t):
+ t = t.strip()[1:-1]
+ # Replace leaves of the form (!), (,), with (! !), (, ,)
+ t = re.sub(r"\((.)\)", r"(\1 \1)", t)
+ # Replace leaves of the form (tag word root) with (tag word)
+ t = re.sub(r"\(([^\s()]+) ([^\s()]+) [^\s()]+\)", r"(\1 \2)", t)
+ return t
+
+ def _parse(self, t):
+ try:
+ return Tree.fromstring(self._normalize(t))
+
+ except ValueError as e:
+ sys.stderr.write("Bad tree detected; trying to recover...\n")
+ # Try to recover, if we can:
+ if e.args == ('mismatched parens',):
+ for n in range(1, 5):
+ try:
+ v = Tree(self._normalize(t+')'*n))
+ sys.stderr.write(" Recovered by adding %d close "
+ "paren(s)\n" % n)
+ return v
+ except ValueError: pass
+ # Try something else:
+ sys.stderr.write(" Recovered by returning a flat parse.\n")
+ #sys.stderr.write(' '.join(t.split())+'\n')
+ return Tree('S', self._tag(t))
+
+ def _tag(self, t, tagset=None):
+ tagged_sent = [(w,t) for (t,w) in TAGWORD.findall(self._normalize(t))]
+ if tagset and tagset != self._tagset:
+ tagged_sent = [(w, map_tag(self._tagset, tagset, t)) for (w,t) in tagged_sent]
+ return tagged_sent
+
+ def _word(self, t):
+ return WORD.findall(self._normalize(t))
+
+class CategorizedBracketParseCorpusReader(CategorizedCorpusReader,
+ BracketParseCorpusReader):
+ """
+ A reader for parsed corpora whose documents are
+ divided into categories based on their file identifiers.
+ @author: Nathan Schneider <nschneid at cs.cmu.edu>
+ """
+ def __init__(self, *args, **kwargs):
+ """
+ Initialize the corpus reader. Categorization arguments
+ (C{cat_pattern}, C{cat_map}, and C{cat_file}) are passed to
+ the L{CategorizedCorpusReader constructor
+ <CategorizedCorpusReader.__init__>}. The remaining arguments
+ are passed to the L{BracketParseCorpusReader constructor
+ <BracketParseCorpusReader.__init__>}.
+ """
+ CategorizedCorpusReader.__init__(self, kwargs)
+ BracketParseCorpusReader.__init__(self, *args, **kwargs)
+
+ def _resolve(self, fileids, categories):
+ if fileids is not None and categories is not None:
+ raise ValueError('Specify fileids or categories, not both')
+ if categories is not None:
+ return self.fileids(categories)
+ else:
+ return fileids
+ def raw(self, fileids=None, categories=None):
+ return BracketParseCorpusReader.raw(
+ self, self._resolve(fileids, categories))
+ def words(self, fileids=None, categories=None):
+ return BracketParseCorpusReader.words(
+ self, self._resolve(fileids, categories))
+ def sents(self, fileids=None, categories=None):
+ return BracketParseCorpusReader.sents(
+ self, self._resolve(fileids, categories))
+ def paras(self, fileids=None, categories=None):
+ return BracketParseCorpusReader.paras(
+ self, self._resolve(fileids, categories))
+ def tagged_words(self, fileids=None, categories=None, tagset=None):
+ return BracketParseCorpusReader.tagged_words(
+ self, self._resolve(fileids, categories), tagset)
+ def tagged_sents(self, fileids=None, categories=None, tagset=None):
+ return BracketParseCorpusReader.tagged_sents(
+ self, self._resolve(fileids, categories), tagset)
+ def tagged_paras(self, fileids=None, categories=None, tagset=None):
+ return BracketParseCorpusReader.tagged_paras(
+ self, self._resolve(fileids, categories), tagset)
+ def parsed_words(self, fileids=None, categories=None):
+ return BracketParseCorpusReader.parsed_words(
+ self, self._resolve(fileids, categories))
+ def parsed_sents(self, fileids=None, categories=None):
+ return BracketParseCorpusReader.parsed_sents(
+ self, self._resolve(fileids, categories))
+ def parsed_paras(self, fileids=None, categories=None):
+ return BracketParseCorpusReader.parsed_paras(
+ self, self._resolve(fileids, categories))
+
+class AlpinoCorpusReader(BracketParseCorpusReader):
+ """
+ Reader for the Alpino Dutch Treebank.
+ """
+ def __init__(self, root, encoding='ISO-8859-1', tagset=None):
+ BracketParseCorpusReader.__init__(self, root, 'alpino\.xml',
+ detect_blocks='blankline',
+ encoding=encoding,
+ tagset=tagset)
+
+ def _normalize(self, t):
+ if t[:10] != "<alpino_ds":
+ return ""
+ # convert XML to sexpr notation
+ t = re.sub(r' <node .*? cat="(\w+)".*>', r"(\1", t)
+ t = re.sub(r' <node .*? pos="(\w+)".*? word="([^"]+)".*/>', r"(\1 \2)", t)
+ t = re.sub(r" </node>", r")", t)
+ t = re.sub(r"<sentence>.*</sentence>", r"", t)
+ t = re.sub(r"</?alpino_ds.*>", r"", t)
+ return t
diff --git a/nltk/corpus/reader/chasen.py b/nltk/corpus/reader/chasen.py
new file mode 100644
index 0000000..dd12228
--- /dev/null
+++ b/nltk/corpus/reader/chasen.py
@@ -0,0 +1,139 @@
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Masato Hagiwara <hagisan at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# For more information, see http://lilyx.net/pages/nltkjapanesecorpus.html
+from __future__ import print_function
+
+import sys
+
+from nltk.corpus.reader import util
+
+from nltk import compat
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class ChasenCorpusReader(CorpusReader):
+
+ def __init__(self, root, fileids, encoding='utf8', sent_splitter=None):
+ self._sent_splitter = sent_splitter
+ CorpusReader.__init__(self, root, fileids, encoding)
+
+ def raw(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def words(self, fileids=None):
+ return concat([ChasenCorpusView(fileid, enc,
+ False, False, False, self._sent_splitter)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tagged_words(self, fileids=None):
+ return concat([ChasenCorpusView(fileid, enc,
+ True, False, False, self._sent_splitter)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def sents(self, fileids=None):
+ return concat([ChasenCorpusView(fileid, enc,
+ False, True, False, self._sent_splitter)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tagged_sents(self, fileids=None):
+ return concat([ChasenCorpusView(fileid, enc,
+ True, True, False, self._sent_splitter)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def paras(self, fileids=None):
+ return concat([ChasenCorpusView(fileid, enc,
+ False, True, True, self._sent_splitter)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tagged_paras(self, fileids=None):
+ return concat([ChasenCorpusView(fileid, enc,
+ True, True, True, self._sent_splitter)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+
+class ChasenCorpusView(StreamBackedCorpusView):
+ """
+ A specialized corpus view for ChasenReader. Similar to ``TaggedCorpusView``,
+ but this'll use fixed sets of word and sentence tokenizer.
+ """
+
+ def __init__(self, corpus_file, encoding,
+ tagged, group_by_sent, group_by_para, sent_splitter=None):
+ self._tagged = tagged
+ self._group_by_sent = group_by_sent
+ self._group_by_para = group_by_para
+ self._sent_splitter = sent_splitter
+ StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
+
+
+ def read_block(self, stream):
+ """Reads one paragraph at a time."""
+ block = []
+ for para_str in read_regexp_block(stream, r".", r"^EOS\n"):
+
+ para = []
+
+ sent = []
+ for line in para_str.splitlines():
+
+ _eos = line.strip() == 'EOS'
+ _cells = line.split('\t')
+ w = (_cells[0], '\t'.join(_cells[1:]))
+ if not _eos: sent.append(w)
+
+ if _eos or (self._sent_splitter and self._sent_splitter(w)):
+ if not self._tagged:
+ sent = [w for (w,t) in sent]
+ if self._group_by_sent:
+ para.append(sent)
+ else:
+ para.extend(sent)
+ sent = []
+
+ if len(sent)>0:
+ if not self._tagged:
+ sent = [w for (w,t) in sent]
+
+ if self._group_by_sent:
+ para.append(sent)
+ else:
+ para.extend(sent)
+
+ if self._group_by_para:
+ block.append(para)
+ else:
+ block.extend(para)
+
+ return block
+
+def demo():
+
+ import nltk
+ from nltk.corpus.util import LazyCorpusLoader
+
+ jeita = LazyCorpusLoader(
+ 'jeita', ChasenCorpusReader, r'.*chasen', encoding='utf-8')
+ print('/'.join( jeita.words()[22100:22140] ))
+
+
+ print('\nEOS\n'.join('\n'.join("%s/%s" % (w[0],w[1].split('\t')[2]) for w in sent)
+ for sent in jeita.tagged_sents()[2170:2173]))
+
+def test():
+
+ from nltk.corpus.util import LazyCorpusLoader
+
+ jeita = LazyCorpusLoader(
+ 'jeita', ChasenCorpusReader, r'.*chasen', encoding='utf-8')
+
+ assert isinstance(jeita.tagged_words()[0][1], compat.string_types)
+
+if __name__ == '__main__':
+ demo()
+ test()
diff --git a/nltk/corpus/reader/childes.py b/nltk/corpus/reader/childes.py
new file mode 100644
index 0000000..a9b3cb0
--- /dev/null
+++ b/nltk/corpus/reader/childes.py
@@ -0,0 +1,482 @@
+# CHILDES XML Corpus Reader
+
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Tomonori Nagano <tnagano at gc.cuny.edu>
+# Alexis Dimitriadis <A.Dimitriadis at uu.nl>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Corpus reader for the XML version of the CHILDES corpus.
+"""
+from __future__ import print_function
+
+__docformat__ = 'epytext en'
+
+import re
+from collections import defaultdict
+
+from nltk.util import flatten
+from nltk.compat import string_types
+
+from nltk.corpus.reader.util import concat
+from nltk.corpus.reader.xmldocs import XMLCorpusReader, ElementTree
+
+# to resolve the namespace issue
+NS = 'http://www.talkbank.org/ns/talkbank'
+
+class CHILDESCorpusReader(XMLCorpusReader):
+ """
+ Corpus reader for the XML version of the CHILDES corpus.
+ The CHILDES corpus is available at ``http://childes.psy.cmu.edu/``. The XML
+ version of CHILDES is located at ``http://childes.psy.cmu.edu/data-xml/``.
+ Copy the needed parts of the CHILDES XML corpus into the NLTK data directory
+ (``nltk_data/corpora/CHILDES/``).
+
+ For access to the file text use the usual nltk functions,
+ ``words()``, ``sents()``, ``tagged_words()`` and ``tagged_sents()``.
+ """
+ def __init__(self, root, fileids, lazy=True):
+ XMLCorpusReader.__init__(self, root, fileids)
+ self._lazy = lazy
+
+ def words(self, fileids=None, speaker='ALL', stem=False,
+ relation=False, strip_space=True, replace=False):
+ """
+ :return: the given file(s) as a list of words
+ :rtype: list(str)
+
+ :param speaker: If specified, select specific speaker(s) defined
+ in the corpus. Default is 'ALL' (all participants). Common choices
+ are 'CHI' (the child), 'MOT' (mother), ['CHI','MOT'] (exclude
+ researchers)
+ :param stem: If true, then use word stems instead of word strings.
+ :param relation: If true, then return tuples of (stem, index,
+ dependent_index)
+ :param strip_space: If true, then strip trailing spaces from word
+ tokens. Otherwise, leave the spaces on the tokens.
+ :param replace: If true, then use the replaced (intended) word instead
+ of the original word (e.g., 'wat' will be replaced with 'watch')
+ """
+ sent=None
+ pos=False
+ return concat([self._get_words(fileid, speaker, sent, stem, relation,
+ pos, strip_space, replace) for fileid in self.abspaths(fileids)])
+
+ def tagged_words(self, fileids=None, speaker='ALL', stem=False,
+ relation=False, strip_space=True, replace=False):
+ """
+ :return: the given file(s) as a list of tagged
+ words and punctuation symbols, encoded as tuples
+ ``(word,tag)``.
+ :rtype: list(tuple(str,str))
+
+ :param speaker: If specified, select specific speaker(s) defined
+ in the corpus. Default is 'ALL' (all participants). Common choices
+ are 'CHI' (the child), 'MOT' (mother), ['CHI','MOT'] (exclude
+ researchers)
+ :param stem: If true, then use word stems instead of word strings.
+ :param relation: If true, then return tuples of (stem, index,
+ dependent_index)
+ :param strip_space: If true, then strip trailing spaces from word
+ tokens. Otherwise, leave the spaces on the tokens.
+ :param replace: If true, then use the replaced (intended) word instead
+ of the original word (e.g., 'wat' will be replaced with 'watch')
+ """
+ sent=None
+ pos=True
+ return concat([self._get_words(fileid, speaker, sent, stem, relation,
+ pos, strip_space, replace) for fileid in self.abspaths(fileids)])
+
+ def sents(self, fileids=None, speaker='ALL', stem=False,
+ relation=None, strip_space=True, replace=False):
+ """
+ :return: the given file(s) as a list of sentences or utterances, each
+ encoded as a list of word strings.
+ :rtype: list(list(str))
+
+ :param speaker: If specified, select specific speaker(s) defined
+ in the corpus. Default is 'ALL' (all participants). Common choices
+ are 'CHI' (the child), 'MOT' (mother), ['CHI','MOT'] (exclude
+ researchers)
+ :param stem: If true, then use word stems instead of word strings.
+ :param relation: If true, then return tuples of ``(str,pos,relation_list)``.
+ If there is manually-annotated relation info, it will return
+ tuples of ``(str,pos,test_relation_list,str,pos,gold_relation_list)``
+ :param strip_space: If true, then strip trailing spaces from word
+ tokens. Otherwise, leave the spaces on the tokens.
+ :param replace: If true, then use the replaced (intended) word instead
+ of the original word (e.g., 'wat' will be replaced with 'watch')
+ """
+ sent=True
+ pos=False
+ return concat([self._get_words(fileid, speaker, sent, stem, relation,
+ pos, strip_space, replace) for fileid in self.abspaths(fileids)])
+
+ def tagged_sents(self, fileids=None, speaker='ALL', stem=False,
+ relation=None, strip_space=True, replace=False):
+ """
+ :return: the given file(s) as a list of
+ sentences, each encoded as a list of ``(word,tag)`` tuples.
+ :rtype: list(list(tuple(str,str)))
+
+ :param speaker: If specified, select specific speaker(s) defined
+ in the corpus. Default is 'ALL' (all participants). Common choices
+ are 'CHI' (the child), 'MOT' (mother), ['CHI','MOT'] (exclude
+ researchers)
+ :param stem: If true, then use word stems instead of word strings.
+ :param relation: If true, then return tuples of ``(str,pos,relation_list)``.
+ If there is manually-annotated relation info, it will return
+ tuples of ``(str,pos,test_relation_list,str,pos,gold_relation_list)``
+ :param strip_space: If true, then strip trailing spaces from word
+ tokens. Otherwise, leave the spaces on the tokens.
+ :param replace: If true, then use the replaced (intended) word instead
+ of the original word (e.g., 'wat' will be replaced with 'watch')
+ """
+ sent=True
+ pos=True
+ return concat([self._get_words(fileid, speaker, sent, stem, relation,
+ pos, strip_space, replace) for fileid in self.abspaths(fileids)])
+
+ def corpus(self, fileids=None):
+ """
+ :return: the given file(s) as a dict of ``(corpus_property_key, value)``
+ :rtype: list(dict)
+ """
+ return [self._get_corpus(fileid) for fileid in self.abspaths(fileids)]
+
+ def _get_corpus(self, fileid):
+ results = dict()
+ xmldoc = ElementTree.parse(fileid).getroot()
+ for key, value in xmldoc.items():
+ results[key] = value
+ return results
+
+ def participants(self, fileids=None):
+ """
+ :return: the given file(s) as a dict of
+ ``(participant_property_key, value)``
+ :rtype: list(dict)
+ """
+ return [self._get_participants(fileid)
+ for fileid in self.abspaths(fileids)]
+
+ def _get_participants(self, fileid):
+ # multidimensional dicts
+ def dictOfDicts():
+ return defaultdict(dictOfDicts)
+
+ xmldoc = ElementTree.parse(fileid).getroot()
+ # getting participants' data
+ pat = dictOfDicts()
+ for participant in xmldoc.findall('.//{%s}Participants/{%s}participant'
+ % (NS,NS)):
+ for (key,value) in participant.items():
+ pat[participant.get('id')][key] = value
+ return pat
+
+ def age(self, fileids=None, speaker='CHI', month=False):
+ """
+ :return: the given file(s) as string or int
+ :rtype: list or int
+
+ :param month: If true, return months instead of year-month-date
+ """
+ return [self._get_age(fileid, speaker, month)
+ for fileid in self.abspaths(fileids)]
+
+ def _get_age(self, fileid, speaker, month):
+ xmldoc = ElementTree.parse(fileid).getroot()
+ for pat in xmldoc.findall('.//{%s}Participants/{%s}participant'
+ % (NS,NS)):
+ try:
+ if pat.get('id') == speaker:
+ age = pat.get('age')
+ if month:
+ age = self.convert_age(age)
+ return age
+ # some files don't have age data
+ except (TypeError, AttributeError) as e:
+ return None
+
+ def convert_age(self, age_year):
+ "Caclculate age in months from a string in CHILDES format"
+ m = re.match("P(\d+)Y(\d+)M?(\d?\d?)D?",age_year)
+ age_month = int(m.group(1))*12 + int(m.group(2))
+ try:
+ if int(m.group(3)) > 15:
+ age_month += 1
+ # some corpora don't have age information?
+ except ValueError as e:
+ pass
+ return age_month
+
+ def MLU(self, fileids=None, speaker='CHI'):
+ """
+ :return: the given file(s) as a floating number
+ :rtype: list(float)
+ """
+ return [self._getMLU(fileid, speaker=speaker)
+ for fileid in self.abspaths(fileids)]
+
+ def _getMLU(self, fileid, speaker):
+ sents = self._get_words(fileid, speaker=speaker, sent=True, stem=True,
+ relation=False, pos=True, strip_space=True, replace=True)
+ results = []
+ lastSent = []
+ numFillers = 0
+ sentDiscount = 0
+ for sent in sents:
+ posList = [pos for (word,pos) in sent]
+ # if any part of the sentence is intelligible
+ if any(pos == 'unk' for pos in posList):
+ next
+ # if the sentence is null
+ elif sent == []:
+ next
+ # if the sentence is the same as the last sent
+ elif sent == lastSent:
+ next
+ else:
+ results.append([word for (word,pos) in sent])
+ # count number of fillers
+ if len(set(['co',None]).intersection(posList)) > 0:
+ numFillers += posList.count('co')
+ numFillers += posList.count(None)
+ sentDiscount += 1
+ lastSent = sent
+ try:
+ thisWordList = flatten(results)
+ # count number of morphemes
+ # (e.g., 'read' = 1 morpheme but 'read-PAST' is 2 morphemes)
+ numWords = float(len(flatten([word.split('-')
+ for word in thisWordList]))) - numFillers
+ numSents = float(len(results)) - sentDiscount
+ mlu = numWords/numSents
+ except ZeroDivisionError:
+ mlu = 0
+ # return {'mlu':mlu,'wordNum':numWords,'sentNum':numSents}
+ return mlu
+
+ def _get_words(self, fileid, speaker, sent, stem, relation, pos,
+ strip_space, replace):
+ if isinstance(speaker, string_types) and speaker != 'ALL': # ensure we have a list of speakers
+ speaker = [ speaker ]
+ xmldoc = ElementTree.parse(fileid).getroot()
+ # processing each xml doc
+ results = []
+ for xmlsent in xmldoc.findall('.//{%s}u' % NS):
+ sents = []
+ # select speakers
+ if speaker == 'ALL' or xmlsent.get('who') in speaker:
+ for xmlword in xmlsent.findall('.//{%s}w' % NS):
+ infl = None ; suffixStem = None
+ # getting replaced words
+ if replace and xmlsent.find('.//{%s}w/{%s}replacement'
+ % (NS,NS)):
+ xmlword = xmlsent.find('.//{%s}w/{%s}replacement/{%s}w'
+ % (NS,NS,NS))
+ elif replace and xmlsent.find('.//{%s}w/{%s}wk' % (NS,NS)):
+ xmlword = xmlsent.find('.//{%s}w/{%s}wk' % (NS,NS))
+ # get text
+ if xmlword.text:
+ word = xmlword.text
+ else:
+ word = ''
+ # strip tailing space
+ if strip_space:
+ word = word.strip()
+ # stem
+ if relation or stem:
+ try:
+ xmlstem = xmlword.find('.//{%s}stem' % NS)
+ word = xmlstem.text
+ except AttributeError as e:
+ pass
+ # if there is an inflection
+ try:
+ xmlinfl = xmlword.find('.//{%s}mor/{%s}mw/{%s}mk'
+ % (NS,NS,NS))
+ word += '-' + xmlinfl.text
+ except:
+ pass
+ # if there is a suffix
+ try:
+ xmlsuffix = xmlword.find('.//{%s}mor/{%s}mor-post/{%s}mw/{%s}stem'
+ % (NS,NS,NS,NS))
+ suffixStem = xmlsuffix.text
+ except AttributeError:
+ suffixStem = ""
+ # pos
+ if relation or pos:
+ try:
+ xmlpos = xmlword.findall(".//{%s}c" % NS)
+ xmlpos2 = xmlword.findall(".//{%s}s" % NS)
+ if xmlpos2 != []:
+ tag = xmlpos[0].text+":"+xmlpos2[0].text
+ else:
+ tag = xmlpos[0].text
+ word = (word,tag)
+ except (AttributeError,IndexError) as e:
+ word = (word,None)
+ if suffixStem:
+ suffixStem = (suffixStem,None)
+ # relational
+ # the gold standard is stored in
+ # <mor></mor><mor type="trn"><gra type="grt">
+ if relation == True:
+ for xmlstem_rel in xmlword.findall('.//{%s}mor/{%s}gra'
+ % (NS,NS)):
+ if not xmlstem_rel.get('type') == 'grt':
+ word = (word[0], word[1],
+ xmlstem_rel.get('index')
+ + "|" + xmlstem_rel.get('head')
+ + "|" + xmlstem_rel.get('relation'))
+ else:
+ word = (word[0], word[1], word[2],
+ word[0], word[1],
+ xmlstem_rel.get('index')
+ + "|" + xmlstem_rel.get('head')
+ + "|" + xmlstem_rel.get('relation'))
+ try:
+ for xmlpost_rel in xmlword.findall('.//{%s}mor/{%s}mor-post/{%s}gra'
+ % (NS,NS,NS)):
+ if not xmlpost_rel.get('type') == 'grt':
+ suffixStem = (suffixStem[0],
+ suffixStem[1],
+ xmlpost_rel.get('index')
+ + "|" + xmlpost_rel.get('head')
+ + "|" + xmlpost_rel.get('relation'))
+ else:
+ suffixStem = (suffixStem[0], suffixStem[1],
+ suffixStem[2], suffixStem[0],
+ suffixStem[1],
+ xmlpost_rel.get('index')
+ + "|" + xmlpost_rel.get('head')
+ + "|" + xmlpost_rel.get('relation'))
+ except:
+ pass
+ sents.append(word)
+ if suffixStem:
+ sents.append(suffixStem)
+ if sent or relation:
+ results.append(sents)
+ else:
+ results.extend(sents)
+ return results
+
+
+ # Ready-to-use browser opener
+
+ """
+ The base URL for viewing files on the childes website. This
+ shouldn't need to be changed, unless CHILDES changes the configuration
+ of their server or unless the user sets up their own corpus webserver.
+ """
+ childes_url_base = r'http://childes.psy.cmu.edu/browser/index.php?url='
+
+
+ def webview_file(self, fileid, urlbase=None):
+ """Map a corpus file to its web version on the CHILDES website,
+ and open it in a web browser.
+
+ The complete URL to be used is:
+ childes.childes_url_base + urlbase + fileid.replace('.xml', '.cha')
+
+ If no urlbase is passed, we try to calculate it. This
+ requires that the childes corpus was set up to mirror the
+ folder hierarchy under childes.psy.cmu.edu/data-xml/, e.g.:
+ nltk_data/corpora/childes/Eng-USA/Cornell/??? or
+ nltk_data/corpora/childes/Romance/Spanish/Aguirre/???
+
+ The function first looks (as a special case) if "Eng-USA" is
+ on the path consisting of <corpus root>+fileid; then if
+ "childes", possibly followed by "data-xml", appears. If neither
+ one is found, we use the unmodified fileid and hope for the best.
+ If this is not right, specify urlbase explicitly, e.g., if the
+ corpus root points to the Cornell folder, urlbase='Eng-USA/Cornell'.
+ """
+
+ import webbrowser, re
+
+ if urlbase:
+ path = urlbase+"/"+fileid
+ else:
+ full = self.root + "/" + fileid
+ full = re.sub(r'\\', '/', full)
+ if '/childes/' in full.lower():
+ # Discard /data-xml/ if present
+ path = re.findall(r'(?i)/childes(?:/data-xml)?/(.*)\.xml', full)[0]
+ elif 'eng-usa' in full.lower():
+ path = 'Eng-USA/' + re.findall(r'/(?i)Eng-USA/(.*)\.xml', full)[0]
+ else:
+ path = fileid
+
+ # Strip ".xml" and add ".cha", as necessary:
+ if path.endswith('.xml'):
+ path = path[:-4]
+
+ if not path.endswith('.cha'):
+ path = path+'.cha'
+
+ url = self.childes_url_base + path
+
+ webbrowser.open_new_tab(url)
+ print("Opening in browser:", url)
+ # Pausing is a good idea, but it's up to the user...
+ # raw_input("Hit Return to continue")
+
+
+
+def demo(corpus_root=None):
+ """
+ The CHILDES corpus should be manually downloaded and saved
+ to ``[NLTK_Data_Dir]/corpora/childes/``
+ """
+ if not corpus_root:
+ from nltk.data import find
+ corpus_root = find('corpora/childes/data-xml/Eng-USA/')
+
+ try:
+ childes = CHILDESCorpusReader(corpus_root, '.*.xml')
+ # describe all corpus
+ for file in childes.fileids()[:5]:
+ corpus = ''
+ corpus_id = ''
+ for (key,value) in childes.corpus(file)[0].items():
+ if key == "Corpus": corpus = value
+ if key == "Id": corpus_id = value
+ print('Reading', corpus,corpus_id,' .....')
+ print("words:", childes.words(file)[:7],"...")
+ print("words with replaced words:", childes.words(file, replace=True)[:7]," ...")
+ print("words with pos tags:", childes.tagged_words(file)[:7]," ...")
+ print("words (only MOT):", childes.words(file, speaker='MOT')[:7], "...")
+ print("words (only CHI):", childes.words(file, speaker='CHI')[:7], "...")
+ print("stemmed words:", childes.words(file, stem=True)[:7]," ...")
+ print("words with relations and pos-tag:", childes.words(file, relation=True)[:5]," ...")
+ print("sentence:", childes.sents(file)[:2]," ...")
+ for (participant, values) in childes.participants(file)[0].items():
+ for (key, value) in values.items():
+ print("\tparticipant", participant, key, ":", value)
+ print("num of sent:", len(childes.sents(file)))
+ print("num of morphemes:", len(childes.words(file, stem=True)))
+ print("age:", childes.age(file))
+ print("age in month:", childes.age(file, month=True))
+ print("MLU:", childes.MLU(file))
+ print()
+
+ except LookupError as e:
+ print("""The CHILDES corpus, or the parts you need, should be manually
+ downloaded from http://childes.psy.cmu.edu/data-xml/ and saved at
+ [NLTK_Data_Dir]/corpora/childes/
+ Alternately, you can call the demo with the path to a portion of the CHILDES corpus, e.g.:
+ demo('/path/to/childes/data-xml/Eng-USA/")
+ """)
+ #corpus_root_http = urllib2.urlopen('http://childes.psy.cmu.edu/data-xml/Eng-USA/Bates.zip')
+ #corpus_root_http_bates = zipfile.ZipFile(cStringIO.StringIO(corpus_root_http.read()))
+ ##this fails
+ #childes = CHILDESCorpusReader(corpus_root_http_bates,corpus_root_http_bates.namelist())
+
+
+if __name__ == "__main__":
+ demo()
diff --git a/nltk/corpus/reader/chunked.py b/nltk/corpus/reader/chunked.py
new file mode 100644
index 0000000..a702917
--- /dev/null
+++ b/nltk/corpus/reader/chunked.py
@@ -0,0 +1,210 @@
+# Natural Language Toolkit: Chunked Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A reader for corpora that contain chunked (and optionally tagged)
+documents.
+"""
+
+import os.path, codecs
+
+import nltk
+from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
+from nltk import compat
+from nltk.tree import Tree
+from nltk.tokenize import *
+from nltk.chunk import tagstr2tree
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class ChunkedCorpusReader(CorpusReader):
+ """
+ Reader for chunked (and optionally tagged) corpora. Paragraphs
+ are split using a block reader. They are then tokenized into
+ sentences using a sentence tokenizer. Finally, these sentences
+ are parsed into chunk trees using a string-to-chunktree conversion
+ function. Each of these steps can be performed using a default
+ function or a custom function. By default, paragraphs are split
+ on blank lines; sentences are listed one per line; and sentences
+ are parsed into chunk trees using ``nltk.chunk.tagstr2tree``.
+ """
+ def __init__(self, root, fileids, extension='',
+ str2chunktree=tagstr2tree,
+ sent_tokenizer=RegexpTokenizer('\n', gaps=True),
+ para_block_reader=read_blankline_block,
+ encoding='utf8'):
+ """
+ :param root: The root directory for this corpus.
+ :param fileids: A list or regexp specifying the fileids in this corpus.
+ """
+ CorpusReader.__init__(self, root, fileids, encoding)
+
+ self._cv_args = (str2chunktree, sent_tokenizer, para_block_reader)
+ """Arguments for corpus views generated by this corpus: a tuple
+ (str2chunktree, sent_tokenizer, para_block_tokenizer)"""
+
+ def raw(self, fileids=None):
+ """
+ :return: the given file(s) as a single string.
+ :rtype: str
+ """
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def words(self, fileids=None):
+ """
+ :return: the given file(s) as a list of words
+ and punctuation symbols.
+ :rtype: list(str)
+ """
+ return concat([ChunkedCorpusView(f, enc, 0, 0, 0, 0, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ sentences or utterances, each encoded as a list of word
+ strings.
+ :rtype: list(list(str))
+ """
+ return concat([ChunkedCorpusView(f, enc, 0, 1, 0, 0, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def paras(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ paragraphs, each encoded as a list of sentences, which are
+ in turn encoded as lists of word strings.
+ :rtype: list(list(list(str)))
+ """
+ return concat([ChunkedCorpusView(f, enc, 0, 1, 1, 0, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def tagged_words(self, fileids=None):
+ """
+ :return: the given file(s) as a list of tagged
+ words and punctuation symbols, encoded as tuples
+ ``(word,tag)``.
+ :rtype: list(tuple(str,str))
+ """
+ return concat([ChunkedCorpusView(f, enc, 1, 0, 0, 0, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def tagged_sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ sentences, each encoded as a list of ``(word,tag)`` tuples.
+
+ :rtype: list(list(tuple(str,str)))
+ """
+ return concat([ChunkedCorpusView(f, enc, 1, 1, 0, 0, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def tagged_paras(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ paragraphs, each encoded as a list of sentences, which are
+ in turn encoded as lists of ``(word,tag)`` tuples.
+ :rtype: list(list(list(tuple(str,str))))
+ """
+ return concat([ChunkedCorpusView(f, enc, 1, 1, 1, 0, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def chunked_words(self, fileids=None):
+ """
+ :return: the given file(s) as a list of tagged
+ words and chunks. Words are encoded as ``(word, tag)``
+ tuples (if the corpus has tags) or word strings (if the
+ corpus has no tags). Chunks are encoded as depth-one
+ trees over ``(word,tag)`` tuples or word strings.
+ :rtype: list(tuple(str,str) and Tree)
+ """
+ return concat([ChunkedCorpusView(f, enc, 1, 0, 0, 1, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def chunked_sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ sentences, each encoded as a shallow Tree. The leaves
+ of these trees are encoded as ``(word, tag)`` tuples (if
+ the corpus has tags) or word strings (if the corpus has no
+ tags).
+ :rtype: list(Tree)
+ """
+ return concat([ChunkedCorpusView(f, enc, 1, 1, 0, 1, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def chunked_paras(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ paragraphs, each encoded as a list of sentences, which are
+ in turn encoded as a shallow Tree. The leaves of these
+ trees are encoded as ``(word, tag)`` tuples (if the corpus
+ has tags) or word strings (if the corpus has no tags).
+ :rtype: list(list(Tree))
+ """
+ return concat([ChunkedCorpusView(f, enc, 1, 1, 1, 1, *self._cv_args)
+ for (f, enc) in self.abspaths(fileids, True)])
+
+ def _read_block(self, stream):
+ return [tagstr2tree(t) for t in read_blankline_block(stream)]
+
+class ChunkedCorpusView(StreamBackedCorpusView):
+ def __init__(self, fileid, encoding, tagged, group_by_sent,
+ group_by_para, chunked, str2chunktree, sent_tokenizer,
+ para_block_reader):
+ StreamBackedCorpusView.__init__(self, fileid, encoding=encoding)
+ self._tagged = tagged
+ self._group_by_sent = group_by_sent
+ self._group_by_para = group_by_para
+ self._chunked = chunked
+ self._str2chunktree = str2chunktree
+ self._sent_tokenizer = sent_tokenizer
+ self._para_block_reader = para_block_reader
+
+ def read_block(self, stream):
+ block = []
+ for para_str in self._para_block_reader(stream):
+ para = []
+ for sent_str in self._sent_tokenizer.tokenize(para_str):
+ sent = self._str2chunktree(sent_str)
+
+ # If requested, throw away the tags.
+ if not self._tagged:
+ sent = self._untag(sent)
+
+ # If requested, throw away the chunks.
+ if not self._chunked:
+ sent = sent.leaves()
+
+ # Add the sentence to `para`.
+ if self._group_by_sent:
+ para.append(sent)
+ else:
+ para.extend(sent)
+
+ # Add the paragraph to `block`.
+ if self._group_by_para:
+ block.append(para)
+ else:
+ block.extend(para)
+
+ # Return the block
+ return block
+
+ def _untag(self, tree):
+ for i, child in enumerate(tree):
+ if isinstance(child, Tree):
+ self._untag(child)
+ elif isinstance(child, tuple):
+ tree[i] = child[0]
+ else:
+ raise ValueError('expected child to be Tree or tuple')
+ return tree
+
diff --git a/nltk/corpus/reader/cmudict.py b/nltk/corpus/reader/cmudict.py
new file mode 100644
index 0000000..bf76eac
--- /dev/null
+++ b/nltk/corpus/reader/cmudict.py
@@ -0,0 +1,95 @@
+# Natural Language Toolkit: Carnegie Mellon Pronouncing Dictionary Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+The Carnegie Mellon Pronouncing Dictionary [cmudict.0.6]
+ftp://ftp.cs.cmu.edu/project/speech/dict/
+Copyright 1998 Carnegie Mellon University
+
+File Format: Each line consists of an uppercased word, a counter
+(for alternative pronunciations), and a transcription. Vowels are
+marked for stress (1=primary, 2=secondary, 0=no stress). E.g.:
+NATURAL 1 N AE1 CH ER0 AH0 L
+
+The dictionary contains 127069 entries. Of these, 119400 words are assigned
+a unique pronunciation, 6830 words have two pronunciations, and 839 words have
+three or more pronunciations. Many of these are fast-speech variants.
+
+Phonemes: There are 39 phonemes, as shown below:
+
+Phoneme Example Translation Phoneme Example Translation
+------- ------- ----------- ------- ------- -----------
+AA odd AA D AE at AE T
+AH hut HH AH T AO ought AO T
+AW cow K AW AY hide HH AY D
+B be B IY CH cheese CH IY Z
+D dee D IY DH thee DH IY
+EH Ed EH D ER hurt HH ER T
+EY ate EY T F fee F IY
+G green G R IY N HH he HH IY
+IH it IH T IY eat IY T
+JH gee JH IY K key K IY
+L lee L IY M me M IY
+N knee N IY NG ping P IH NG
+OW oat OW T OY toy T OY
+P pee P IY R read R IY D
+S sea S IY SH she SH IY
+T tea T IY TH theta TH EY T AH
+UH hood HH UH D UW two T UW
+V vee V IY W we W IY
+Y yield Y IY L D Z zee Z IY
+ZH seizure S IY ZH ER
+"""
+
+import codecs
+
+from nltk import compat
+from nltk.util import Index
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class CMUDictCorpusReader(CorpusReader):
+ def entries(self):
+ """
+ :return: the cmudict lexicon as a list of entries
+ containing (word, transcriptions) tuples.
+ """
+ return concat([StreamBackedCorpusView(fileid, read_cmudict_block,
+ encoding=enc)
+ for fileid, enc in self.abspaths(None, True)])
+
+ def raw(self):
+ """
+ :return: the cmudict lexicon as a raw string.
+ """
+ fileids = self._fileids
+ if isinstance(fileids, compat.string_types):
+ fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def words(self):
+ """
+ :return: a list of all words defined in the cmudict lexicon.
+ """
+ return [word.lower() for (word, _) in self.entries()]
+
+ def dict(self):
+ """
+ :return: the cmudict lexicon as a dictionary, whose keys are
+ lowercase words and whose values are lists of pronunciations.
+ """
+ return dict(Index(self.entries()))
+
+def read_cmudict_block(stream):
+ entries = []
+ while len(entries) < 100: # Read 100 at a time.
+ line = stream.readline()
+ if line == '': return entries # end of file.
+ pieces = line.split()
+ entries.append( (pieces[0].lower(), pieces[2:]) )
+ return entries
diff --git a/nltk/corpus/reader/conll.py b/nltk/corpus/reader/conll.py
new file mode 100644
index 0000000..ce55c00
--- /dev/null
+++ b/nltk/corpus/reader/conll.py
@@ -0,0 +1,521 @@
+# Natural Language Toolkit: CONLL Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Read CoNLL-style chunk fileids.
+"""
+
+from __future__ import unicode_literals
+
+import os
+import codecs
+import textwrap
+
+from nltk import compat
+from nltk.tree import Tree
+from nltk.util import LazyMap, LazyConcatenation
+from nltk.tag import map_tag
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class ConllCorpusReader(CorpusReader):
+ """
+ A corpus reader for CoNLL-style files. These files consist of a
+ series of sentences, separated by blank lines. Each sentence is
+ encoded using a table (or "grid") of values, where each line
+ corresponds to a single word, and each column corresponds to an
+ annotation type. The set of columns used by CoNLL-style files can
+ vary from corpus to corpus; the ``ConllCorpusReader`` constructor
+ therefore takes an argument, ``columntypes``, which is used to
+ specify the columns that are used by a given corpus.
+
+ @todo: Add support for reading from corpora where different
+ parallel files contain different columns.
+ @todo: Possibly add caching of the grid corpus view? This would
+ allow the same grid view to be used by different data access
+ methods (eg words() and parsed_sents() could both share the
+ same grid corpus view object).
+ @todo: Better support for -DOCSTART-. Currently, we just ignore
+ it, but it could be used to define methods that retrieve a
+ document at a time (eg parsed_documents()).
+ """
+
+ #/////////////////////////////////////////////////////////////////
+ # Column Types
+ #/////////////////////////////////////////////////////////////////
+
+ WORDS = 'words' #: column type for words
+ POS = 'pos' #: column type for part-of-speech tags
+ TREE = 'tree' #: column type for parse trees
+ CHUNK = 'chunk' #: column type for chunk structures
+ NE = 'ne' #: column type for named entities
+ SRL = 'srl' #: column type for semantic role labels
+ IGNORE = 'ignore' #: column type for column that should be ignored
+
+ #: A list of all column types supported by the conll corpus reader.
+ COLUMN_TYPES = (WORDS, POS, TREE, CHUNK, NE, SRL, IGNORE)
+
+ #/////////////////////////////////////////////////////////////////
+ # Constructor
+ #/////////////////////////////////////////////////////////////////
+
+ def __init__(self, root, fileids, columntypes,
+ chunk_types=None, root_label='S', pos_in_tree=False,
+ srl_includes_roleset=True, encoding='utf8',
+ tree_class=Tree, tagset=None):
+ for columntype in columntypes:
+ if columntype not in self.COLUMN_TYPES:
+ raise ValueError('Bad column type %r' % columntype)
+ if isinstance(chunk_types, compat.string_types):
+ chunk_types = [chunk_types]
+ self._chunk_types = chunk_types
+ self._colmap = dict((c,i) for (i,c) in enumerate(columntypes))
+ self._pos_in_tree = pos_in_tree
+ self._root_label = root_label # for chunks
+ self._srl_includes_roleset = srl_includes_roleset
+ self._tree_class = tree_class
+ CorpusReader.__init__(self, root, fileids, encoding)
+ self._tagset = tagset
+
+ #/////////////////////////////////////////////////////////////////
+ # Data Access Methods
+ #/////////////////////////////////////////////////////////////////
+
+ def raw(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def words(self, fileids=None):
+ self._require(self.WORDS)
+ return LazyConcatenation(LazyMap(self._get_words, self._grids(fileids)))
+
+ def sents(self, fileids=None):
+ self._require(self.WORDS)
+ return LazyMap(self._get_words, self._grids(fileids))
+
+ def tagged_words(self, fileids=None, tagset=None):
+ self._require(self.WORDS, self.POS)
+ def get_tagged_words(grid):
+ return self._get_tagged_words(grid, tagset)
+ return LazyConcatenation(LazyMap(get_tagged_words,
+ self._grids(fileids)))
+
+ def tagged_sents(self, fileids=None, tagset=None):
+ self._require(self.WORDS, self.POS)
+ def get_tagged_words(grid):
+ return self._get_tagged_words(grid, tagset)
+ return LazyMap(get_tagged_words, self._grids(fileids))
+
+ def chunked_words(self, fileids=None, chunk_types=None,
+ tagset=None):
+ self._require(self.WORDS, self.POS, self.CHUNK)
+ if chunk_types is None: chunk_types = self._chunk_types
+ def get_chunked_words(grid): # capture chunk_types as local var
+ return self._get_chunked_words(grid, chunk_types, tagset)
+ return LazyConcatenation(LazyMap(get_chunked_words,
+ self._grids(fileids)))
+
+ def chunked_sents(self, fileids=None, chunk_types=None,
+ tagset=None):
+ self._require(self.WORDS, self.POS, self.CHUNK)
+ if chunk_types is None: chunk_types = self._chunk_types
+ def get_chunked_words(grid): # capture chunk_types as local var
+ return self._get_chunked_words(grid, chunk_types, tagset)
+ return LazyMap(get_chunked_words, self._grids(fileids))
+
+ def parsed_sents(self, fileids=None, pos_in_tree=None, tagset=None):
+ self._require(self.WORDS, self.POS, self.TREE)
+ if pos_in_tree is None: pos_in_tree = self._pos_in_tree
+ def get_parsed_sent(grid): # capture pos_in_tree as local var
+ return self._get_parsed_sent(grid, pos_in_tree, tagset)
+ return LazyMap(get_parsed_sent, self._grids(fileids))
+
+ def srl_spans(self, fileids=None):
+ self._require(self.SRL)
+ return LazyMap(self._get_srl_spans, self._grids(fileids))
+
+ def srl_instances(self, fileids=None, pos_in_tree=None, flatten=True):
+ self._require(self.WORDS, self.POS, self.TREE, self.SRL)
+ if pos_in_tree is None: pos_in_tree = self._pos_in_tree
+ def get_srl_instances(grid): # capture pos_in_tree as local var
+ return self._get_srl_instances(grid, pos_in_tree)
+ result = LazyMap(get_srl_instances, self._grids(fileids))
+ if flatten: result = LazyConcatenation(result)
+ return result
+
+ def iob_words(self, fileids=None, tagset=None):
+ """
+ :return: a list of word/tag/IOB tuples
+ :rtype: list(tuple)
+ :param fileids: the list of fileids that make up this corpus
+ :type fileids: None or str or list
+ """
+ self._require(self.WORDS, self.POS, self.CHUNK)
+ def get_iob_words(grid):
+ return self._get_iob_words(grid, tagset)
+ return LazyConcatenation(LazyMap(get_iob_words, self._grids(fileids)))
+
+ def iob_sents(self, fileids=None, tagset=None):
+ """
+ :return: a list of lists of word/tag/IOB tuples
+ :rtype: list(list)
+ :param fileids: the list of fileids that make up this corpus
+ :type fileids: None or str or list
+ """
+ self._require(self.WORDS, self.POS, self.CHUNK)
+ def get_iob_words(grid):
+ return self._get_iob_words(grid, tagset)
+ return LazyMap(get_iob_words, self._grids(fileids))
+
+ #/////////////////////////////////////////////////////////////////
+ # Grid Reading
+ #/////////////////////////////////////////////////////////////////
+
+ def _grids(self, fileids=None):
+ # n.b.: we could cache the object returned here (keyed on
+ # fileids), which would let us reuse the same corpus view for
+ # different things (eg srl and parse trees).
+ return concat([StreamBackedCorpusView(fileid, self._read_grid_block,
+ encoding=enc)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def _read_grid_block(self, stream):
+ grids = []
+ for block in read_blankline_block(stream):
+ block = block.strip()
+ if not block: continue
+
+ grid = [line.split() for line in block.split('\n')]
+
+ # If there's a docstart row, then discard. ([xx] eventually it
+ # would be good to actually use it)
+ if grid[0][self._colmap.get('words', 0)] == '-DOCSTART-':
+ del grid[0]
+
+ # Check that the grid is consistent.
+ for row in grid:
+ if len(row) != len(grid[0]):
+ raise ValueError('Inconsistent number of columns:\n%s'
+ % block)
+ grids.append(grid)
+ return grids
+
+ #/////////////////////////////////////////////////////////////////
+ # Transforms
+ #/////////////////////////////////////////////////////////////////
+ # given a grid, transform it into some representation (e.g.,
+ # a list of words or a parse tree).
+
+ def _get_words(self, grid):
+ return self._get_column(grid, self._colmap['words'])
+
+ def _get_tagged_words(self, grid, tagset=None):
+ pos_tags = self._get_column(grid, self._colmap['pos'])
+ if tagset and tagset != self._tagset:
+ pos_tags = [map_tag(self._tagset, tagset, t) for t in pos_tags]
+ return list(zip(self._get_column(grid, self._colmap['words']), pos_tags))
+
+ def _get_iob_words(self, grid, tagset=None):
+ pos_tags = self._get_column(grid, self._colmap['pos'])
+ if tagset and tagset != self._tagset:
+ pos_tags = [map_tag(self._tagset, tagset, t) for t in pos_tags]
+ return list(zip(self._get_column(grid, self._colmap['words']), pos_tags,
+ self._get_column(grid, self._colmap['chunk'])))
+
+ def _get_chunked_words(self, grid, chunk_types, tagset=None):
+ # n.b.: this method is very similar to conllstr2tree.
+ words = self._get_column(grid, self._colmap['words'])
+ pos_tags = self._get_column(grid, self._colmap['pos'])
+ if tagset and tagset != self._tagset:
+ pos_tags = [map_tag(self._tagset, tagset, t) for t in pos_tags]
+ chunk_tags = self._get_column(grid, self._colmap['chunk'])
+
+ stack = [Tree(self._root_label, [])]
+
+ for (word, pos_tag, chunk_tag) in zip(words, pos_tags, chunk_tags):
+ if chunk_tag == 'O':
+ state, chunk_type = 'O', ''
+ else:
+ (state, chunk_type) = chunk_tag.split('-')
+ # If it's a chunk we don't care about, treat it as O.
+ if chunk_types is not None and chunk_type not in chunk_types:
+ state = 'O'
+ # Treat a mismatching I like a B.
+ if state == 'I' and chunk_type != stack[-1].label():
+ state = 'B'
+ # For B or I: close any open chunks
+ if state in 'BO' and len(stack) == 2:
+ stack.pop()
+ # For B: start a new chunk.
+ if state == 'B':
+ new_chunk = Tree(chunk_type, [])
+ stack[-1].append(new_chunk)
+ stack.append(new_chunk)
+ # Add the word token.
+ stack[-1].append((word, pos_tag))
+
+ return stack[0]
+
+ def _get_parsed_sent(self, grid, pos_in_tree, tagset=None):
+ words = self._get_column(grid, self._colmap['words'])
+ pos_tags = self._get_column(grid, self._colmap['pos'])
+ if tagset and tagset != self._tagset:
+ pos_tags = [map_tag(self._tagset, tagset, t) for t in pos_tags]
+ parse_tags = self._get_column(grid, self._colmap['tree'])
+
+ treestr = ''
+ for (word, pos_tag, parse_tag) in zip(words, pos_tags, parse_tags):
+ if word == '(': word = '-LRB-'
+ if word == ')': word = '-RRB-'
+ if pos_tag == '(': pos_tag = '-LRB-'
+ if pos_tag == ')': pos_tag = '-RRB-'
+ (left, right) = parse_tag.split('*')
+ right = right.count(')')*')' # only keep ')'.
+ treestr += '%s (%s %s) %s' % (left, pos_tag, word, right)
+ try:
+ tree = self._tree_class.parse(treestr)
+ except (ValueError, IndexError):
+ tree = self._tree_class.parse('(%s %s)' %
+ (self._root_label, treestr))
+
+ if not pos_in_tree:
+ for subtree in tree.subtrees():
+ for i, child in enumerate(subtree):
+ if (isinstance(child, Tree) and len(child)==1 and
+ isinstance(child[0], compat.string_types)):
+ subtree[i] = (child[0], child.label())
+
+ return tree
+
+ def _get_srl_spans(self, grid):
+ """
+ list of list of (start, end), tag) tuples
+ """
+ if self._srl_includes_roleset:
+ predicates = self._get_column(grid, self._colmap['srl']+1)
+ start_col = self._colmap['srl']+2
+ else:
+ predicates = self._get_column(grid, self._colmap['srl'])
+ start_col = self._colmap['srl']+1
+
+ # Count how many predicates there are. This tells us how many
+ # columns to expect for SRL data.
+ num_preds = len([p for p in predicates if p != '-'])
+
+ spanlists = []
+ for i in range(num_preds):
+ col = self._get_column(grid, start_col+i)
+ spanlist = []
+ stack = []
+ for wordnum, srl_tag in enumerate(col):
+ (left, right) = srl_tag.split('*')
+ for tag in left.split('('):
+ if tag:
+ stack.append((tag, wordnum))
+ for i in range(right.count(')')):
+ (tag, start) = stack.pop()
+ spanlist.append( ((start, wordnum+1), tag) )
+ spanlists.append(spanlist)
+
+ return spanlists
+
+ def _get_srl_instances(self, grid, pos_in_tree):
+ tree = self._get_parsed_sent(grid, pos_in_tree)
+ spanlists = self._get_srl_spans(grid)
+ if self._srl_includes_roleset:
+ predicates = self._get_column(grid, self._colmap['srl']+1)
+ rolesets = self._get_column(grid, self._colmap['srl'])
+ else:
+ predicates = self._get_column(grid, self._colmap['srl'])
+ rolesets = [None] * len(predicates)
+
+ instances = ConllSRLInstanceList(tree)
+ for wordnum, predicate in enumerate(predicates):
+ if predicate == '-': continue
+ # Decide which spanlist to use. Don't assume that they're
+ # sorted in the same order as the predicates (even though
+ # they usually are).
+ for spanlist in spanlists:
+ for (start, end), tag in spanlist:
+ if wordnum in range(start,end) and tag in ('V', 'C-V'):
+ break
+ else: continue
+ break
+ else:
+ raise ValueError('No srl column found for %r' % predicate)
+ instances.append(ConllSRLInstance(tree, wordnum, predicate,
+ rolesets[wordnum], spanlist))
+
+ return instances
+
+ #/////////////////////////////////////////////////////////////////
+ # Helper Methods
+ #/////////////////////////////////////////////////////////////////
+
+ def _require(self, *columntypes):
+ for columntype in columntypes:
+ if columntype not in self._colmap:
+ raise ValueError('This corpus does not contain a %s '
+ 'column.' % columntype)
+
+ @staticmethod
+ def _get_column(grid, column_index):
+ return [grid[i][column_index] for i in range(len(grid))]
+
+
+ at compat.python_2_unicode_compatible
+class ConllSRLInstance(object):
+ """
+ An SRL instance from a CoNLL corpus, which identifies and
+ providing labels for the arguments of a single verb.
+ """
+ # [xx] add inst.core_arguments, inst.argm_arguments?
+
+ def __init__(self, tree, verb_head, verb_stem, roleset, tagged_spans):
+ self.verb = []
+ """A list of the word indices of the words that compose the
+ verb whose arguments are identified by this instance.
+ This will contain multiple word indices when multi-word
+ verbs are used (e.g. 'turn on')."""
+
+ self.verb_head = verb_head
+ """The word index of the head word of the verb whose arguments
+ are identified by this instance. E.g., for a sentence that
+ uses the verb 'turn on,' ``verb_head`` will be the word index
+ of the word 'turn'."""
+
+ self.verb_stem = verb_stem
+
+ self.roleset = roleset
+
+ self.arguments = []
+ """A list of ``(argspan, argid)`` tuples, specifying the location
+ and type for each of the arguments identified by this
+ instance. ``argspan`` is a tuple ``start, end``, indicating
+ that the argument consists of the ``words[start:end]``."""
+
+ self.tagged_spans = tagged_spans
+ """A list of ``(span, id)`` tuples, specifying the location and
+ type for each of the arguments, as well as the verb pieces,
+ that make up this instance."""
+
+ self.tree = tree
+ """The parse tree for the sentence containing this instance."""
+
+ self.words = tree.leaves()
+ """A list of the words in the sentence containing this
+ instance."""
+
+ # Fill in the self.verb and self.arguments values.
+ for (start, end), tag in tagged_spans:
+ if tag in ('V', 'C-V'):
+ self.verb += list(range(start, end))
+ else:
+ self.arguments.append( ((start, end), tag) )
+
+ def __repr__(self):
+ plural = len(self.arguments)!=1 and 's' or ''
+ return '<ConllSRLInstance for %r with %d argument%s>' % (
+ (self.verb_stem, len(self.arguments), plural))
+
+ def pprint(self):
+ verbstr = ' '.join(self.words[i][0] for i in self.verb)
+ hdr = 'SRL for %r (stem=%r):\n' % (verbstr, self.verb_stem)
+ s = ''
+ for i, word in enumerate(self.words):
+ if isinstance(word, tuple): word = word[0]
+ for (start, end), argid in self.arguments:
+ if i == start: s += '[%s ' % argid
+ if i == end: s += '] '
+ if i in self.verb: word = '<<%s>>' % word
+ s += word + ' '
+ return hdr + textwrap.fill(s.replace(' ]', ']'),
+ initial_indent=' ',
+ subsequent_indent=' ')
+
+ at compat.python_2_unicode_compatible
+class ConllSRLInstanceList(list):
+ """
+ Set of instances for a single sentence
+ """
+ def __init__(self, tree, instances=()):
+ self.tree = tree
+ list.__init__(self, instances)
+
+ def __str__(self):
+ return self.pprint()
+
+ def pprint(self, include_tree=False):
+ # Sanity check: trees should be the same
+ for inst in self:
+ if inst.tree != self.tree:
+ raise ValueError('Tree mismatch!')
+
+ # If desired, add trees:
+ if include_tree:
+ words = self.tree.leaves()
+ pos = [None] * len(words)
+ synt = ['*'] * len(words)
+ self._tree2conll(self.tree, 0, words, pos, synt)
+
+ s = ''
+ for i in range(len(words)):
+ # optional tree columns
+ if include_tree:
+ s += '%-20s ' % words[i]
+ s += '%-8s ' % pos[i]
+ s += '%15s*%-8s ' % tuple(synt[i].split('*'))
+
+ # verb head column
+ for inst in self:
+ if i == inst.verb_head:
+ s += '%-20s ' % inst.verb_stem
+ break
+ else:
+ s += '%-20s ' % '-'
+ # Remaining columns: self
+ for inst in self:
+ argstr = '*'
+ for (start, end), argid in inst.tagged_spans:
+ if i==start: argstr = '(%s%s' % (argid, argstr)
+ if i==(end-1): argstr += ')'
+ s += '%-12s ' % argstr
+ s += '\n'
+ return s
+
+ def _tree2conll(self, tree, wordnum, words, pos, synt):
+ assert isinstance(tree, Tree)
+ if len(tree) == 1 and isinstance(tree[0], compat.string_types):
+ pos[wordnum] = tree.label()
+ assert words[wordnum] == tree[0]
+ return wordnum+1
+ elif len(tree) == 1 and isinstance(tree[0], tuple):
+ assert len(tree[0]) == 2
+ pos[wordnum], pos[wordnum] = tree[0]
+ return wordnum+1
+ else:
+ synt[wordnum] = '(%s%s' % (tree.label(), synt[wordnum])
+ for child in tree:
+ wordnum = self._tree2conll(child, wordnum, words,
+ pos, synt)
+ synt[wordnum-1] += ')'
+ return wordnum
+
+class ConllChunkCorpusReader(ConllCorpusReader):
+ """
+ A ConllCorpusReader whose data file contains three columns: words,
+ pos, and chunk.
+ """
+ def __init__(self, root, fileids, chunk_types, encoding='utf8',
+ tagset=None):
+ ConllCorpusReader.__init__(
+ self, root, fileids, ('words', 'pos', 'chunk'),
+ chunk_types=chunk_types, encoding=encoding,
+ tagset=tagset)
diff --git a/nltk/corpus/reader/dependency.py b/nltk/corpus/reader/dependency.py
new file mode 100644
index 0000000..a43553b
--- /dev/null
+++ b/nltk/corpus/reader/dependency.py
@@ -0,0 +1,101 @@
+# Natural Language Toolkit: Dependency Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Kepa Sarasola <kepa.sarasola at ehu.es>
+# Iker Manterola <returntothehangar at hotmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import codecs
+
+from nltk.parse import DependencyGraph
+from nltk.tokenize import *
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class DependencyCorpusReader(SyntaxCorpusReader):
+
+ def __init__(self, root, fileids, encoding='utf8',
+ word_tokenizer=TabTokenizer(),
+ sent_tokenizer=RegexpTokenizer('\n', gaps=True),
+ para_block_reader=read_blankline_block):
+
+ CorpusReader.__init__(self, root, fileids, encoding)
+
+ #########################################################
+
+ def raw(self, fileids=None):
+ """
+ :return: the given file(s) as a single string.
+ :rtype: str
+ """
+ result = []
+ for fileid, encoding in self.abspaths(fileids, include_encoding=True):
+ if isinstance(fileid, PathPointer):
+ result.append(fileid.open(encoding=encoding).read())
+ else:
+ with codecs.open(fileid, "r", encoding) as fp:
+ result.append(fp.read())
+ return concat(result)
+
+ def words(self, fileids=None):
+ return concat([DependencyCorpusView(fileid, False, False, False, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+
+ def tagged_words(self, fileids=None):
+ return concat([DependencyCorpusView(fileid, True, False, False, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+
+ def sents(self, fileids=None):
+ return concat([DependencyCorpusView(fileid, False, True, False, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+
+ def tagged_sents(self, fileids=None):
+ return concat([DependencyCorpusView(fileid, True, True, False, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+
+ def parsed_sents(self, fileids=None):
+ sents=concat([DependencyCorpusView(fileid, False, True, True, encoding=enc)
+ for fileid, enc in self.abspaths(fileids, include_encoding=True)])
+ return [DependencyGraph(sent) for sent in sents]
+
+
+class DependencyCorpusView(StreamBackedCorpusView):
+ _DOCSTART = '-DOCSTART- -DOCSTART- O\n' #dokumentu hasiera definitzen da
+
+ def __init__(self, corpus_file, tagged, group_by_sent, dependencies,
+ chunk_types=None, encoding='utf8'):
+ self._tagged = tagged
+ self._dependencies = dependencies
+ self._group_by_sent = group_by_sent
+ self._chunk_types = chunk_types
+ StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
+
+ def read_block(self, stream):
+ # Read the next sentence.
+ sent = read_blankline_block(stream)[0].strip()
+ # Strip off the docstart marker, if present.
+ if sent.startswith(self._DOCSTART):
+ sent = sent[len(self._DOCSTART):].lstrip()
+
+ # extract word and tag from any of the formats
+ if not self._dependencies:
+ lines = [line.split('\t') for line in sent.split('\n')]
+ if len(lines[0]) == 3 or len(lines[0]) == 4:
+ sent = [(line[0], line[1]) for line in lines]
+ elif len(lines[0]) == 10:
+ sent = [(line[1], line[4]) for line in lines]
+ else:
+ raise ValueError('Unexpected number of fields in dependency tree file')
+
+ # discard tags if they weren't requested
+ if not self._tagged:
+ sent = [word for (word, tag) in sent]
+
+ # Return the result.
+ if self._group_by_sent:
+ return [sent]
+ else:
+ return list(sent)
diff --git a/nltk/corpus/reader/framenet.py b/nltk/corpus/reader/framenet.py
new file mode 100644
index 0000000..567f92e
--- /dev/null
+++ b/nltk/corpus/reader/framenet.py
@@ -0,0 +1,2062 @@
+# Natural Language Toolkit: Framenet Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Authors: Chuck Wooters <wooters at icsi.berkeley.edu>,
+# Nathan Schneider <nschneid at cs.cmu.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+"""
+Corpus reader for the Framenet 1.5 Corpus.
+"""
+
+__docformat__ = 'epytext en'
+
+import os, sys
+import re
+import textwrap
+from collections import defaultdict
+from pprint import pprint, pformat
+from nltk.internals import ElementWrapper
+from nltk.corpus.reader import XMLCorpusReader, XMLCorpusView
+from nltk.compat import text_type, string_types, python_2_unicode_compatible
+from nltk.util import AbstractLazySequence, LazyMap
+
+
+def _pretty_longstring(defstr, prefix='', wrap_at=65):
+
+ """
+ Helper function for pretty-printing a long string.
+
+ :param defstr: The string to be printed.
+ :type defstr: str
+ :return: A nicely formated string representation of the long string.
+ :rtype: str
+ """
+
+ outstr = ""
+ for line in textwrap.fill(defstr, wrap_at).split('\n'):
+ outstr += prefix + line + '\n'
+ return outstr
+
+def _pretty_any(obj):
+
+ """
+ Helper function for pretty-printing any AttrDict object.
+
+ :param obj: The obj to be printed.
+ :type obj: AttrDict
+ :return: A nicely formated string representation of the AttrDict object.
+ :rtype: str
+ """
+
+ outstr = ""
+ for k in obj:
+ if isinstance(obj[k], string_types) and len(obj[k]) > 65:
+ outstr += "[{0}]\n".format(k)
+ outstr += "{0}".format(_pretty_longstring(obj[k], prefix=' '))
+ outstr += '\n'
+ else:
+ outstr += "[{0}] {1}\n".format(k, obj[k])
+
+ return outstr
+
+def _pretty_semtype(st):
+
+ """
+ Helper function for pretty-printing a semantic type.
+
+ :param st: The semantic type to be printed.
+ :type st: AttrDict
+ :return: A nicely formated string representation of the semantic type.
+ :rtype: str
+ """
+
+ semkeys = st.keys()
+ if len(semkeys) == 1: return "<None>"
+
+ outstr = ""
+ outstr += "semantic type ({0.ID}): {0.name}\n".format(st)
+ if 'abbrev' in semkeys:
+ outstr += "[abbrev] {0}\n".format(st.abbrev)
+ if 'definition' in semkeys:
+ outstr += "[definition]\n"
+ outstr += _pretty_longstring(st.definition,' ')
+ outstr += "[rootType] {0}({1})\n".format(st.rootType.name, st.rootType.ID)
+ if st.superType is None:
+ outstr += "[superType] <None>\n"
+ else:
+ outstr += "[superType] {0}({1})\n".format(st.superType.name, st.superType.ID)
+ outstr += "[subTypes] {0} subtypes\n".format(len(st.subTypes))
+ outstr += " " + ", ".join('{0}({1})'.format(x.name, x.ID) for x in st.subTypes) + '\n'*(len(st.subTypes)>0)
+ return outstr
+
+def _pretty_frame_relation_type(freltyp):
+
+ """
+ Helper function for pretty-printing a frame relation type.
+
+ :param freltyp: The frame relation type to be printed.
+ :type freltyp: AttrDict
+ :return: A nicely formated string representation of the frame relation type.
+ :rtype: str
+ """
+ outstr = "<frame relation type ({0.ID}): {0.superFrameName} -- {0.name} -> {0.subFrameName}>".format(freltyp)
+ return outstr
+
+def _pretty_frame_relation(frel):
+
+ """
+ Helper function for pretty-printing a frame relation.
+
+ :param frel: The frame relation to be printed.
+ :type frel: AttrDict
+ :return: A nicely formated string representation of the frame relation.
+ :rtype: str
+ """
+ outstr = "<{0.type.superFrameName}={0.superFrameName} -- {0.type.name} -> {0.type.subFrameName}={0.subFrameName}>".format(frel)
+ return outstr
+
+def _pretty_fe_relation(ferel):
+
+ """
+ Helper function for pretty-printing an FE relation.
+
+ :param ferel: The FE relation to be printed.
+ :type ferel: AttrDict
+ :return: A nicely formated string representation of the FE relation.
+ :rtype: str
+ """
+ outstr = "<{0.type.superFrameName}={0.frameRelation.superFrameName}.{0.superFEName} -- {0.type.name} -> {0.type.subFrameName}={0.frameRelation.subFrameName}.{0.subFEName}>".format(ferel)
+ return outstr
+
+def _pretty_lu(lu):
+
+ """
+ Helper function for pretty-printing a lexical unit.
+
+ :param lu: The lu to be printed.
+ :type lu: AttrDict
+ :return: A nicely formated string representation of the lexical unit.
+ :rtype: str
+ """
+
+ lukeys = lu.keys()
+ outstr = ""
+ outstr += "lexical unit ({0.ID}): {0.name}\n\n".format(lu)
+ if 'definition' in lukeys:
+ outstr += "[definition]\n"
+ outstr += _pretty_longstring(lu.definition,' ')
+ if 'frame' in lukeys:
+ outstr += "\n[frame] {0}({1})\n".format(lu.frame.name,lu.frame.ID)
+ if 'incorporatedFE' in lukeys:
+ outstr += "\n[incorporatedFE] {0}\n".format(lu.incorporatedFE)
+ if 'POS' in lukeys:
+ outstr += "\n[POS] {0}\n".format(lu.POS)
+ if 'status' in lukeys:
+ outstr += "\n[status] {0}\n".format(lu.status)
+ if 'totalAnnotated' in lukeys:
+ outstr += "\n[totalAnnotated] {0} annotated examples\n".format(lu.totalAnnotated)
+ if 'lexemes' in lukeys:
+ outstr += "\n[lexemes] {0}\n".format(' '.join('{0}/{1}'.format(lex.name,lex.POS) for lex in lu.lexemes))
+ if 'semTypes' in lukeys:
+ outstr += "\n[semTypes] {0} semantic types\n".format(len(lu.semTypes))
+ outstr += " "*(len(lu.semTypes)>0) + ", ".join('{0}({1})'.format(x.name, x.ID) for x in lu.semTypes) + '\n'*(len(lu.semTypes)>0)
+ if 'subCorpus' in lukeys:
+ subc = [x.name for x in lu.subCorpus]
+ outstr += "\n[subCorpus] {0} subcorpora\n".format(len(lu.subCorpus))
+ for line in textwrap.fill(", ".join(sorted(subc)), 60).split('\n'):
+ outstr += " {0}\n".format(line)
+
+ return outstr
+
+def _pretty_fe(fe):
+
+ """
+ Helper function for pretty-printing a frame element.
+
+ :param fe: The frame element to be printed.
+ :type fe: AttrDict
+ :return: A nicely formated string representation of the frame element.
+ :rtype: str
+ """
+ fekeys = fe.keys()
+ outstr = ""
+ outstr += "frame element ({0.ID}): {0.name}\n of {1.name}({1.ID})\n".format(fe, fe.frame)
+ if 'definition' in fekeys:
+ outstr += "[definition]\n"
+ outstr += _pretty_longstring(fe.definition,' ')
+ if 'abbrev' in fekeys:
+ outstr += "[abbrev] {0}\n".format(fe.abbrev)
+ if 'coreType' in fekeys:
+ outstr += "[coreType] {0}\n".format(fe.coreType)
+ if 'requiresFE' in fekeys:
+ outstr += "[requiresFE] "
+ if fe.requiresFE is None:
+ outstr += "<None>\n"
+ else:
+ outstr += "{0}({1})\n".format(fe.requiresFE.name, fe.requiresFE.ID)
+ if 'excludesFE' in fekeys:
+ outstr += "[excludesFE] "
+ if fe.excludesFE is None:
+ outstr += "<None>\n"
+ else:
+ outstr += "{0}({1})\n".format(fe.excludesFE.name, fe.excludesFE.ID)
+ if 'semType' in fekeys:
+ outstr += "[semType] "
+ if fe.semType is None:
+ outstr += "<None>\n"
+ else:
+ outstr += "\n " + "{0}({1})".format(fe.semType.name, fe.semType.ID) + '\n'
+
+ return outstr
+
+def _pretty_frame(frame):
+
+ """
+ Helper function for pretty-printing a frame.
+
+ :param frame: The frame to be printed.
+ :type frame: AttrDict
+ :return: A nicely formated string representation of the frame.
+ :rtype: str
+ """
+
+ outstr = ""
+ outstr += "frame ({0.ID}): {0.name}\n\n".format(frame)
+ outstr += "[definition]\n"
+ outstr += _pretty_longstring(frame.definition, ' ') + '\n'
+
+ outstr += "[semTypes] {0} semantic types\n".format(len(frame.semTypes))
+ outstr += " "*(len(frame.semTypes)>0) + ", ".join("{0}({1})".format(x.name, x.ID) for x in frame.semTypes) + '\n'*(len(frame.semTypes)>0)
+
+ outstr += "\n[frameRelations] {0} frame relations\n".format(len(frame.frameRelations))
+ outstr += ' ' + '\n '.join(repr(frel) for frel in frame.frameRelations) + '\n'
+
+ outstr += "\n[lexUnit] {0} lexical units\n".format(len(frame.lexUnit))
+ lustrs = []
+ for luName,lu in sorted(frame.lexUnit.items()):
+ tmpstr = '{0} ({1})'.format(luName, lu.ID)
+ lustrs.append(tmpstr)
+ outstr += "{0}\n".format(_pretty_longstring(', '.join(lustrs),prefix=' '))
+
+ outstr += "\n[FE] {0} frame elements\n".format(len(frame.FE))
+ fes = {}
+ for feName,fe in sorted(frame.FE.items()):
+ try:
+ fes[fe.coreType].append("{0} ({1})".format(feName, fe.ID))
+ except KeyError:
+ fes[fe.coreType] = []
+ fes[fe.coreType].append("{0} ({1})".format(feName, fe.ID))
+ for ct in sorted(fes.keys(), key=lambda ct2: ['Core','Core-Unexpressed','Peripheral','Extra-Thematic'].index(ct2)):
+ outstr += "{0:>16}: {1}\n".format(ct, ', '.join(sorted(fes[ct])))
+
+ outstr += "\n[FEcoreSets] {0} frame element core sets\n".format(len(frame.FEcoreSets))
+ outstr += " " + '\n '.join(", ".join([x.name for x in coreSet]) for coreSet in frame.FEcoreSets) + '\n'
+
+ return outstr
+
+class FramenetError(Exception):
+
+ """An exception class for framenet-related errors."""
+
+ at python_2_unicode_compatible
+class AttrDict(dict):
+
+ """A class that wraps a dict and allows accessing the keys of the
+ dict as if they were attributes. Taken from here:
+ http://stackoverflow.com/a/14620633/8879
+
+ >>> foo = {'a':1, 'b':2, 'c':3}
+ >>> bar = AttrDict(foo)
+ >>> pprint(dict(bar))
+ {'a': 1, 'b': 2, 'c': 3}
+ >>> bar.b
+ 2
+ >>> bar.d = 4
+ >>> pprint(dict(bar))
+ {'a': 1, 'b': 2, 'c': 3, 'd': 4}
+ """
+
+ def __init__(self, *args, **kwargs):
+ super(AttrDict, self).__init__(*args, **kwargs)
+ #self.__dict__ = self
+
+ def __setattr__(self, name, value):
+ self[name] = value
+ def __getattr__(self, name):
+ if name=='_short_repr':
+ return self._short_repr
+ return self[name]
+ def __getitem__(self, name):
+ v = super(AttrDict,self).__getitem__(name)
+ if isinstance(v,Future):
+ return v._data()
+ return v
+
+ def _short_repr(self):
+ if '_type' in self:
+ if self['_type'].endswith('relation'):
+ return self.__repr__()
+ try:
+ return "<{0} ID={1} name={2}>".format(self['_type'], self['ID'], self['name'])
+ except KeyError: # no ID--e.g., for _type=lusubcorpus
+ return "<{0} name={1}>".format(self['_type'], self['name'])
+ else:
+ return self.__repr__()
+
+ def _str(self):
+ outstr = ""
+
+ if not '_type' in self:
+ outstr = _pretty_any(self)
+ elif self['_type'] == 'frame':
+ outstr = _pretty_frame(self)
+ elif self['_type'] == 'fe':
+ outstr = _pretty_fe(self)
+ elif self['_type'] == 'lu':
+ outstr = _pretty_lu(self)
+ elif self['_type'] == 'semtype':
+ outstr = _pretty_semtype(self)
+ elif self['_type'] == 'framerelationtype':
+ outstr = _pretty_frame_relation_type(self)
+ elif self['_type'] == 'framerelation':
+ outstr = _pretty_frame_relation(self)
+ elif self['_type'] == 'ferelation':
+ outstr = _pretty_fe_relation(self)
+ else:
+ outstr = _pretty_any(self)
+
+ # ensure result is unicode string prior to applying the
+ # @python_2_unicode_compatible decorator (because non-ASCII characters
+ # could in principle occur in the data and would trigger an encoding error when
+ # passed as arguments to str.format()).
+ # assert isinstance(outstr, unicode) # not in Python 3.2
+ return outstr
+
+ def __str__(self):
+ return self._str()
+ def __repr__(self):
+ return self.__str__()
+
+class Future(object):
+ """
+ Wraps and acts as a proxy for a value to be loaded lazily (on demand).
+ Adapted from https://gist.github.com/sergey-miryanov/2935416
+ """
+ def __init__(self, loader, *args, **kwargs):
+ """
+ :param loader: when called with no arguments, returns the value to be stored
+ :type loader: callable
+ """
+ super (Future, self).__init__(*args, **kwargs)
+ self._loader = loader
+ self._d = None
+ def _data(self):
+ if callable(self._loader):
+ self._d = self._loader()
+ self._loader = None # the data is now cached
+ return self._d
+
+ def __nonzero__(self):
+ return bool(self._data())
+ def __len__(self):
+ return len(self._data())
+
+ def __setitem__(self, key, value):
+ return self._data ().__setitem__(key, value)
+ def __getitem__(self, key):
+ return self._data ().__getitem__(key)
+ def __getattr__(self, key):
+ return self._data().__getattr__(key)
+
+ def __str__(self):
+ return self._data().__str__()
+ def __repr__(self):
+ return self._data().__repr__()
+
+
+ at python_2_unicode_compatible
+class PrettyDict(AttrDict):
+ """
+ Displays an abbreviated repr of values where possible.
+ Inherits from AttrDict, so a callable value will
+ be lazily converted to an actual value.
+ """
+ def __init__(self, *args, **kwargs):
+ _BREAK_LINES = kwargs.pop('breakLines', False)
+ super(PrettyDict, self).__init__(*args, **kwargs)
+ dict.__setattr__(self, '_BREAK_LINES', _BREAK_LINES)
+ def __repr__(self):
+ parts = []
+ for k,v in sorted(self.items()):
+ kv = repr(k)+': '
+ try:
+ kv += v._short_repr()
+ except AttributeError:
+ kv += repr(v)
+ parts.append(kv)
+ return '{'+(',\n ' if self._BREAK_LINES else ', ').join(parts)+'}'
+
+ at python_2_unicode_compatible
+class PrettyList(list):
+ """
+ Displays an abbreviated repr of only the first several elements, not the whole list.
+ """
+ # from nltk.util
+ def __init__(self, *args, **kwargs):
+ self._MAX_REPR_SIZE = kwargs.pop('maxReprSize', 60)
+ self._BREAK_LINES = kwargs.pop('breakLines', False)
+ super(PrettyList, self).__init__(*args, **kwargs)
+ def __repr__(self):
+ """
+ Return a string representation for this corpus view that is
+ similar to a list's representation; but if it would be more
+ than 60 characters long, it is truncated.
+ """
+ pieces = []
+ length = 5
+
+ for elt in self:
+ pieces.append(elt._short_repr()) # key difference from inherited version: call to _short_repr()
+ length += len(pieces[-1]) + 2
+ if self._MAX_REPR_SIZE and length > self._MAX_REPR_SIZE and len(pieces) > 2:
+ return "[%s, ...]" % text_type(',\n ' if self._BREAK_LINES else ', ').join(pieces[:-1])
+ return "[%s]" % text_type(',\n ' if self._BREAK_LINES else ', ').join(pieces)
+
+ at python_2_unicode_compatible
+class PrettyLazyMap(LazyMap):
+ """
+ Displays an abbreviated repr of only the first several elements, not the whole list.
+ """
+ # from nltk.util
+ _MAX_REPR_SIZE = 60
+ def __repr__(self):
+ """
+ Return a string representation for this corpus view that is
+ similar to a list's representation; but if it would be more
+ than 60 characters long, it is truncated.
+ """
+ pieces = []
+ length = 5
+ for elt in self:
+ pieces.append(elt._short_repr()) # key difference from inherited version: call to _short_repr()
+ length += len(pieces[-1]) + 2
+ if length > self._MAX_REPR_SIZE and len(pieces) > 2:
+ return "[%s, ...]" % text_type(', ').join(pieces[:-1])
+ else:
+ return "[%s]" % text_type(', ').join(pieces)
+
+class FramenetCorpusReader(XMLCorpusReader):
+ """A corpus reader for the Framenet Corpus.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> fn.lu(3238).frame.lexUnit['glint.v'] is fn.lu(3238)
+ True
+ >>> fn.frame_by_name('Replacing') is fn.lus('replace.v')[0].frame
+ True
+ >>> fn.lus('prejudice.n')[0].frame.frameRelations == fn.frame_relations('Partiality')
+ True
+ """
+
+ _bad_statuses = ['Problem']
+ """
+ When loading LUs for a frame, those whose status is in this list will be ignored.
+ Due to caching, if user code modifies this, it should do so before loading any data.
+ 'Problem' should always be listed for FrameNet 1.5, as these LUs are not included
+ in the XML index.
+ """
+
+ def __init__(self, root, fileids):
+ XMLCorpusReader.__init__(self, root, fileids)
+
+ # framenet corpus sub dirs
+ # sub dir containing the xml files for frames
+ self._frame_dir = "frame"
+ # sub dir containing the xml files for lexical units
+ self._lu_dir = "lu"
+ # sub dir containing the xml files for fulltext annotation files
+ self._fulltext_dir = "fulltext"
+
+ # Indexes used for faster look-ups
+ self._frame_idx = None
+ self._cached_frames = {} # name -> ID
+ self._lu_idx = None
+ self._fulltext_idx = None
+ self._semtypes = None
+ self._freltyp_idx = None # frame relation types (Inheritance, Using, etc.)
+ self._frel_idx = None # frame-to-frame relation instances
+ self._ferel_idx = None # FE-to-FE relation instances
+ self._frel_f_idx = None # frame-to-frame relations associated with each frame
+
+ def _buildframeindex(self):
+ # The total number of Frames in Framenet is fairly small (~1200) so
+ # this index should not be very large
+ if not self._frel_idx:
+ self._buildrelationindex() # always load frame relations before frames,
+ # otherwise weird ordering effects might result in incomplete information
+ self._frame_idx = {}
+ for f in XMLCorpusView(self.abspath("frameIndex.xml"),
+ 'frameIndex/frame', self._handle_elt):
+ self._frame_idx[f['ID']] = f
+
+ def _buildcorpusindex(self):
+ # The total number of fulltext annotated documents in Framenet
+ # is fairly small (~90) so this index should not be very large
+ self._fulltext_idx = {}
+ for doclist in XMLCorpusView(self.abspath("fulltextIndex.xml"),
+ 'fulltextIndex/corpus',
+ self._handle_fulltextindex_elt):
+ for doc in doclist:
+ self._fulltext_idx[doc.ID] = doc
+
+ def _buildluindex(self):
+ # The number of LUs in Framenet is about 13,000 so this index
+ # should not be very large
+ self._lu_idx = {}
+ for lu in XMLCorpusView(self.abspath("luIndex.xml"),
+ 'luIndex/lu', self._handle_elt):
+ self._lu_idx[lu['ID']] = lu # populate with LU index entries. if any of these
+ # are looked up they will be replaced by full LU objects.
+
+ def _buildrelationindex(self):
+ #print('building relation index...', file=sys.stderr)
+ freltypes = PrettyList(x for x in XMLCorpusView(self.abspath("frRelation.xml"),
+ 'frameRelations/frameRelationType',
+ self._handle_framerelationtype_elt))
+ self._freltyp_idx = {}
+ self._frel_idx = {}
+ self._frel_f_idx = defaultdict(set)
+ self._ferel_idx = {}
+
+ for freltyp in freltypes:
+ self._freltyp_idx[freltyp.ID] = freltyp
+ for frel in freltyp.frameRelations:
+ supF = frel.superFrame = frel[freltyp.superFrameName] = Future((lambda fID: lambda: self.frame_by_id(fID))(frel.supID))
+ subF = frel.subFrame = frel[freltyp.subFrameName] = Future((lambda fID: lambda: self.frame_by_id(fID))(frel.subID))
+ self._frel_idx[frel.ID] = frel
+ self._frel_f_idx[frel.supID].add(frel.ID)
+ self._frel_f_idx[frel.subID].add(frel.ID)
+ for ferel in frel.feRelations:
+ ferel.superFrame = supF
+ ferel.subFrame = subF
+ ferel.superFE = Future((lambda fer: lambda: fer.superFrame.FE[fer.superFEName])(ferel))
+ ferel.subFE = Future((lambda fer: lambda: fer.subFrame.FE[fer.subFEName])(ferel))
+ self._ferel_idx[ferel.ID] = ferel
+ #print('...done building relation index', file=sys.stderr)
+
+ def readme(self):
+ """
+ Return the contents of the corpus README.txt (or README) file.
+ """
+ try:
+ return self.open("README.txt").read()
+ except IOError:
+ return self.open("README").read()
+
+ def buildindexes(self):
+ """
+ Build the internal indexes to make look-ups faster.
+ """
+ # Frames
+ self._buildframeindex()
+ # LUs
+ self._buildluindex()
+ # Fulltext annotation corpora index
+ self._buildcorpusindex()
+ # frame and FE relations
+ self._buildrelationindex()
+
+ def annotated_document(self, fn_docid):
+ """
+ Returns the annotated document whose id number is
+ ``fn_docid``. This id number can be obtained by calling the
+ Documents() function.
+
+ The dict that is returned from this function will contain the
+ following keys:
+
+ - '_type' : 'fulltextannotation'
+ - 'sentence' : a list of sentences in the document
+ - Each item in the list is a dict containing the following keys:
+ - 'ID' : the ID number of the sentence
+ - '_type' : 'sentence'
+ - 'text' : the text of the sentence
+ - 'paragNo' : the paragraph number
+ - 'sentNo' : the sentence number
+ - 'docID' : the document ID number
+ - 'corpID' : the corpus ID number
+ - 'aPos' : the annotation position
+ - 'annotationSet' : a list of annotation layers for the sentence
+ - Each item in the list is a dict containing the following keys:
+ - 'ID' : the ID number of the annotation set
+ - '_type' : 'annotationset'
+ - 'status' : either 'MANUAL' or 'UNANN'
+ - 'luName' : (only if status is 'MANUAL')
+ - 'luID' : (only if status is 'MANUAL')
+ - 'frameID' : (only if status is 'MANUAL')
+ - 'frameName': (only if status is 'MANUAL')
+ - 'layer' : a list of labels for the layer
+ - Each item in the layer is a dict containing the
+ following keys:
+ - '_type': 'layer'
+ - 'rank'
+ - 'name'
+ - 'label' : a list of labels in the layer
+ - Each item is a dict containing the following keys:
+ - 'start'
+ - 'end'
+ - 'name'
+ - 'feID' (optional)
+
+ :param fn_docid: The Framenet id number of the document
+ :type fn_docid: int
+ :return: Information about the annotated document
+ :rtype: dict
+ """
+ try:
+ xmlfname = self._fulltext_idx[fn_docid].filename
+ except TypeError: # happens when self._fulltext_idx == None
+ # build the index
+ self._buildcorpusindex()
+ xmlfname = self._fulltext_idx[fn_docid].filename
+ except KeyError: # probably means that fn_docid was not in the index
+ raise FramenetError("Unknown document id: {0}".format(fn_docid))
+
+ # construct the path name for the xml file containing the document info
+ locpath = os.path.join(
+ "{0}".format(self._root), self._fulltext_dir, xmlfname)
+
+ # Grab the top-level xml element containing the fulltext annotation
+ elt = XMLCorpusView(locpath, 'fullTextAnnotation')[0]
+ return self._handle_fulltextannotation_elt(elt)
+
+ def frame_by_id(self, fn_fid, ignorekeys=[]):
+ """
+ Get the details for the specified Frame using the frame's id
+ number.
+
+ Usage examples:
+
+ >>> from nltk.corpus import framenet as fn
+ >>> f = fn.frame_by_id(256)
+ >>> f.ID
+ 256
+ >>> f.name
+ 'Medical_specialties'
+ >>> f.definition
+ "This frame includes words that name ..."
+
+ :param fn_fid: The Framenet id number of the frame
+ :type fn_fid: int
+ :param ignorekeys: The keys to ignore. These keys will not be
+ included in the output. (optional)
+ :type ignorekeys: list(str)
+ :return: Information about a frame
+ :rtype: dict
+
+ Also see the ``frame()`` function for details about what is
+ contained in the dict that is returned.
+ """
+
+ # get the name of the frame with this id number
+ try:
+ fentry = self._frame_idx[fn_fid]
+ if '_type' in fentry:
+ return fentry # full frame object is cached
+ name = fentry['name']
+ except TypeError:
+ self._buildframeindex()
+ name = self._frame_idx[fn_fid]['name']
+ except KeyError:
+ raise FramenetError('Unknown frame id: {0}'.format(fn_fid))
+
+ return self.frame_by_name(name, ignorekeys, check_cache=False)
+
+ def frame_by_name(self, fn_fname, ignorekeys=[], check_cache=True):
+ """
+ Get the details for the specified Frame using the frame's name.
+
+ Usage examples:
+
+ >>> from nltk.corpus import framenet as fn
+ >>> f = fn.frame_by_name('Medical_specialties')
+ >>> f.ID
+ 256
+ >>> f.name
+ 'Medical_specialties'
+ >>> f.definition
+ "This frame includes words that name ..."
+
+ :param fn_fname: The name of the frame
+ :type fn_fname: str
+ :param ignorekeys: The keys to ignore. These keys will not be
+ included in the output. (optional)
+ :type ignorekeys: list(str)
+ :return: Information about a frame
+ :rtype: dict
+
+ Also see the ``frame()`` function for details about what is
+ contained in the dict that is returned.
+ """
+
+ if check_cache and fn_fname in self._cached_frames:
+ return self._frame_idx[self._cached_frames[fn_fname]]
+ elif not self._frame_idx:
+ self._buildframeindex()
+
+ # construct the path name for the xml file containing the Frame info
+ locpath = os.path.join(
+ "{0}".format(self._root), self._frame_dir, fn_fname + ".xml")
+ #print(locpath, file=sys.stderr)
+ # Grab the xml for the frame
+ try:
+ elt = XMLCorpusView(locpath, 'frame')[0]
+ except IOError:
+ raise FramenetError('Unknown frame: {0}'.format(fn_fname))
+
+ fentry = self._handle_frame_elt(elt, ignorekeys)
+ assert fentry
+
+ # INFERENCE RULE: propagate lexical semtypes from the frame to all its LUs
+ for st in fentry.semTypes:
+ if st.rootType.name=='Lexical_type':
+ for lu in fentry.lexUnit.values():
+ if st not in lu.semTypes:
+ lu.semTypes.append(st)
+
+
+ self._frame_idx[fentry.ID] = fentry
+ self._cached_frames[fentry.name] = fentry.ID
+ '''
+ # now set up callables to resolve the LU pointers lazily.
+ # (could also do this here--caching avoids infinite recursion.)
+ for luName,luinfo in fentry.lexUnit.items():
+ fentry.lexUnit[luName] = (lambda luID: Future(lambda: self.lu(luID)))(luinfo.ID)
+ '''
+ return fentry
+
+ def frame(self, fn_fid_or_fname, ignorekeys=[]):
+ """
+ Get the details for the specified Frame using the frame's name
+ or id number.
+
+ Usage examples:
+
+ >>> from nltk.corpus import framenet as fn
+ >>> f = fn.frame(256)
+ >>> f.name
+ 'Medical_specialties'
+ >>> f = fn.frame('Medical_specialties')
+ >>> f.ID
+ 256
+ >>> # ensure non-ASCII character in definition doesn't trigger an encoding error:
+ >>> fn.frame('Imposing_obligation')
+ frame (1494): Imposing_obligation...
+
+ The dict that is returned from this function will contain the
+ following information about the Frame:
+
+ - 'name' : the name of the Frame (e.g. 'Birth', 'Apply_heat', etc.)
+ - 'definition' : textual definition of the Frame
+ - 'ID' : the internal ID number of the Frame
+ - 'semTypes' : a list of semantic types for this frame
+ - Each item in the list is a dict containing the following keys:
+ - 'name' : can be used with the semtype() function
+ - 'ID' : can be used with the semtype() function
+
+ - 'lexUnit' : a dict containing all of the LUs for this frame.
+ The keys in this dict are the names of the LUs and
+ the value for each key is itself a dict containing
+ info about the LU (see the lu() function for more info.)
+
+ - 'FE' : a dict containing the Frame Elements that are part of this frame
+ The keys in this dict are the names of the FEs (e.g. 'Body_system')
+ and the values are dicts containing the following keys
+ - 'definition' : The definition of the FE
+ - 'name' : The name of the FE e.g. 'Body_system'
+ - 'ID' : The id number
+ - '_type' : 'fe'
+ - 'abbrev' : Abbreviation e.g. 'bod'
+ - 'coreType' : one of "Core", "Peripheral", or "Extra-Thematic"
+ - 'semType' : if not None, a dict with the following two keys:
+ - 'name' : name of the semantic type. can be used with
+ the semtype() function
+ - 'ID' : id number of the semantic type. can be used with
+ the semtype() function
+ - 'requiresFE' : if not None, a dict with the following two keys:
+ - 'name' : the name of another FE in this frame
+ - 'ID' : the id of the other FE in this frame
+ - 'excludesFE' : if not None, a dict with the following two keys:
+ - 'name' : the name of another FE in this frame
+ - 'ID' : the id of the other FE in this frame
+
+ - 'frameRelation' : a list of objects describing frame relations
+ - 'FEcoreSets' : a list of Frame Element core sets for this frame
+ - Each item in the list is a list of FE objects
+
+ :param fn_fid_or_fname: The Framenet name or id number of the frame
+ :type fn_fid_or_fname: int or str
+ :param ignorekeys: The keys to ignore. These keys will not be
+ included in the output. (optional)
+ :type ignorekeys: list(str)
+ :return: Information about a frame
+ :rtype: dict
+ """
+
+ # get the frame info by name or id number
+ if isinstance(fn_fid_or_fname, string_types):
+ f = self.frame_by_name(fn_fid_or_fname, ignorekeys)
+ else:
+ f = self.frame_by_id(fn_fid_or_fname, ignorekeys)
+
+ return f
+
+ def frames_by_lemma(self, pat):
+ """
+ Returns a list of all frames that contain LUs in which the
+ ``name`` attribute of the LU matchs the given regular expression
+ ``pat``. Note that LU names are composed of "lemma.POS", where
+ the "lemma" part can be made up of either a single lexeme
+ (e.g. 'run') or multiple lexemes (e.g. 'a little').
+
+ Note: if you are going to be doing a lot of this type of
+ searching, you'd want to build an index that maps from lemmas to
+ frames because each time frames_by_lemma() is called, it has to
+ search through ALL of the frame XML files in the db.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> fn.frames_by_lemma(r'(?i)a little')
+ [<frame ID=189 name=Quantity>, <frame ID=2001 name=Degree>]
+
+ :return: A list of frame objects.
+ :rtype: list(AttrDict)
+ """
+ return PrettyList(f for f in self.frames() if any(re.search(pat, luName) for luName in f.lexUnit))
+
+ def lu_basic(self, fn_luid):
+ """
+ Returns basic information about the LU whose id is
+ ``fn_luid``. This is basically just a wrapper around the
+ ``lu()`` function with "subCorpus" info excluded.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> PrettyDict(fn.lu_basic(256), breakLines=True)
+ {'ID': 256,
+ 'POS': 'V',
+ '_type': 'lu',
+ 'definition': 'COD: be aware of beforehand; predict.',
+ 'frame': <frame ID=26 name=Expectation>,
+ 'lemmaID': 15082,
+ 'lexemes': [{'POS': 'V', 'breakBefore': 'false', 'headword': 'false', 'name': 'foresee', 'order': 1}],
+ 'name': 'foresee.v',
+ 'semTypes': [],
+ 'sentenceCount': {'annotated': 44, 'total': 227},
+ 'status': 'FN1_Sent'}
+
+ :param fn_luid: The id number of the desired LU
+ :type fn_luid: int
+ :return: Basic information about the lexical unit
+ :rtype: dict
+ """
+ return self.lu(fn_luid, ignorekeys=['subCorpus'])
+
+ def lu(self, fn_luid, ignorekeys=[]):
+ """
+ Get information about a specific Lexical Unit using the id number
+ ``fn_luid``. This function reads the LU information from the xml
+ file on disk each time it is called. You may want to cache this
+ info if you plan to call this function with the same id number
+ multiple times.
+
+ Usage examples:
+
+ >>> from nltk.corpus import framenet as fn
+ >>> fn.lu(256).name
+ 'foresee.v'
+ >>> fn.lu(256).definition
+ 'COD: be aware of beforehand; predict.'
+ >>> fn.lu(256).frame.name
+ 'Expectation'
+ >>> pprint(list(map(PrettyDict, fn.lu(256).lexemes)))
+ [{'POS': 'V', 'breakBefore': 'false', 'headword': 'false', 'name': 'foresee', 'order': 1}]
+
+ The dict that is returned from this function will contain most of the
+ following information about the LU. Note that some LUs do not contain
+ all of these pieces of information - particularly 'totalAnnotated' and
+ 'incorporatedFE' may be missing in some LUs:
+
+ - 'name' : the name of the LU (e.g. 'merger.n')
+ - 'definition' : textual definition of the LU
+ - 'ID' : the internal ID number of the LU
+ - '_type' : 'lu'
+ - 'status' : e.g. 'Created'
+ - 'frame' : Frame that this LU belongs to
+ - 'POS' : the part of speech of this LU (e.g. 'N')
+ - 'totalAnnotated' : total number of examples annotated with this LU
+ - 'incorporatedFE' : FE that incorporates this LU (e.g. 'Ailment')
+ - 'sentenceCount' : a dict with the following two keys:
+ - 'annotated': number of sentences annotated with this LU
+ - 'total' : total number of sentences with this LU
+
+ - 'lexemes' : a list of dicts describing the lemma of this LU.
+ Each dict in the list contains these keys:
+ - 'POS' : part of speech e.g. 'N'
+ - 'name' : either single-lexeme e.g. 'merger' or
+ multi-lexeme e.g. 'a little'
+ - 'order': the order of the lexeme in the lemma (starting from 1)
+ - 'headword': a boolean ('true' or 'false')
+ - 'breakBefore': Can this lexeme be separated from the previous lexeme?
+ Consider: "take over.v" as in:
+ Germany took over the Netherlands in 2 days.
+ Germany took the Netherlands over in 2 days.
+ In this case, 'breakBefore' would be "true" for the lexeme
+ "over". Contrast this with "take after.v" as in:
+ Mary takes after her grandmother.
+ *Mary takes her grandmother after.
+ In this case, 'breakBefore' would be "false" for the lexeme "after"
+
+ - 'lemmaID' : Can be used to connect lemmas in different LUs
+ - 'semTypes' : a list of semantic type objects for this LU
+ - 'subCorpus' : a list of subcorpora
+ - Each item in the list is a dict containing the following keys:
+ - 'name' :
+ - 'sentence' : a list of sentences in the subcorpus
+ - each item in the list is a dict with the following keys:
+ - 'ID':
+ - 'sentNo':
+ - 'text': the text of the sentence
+ - 'aPos':
+ - 'annotationSet': a list of annotation sets
+ - each item in the list is a dict with the following keys:
+ - 'ID':
+ - 'status':
+ - 'layer': a list of layers
+ - each layer is a dict containing the following keys:
+ - 'name': layer name (e.g. 'BNC')
+ - 'rank':
+ - 'label': a list of labels for the layer
+ - each label is a dict containing the following keys:
+ - 'start': start pos of label in sentence 'text' (0-based)
+ - 'end': end pos of label in sentence 'text' (0-based)
+ - 'name': name of label (e.g. 'NN1')
+
+ Under the hood, this implementation looks up the lexical unit information
+ in the *frame* definition file. That file does not contain
+ corpus annotations, so the LU files will be accessed on demand if those are
+ needed. In principle, valence patterns could be loaded here too,
+ though these are not currently supported.
+
+ :param fn_luid: The id number of the lexical unit
+ :type fn_luid: int
+ :param ignorekeys: The keys to ignore. These keys will not be
+ included in the output. (optional)
+ :type ignorekeys: list(str)
+ :return: All information about the lexical unit
+ :rtype: dict
+ """
+ # look for this LU in cache
+ if not self._lu_idx:
+ self._buildluindex()
+ luinfo = self._lu_idx[fn_luid]
+ if '_type' not in luinfo:
+ # we only have an index entry for the LU. loading the frame will replace this.
+ f = self.frame_by_id(luinfo.frameID)
+ luinfo = self._lu_idx[fn_luid]
+ if ignorekeys:
+ return AttrDict(dict((k, v) for k, v in luinfo.items() if k not in ignorekeys))
+
+ return luinfo
+
+ def _lu_file(self, lu, ignorekeys=[]):
+ """
+ Augment the LU information that was loaded from the frame file
+ with additional information from the LU file.
+ """
+ fn_luid = lu.ID
+
+ fname = "lu{0}.xml".format(fn_luid)
+ locpath = os.path.join("{0}".format(self._root), self._lu_dir, fname)
+ #print(locpath, file=sys.stderr)
+ if not self._lu_idx:
+ self._buildluindex()
+
+ try:
+ elt = XMLCorpusView(locpath, 'lexUnit')[0]
+ except IOError:
+ raise FramenetError('Unknown LU id: {0}'.format(fn_luid))
+
+ lu2 = self._handle_lexunit_elt(elt, ignorekeys)
+ lu.subCorpus = lu2.subCorpus
+
+ return lu.subCorpus
+
+ def _loadsemtypes(self):
+ """Create the semantic types index."""
+ self._semtypes = AttrDict()
+ semtypeXML = [x for x in XMLCorpusView(self.abspath("semTypes.xml"),
+ 'semTypes/semType',
+ self._handle_semtype_elt)]
+ for st in semtypeXML:
+ n = st['name']
+ a = st['abbrev']
+ i = st['ID']
+ # Both name and abbrev should be able to retrieve the
+ # ID. The ID will retrieve the semantic type dict itself.
+ self._semtypes[n] = i
+ self._semtypes[a] = i
+ self._semtypes[i] = st
+ # now that all individual semtype XML is loaded, we can link them together
+ roots = []
+ for st in self.semtypes():
+ if st.superType:
+ st.superType = self.semtype(st.superType.supID)
+ st.superType.subTypes.append(st)
+ else:
+ if st not in roots: roots.append(st)
+ st.rootType = st
+ queue = list(roots)
+ assert queue
+ while queue:
+ st = queue.pop(0)
+ for child in st.subTypes:
+ child.rootType = st.rootType
+ queue.append(child)
+ #self.propagate_semtypes() # apply inferencing over FE relations
+
+ def propagate_semtypes(self):
+ """
+ Apply inference rules to distribute semtypes over relations between FEs.
+ For FrameNet 1.5, this results in 1011 semtypes being propagated.
+ (Not done by default because it requires loading all frame files,
+ which takes several seconds. If this needed to be fast, it could be rewritten
+ to traverse the neighboring relations on demand for each FE semtype.)
+
+ >>> from nltk.corpus import framenet as fn
+ >>> sum(1 for f in fn.frames() for fe in f.FE.values() if fe.semType)
+ 4241
+ >>> fn.propagate_semtypes()
+ >>> sum(1 for f in fn.frames() for fe in f.FE.values() if fe.semType)
+ 5252
+ """
+ if not self._semtypes:
+ self._loadsemtypes()
+ if not self._ferel_idx:
+ self._buildrelationindex()
+ changed = True
+ i = 0
+ nPropagations = 0
+ while changed:
+ # make a pass and see if anything needs to be propagated
+ i += 1
+ changed = False
+ for ferel in self.fe_relations():
+ superST = ferel.superFE.semType
+ subST = ferel.subFE.semType
+ try:
+ if superST and superST is not subST:
+ # propagate downward
+ assert subST is None or self.semtype_inherits(subST, superST),(superST.name,ferel,subST.name)
+ if subST is None:
+ ferel.subFE.semType = subST = superST
+ changed = True
+ nPropagations += 1
+ if ferel.type.name in ['Perspective_on', 'Subframe', 'Precedes'] and subST \
+ and subST is not superST:
+ # propagate upward
+ assert superST is None,(superST.name,ferel,subST.name)
+ ferel.superFE.semType = superST = subST
+ changed = True
+ nPropagations += 1
+ except AssertionError as ex:
+ # bug in the data! ignore
+ #print(ex, file=sys.stderr)
+ continue
+ #print(i, nPropagations, file=sys.stderr)
+
+ def semtype(self, key):
+ """
+ >>> from nltk.corpus import framenet as fn
+ >>> fn.semtype(233).name
+ 'Temperature'
+ >>> fn.semtype(233).abbrev
+ 'Temp'
+ >>> fn.semtype('Temperature').ID
+ 233
+
+ :param key: The name, abbreviation, or id number of the semantic type
+ :type key: string or int
+ :return: Information about a semantic type
+ :rtype: dict
+ """
+ if isinstance(key, int):
+ stid = key
+ else:
+ try:
+ stid = self._semtypes[key]
+ except TypeError:
+ self._loadsemtypes()
+ stid = self._semtypes[key]
+
+ try:
+ st = self._semtypes[stid]
+ except TypeError:
+ self._loadsemtypes()
+ st = self._semtypes[stid]
+
+ return st
+
+ def semtype_inherits(self, st, superST):
+ if not isinstance(st, dict):
+ st = self.semtype(st)
+ if not isinstance(superST, dict):
+ superST = self.semtype(superST)
+ par = st.superType
+ while par:
+ if par is superST:
+ return True
+ par = par.superType
+ return False
+
+ def frames(self, name=None):
+ """
+ Obtain details for a specific frame.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> len(fn.frames())
+ 1019
+ >>> PrettyList(fn.frames(r'(?i)medical'), maxReprSize=0, breakLines=True)
+ [<frame ID=256 name=Medical_specialties>,
+ <frame ID=257 name=Medical_instruments>,
+ <frame ID=255 name=Medical_professionals>,
+ <frame ID=239 name=Medical_conditions>]
+
+ A brief intro to Frames (excerpted from "FrameNet II: Extended
+ Theory and Practice" by Ruppenhofer et. al., 2010):
+
+ A Frame is a script-like conceptual structure that describes a
+ particular type of situation, object, or event along with the
+ participants and props that are needed for that Frame. For
+ example, the "Apply_heat" frame describes a common situation
+ involving a Cook, some Food, and a Heating_Instrument, and is
+ evoked by words such as bake, blanch, boil, broil, brown,
+ simmer, steam, etc.
+
+ We call the roles of a Frame "frame elements" (FEs) and the
+ frame-evoking words are called "lexical units" (LUs).
+
+ FrameNet includes relations between Frames. Several types of
+ relations are defined, of which the most important are:
+
+ - Inheritance: An IS-A relation. The child frame is a subtype
+ of the parent frame, and each FE in the parent is bound to
+ a corresponding FE in the child. An example is the
+ "Revenge" frame which inherits from the
+ "Rewards_and_punishments" frame.
+
+ - Using: The child frame presupposes the parent frame as
+ background, e.g the "Speed" frame "uses" (or presupposes)
+ the "Motion" frame; however, not all parent FEs need to be
+ bound to child FEs.
+
+ - Subframe: The child frame is a subevent of a complex event
+ represented by the parent, e.g. the "Criminal_process" frame
+ has subframes of "Arrest", "Arraignment", "Trial", and
+ "Sentencing".
+
+ - Perspective_on: The child frame provides a particular
+ perspective on an un-perspectivized parent frame. A pair of
+ examples consists of the "Hiring" and "Get_a_job" frames,
+ which perspectivize the "Employment_start" frame from the
+ Employer's and the Employee's point of view, respectively.
+
+ :param name: A regular expression pattern used to match against
+ Frame names. If 'name' is None, then a list of all
+ Framenet Frames will be returned.
+ :type name: str
+ :return: A list of matching Frames (or all Frames).
+ :rtype: list(AttrDict)
+ """
+ try:
+ fIDs = list(self._frame_idx.keys())
+ except AttributeError:
+ self._buildframeindex()
+ fIDs = list(self._frame_idx.keys())
+
+ if name is not None:
+ return PrettyList(self.frame(fID) for fID,finfo in self.frame_ids_and_names(name).items())
+ else:
+ return PrettyLazyMap(self.frame, fIDs)
+
+ def frame_ids_and_names(self, name=None):
+ """
+ Uses the frame index, which is much faster than looking up each frame definition
+ if only the names and IDs are needed.
+ """
+ if not self._frame_idx:
+ self._buildframeindex()
+ return dict((fID, finfo.name) for fID,finfo in self._frame_idx.items() if name is None or re.search(name, finfo.name) is not None)
+
+ def lus(self, name=None):
+ """
+ Obtain details for a specific lexical unit.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> len(fn.lus())
+ 11829
+ >>> PrettyList(fn.lus(r'(?i)a little'), maxReprSize=0, breakLines=True)
+ [<lu ID=14744 name=a little bit.adv>,
+ <lu ID=14733 name=a little.n>,
+ <lu ID=14743 name=a little.adv>]
+
+ A brief intro to Lexical Units (excerpted from "FrameNet II:
+ Extended Theory and Practice" by Ruppenhofer et. al., 2010):
+
+ A lexical unit (LU) is a pairing of a word with a meaning. For
+ example, the "Apply_heat" Frame describes a common situation
+ involving a Cook, some Food, and a Heating Instrument, and is
+ _evoked_ by words such as bake, blanch, boil, broil, brown,
+ simmer, steam, etc. These frame-evoking words are the LUs in the
+ Apply_heat frame. Each sense of a polysemous word is a different
+ LU.
+
+ We have used the word "word" in talking about LUs. The reality
+ is actually rather complex. When we say that the word "bake" is
+ polysemous, we mean that the lemma "bake.v" (which has the
+ word-forms "bake", "bakes", "baked", and "baking") is linked to
+ three different frames:
+
+ - Apply_heat: "Michelle baked the potatoes for 45 minutes."
+
+ - Cooking_creation: "Michelle baked her mother a cake for her birthday."
+
+ - Absorb_heat: "The potatoes have to bake for more than 30 minutes."
+
+ These constitute three different LUs, with different
+ definitions.
+
+ Multiword expressions such as "given name" and hyphenated words
+ like "shut-eye" can also be LUs. Idiomatic phrases such as
+ "middle of nowhere" and "give the slip (to)" are also defined as
+ LUs in the appropriate frames ("Isolated_places" and "Evading",
+ respectively), and their internal structure is not analyzed.
+
+ Framenet provides multiple annotated examples of each sense of a
+ word (i.e. each LU). Moreover, the set of examples
+ (approximately 20 per LU) illustrates all of the combinatorial
+ possibilities of the lexical unit.
+
+ Each LU is linked to a Frame, and hence to the other words which
+ evoke that Frame. This makes the FrameNet database similar to a
+ thesaurus, grouping together semantically similar words.
+
+ In the simplest case, frame-evoking words are verbs such as
+ "fried" in:
+
+ "Matilde fried the catfish in a heavy iron skillet."
+
+ Sometimes event nouns may evoke a Frame. For example,
+ "reduction" evokes "Cause_change_of_scalar_position" in:
+
+ "...the reduction of debt levels to $665 million from $2.6 billion."
+
+ Adjectives may also evoke a Frame. For example, "asleep" may
+ evoke the "Sleep" frame as in:
+
+ "They were asleep for hours."
+
+ Many common nouns, such as artifacts like "hat" or "tower",
+ typically serve as dependents rather than clearly evoking their
+ own frames.
+
+ :param name: A regular expression pattern used to search the LU
+ names. Note that LU names take the form of a dotted
+ string (e.g. "run.v" or "a little.adv") in which a
+ lemma preceeds the "." and a POS follows the
+ dot. The lemma may be composed of a single lexeme
+ (e.g. "run") or of multiple lexemes (e.g. "a
+ little"). If 'name' is not given, then all LUs will
+ be returned.
+
+ The valid POSes are:
+
+ v - verb
+ n - noun
+ a - adjective
+ adv - adverb
+ prep - preposition
+ num - numbers
+ intj - interjection
+ art - article
+ c - conjunction
+ scon - subordinating conjunction
+
+ :type name: str
+ :return: A list of selected (or all) lexical units
+ :rtype: list of LU objects (dicts). See the lu() function for info
+ about the specifics of LU objects.
+
+ """
+ try:
+ luIDs = list(self._lu_idx.keys())
+ except AttributeError:
+ self._buildluindex()
+ luIDs = list(self._lu_idx.keys())
+
+ if name is not None:
+ return PrettyList(self.lu(luID) for luID,luName in self.lu_ids_and_names(name).items())
+ else:
+ return PrettyLazyMap(self.lu, luIDs)
+
+ def lu_ids_and_names(self, name=None):
+ """
+ Uses the LU index, which is much faster than looking up each LU definition
+ if only the names and IDs are needed.
+ """
+ if not self._lu_idx:
+ self._buildluindex()
+ return dict((luID, luinfo.name) for luID,luinfo in self._lu_idx.items() if name is None or re.search(name, luinfo.name) is not None)
+
+ def documents(self, name=None):
+ """
+ Return a list of the annotated documents in Framenet.
+
+ Details for a specific annotated document can be obtained using this
+ class's annotated_document() function and pass it the value of the 'ID' field.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> len(fn.documents())
+ 78
+ >>> set([x.corpname for x in fn.documents()])==set(['ANC', 'C-4', 'KBEval', \
+ 'LUCorpus-v0.3', 'Miscellaneous', 'NTI', 'PropBank', 'QA', 'SemAnno'])
+ True
+
+ :param name: A regular expression pattern used to search the
+ file name of each annotated document. The document's
+ file name contains the name of the corpus that the
+ document is from, followed by two underscores "__"
+ followed by the document name. So, for example, the
+ file name "LUCorpus-v0.3__20000410_nyt-NEW.xml" is
+ from the corpus named "LUCorpus-v0.3" and the
+ document name is "20000410_nyt-NEW.xml".
+ :type name: str
+ :return: A list of selected (or all) annotated documents
+ :rtype: list of dicts, where each dict object contains the following
+ keys:
+
+ - 'name'
+ - 'ID'
+ - 'corpid'
+ - 'corpname'
+ - 'description'
+ - 'filename'
+ """
+ try:
+ ftlist = PrettyList(self._fulltext_idx.values())
+ except AttributeError:
+ self._buildcorpusindex()
+ ftlist = PrettyList(self._fulltext_idx.values())
+
+ if name is None:
+ return ftlist
+ else:
+ return PrettyList(x for x in ftlist if re.search(name, x['filename']) is not None)
+
+ def frame_relation_types(self):
+ """
+ Obtain a list of frame relation types.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> frts = list(fn.frame_relation_types())
+ >>> isinstance(frts, list)
+ True
+ >>> len(frts)
+ 9
+ >>> PrettyDict(frts[0], breakLines=True)
+ {'ID': 1,
+ '_type': 'framerelationtype',
+ 'frameRelations': [<Parent=Event -- Inheritance -> Child=Change_of_consistency>, <Parent=Event -- Inheritance -> Child=Rotting>, ...],
+ 'name': 'Inheritance',
+ 'subFrameName': 'Child',
+ 'superFrameName': 'Parent'}
+
+ :return: A list of all of the frame relation types in framenet
+ :rtype: list(dict)
+ """
+ if not self._freltyp_idx:
+ self._buildrelationindex()
+ return self._freltyp_idx.values()
+
+ def frame_relations(self, frame=None, frame2=None, type=None):
+ """
+ :param frame: (optional) frame object, name, or ID; only relations involving
+ this frame will be returned
+ :param frame2: (optional; 'frame' must be a different frame) only show relations
+ between the two specified frames, in either direction
+ :param type: (optional) frame relation type (name or object); show only relations
+ of this type
+ :type frame: int or str or AttrDict
+ :return: A list of all of the frame relations in framenet
+ :rtype: list(dict)
+
+ >>> from nltk.corpus import framenet as fn
+ >>> frels = fn.frame_relations()
+ >>> isinstance(frels, list)
+ True
+ >>> len(frels)
+ 1676
+ >>> PrettyList(fn.frame_relations('Cooking_creation'), maxReprSize=0, breakLines=True)
+ [<Parent=Intentionally_create -- Inheritance -> Child=Cooking_creation>,
+ <Parent=Apply_heat -- Using -> Child=Cooking_creation>,
+ <MainEntry=Apply_heat -- See_also -> ReferringEntry=Cooking_creation>]
+ >>> PrettyList(fn.frame_relations(373), breakLines=True)
+ [<Parent=Topic -- Using -> Child=Communication>,
+ <Source=Discussion -- ReFraming_Mapping -> Target=Topic>, ...]
+ >>> PrettyList(fn.frame_relations(fn.frame('Cooking_creation')), breakLines=True)
+ [<Parent=Intentionally_create -- Inheritance -> Child=Cooking_creation>,
+ <Parent=Apply_heat -- Using -> Child=Cooking_creation>, ...]
+ >>> PrettyList(fn.frame_relations('Cooking_creation', type='Inheritance'))
+ [<Parent=Intentionally_create -- Inheritance -> Child=Cooking_creation>]
+ >>> PrettyList(fn.frame_relations('Cooking_creation', 'Apply_heat'), breakLines=True)
+ [<Parent=Apply_heat -- Using -> Child=Cooking_creation>,
+ <MainEntry=Apply_heat -- See_also -> ReferringEntry=Cooking_creation>]
+ """
+ relation_type = type
+
+ if not self._frel_idx:
+ self._buildrelationindex()
+
+ rels = None
+
+ if relation_type is not None:
+ if not isinstance(relation_type, dict):
+ type = [rt for rt in self.frame_relation_types() if rt.name==type][0]
+ assert isinstance(type,dict)
+
+ # lookup by 'frame'
+ if frame is not None:
+ if isinstance(frame,dict) and 'frameRelations' in frame:
+ rels = PrettyList(frame.frameRelations)
+ else:
+ if not isinstance(frame, int):
+ if isinstance(frame, dict):
+ frame = frame.ID
+ else:
+ frame = self.frame_by_name(frame).ID
+ rels = [self._frel_idx[frelID] for frelID in self._frel_f_idx[frame]]
+
+ # filter by 'type'
+ if type is not None:
+ rels = [rel for rel in rels if rel.type is type]
+ elif type is not None:
+ # lookup by 'type'
+ rels = type.frameRelations
+ else:
+ rels = self._frel_idx.values()
+
+ # filter by 'frame2'
+ if frame2 is not None:
+ if frame is None:
+ raise FramenetError("frame_relations(frame=None, frame2=<value>) is not allowed")
+ if not isinstance(frame2, int):
+ if isinstance(frame2, dict):
+ frame2 = frame2.ID
+ else:
+ frame2 = self.frame_by_name(frame2).ID
+ if frame==frame2:
+ raise FramenetError("The two frame arguments to frame_relations() must be different frames")
+ rels = [rel for rel in rels if rel.superFrame.ID==frame2 or rel.subFrame.ID==frame2]
+
+ return PrettyList(sorted(rels,
+ key=lambda frel: (frel.type.ID, frel.superFrameName, frel.subFrameName)))
+
+ def fe_relations(self):
+ """
+ Obtain a list of frame element relations.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> ferels = fn.fe_relations()
+ >>> isinstance(ferels, list)
+ True
+ >>> len(ferels)
+ 10020
+ >>> PrettyDict(ferels[0], breakLines=True)
+ {'ID': 14642,
+ '_type': 'ferelation',
+ 'frameRelation': <Parent=Abounding_with -- Inheritance -> Child=Lively_place>,
+ 'subFE': <fe ID=11370 name=Degree>,
+ 'subFEName': 'Degree',
+ 'subFrame': <frame ID=1904 name=Lively_place>,
+ 'subID': 11370,
+ 'supID': 2271,
+ 'superFE': <fe ID=2271 name=Degree>,
+ 'superFEName': 'Degree',
+ 'superFrame': <frame ID=262 name=Abounding_with>,
+ 'type': <framerelationtype ID=1 name=Inheritance>}
+
+ :return: A list of all of the frame element relations in framenet
+ :rtype: list(dict)
+ """
+ if not self._ferel_idx:
+ self._buildrelationindex()
+ return PrettyList(sorted(self._ferel_idx.values(),
+ key=lambda ferel: (ferel.type.ID, ferel.frameRelation.superFrameName,
+ ferel.superFEName, ferel.frameRelation.subFrameName, ferel.subFEName)))
+
+ def semtypes(self):
+ """
+ Obtain a list of semantic types.
+
+ >>> from nltk.corpus import framenet as fn
+ >>> stypes = fn.semtypes()
+ >>> len(stypes)
+ 73
+ >>> sorted(stypes[0].keys())
+ ['ID', '_type', 'abbrev', 'definition', 'name', 'rootType', 'subTypes', 'superType']
+
+ :return: A list of all of the semantic types in framenet
+ :rtype: list(dict)
+ """
+ if not self._semtypes:
+ self._loadsemtypes()
+ return PrettyList(self._semtypes[i] for i in self._semtypes if isinstance(i, int))
+
+ def _load_xml_attributes(self, d, elt):
+ """
+ Extracts a subset of the attributes from the given element and
+ returns them in a dictionary.
+
+ :param d: A dictionary in which to store the attributes.
+ :type d: dict
+ :param elt: An ElementTree Element
+ :type elt: Element
+ :return: Returns the input dict ``d`` possibly including attributes from ``elt``
+ :rtype: dict
+ """
+
+ d = type(d)(d)
+
+ try:
+ attr_dict = elt.attrib
+ except AttributeError:
+ return d
+
+ if attr_dict is None:
+ return d
+
+ # Ignore these attributes when loading attributes from an xml node
+ ignore_attrs = ['cBy', 'cDate', 'mDate', 'xsi',
+ 'schemaLocation', 'xmlns', 'bgColor', 'fgColor']
+
+ for attr in attr_dict:
+
+ if any(attr.endswith(x) for x in ignore_attrs):
+ continue
+
+ val = attr_dict[attr]
+ if val.isdigit():
+ d[attr] = int(val)
+ else:
+ d[attr] = val
+
+ return d
+
+ def _strip_tags(self, data):
+ """
+ Gets rid of all tags and newline characters from the given input
+
+ :return: A cleaned-up version of the input string
+ :rtype: str
+ """
+
+ try:
+ data = data.replace('<t>', '')
+ data = data.replace('</t>', '')
+ data = re.sub('<fex name="[^"]+">', '', data)
+ data = data.replace('</fex>', '')
+ data = data.replace('<fen>', '')
+ data = data.replace('</fen>', '')
+ data = data.replace('<m>', '')
+ data = data.replace('</m>', '')
+ data = data.replace('<ment>', '')
+ data = data.replace('</ment>', '')
+ data = data.replace('<ex>', "'")
+ data = data.replace('</ex>', "'")
+ data = data.replace('<gov>', '')
+ data = data.replace('</gov>', '')
+ data = data.replace('<x>', '')
+ data = data.replace('</x>', '')
+
+ # Get rid of <def-root> and </def-root> tags
+ data = data.replace('<def-root>', '')
+ data = data.replace('</def-root>', '')
+
+ data = data.replace('\n', ' ')
+ except AttributeError:
+ pass
+
+ return data
+
+ def _handle_elt(self, elt, tagspec=None):
+ """Extracts and returns the attributes of the given element"""
+ return self._load_xml_attributes(AttrDict(), elt)
+
+ def _handle_fulltextindex_elt(self, elt, tagspec=None):
+ """
+ Extracts corpus/document info from the fulltextIndex.xml file.
+
+ Note that this function "flattens" the information contained
+ in each of the "corpus" elements, so that each "document"
+ element will contain attributes for the corpus and
+ corpusid. Also, each of the "document" items will contain a
+ new attribute called "filename" that is the base file name of
+ the xml file for the document in the "fulltext" subdir of the
+ Framenet corpus.
+ """
+ ftinfo = self._load_xml_attributes(AttrDict(), elt)
+ corpname = ftinfo.name
+ corpid = ftinfo.ID
+ retlist = []
+ for sub in elt:
+ if sub.tag.endswith('document'):
+ doc = self._load_xml_attributes(AttrDict(), sub)
+ if 'name' in doc:
+ docname = doc.name
+ else:
+ docname = doc.description
+ doc.filename = "{0}__{1}.xml".format(corpname, docname)
+ doc.corpname = corpname
+ doc.corpid = corpid
+ retlist.append(doc)
+
+ return retlist
+
+ def _handle_frame_elt(self, elt, ignorekeys=[]):
+ """Load the info for a Frame from an frame xml file"""
+ frinfo = self._load_xml_attributes(AttrDict(), elt)
+
+ frinfo['_type'] = 'frame'
+ frinfo['definition'] = ""
+ frinfo['FE'] = PrettyDict()
+ frinfo['FEcoreSets'] = []
+ frinfo['lexUnit'] = PrettyDict()
+ frinfo['semTypes'] = []
+ for k in ignorekeys:
+ if k in frinfo:
+ del frinfo[k]
+
+ for sub in elt:
+ if sub.tag.endswith('definition') and 'definition' not in ignorekeys:
+ frinfo['definition'] = self._strip_tags(sub.text)
+ elif sub.tag.endswith('FE') and 'FE' not in ignorekeys:
+ feinfo = self._handle_fe_elt(sub)
+ frinfo['FE'][feinfo.name] = feinfo
+ feinfo['frame'] = frinfo # backpointer
+ elif sub.tag.endswith('FEcoreSet') and 'FEcoreSet' not in ignorekeys:
+ coreset = self._handle_fecoreset_elt(sub)
+ # assumes all FEs have been loaded before coresets
+ frinfo['FEcoreSets'].append(PrettyList(frinfo['FE'][fe.name] for fe in coreset))
+ elif sub.tag.endswith('lexUnit') and 'lexUnit' not in ignorekeys:
+ luentry = self._handle_framelexunit_elt(sub)
+ if luentry['status'] in self._bad_statuses:
+ # problematic LU entry; ignore it
+ continue
+ luentry['frame'] = frinfo
+ luentry['subCorpus'] = Future((lambda lu: lambda: self._lu_file(lu))(luentry))
+ frinfo['lexUnit'][luentry.name] = luentry
+ if not self._lu_idx:
+ self._buildluindex()
+ self._lu_idx[luentry.ID] = luentry
+ elif sub.tag.endswith('semType') and 'semTypes' not in ignorekeys:
+ semtypeinfo = self._load_xml_attributes(AttrDict(), sub)
+ frinfo['semTypes'].append(self.semtype(semtypeinfo.ID))
+
+ frinfo['frameRelations'] = self.frame_relations(frame=frinfo)
+
+ # resolve 'requires' and 'excludes' links between FEs of this frame
+ for fe in frinfo.FE.values():
+ if fe.requiresFE:
+ name, ID = fe.requiresFE.name, fe.requiresFE.ID
+ fe.requiresFE = frinfo.FE[name]
+ assert fe.requiresFE.ID==ID
+ if fe.excludesFE:
+ name, ID = fe.excludesFE.name, fe.excludesFE.ID
+ fe.excludesFE = frinfo.FE[name]
+ assert fe.excludesFE.ID==ID
+
+ return frinfo
+
+ def _handle_fecoreset_elt(self, elt):
+ """Load fe coreset info from xml."""
+ info = self._load_xml_attributes(AttrDict(), elt)
+ tmp = []
+ for sub in elt:
+ tmp.append(self._load_xml_attributes(AttrDict(), sub))
+
+ return tmp
+
+ def _handle_framerelationtype_elt(self, elt, *args):
+ """Load frame-relation element and its child fe-relation elements from frRelation.xml."""
+ info = self._load_xml_attributes(AttrDict(), elt)
+ info['_type'] = 'framerelationtype'
+ info['frameRelations'] = PrettyList()
+
+ for sub in elt:
+ if sub.tag.endswith('frameRelation'):
+ frel = self._handle_framerelation_elt(sub)
+ frel['type'] = info # backpointer
+ for ferel in frel.feRelations:
+ ferel['type'] = info
+ info['frameRelations'].append(frel)
+
+ return info
+
+ def _handle_framerelation_elt(self, elt):
+ """Load frame-relation element and its child fe-relation elements from frRelation.xml."""
+ info = self._load_xml_attributes(AttrDict(), elt)
+ assert info['superFrameName']!=info['subFrameName'],(elt,info)
+ info['_type'] = 'framerelation'
+ info['feRelations'] = PrettyList()
+
+ for sub in elt:
+ if sub.tag.endswith('FERelation'):
+ ferel = self._handle_elt(sub)
+ ferel['_type'] = 'ferelation'
+ ferel['frameRelation'] = info # backpointer
+ info['feRelations'].append(ferel)
+
+ return info
+
+ def _handle_fulltextannotation_elt(self, elt):
+ """Load full annotation info for a document from its xml
+ file. The main element (fullTextAnnotation) contains a 'header'
+ element (which we ignore here) and a bunch of 'sentence'
+ elements."""
+ info = AttrDict()
+ info['_type'] = 'fulltextannotation'
+ info['sentence'] = []
+
+ for sub in elt:
+ if sub.tag.endswith('header'):
+ continue # not used
+ elif sub.tag.endswith('sentence'):
+ s = self._handle_fulltext_sentence_elt(sub)
+ info['sentence'].append(s)
+
+ return info
+
+ def _handle_fulltext_sentence_elt(self, elt):
+ """Load information from the given 'sentence' element. Each
+ 'sentence' element contains a "text" and an "annotationSet" sub
+ element."""
+ info = self._load_xml_attributes(AttrDict(), elt)
+ info['_type'] = "sentence"
+ info['annotationSet'] = []
+ info['text'] = ""
+
+ for sub in elt:
+ if sub.tag.endswith('text'):
+ info['text'] = self._strip_tags(sub.text)
+ elif sub.tag.endswith('annotationSet'):
+ a = self._handle_fulltextannotationset_elt(sub)
+ info['annotationSet'].append(a)
+
+ return info
+
+ def _handle_fulltextannotationset_elt(self, elt):
+ """Load information from the given 'annotationSet' element. Each
+ 'annotationSet' contains several "layer" elements."""
+ info = self._load_xml_attributes(AttrDict(), elt)
+ info['_type'] = "annotationset"
+ info['layer'] = []
+
+ for sub in elt:
+ if sub.tag.endswith('layer'):
+ l = self._handle_fulltextlayer_elt(sub)
+ info['layer'].append(l)
+
+ return info
+
+ def _handle_fulltextlayer_elt(self, elt):
+ """Load information from the given 'layer' element. Each
+ 'layer' contains several "label" elements."""
+ info = self._load_xml_attributes(AttrDict(), elt)
+ info['_type'] = 'layer'
+ info['label'] = []
+
+ for sub in elt:
+ if sub.tag.endswith('label'):
+ l = self._load_xml_attributes(AttrDict(), sub)
+ info['label'].append(l)
+
+ return info
+
+ def _handle_framelexunit_elt(self, elt):
+ """Load the lexical unit info from an xml element in a frame's xml file."""
+ luinfo = AttrDict()
+ luinfo['_type'] = 'lu'
+ luinfo = self._load_xml_attributes(luinfo, elt)
+ luinfo["definition"] = ""
+ luinfo["sentenceCount"] = PrettyDict()
+ luinfo['lexemes'] = PrettyList() # multiword LUs have multiple lexemes
+ luinfo['semTypes'] = PrettyList() # an LU can have multiple semtypes
+
+ for sub in elt:
+ if sub.tag.endswith('definition'):
+ luinfo['definition'] = self._strip_tags(sub.text)
+ elif sub.tag.endswith('sentenceCount'):
+ luinfo['sentenceCount'] = self._load_xml_attributes(
+ PrettyDict(), sub)
+ elif sub.tag.endswith('lexeme'):
+ luinfo['lexemes'].append(self._load_xml_attributes(PrettyDict(), sub))
+ elif sub.tag.endswith('semType'):
+ semtypeinfo = self._load_xml_attributes(PrettyDict(), sub)
+ luinfo['semTypes'].append(self.semtype(semtypeinfo.ID))
+
+ return luinfo
+
+ def _handle_lexunit_elt(self, elt, ignorekeys):
+ """
+ Load full info for a lexical unit from its xml file.
+ This should only be called when accessing corpus annotations
+ (which are not included in frame files).
+ """
+ luinfo = self._load_xml_attributes(AttrDict(), elt)
+ luinfo['_type'] = 'lu'
+ luinfo['definition'] = ""
+ luinfo['subCorpus'] = PrettyList()
+ luinfo['lexemes'] = PrettyList() # multiword LUs have multiple lexemes
+ luinfo['semTypes'] = PrettyList() # an LU can have multiple semtypes
+ for k in ignorekeys:
+ if k in luinfo:
+ del luinfo[k]
+
+ for sub in elt:
+ if sub.tag.endswith('header'):
+ continue # not used
+ elif sub.tag.endswith('valences'):
+ continue # not used
+ elif sub.tag.endswith('definition') and 'definition' not in ignorekeys:
+ luinfo['definition'] = self._strip_tags(sub.text)
+ elif sub.tag.endswith('subCorpus') and 'subCorpus' not in ignorekeys:
+ sc = self._handle_lusubcorpus_elt(sub)
+ if sc is not None:
+ luinfo['subCorpus'].append(sc)
+ elif sub.tag.endswith('lexeme') and 'lexeme' not in ignorekeys:
+ luinfo['lexemes'].append(self._load_xml_attributes(PrettyDict(), sub))
+ elif sub.tag.endswith('semType') and 'semType' not in ignorekeys:
+ semtypeinfo = self._load_xml_attributes(AttrDict(), sub)
+ luinfo['semTypes'].append(self.semtype(semtypeinfo.ID))
+
+ return luinfo
+
+ def _handle_lusubcorpus_elt(self, elt):
+ """Load a subcorpus of a lexical unit from the given xml."""
+ sc = AttrDict()
+ try:
+ sc['name'] = str(elt.get('name'))
+ except AttributeError:
+ return None
+ sc['_type'] = "lusubcorpus"
+ sc['sentence'] = []
+
+ for sub in elt:
+ if sub.tag.endswith('sentence'):
+ s = self._handle_lusentence_elt(sub)
+ if s is not None:
+ sc['sentence'].append(s)
+
+ return sc
+
+ def _handle_lusentence_elt(self, elt):
+ """Load a sentence from a subcorpus of an LU from xml."""
+ info = self._load_xml_attributes(AttrDict(), elt)
+ info['_type'] = 'lusentence'
+ info['annotationSet'] = []
+ for sub in elt:
+ if sub.tag.endswith('text'):
+ info['text'] = self._strip_tags(sub.text)
+ elif sub.tag.endswith('annotationSet'):
+ annset = self._handle_luannotationset_elt(sub)
+ if annset is not None:
+ info['annotationSet'].append(annset)
+ return info
+
+ def _handle_luannotationset_elt(self, elt):
+ """Load an annotation set from a sentence in an subcorpus of an LU"""
+ info = self._load_xml_attributes(AttrDict(), elt)
+ info['_type'] = 'luannotationset'
+ info['layer'] = []
+ for sub in elt:
+ if sub.tag.endswith('layer'):
+ l = self._handle_lulayer_elt(sub)
+ if l is not None:
+ info['layer'].append(l)
+ return info
+
+ def _handle_lulayer_elt(self, elt):
+ """Load a layer from an annotation set"""
+ layer = self._load_xml_attributes(AttrDict(), elt)
+ layer['_type'] = 'lulayer'
+ layer['label'] = []
+
+ for sub in elt:
+ if sub.tag.endswith('label'):
+ l = self._load_xml_attributes(AttrDict(), sub)
+ if l is not None:
+ layer['label'].append(l)
+ return layer
+
+ def _handle_fe_elt(self, elt):
+ feinfo = self._load_xml_attributes(AttrDict(), elt)
+ feinfo['_type'] = 'fe'
+ feinfo['definition'] = ""
+ feinfo['semType'] = None
+ feinfo['requiresFE'] = None
+ feinfo['excludesFE'] = None
+ for sub in elt:
+ if sub.tag.endswith('definition'):
+ feinfo['definition'] = self._strip_tags(sub.text)
+ elif sub.tag.endswith('semType'):
+ stinfo = self._load_xml_attributes(AttrDict(), sub)
+ feinfo['semType'] = self.semtype(stinfo.ID)
+ elif sub.tag.endswith('requiresFE'):
+ feinfo['requiresFE'] = self._load_xml_attributes(AttrDict(), sub)
+ elif sub.tag.endswith('excludesFE'):
+ feinfo['excludesFE'] = self._load_xml_attributes(AttrDict(), sub)
+
+ return feinfo
+
+ def _handle_semtype_elt(self, elt, tagspec=None):
+ semt = self._load_xml_attributes(AttrDict(), elt)
+ semt['_type'] = 'semtype'
+ semt['superType'] = None
+ semt['subTypes'] = PrettyList()
+ for sub in elt:
+ if sub.text is not None:
+ semt['definition'] = self._strip_tags(sub.text)
+ else:
+ supertypeinfo = self._load_xml_attributes(AttrDict(), sub)
+ semt['superType'] = supertypeinfo
+ # the supertype may not have been loaded yet
+
+ return semt
+
+
+#
+# Demo
+#
+def demo():
+ from nltk.corpus import framenet as fn
+
+ #
+ # It is not necessary to explicitly build the indexes by calling
+ # buildindexes(). We do this here just for demo purposes. If the
+ # indexes are not built explicitely, they will be built as needed.
+ #
+ print('Building the indexes...')
+ fn.buildindexes()
+
+ #
+ # Get some statistics about the corpus
+ #
+ print('Number of Frames:', len(fn.frames()))
+ print('Number of Lexical Units:', len(fn.lus()))
+ print('Number of annotated documents:', len(fn.documents()))
+ print()
+
+ #
+ # Frames
+ #
+ print('getting frames whose name matches the (case insensitive) regex: "(?i)medical"')
+ medframes = fn.frames(r'(?i)medical')
+ print(
+ 'Found {0} Frames whose name matches "(?i)medical":'.format(len(medframes)))
+ print([(f.name, f.ID) for f in medframes])
+
+ #
+ # store the first frame in the list of frames
+ #
+ tmp_id = medframes[0].ID
+ m_frame = fn.frame(tmp_id) # reads all info for the frame
+
+ #
+ # get the frame relations
+ #
+ print(
+ '\nNumber of frame relations for the "{0}" ({1}) frame:'.format(m_frame.name,
+ m_frame.ID),
+ len(m_frame.frameRelations))
+ for fr in m_frame.frameRelations:
+ print(' ', fr)
+
+ #
+ # get the names of the Frame Elements
+ #
+ print(
+ '\nNumber of Frame Elements in the "{0}" frame:'.format(m_frame.name),
+ len(m_frame.FE))
+ print(' ', [x for x in m_frame.FE])
+
+ #
+ # get the names of the "Core" Frame Elements
+ #
+ print(
+ '\nThe "core" Frame Elements in the "{0}" frame:'.format(m_frame.name))
+ print(' ', [x.name for x in m_frame.FE.values() if x.coreType == "Core"])
+
+ #
+ # get all of the Lexical Units that are incorporated in the
+ # 'Ailment' FE of the 'Medical_conditions' frame (id=239)
+ #
+ print('\nAll Lexical Units that are incorporated in the "Ailment" FE:')
+ m_frame = fn.frame(239)
+ ailment_lus = [x for x in m_frame.lexUnit.values() if 'incorporatedFE' in x and x.incorporatedFE == 'Ailment']
+ print(' ', [x.name for x in ailment_lus])
+
+ #
+ # get all of the Lexical Units for the frame
+ #
+ print('\nNumber of Lexical Units in the "{0}" frame:'.format(m_frame.name),
+ len(m_frame.lexUnit))
+ print(' ', [x.name for x in m_frame.lexUnit.values()][:5], '...')
+
+ #
+ # get basic info on the second LU in the frame
+ #
+ tmp_id = m_frame.lexUnit['ailment.n'].ID # grab the id of the specified LU
+ luinfo = fn.lu_basic(tmp_id) # get basic info on the LU
+ print('\nInformation on the LU: {0}'.format(luinfo.name))
+ pprint(luinfo)
+
+ #
+ # Get a list of all of the corpora used for fulltext annotation
+ #
+ print('\nNames of all of the corpora used for fulltext annotation:')
+ allcorpora = set([x.corpname for x in fn.documents()])
+ pprint(list(allcorpora))
+
+ #
+ # Get the names of the annotated documents in the first corpus
+ #
+ firstcorp = list(allcorpora)[0]
+ firstcorp_docs = fn.documents(firstcorp)
+ print(
+ '\nNames of the annotated documents in the "{0}" corpus:'.format(firstcorp))
+ pprint([x.filename for x in firstcorp_docs])
+
+ #
+ # Search for frames containing LUs whose name attribute matches a
+ # regexp pattern.
+ #
+ # Note: if you were going to be doing a lot of this type of
+ # searching, you'd want to build an index that maps from
+ # lemmas to frames because each time frames_by_lemma() is
+ # called, it has to search through ALL of the frame XML files
+ # in the db.
+ print('\nSearching for all Frames that have a lemma that matches the regexp: "^run.v$":')
+ pprint(fn.frames_by_lemma(r'^run.v$'))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/corpus/reader/ieer.py b/nltk/corpus/reader/ieer.py
new file mode 100644
index 0000000..f73f4ec
--- /dev/null
+++ b/nltk/corpus/reader/ieer.py
@@ -0,0 +1,111 @@
+# Natural Language Toolkit: IEER Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Corpus reader for the Information Extraction and Entity Recognition Corpus.
+
+NIST 1999 Information Extraction: Entity Recognition Evaluation
+http://www.itl.nist.gov/iad/894.01/tests/ie-er/er_99/er_99.htm
+
+This corpus contains the NEWSWIRE development test data for the
+NIST 1999 IE-ER Evaluation. The files were taken from the
+subdirectory: /ie_er_99/english/devtest/newswire/*.ref.nwt
+and filenames were shortened.
+
+The corpus contains the following files: APW_19980314, APW_19980424,
+APW_19980429, NYT_19980315, NYT_19980403, and NYT_19980407.
+"""
+from __future__ import unicode_literals
+
+import nltk
+from nltk import compat
+from nltk.corpus.reader.api import *
+
+#: A dictionary whose keys are the names of documents in this corpus;
+#: and whose values are descriptions of those documents' contents.
+titles = {
+ 'APW_19980314': 'Associated Press Weekly, 14 March 1998',
+ 'APW_19980424': 'Associated Press Weekly, 24 April 1998',
+ 'APW_19980429': 'Associated Press Weekly, 29 April 1998',
+ 'NYT_19980315': 'New York Times, 15 March 1998',
+ 'NYT_19980403': 'New York Times, 3 April 1998',
+ 'NYT_19980407': 'New York Times, 7 April 1998',
+ }
+
+#: A list of all documents in this corpus.
+documents = sorted(titles)
+
+ at compat.python_2_unicode_compatible
+class IEERDocument(object):
+ def __init__(self, text, docno=None, doctype=None,
+ date_time=None, headline=''):
+ self.text = text
+ self.docno = docno
+ self.doctype = doctype
+ self.date_time = date_time
+ self.headline = headline
+
+ def __repr__(self):
+ if self.headline:
+ headline = ' '.join(self.headline.leaves())
+ else:
+ headline = ' '.join([w for w in self.text.leaves()
+ if w[:1] != '<'][:12])+'...'
+ if self.docno is not None:
+ return '<IEERDocument %s: %r>' % (self.docno, headline)
+ else:
+ return '<IEERDocument: %r>' % headline
+
+class IEERCorpusReader(CorpusReader):
+ """
+ """
+ def raw(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def docs(self, fileids=None):
+ return concat([StreamBackedCorpusView(fileid, self._read_block,
+ encoding=enc)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def parsed_docs(self, fileids=None):
+ return concat([StreamBackedCorpusView(fileid,
+ self._read_parsed_block,
+ encoding=enc)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def _read_parsed_block(self,stream):
+ # TODO: figure out while empty documents are being returned
+ return [self._parse(doc) for doc in self._read_block(stream)
+ if self._parse(doc).docno is not None]
+
+ def _parse(self, doc):
+ val = nltk.chunk.ieerstr2tree(doc, root_label="DOCUMENT")
+ if isinstance(val, dict):
+ return IEERDocument(**val)
+ else:
+ return IEERDocument(val)
+
+ def _read_block(self, stream):
+ out = []
+ # Skip any preamble.
+ while True:
+ line = stream.readline()
+ if not line: break
+ if line.strip() == '<DOC>': break
+ out.append(line)
+ # Read the document
+ while True:
+ line = stream.readline()
+ if not line: break
+ out.append(line)
+ if line.strip() == '</DOC>': break
+ # Return the document
+ return ['\n'.join(out)]
+
diff --git a/nltk/corpus/reader/indian.py b/nltk/corpus/reader/indian.py
new file mode 100644
index 0000000..6134ac9
--- /dev/null
+++ b/nltk/corpus/reader/indian.py
@@ -0,0 +1,88 @@
+# Natural Language Toolkit: Indian Language POS-Tagged Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Indian Language POS-Tagged Corpus
+Collected by A Kumaran, Microsoft Research, India
+Distributed with permission
+
+Contents:
+ - Bangla: IIT Kharagpur
+ - Hindi: Microsoft Research India
+ - Marathi: IIT Bombay
+ - Telugu: IIIT Hyderabad
+"""
+
+import codecs
+
+from nltk import compat
+from nltk.tag import str2tuple, map_tag
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class IndianCorpusReader(CorpusReader):
+ """
+ List of words, one per line. Blank lines are ignored.
+ """
+ def words(self, fileids=None):
+ return concat([IndianCorpusView(fileid, enc,
+ False, False)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tagged_words(self, fileids=None, tagset=None):
+ if tagset and tagset != self._tagset:
+ tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
+ else:
+ tag_mapping_function = None
+ return concat([IndianCorpusView(fileid, enc,
+ True, False, tag_mapping_function)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def sents(self, fileids=None):
+ return concat([IndianCorpusView(fileid, enc,
+ False, True)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tagged_sents(self, fileids=None, tagset=None):
+ if tagset and tagset != self._tagset:
+ tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
+ else:
+ tag_mapping_function = None
+ return concat([IndianCorpusView(fileid, enc,
+ True, True, tag_mapping_function)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def raw(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+
+class IndianCorpusView(StreamBackedCorpusView):
+ def __init__(self, corpus_file, encoding, tagged,
+ group_by_sent, tag_mapping_function=None):
+ self._tagged = tagged
+ self._group_by_sent = group_by_sent
+ self._tag_mapping_function = tag_mapping_function
+ StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
+
+ def read_block(self, stream):
+ line = stream.readline()
+ if line.startswith('<'):
+ return []
+ sent = [str2tuple(word, sep='_') for word in line.split()]
+ if self._tag_mapping_function:
+ sent = [(w, self._tag_mapping_function(t)) for (w,t) in sent]
+ if not self._tagged: sent = [w for (w,t) in sent]
+ if self._group_by_sent:
+ return [sent]
+ else:
+ return sent
+
+
diff --git a/nltk/corpus/reader/ipipan.py b/nltk/corpus/reader/ipipan.py
new file mode 100644
index 0000000..0d11ea8
--- /dev/null
+++ b/nltk/corpus/reader/ipipan.py
@@ -0,0 +1,331 @@
+# Natural Language Toolkit: IPI PAN Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Konrad Goluchowski <kodie at mimuw.edu.pl>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import functools
+
+from nltk import compat
+from nltk.corpus.reader.util import StreamBackedCorpusView, concat
+from nltk.corpus.reader.api import CorpusReader
+
+def _parse_args(fun):
+ @functools.wraps(fun)
+ def decorator(self, fileids=None, **kwargs):
+ kwargs.pop('tags', None)
+ if not fileids:
+ fileids = self.fileids()
+ return fun(self, fileids, **kwargs)
+ return decorator
+
+class IPIPANCorpusReader(CorpusReader):
+ """
+ Corpus reader designed to work with corpus created by IPI PAN.
+ See http://korpus.pl/en/ for more details about IPI PAN corpus.
+
+ The corpus includes information about text domain, channel and categories.
+ You can access possible values using ``domains()``, ``channels()`` and
+ ``categories()``. You can use also this metadata to filter files, e.g.:
+ ``fileids(channel='prasa')``, ``fileids(categories='publicystyczny')``.
+
+ The reader supports methods: words, sents, paras and their tagged versions.
+ You can get part of speech instead of full tag by giving "simplify_tags=True"
+ parameter, e.g.: ``tagged_sents(simplify_tags=True)``.
+
+ Also you can get all tags disambiguated tags specifying parameter
+ "one_tag=False", e.g.: ``tagged_paras(one_tag=False)``.
+
+ You can get all tags that were assigned by a morphological analyzer specifying
+ parameter "disamb_only=False", e.g. ``tagged_words(disamb_only=False)``.
+
+ The IPIPAN Corpus contains tags indicating if there is a space between two
+ tokens. To add special "no space" markers, you should specify parameter
+ "append_no_space=True", e.g. ``tagged_words(append_no_space=True)``.
+ As a result in place where there should be no space between two tokens new
+ pair ('', 'no-space') will be inserted (for tagged data) and just '' for
+ methods without tags.
+
+ The corpus reader can also try to append spaces between words. To enable this
+ option, specify parameter "append_space=True", e.g. ``words(append_space=True)``.
+ As a result either ' ' or (' ', 'space') will be inserted between tokens.
+
+ By default, xml entities like " and & are replaced by corresponding
+ characters. You can turn off this feature, specifying parameter
+ "replace_xmlentities=False", e.g. ``words(replace_xmlentities=False)``.
+ """
+
+ def __init__(self, root, fileids):
+ CorpusReader.__init__(self, root, fileids, None, None)
+
+ def raw(self, fileids=None):
+ if not fileids:
+ fileids = self.fileids()
+
+ filecontents = []
+ for fileid in self._list_morph_files(fileids):
+ with open(fileid, 'r') as infile:
+ filecontents.append(infile.read())
+ return ''.join(filecontents)
+
+ def channels(self, fileids=None):
+ if not fileids:
+ fileids = self.fileids()
+ return self._parse_header(fileids, 'channel')
+
+ def domains(self, fileids=None):
+ if not fileids:
+ fileids = self.fileids()
+ return self._parse_header(fileids, 'domain')
+
+ def categories(self, fileids=None):
+ if not fileids:
+ fileids = self.fileids()
+ return [self._map_category(cat)
+ for cat in self._parse_header(fileids, 'keyTerm')]
+
+ def fileids(self, channels=None, domains=None, categories=None):
+ if channels is not None and domains is not None and \
+ categories is not None:
+ raise ValueError('You can specify only one of channels, domains '
+ 'and categories parameter at once')
+ if channels is None and domains is None and \
+ categories is None:
+ return CorpusReader.fileids(self)
+ if isinstance(channels, compat.string_types):
+ channels = [channels]
+ if isinstance(domains, compat.string_types):
+ domains = [domains]
+ if isinstance(categories, compat.string_types):
+ categories = [categories]
+ if channels:
+ return self._list_morph_files_by('channel', channels)
+ elif domains:
+ return self._list_morph_files_by('domain', domains)
+ else:
+ return self._list_morph_files_by('keyTerm', categories,
+ map=self._map_category)
+
+ @_parse_args
+ def sents(self, fileids=None, **kwargs):
+ return concat([self._view(fileid,
+ mode=IPIPANCorpusView.SENTS_MODE, tags=False, **kwargs)
+ for fileid in self._list_morph_files(fileids)])
+
+ @_parse_args
+ def paras(self, fileids=None, **kwargs):
+ return concat([self._view(fileid,
+ mode=IPIPANCorpusView.PARAS_MODE, tags=False, **kwargs)
+ for fileid in self._list_morph_files(fileids)])
+
+ @_parse_args
+ def words(self, fileids=None, **kwargs):
+ return concat([self._view(fileid, tags=False, **kwargs)
+ for fileid in self._list_morph_files(fileids)])
+
+ @_parse_args
+ def tagged_sents(self, fileids=None, **kwargs):
+ return concat([self._view(fileid, mode=IPIPANCorpusView.SENTS_MODE,
+ **kwargs)
+ for fileid in self._list_morph_files(fileids)])
+
+ @_parse_args
+ def tagged_paras(self, fileids=None, **kwargs):
+ return concat([self._view(fileid, mode=IPIPANCorpusView.PARAS_MODE,
+ **kwargs)
+ for fileid in self._list_morph_files(fileids)])
+
+ @_parse_args
+ def tagged_words(self, fileids=None, **kwargs):
+ return concat([self._view(fileid, **kwargs)
+ for fileid in self._list_morph_files(fileids)])
+
+ def _list_morph_files(self, fileids):
+ return [f for f in self.abspaths(fileids)]
+
+ def _list_header_files(self, fileids):
+ return [f.replace('morph.xml', 'header.xml')
+ for f in self._list_morph_files(fileids)]
+
+ def _parse_header(self, fileids, tag):
+ values = set()
+ for f in self._list_header_files(fileids):
+ values_list = self._get_tag(f, tag)
+ for v in values_list:
+ values.add(v)
+ return list(values)
+
+ def _list_morph_files_by(self, tag, values, map=None):
+ fileids = self.fileids()
+ ret_fileids = set()
+ for f in fileids:
+ fp = self.abspath(f).replace('morph.xml', 'header.xml')
+ values_list = self._get_tag(fp, tag)
+ for value in values_list:
+ if map is not None:
+ value = map(value)
+ if value in values:
+ ret_fileids.add(f)
+ return list(ret_fileids)
+
+ def _get_tag(self, f, tag):
+ tags = []
+ with open(f, 'r') as infile:
+ header = infile.read()
+ tag_end = 0
+ while True:
+ tag_pos = header.find('<'+tag, tag_end)
+ if tag_pos < 0: return tags
+ tag_end = header.find('</'+tag+'>', tag_pos)
+ tags.append(header[tag_pos+len(tag)+2:tag_end])
+
+ def _map_category(self, cat):
+ pos = cat.find('>')
+ if pos == -1:
+ return cat
+ else:
+ return cat[pos+1:]
+
+ def _view(self, filename, **kwargs):
+ tags = kwargs.pop('tags', True)
+ mode = kwargs.pop('mode', 0)
+ simplify_tags = kwargs.pop('simplify_tags', False)
+ one_tag = kwargs.pop('one_tag', True)
+ disamb_only = kwargs.pop('disamb_only', True)
+ append_no_space = kwargs.pop('append_no_space', False)
+ append_space = kwargs.pop('append_space', False)
+ replace_xmlentities = kwargs.pop('replace_xmlentities', True)
+
+ if len(kwargs) > 0:
+ raise ValueError('Unexpected arguments: %s' % kwargs.keys())
+ if not one_tag and not disamb_only:
+ raise ValueError('You cannot specify both one_tag=False and '
+ 'disamb_only=False')
+ if not tags and (simplify_tags or not one_tag or not disamb_only):
+ raise ValueError('You cannot specify simplify_tags, one_tag or '
+ 'disamb_only with functions other than tagged_*')
+
+ return IPIPANCorpusView(filename,
+ tags=tags, mode=mode, simplify_tags=simplify_tags,
+ one_tag=one_tag, disamb_only=disamb_only,
+ append_no_space=append_no_space,
+ append_space=append_space,
+ replace_xmlentities=replace_xmlentities
+ )
+
+
+class IPIPANCorpusView(StreamBackedCorpusView):
+
+ WORDS_MODE = 0
+ SENTS_MODE = 1
+ PARAS_MODE = 2
+
+ def __init__(self, filename, startpos=0, **kwargs):
+ StreamBackedCorpusView.__init__(self, filename, None, startpos, None)
+ self.in_sentence = False
+ self.position = 0
+
+ self.show_tags = kwargs.pop('tags', True)
+ self.disamb_only = kwargs.pop('disamb_only', True)
+ self.mode = kwargs.pop('mode', IPIPANCorpusView.WORDS_MODE)
+ self.simplify_tags = kwargs.pop('simplify_tags', False)
+ self.one_tag = kwargs.pop('one_tag', True)
+ self.append_no_space = kwargs.pop('append_no_space', False)
+ self.append_space = kwargs.pop('append_space', False)
+ self.replace_xmlentities = kwargs.pop('replace_xmlentities', True)
+
+ def read_block(self, stream):
+ sentence = []
+ sentences = []
+ space = False
+ no_space = False
+
+ tags = set()
+
+ lines = self._read_data(stream)
+
+ while True:
+
+ # we may have only part of last line
+ if len(lines) <= 1:
+ self._seek(stream)
+ lines = self._read_data(stream)
+
+ if lines == ['']:
+ assert not sentences
+ return []
+
+ line = lines.pop()
+ self.position += len(line) + 1
+
+ if line.startswith('<chunk type="s"'):
+ self.in_sentence = True
+ elif line.startswith('<chunk type="p"'):
+ pass
+ elif line.startswith('<tok'):
+ if self.append_space and space and not no_space:
+ self._append_space(sentence)
+ space = True
+ no_space = False
+ orth = ""
+ tags = set()
+ elif line.startswith('</chunk'):
+ if self.in_sentence:
+ self.in_sentence = False
+ self._seek(stream)
+ if self.mode == self.SENTS_MODE:
+ return [sentence]
+ elif self.mode == self.WORDS_MODE:
+ if self.append_space:
+ self._append_space(sentence)
+ return sentence
+ else:
+ sentences.append(sentence)
+ elif self.mode == self.PARAS_MODE:
+ self._seek(stream)
+ return [sentences]
+ elif line.startswith('<orth'):
+ orth = line[6:-7]
+ if self.replace_xmlentities:
+ orth = orth.replace('"', '"').replace('&', '&')
+ elif line.startswith('<lex'):
+ if not self.disamb_only or line.find('disamb=') != -1:
+ tag = line[line.index('<ctag')+6 : line.index('</ctag') ]
+ tags.add(tag)
+ elif line.startswith('</tok'):
+ if self.show_tags:
+ if self.simplify_tags:
+ tags = [t.split(':')[0] for t in tags]
+ if not self.one_tag or not self.disamb_only:
+ sentence.append((orth, tuple(tags)))
+ else:
+ sentence.append((orth, tags.pop()))
+ else:
+ sentence.append(orth)
+ elif line.startswith('<ns/>'):
+ if self.append_space:
+ no_space = True
+ if self.append_no_space:
+ if self.show_tags:
+ sentence.append(('', 'no-space'))
+ else:
+ sentence.append('')
+ elif line.startswith('</cesAna'):
+ pass
+
+ def _read_data(self, stream):
+ self.position = stream.tell()
+ buff = stream.read(4096)
+ lines = buff.split('\n')
+ lines.reverse()
+ return lines
+
+ def _seek(self, stream):
+ stream.seek(self.position)
+
+ def _append_space(self, sentence):
+ if self.show_tags:
+ sentence.append((' ', 'space'))
+ else:
+ sentence.append(' ')
diff --git a/nltk/corpus/reader/knbc.py b/nltk/corpus/reader/knbc.py
new file mode 100644
index 0000000..7f56b43
--- /dev/null
+++ b/nltk/corpus/reader/knbc.py
@@ -0,0 +1,164 @@
+#! /usr/bin/env python
+# KNB Corpus reader
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Masato Hagiwara <hagisan at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# For more information, see http://lilyx.net/pages/nltkjapanesecorpus.html
+from __future__ import print_function
+
+import sys
+
+from nltk import compat
+from nltk.tree import bracket_parse, Tree
+from nltk.parse import DependencyGraph
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+# default function to convert morphlist to str for tree representation
+_morphs2str_default = lambda morphs: '/'.join(m[0] for m in morphs if m[0] != 'EOS')
+
+class KNBCorpusReader(SyntaxCorpusReader):
+ """
+ This class implements:
+ - ``__init__``, which specifies the location of the corpus
+ and a method for detecting the sentence blocks in corpus files.
+ - ``_read_block``, which reads a block from the input stream.
+ - ``_word``, which takes a block and returns a list of list of words.
+ - ``_tag``, which takes a block and returns a list of list of tagged
+ words.
+ - ``_parse``, which takes a block and returns a list of parsed
+ sentences.
+
+ The structure of tagged words:
+ tagged_word = (word(str), tags(tuple))
+ tags = (surface, reading, lemma, pos1, posid1, pos2, posid2, pos3, posid3, others ...)
+ """
+
+ def __init__(self, root, fileids, encoding='utf8', morphs2str=_morphs2str_default):
+ """
+ Initialize KNBCorpusReader
+ morphs2str is a function to convert morphlist to str for tree representation
+ for _parse()
+ """
+ CorpusReader.__init__(self, root, fileids, encoding)
+ self.morphs2str = morphs2str
+
+ def _read_block(self, stream):
+ # blocks are split by blankline (or EOF) - default
+ return read_blankline_block(stream)
+
+ def _word(self, t):
+ res = []
+ for line in t.splitlines():
+ # ignore the Bunsets headers
+ if not re.match(r"EOS|\*|\#|\+", line):
+ cells = line.strip().split(" ")
+ res.append(cells[0])
+
+ return res
+
+ # ignores tagset argument
+ def _tag(self, t, tagset=None):
+ res = []
+ for line in t.splitlines():
+ # ignore the Bunsets headers
+ if not re.match(r"EOS|\*|\#|\+", line):
+ cells = line.strip().split(" ")
+ # convert cells to morph tuples
+ res.append( (cells[0], ' '.join(cells[1:])) )
+
+ return res
+
+ def _parse(self, t):
+ dg = DependencyGraph()
+ i = 0
+ for line in t.splitlines():
+ if line.startswith("*") or line.startswith("+"):
+ # start of bunsetsu or tag
+
+ cells = line.strip().split(" ", 3)
+ m = re.match(r"([\-0-9]*)([ADIP])", cells[1])
+
+ assert m is not None
+
+ node = dg.nodelist[i]
+ node['address'] = i
+ node['rel'] = m.group(2) # dep_type
+
+ node['word'] = []
+
+ dep_parent = int(m.group(1))
+
+ while len(dg.nodelist) < i+1 or len(dg.nodelist) < dep_parent+1:
+ dg.nodelist.append({'word':[], 'deps':[]})
+
+ if dep_parent == -1:
+ dg.root = node
+ else:
+ dg.nodelist[dep_parent]['deps'].append(i)
+
+ i += 1
+ elif not line.startswith("#"):
+ # normal morph
+ cells = line.strip().split(" ")
+ # convert cells to morph tuples
+ morph = ( cells[0], ' '.join(cells[1:]) )
+ dg.nodelist[i-1]['word'].append(morph)
+
+ if self.morphs2str:
+ for node in dg.nodelist:
+ node['word'] = self.morphs2str(node['word'])
+
+ return dg.tree()
+
+######################################################################
+# Demo
+######################################################################
+
+def demo():
+
+ import nltk
+ from nltk.corpus.util import LazyCorpusLoader
+
+ root = nltk.data.find('corpora/knbc/corpus1')
+ fileids = [f for f in find_corpus_fileids(FileSystemPathPointer(root), ".*")
+ if re.search(r"\d\-\d\-[\d]+\-[\d]+", f)]
+
+ def _knbc_fileids_sort(x):
+ cells = x.split('-')
+ return (cells[0], int(cells[1]), int(cells[2]), int(cells[3]))
+
+ knbc = LazyCorpusLoader('knbc/corpus1', KNBCorpusReader,
+ sorted(fileids, key=_knbc_fileids_sort), encoding='euc-jp')
+
+ print(knbc.fileids()[:10])
+ print(''.join( knbc.words()[:100] ))
+
+ print('\n\n'.join( '%s' % tree for tree in knbc.parsed_sents()[:2] ))
+
+ knbc.morphs2str = lambda morphs: '/'.join(
+ "%s(%s)"%(m[0], m[1].split(' ')[2]) for m in morphs if m[0] != 'EOS'
+ ).encode('utf-8')
+
+ print('\n\n'.join( '%s' % tree for tree in knbc.parsed_sents()[:2] ))
+
+ print('\n'.join( ' '.join("%s/%s"%(w[0], w[1].split(' ')[2]) for w in sent)
+ for sent in knbc.tagged_sents()[0:2] ))
+
+def test():
+
+ from nltk.corpus.util import LazyCorpusLoader
+
+ knbc = LazyCorpusLoader(
+ 'knbc/corpus1', KNBCorpusReader, r'.*/KN.*', encoding='euc-jp')
+ assert isinstance(knbc.words()[0], compat.string_types)
+ assert isinstance(knbc.sents()[0][0], compat.string_types)
+ assert isinstance(knbc.tagged_words()[0], tuple)
+ assert isinstance(knbc.tagged_sents()[0][0], tuple)
+
+if __name__ == '__main__':
+ demo()
+ # test()
diff --git a/nltk/corpus/reader/lin.py b/nltk/corpus/reader/lin.py
new file mode 100644
index 0000000..d8646e9
--- /dev/null
+++ b/nltk/corpus/reader/lin.py
@@ -0,0 +1,156 @@
+# Natural Language Toolkit: Lin's Thesaurus
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Dan Blanchard <dblanchard at ets.org>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.txt
+from __future__ import print_function
+
+import re
+from collections import defaultdict
+from functools import reduce
+
+from nltk.corpus.reader import CorpusReader
+
+
+class LinThesaurusCorpusReader(CorpusReader):
+ """ Wrapper for the LISP-formatted thesauruses distributed by Dekang Lin. """
+
+ # Compiled regular expression for extracting the key from the first line of each
+ # thesaurus entry
+ _key_re = re.compile(r'\("?([^"]+)"? \(desc [0-9.]+\).+')
+
+ @staticmethod
+ def __defaultdict_factory():
+ ''' Factory for creating defaultdict of defaultdict(dict)s '''
+ return defaultdict(dict)
+
+ def __init__(self, root, badscore=0.0):
+ '''
+ Initialize the thesaurus.
+
+ :param root: root directory containing thesaurus LISP files
+ :type root: C{string}
+ :param badscore: the score to give to words which do not appear in each other's sets of synonyms
+ :type badscore: C{float}
+ '''
+
+ super(LinThesaurusCorpusReader, self).__init__(root, r'sim[A-Z]\.lsp')
+ self._thesaurus = defaultdict(LinThesaurusCorpusReader.__defaultdict_factory)
+ self._badscore = badscore
+ for path, encoding, fileid in self.abspaths(include_encoding=True, include_fileid=True):
+ with open(path) as lin_file:
+ first = True
+ for line in lin_file:
+ line = line.strip()
+ # Start of entry
+ if first:
+ key = LinThesaurusCorpusReader._key_re.sub(r'\1', line)
+ first = False
+ # End of entry
+ elif line == '))':
+ first = True
+ # Lines with pairs of ngrams and scores
+ else:
+ split_line = line.split('\t')
+ if len(split_line) == 2:
+ ngram, score = split_line
+ self._thesaurus[fileid][key][ngram.strip('"')] = float(score)
+
+ def similarity(self, ngram1, ngram2, fileid=None):
+ '''
+ Returns the similarity score for two ngrams.
+
+ :param ngram1: first ngram to compare
+ :type ngram1: C{string}
+ :param ngram2: second ngram to compare
+ :type ngram2: C{string}
+ :param fileid: thesaurus fileid to search in. If None, search all fileids.
+ :type fileid: C{string}
+ :return: If fileid is specified, just the score for the two ngrams; otherwise,
+ list of tuples of fileids and scores.
+ '''
+ # Entries don't contain themselves, so make sure similarity between item and itself is 1.0
+ if ngram1 == ngram2:
+ if fileid:
+ return 1.0
+ else:
+ return [(fid, 1.0) for fid in self._fileids]
+ else:
+ if fileid:
+ return self._thesaurus[fileid][ngram1][ngram2] if ngram2 in self._thesaurus[fileid][ngram1] else self._badscore
+ else:
+ return [(fid, (self._thesaurus[fid][ngram1][ngram2] if ngram2 in self._thesaurus[fid][ngram1]
+ else self._badscore)) for fid in self._fileids]
+
+ def scored_synonyms(self, ngram, fileid=None):
+ '''
+ Returns a list of scored synonyms (tuples of synonyms and scores) for the current ngram
+
+ :param ngram: ngram to lookup
+ :type ngram: C{string}
+ :param fileid: thesaurus fileid to search in. If None, search all fileids.
+ :type fileid: C{string}
+ :return: If fileid is specified, list of tuples of scores and synonyms; otherwise,
+ list of tuples of fileids and lists, where inner lists consist of tuples of
+ scores and synonyms.
+ '''
+ if fileid:
+ return self._thesaurus[fileid][ngram].items()
+ else:
+ return [(fileid, self._thesaurus[fileid][ngram].items()) for fileid in self._fileids]
+
+ def synonyms(self, ngram, fileid=None):
+ '''
+ Returns a list of synonyms for the current ngram.
+
+ :param ngram: ngram to lookup
+ :type ngram: C{string}
+ :param fileid: thesaurus fileid to search in. If None, search all fileids.
+ :type fileid: C{string}
+ :return: If fileid is specified, list of synonyms; otherwise, list of tuples of fileids and
+ lists, where inner lists contain synonyms.
+ '''
+ if fileid:
+ return self._thesaurus[fileid][ngram].keys()
+ else:
+ return [(fileid, self._thesaurus[fileid][ngram].keys()) for fileid in self._fileids]
+
+ def __contains__(self, ngram):
+ '''
+ Determines whether or not the given ngram is in the thesaurus.
+
+ :param ngram: ngram to lookup
+ :type ngram: C{string}
+ :return: whether the given ngram is in the thesaurus.
+ '''
+ return reduce(lambda accum, fileid: accum or (ngram in self._thesaurus[fileid]), self._fileids, False)
+
+
+######################################################################
+# Demo
+######################################################################
+
+def demo():
+ from nltk.corpus import lin_thesaurus as thes
+
+ word1 = "business"
+ word2 = "enterprise"
+ print("Getting synonyms for " + word1)
+ print(thes.synonyms(word1))
+
+ print("Getting scored synonyms for " + word1)
+ print(thes.synonyms(word1))
+
+ print("Getting synonyms from simN.lsp (noun subsection) for " + word1)
+ print(thes.synonyms(word1, fileid="simN.lsp"))
+
+ print("Getting synonyms from simN.lsp (noun subsection) for " + word1)
+ print(thes.synonyms(word1, fileid="simN.lsp"))
+
+ print("Similarity score for %s and %s:" % (word1, word2))
+ print(thes.similarity(word1, word2))
+
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/corpus/reader/nombank.py b/nltk/corpus/reader/nombank.py
new file mode 100644
index 0000000..dc033ea
--- /dev/null
+++ b/nltk/corpus/reader/nombank.py
@@ -0,0 +1,421 @@
+# Natural Language Toolkit: NomBank Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Authors: Paul Bedaride <paul.bedaride at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import unicode_literals
+
+from nltk.tree import Tree
+from xml.etree import ElementTree
+from nltk.internals import raise_unorderable_types
+from nltk.compat import total_ordering, python_2_unicode_compatible, string_types
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class NombankCorpusReader(CorpusReader):
+ """
+ Corpus reader for the nombank corpus, which augments the Penn
+ Treebank with information about the predicate argument structure
+ of every noun instance. The corpus consists of two parts: the
+ predicate-argument annotations themselves, and a set of "frameset
+ files" which define the argument labels used by the annotations,
+ on a per-noun basis. Each "frameset file" contains one or more
+ predicates, such as ``'turn'`` or ``'turn_on'``, each of which is
+ divided into coarse-grained word senses called "rolesets". For
+ each "roleset", the frameset file provides descriptions of the
+ argument roles, along with examples.
+ """
+ def __init__(self, root, nomfile, framefiles='',
+ nounsfile=None, parse_fileid_xform=None,
+ parse_corpus=None, encoding='utf8'):
+ """
+ :param root: The root directory for this corpus.
+ :param nomfile: The name of the file containing the predicate-
+ argument annotations (relative to ``root``).
+ :param framefiles: A list or regexp specifying the frameset
+ fileids for this corpus.
+ :param parse_fileid_xform: A transform that should be applied
+ to the fileids in this corpus. This should be a function
+ of one argument (a fileid) that returns a string (the new
+ fileid).
+ :param parse_corpus: The corpus containing the parse trees
+ corresponding to this corpus. These parse trees are
+ necessary to resolve the tree pointers used by nombank.
+ """
+ # If framefiles is specified as a regexp, expand it.
+ if isinstance(framefiles, string_types):
+ framefiles = find_corpus_fileids(root, framefiles)
+ framefiles = list(framefiles)
+ # Initialze the corpus reader.
+ CorpusReader.__init__(self, root, [nomfile, nounsfile] + framefiles,
+ encoding)
+
+ # Record our frame fileids & nom file.
+ self._nomfile = nomfile
+ self._framefiles = framefiles
+ self._nounsfile = nounsfile
+ self._parse_fileid_xform = parse_fileid_xform
+ self._parse_corpus = parse_corpus
+
+ def raw(self, fileids=None):
+ """
+ :return: the text contents of the given fileids, as a single string.
+ """
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def instances(self, baseform=None):
+ """
+ :return: a corpus view that acts as a list of
+ ``NombankInstance`` objects, one for each noun in the corpus.
+ """
+ kwargs = {}
+ if baseform is not None:
+ kwargs['instance_filter'] = lambda inst: inst.baseform==baseform
+ return StreamBackedCorpusView(self.abspath(self._nomfile),
+ lambda stream: self._read_instance_block(stream, **kwargs),
+ encoding=self.encoding(self._nomfile))
+
+ def lines(self):
+ """
+ :return: a corpus view that acts as a list of strings, one for
+ each line in the predicate-argument annotation file.
+ """
+ return StreamBackedCorpusView(self.abspath(self._nomfile),
+ read_line_block,
+ encoding=self.encoding(self._nomfile))
+
+ def roleset(self, roleset_id):
+ """
+ :return: the xml description for the given roleset.
+ """
+ baseform = roleset_id.split('.')[0]
+ baseform = baseform.replace('perc-sign','%')
+ baseform = baseform.replace('oneslashonezero', '1/10').replace('1/10','1-slash-10')
+ framefile = 'frames/%s.xml' % baseform
+ if framefile not in self._framefiles:
+ raise ValueError('Frameset file for %s not found' %
+ roleset_id)
+
+ # n.b.: The encoding for XML fileids is specified by the file
+ # itself; so we ignore self._encoding here.
+ etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
+ for roleset in etree.findall('predicate/roleset'):
+ if roleset.attrib['id'] == roleset_id:
+ return roleset
+ else:
+ raise ValueError('Roleset %s not found in %s' %
+ (roleset_id, framefile))
+
+ def rolesets(self, baseform=None):
+ """
+ :return: list of xml descriptions for rolesets.
+ """
+ if baseform is not None:
+ framefile = 'frames/%s.xml' % baseform
+ if framefile not in self._framefiles:
+ raise ValueError('Frameset file for %s not found' %
+ baseform)
+ framefiles = [framefile]
+ else:
+ framefiles = self._framefiles
+
+ rsets = []
+ for framefile in framefiles:
+ # n.b.: The encoding for XML fileids is specified by the file
+ # itself; so we ignore self._encoding here.
+ etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
+ rsets.append(etree.findall('predicate/roleset'))
+ return LazyConcatenation(rsets)
+
+ def nouns(self):
+ """
+ :return: a corpus view that acts as a list of all noun lemmas
+ in this corpus (from the nombank.1.0.words file).
+ """
+ return StreamBackedCorpusView(self.abspath(self._nounsfile),
+ read_line_block,
+ encoding=self.encoding(self._nounsfile))
+
+ def _read_instance_block(self, stream, instance_filter=lambda inst: True):
+ block = []
+
+ # Read 100 at a time.
+ for i in range(100):
+ line = stream.readline().strip()
+ if line:
+ inst = NombankInstance.parse(
+ line, self._parse_fileid_xform,
+ self._parse_corpus)
+ if instance_filter(inst):
+ block.append(inst)
+
+ return block
+
+######################################################################
+#{ Nombank Instance & related datatypes
+######################################################################
+
+ at python_2_unicode_compatible
+class NombankInstance(object):
+
+ def __init__(self, fileid, sentnum, wordnum, baseform, sensenumber,
+ predicate, predid, arguments, parse_corpus=None):
+
+ self.fileid = fileid
+ """The name of the file containing the parse tree for this
+ instance's sentence."""
+
+ self.sentnum = sentnum
+ """The sentence number of this sentence within ``fileid``.
+ Indexing starts from zero."""
+
+ self.wordnum = wordnum
+ """The word number of this instance's predicate within its
+ containing sentence. Word numbers are indexed starting from
+ zero, and include traces and other empty parse elements."""
+
+ self.baseform = baseform
+ """The baseform of the predicate."""
+
+ self.sensenumber = sensenumber
+ """The sense number of the predicate."""
+
+ self.predicate = predicate
+ """A ``NombankTreePointer`` indicating the position of this
+ instance's predicate within its containing sentence."""
+
+ self.predid = predid
+ """Identifier of the predicate."""
+
+ self.arguments = tuple(arguments)
+ """A list of tuples (argloc, argid), specifying the location
+ and identifier for each of the predicate's argument in the
+ containing sentence. Argument identifiers are strings such as
+ ``'ARG0'`` or ``'ARGM-TMP'``. This list does *not* contain
+ the predicate."""
+
+ self.parse_corpus = parse_corpus
+ """A corpus reader for the parse trees corresponding to the
+ instances in this nombank corpus."""
+
+ @property
+ def roleset(self):
+ """The name of the roleset used by this instance's predicate.
+ Use ``nombank.roleset() <NombankCorpusReader.roleset>`` to
+ look up information about the roleset."""
+ r = self.baseform.replace('%', 'perc-sign')
+ r = r.replace('1/10', '1-slash-10').replace('1-slash-10', 'oneslashonezero')
+ return '%s.%s'%(r, self.sensenumber)
+
+ def __repr__(self):
+ return ('<NombankInstance: %s, sent %s, word %s>' %
+ (self.fileid, self.sentnum, self.wordnum))
+
+ def __str__(self):
+ s = '%s %s %s %s %s' % (self.fileid, self.sentnum, self.wordnum,
+ self.baseform, self.sensenumber)
+ items = self.arguments + ((self.predicate, 'rel'),)
+ for (argloc, argid) in sorted(items):
+ s += ' %s-%s' % (argloc, argid)
+ return s
+
+ def _get_tree(self):
+ if self.parse_corpus is None: return None
+ if self.fileid not in self.parse_corpus.fileids(): return None
+ return self.parse_corpus.parsed_sents(self.fileid)[self.sentnum]
+ tree = property(_get_tree, doc="""
+ The parse tree corresponding to this instance, or None if
+ the corresponding tree is not available.""")
+
+ @staticmethod
+ def parse(s, parse_fileid_xform=None, parse_corpus=None):
+ pieces = s.split()
+ if len(pieces) < 6:
+ raise ValueError('Badly formatted nombank line: %r' % s)
+
+ # Divide the line into its basic pieces.
+ (fileid, sentnum, wordnum,
+ baseform, sensenumber) = pieces[:5]
+
+ args = pieces[5:]
+ rel = [args.pop(i) for i,p in enumerate(args) if '-rel' in p]
+ if len(rel) != 1:
+ raise ValueError('Badly formatted nombank line: %r' % s)
+
+ # Apply the fileid selector, if any.
+ if parse_fileid_xform is not None:
+ fileid = parse_fileid_xform(fileid)
+
+ # Convert sentence & word numbers to ints.
+ sentnum = int(sentnum)
+ wordnum = int(wordnum)
+
+ # Parse the predicate location.
+
+ predloc, predid = rel[0].split('-', 1)
+ predicate = NombankTreePointer.parse(predloc)
+
+ # Parse the arguments.
+ arguments = []
+ for arg in args:
+ argloc, argid = arg.split('-', 1)
+ arguments.append( (NombankTreePointer.parse(argloc), argid) )
+
+ # Put it all together.
+ return NombankInstance(fileid, sentnum, wordnum, baseform, sensenumber,
+ predicate, predid, arguments, parse_corpus)
+
+class NombankPointer(object):
+ """
+ A pointer used by nombank to identify one or more constituents in
+ a parse tree. ``NombankPointer`` is an abstract base class with
+ three concrete subclasses:
+
+ - ``NombankTreePointer`` is used to point to single constituents.
+ - ``NombankSplitTreePointer`` is used to point to 'split'
+ constituents, which consist of a sequence of two or more
+ ``NombankTreePointer`` pointers.
+ - ``NombankChainTreePointer`` is used to point to entire trace
+ chains in a tree. It consists of a sequence of pieces, which
+ can be ``NombankTreePointer`` or ``NombankSplitTreePointer`` pointers.
+ """
+ def __init__(self):
+ if self.__class__ == NombankPointer:
+ raise NotImplementedError()
+
+ at python_2_unicode_compatible
+class NombankChainTreePointer(NombankPointer):
+ def __init__(self, pieces):
+ self.pieces = pieces
+ """A list of the pieces that make up this chain. Elements may
+ be either ``NombankSplitTreePointer`` or
+ ``NombankTreePointer`` pointers."""
+
+ def __str__(self):
+ return '*'.join('%s' % p for p in self.pieces)
+ def __repr__(self):
+ return '<NombankChainTreePointer: %s>' % self
+ def select(self, tree):
+ if tree is None: raise ValueError('Parse tree not avaialable')
+ return Tree('*CHAIN*', [p.select(tree) for p in self.pieces])
+
+ at python_2_unicode_compatible
+class NombankSplitTreePointer(NombankPointer):
+ def __init__(self, pieces):
+ self.pieces = pieces
+ """A list of the pieces that make up this chain. Elements are
+ all ``NombankTreePointer`` pointers."""
+
+ def __str__(self):
+ return ','.join('%s' % p for p in self.pieces)
+ def __repr__(self):
+ return '<NombankSplitTreePointer: %s>' % self
+ def select(self, tree):
+ if tree is None: raise ValueError('Parse tree not avaialable')
+ return Tree('*SPLIT*', [p.select(tree) for p in self.pieces])
+
+ at total_ordering
+ at python_2_unicode_compatible
+class NombankTreePointer(NombankPointer):
+ """
+ wordnum:height*wordnum:height*...
+ wordnum:height,
+
+ """
+ def __init__(self, wordnum, height):
+ self.wordnum = wordnum
+ self.height = height
+
+ @staticmethod
+ def parse(s):
+ # Deal with chains (xx*yy*zz)
+ pieces = s.split('*')
+ if len(pieces) > 1:
+ return NombankChainTreePointer([NombankTreePointer.parse(elt)
+ for elt in pieces])
+
+ # Deal with split args (xx,yy,zz)
+ pieces = s.split(',')
+ if len(pieces) > 1:
+ return NombankSplitTreePointer([NombankTreePointer.parse(elt)
+ for elt in pieces])
+
+ # Deal with normal pointers.
+ pieces = s.split(':')
+ if len(pieces) != 2: raise ValueError('bad nombank pointer %r' % s)
+ return NombankTreePointer(int(pieces[0]), int(pieces[1]))
+
+ def __str__(self):
+ return '%s:%s' % (self.wordnum, self.height)
+
+ def __repr__(self):
+ return 'NombankTreePointer(%d, %d)' % (self.wordnum, self.height)
+
+ def __eq__(self, other):
+ while isinstance(other, (NombankChainTreePointer,
+ NombankSplitTreePointer)):
+ other = other.pieces[0]
+
+ if not isinstance(other, NombankTreePointer):
+ return self is other
+
+ return (self.wordnum == other.wordnum and self.height == other.height)
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ while isinstance(other, (NombankChainTreePointer,
+ NombankSplitTreePointer)):
+ other = other.pieces[0]
+
+ if not isinstance(other, NombankTreePointer):
+ return id(self) < id(other)
+
+ return (self.wordnum, -self.height) < (other.wordnum, -other.height)
+
+ def select(self, tree):
+ if tree is None: raise ValueError('Parse tree not avaialable')
+ return tree[self.treepos(tree)]
+
+ def treepos(self, tree):
+ """
+ Convert this pointer to a standard 'tree position' pointer,
+ given that it points to the given tree.
+ """
+ if tree is None: raise ValueError('Parse tree not avaialable')
+ stack = [tree]
+ treepos = []
+
+ wordnum = 0
+ while True:
+ #print treepos
+ #print stack[-1]
+ # tree node:
+ if isinstance(stack[-1], Tree):
+ # Select the next child.
+ if len(treepos) < len(stack):
+ treepos.append(0)
+ else:
+ treepos[-1] += 1
+ # Update the stack.
+ if treepos[-1] < len(stack[-1]):
+ stack.append(stack[-1][treepos[-1]])
+ else:
+ # End of node's child list: pop up a level.
+ stack.pop()
+ treepos.pop()
+ # word node:
+ else:
+ if wordnum == self.wordnum:
+ return tuple(treepos[:len(treepos)-self.height-1])
+ else:
+ wordnum += 1
+ stack.pop()
+
diff --git a/nltk/corpus/reader/nps_chat.py b/nltk/corpus/reader/nps_chat.py
new file mode 100644
index 0000000..0a2ede5
--- /dev/null
+++ b/nltk/corpus/reader/nps_chat.py
@@ -0,0 +1,73 @@
+# Natural Language Toolkit: NPS Chat Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import unicode_literals
+
+import re
+import textwrap
+
+from nltk.util import LazyConcatenation
+from nltk.internals import ElementWrapper
+from nltk.tag import map_tag
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.xmldocs import *
+
+class NPSChatCorpusReader(XMLCorpusReader):
+
+ def __init__(self, root, fileids, wrap_etree=False, tagset=None):
+ XMLCorpusReader.__init__(self, root, fileids, wrap_etree)
+ self._tagset = tagset
+
+ def xml_posts(self, fileids=None):
+ if self._wrap_etree:
+ return concat([XMLCorpusView(fileid, 'Session/Posts/Post',
+ self._wrap_elt)
+ for fileid in self.abspaths(fileids)])
+ else:
+ return concat([XMLCorpusView(fileid, 'Session/Posts/Post')
+ for fileid in self.abspaths(fileids)])
+
+ def posts(self, fileids=None):
+ return concat([XMLCorpusView(fileid, 'Session/Posts/Post/terminals',
+ self._elt_to_words)
+ for fileid in self.abspaths(fileids)])
+
+ def tagged_posts(self, fileids=None, tagset=None):
+ def reader(elt, handler):
+ return self._elt_to_tagged_words(elt, handler, tagset)
+ return concat([XMLCorpusView(fileid, 'Session/Posts/Post/terminals',
+ reader)
+ for fileid in self.abspaths(fileids)])
+
+ def words(self, fileids=None):
+ return LazyConcatenation(self.posts(fileids))
+
+ def tagged_words(self, fileids=None, tagset=None):
+ return LazyConcatenation(self.tagged_posts(fileids, tagset))
+
+ def _wrap_elt(self, elt, handler):
+ return ElementWrapper(elt)
+
+ def _elt_to_words(self, elt, handler):
+ return [self._simplify_username(t.attrib['word'])
+ for t in elt.findall('t')]
+
+ def _elt_to_tagged_words(self, elt, handler, tagset=None):
+ tagged_post = [(self._simplify_username(t.attrib['word']),
+ t.attrib['pos']) for t in elt.findall('t')]
+ if tagset and tagset != self._tagset:
+ tagged_post = [(w, map_tag(self._tagset, tagset, t)) for (w, t) in tagged_post]
+ return tagged_post
+
+ @staticmethod
+ def _simplify_username(word):
+ if 'User' in word:
+ word = 'U' + word.split('User', 1)[1]
+ elif isinstance(word, bytes):
+ word = word.decode('ascii')
+ return word
diff --git a/nltk/corpus/reader/pl196x.py b/nltk/corpus/reader/pl196x.py
new file mode 100644
index 0000000..aa86698
--- /dev/null
+++ b/nltk/corpus/reader/pl196x.py
@@ -0,0 +1,278 @@
+# Natural Language Toolkit:
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Piotr Kasprzyk <p.j.kasprzyk at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import os
+import re
+
+from nltk import compat
+from nltk import tokenize, tree
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.xmldocs import XMLCorpusReader
+
+# (?:something) -- non-capturing parentheses!
+
+PARA = re.compile(r'<p(?: [^>]*){0,1}>(.*?)</p>')
+SENT = re.compile(r'<s(?: [^>]*){0,1}>(.*?)</s>')
+
+TAGGEDWORD = re.compile(r'<([wc](?: [^>]*){0,1}>)(.*?)</[wc]>')
+WORD = re.compile(r'<[wc](?: [^>]*){0,1}>(.*?)</[wc]>')
+
+TYPE = re.compile(r'type="(.*?)"')
+ANA = re.compile(r'ana="(.*?)"')
+
+TEXTID = re.compile(r'text id="(.*?)"')
+
+
+class TEICorpusView(StreamBackedCorpusView):
+ def __init__(self, corpus_file,
+ tagged, group_by_sent, group_by_para,
+ tagset=None, headLen=0, textids=None):
+ self._tagged = tagged
+ self._textids = textids
+
+ self._group_by_sent = group_by_sent
+ self._group_by_para = group_by_para
+ # WARNING -- skip header
+ StreamBackedCorpusView.__init__(self, corpus_file, startpos=headLen)
+
+ _pagesize = 4096
+
+ def read_block(self, stream):
+ block = stream.readlines(self._pagesize)
+ block = concat(block)
+ while (block.count('<text id') > block.count('</text>')) \
+ or block.count('<text id') == 0:
+ tmp = stream.readline()
+ if len(tmp) <= 0:
+ break
+ block += tmp
+
+ block = block.replace('\n','')
+
+ textids = TEXTID.findall(block)
+ if self._textids:
+ for tid in textids:
+ if tid not in self._textids:
+ beg = block.find(tid)-1
+ end = block[beg: ].find('</text>')+len('</text>')
+ block = block[ :beg]+block[beg+end: ]
+
+ output = []
+ for para_str in PARA.findall(block):
+ para = []
+ for sent_str in SENT.findall(para_str):
+ if not self._tagged:
+ sent = WORD.findall(sent_str)
+ else:
+ sent = list(map(self._parse_tag, TAGGEDWORD.findall(sent_str)))
+ if self._group_by_sent:
+ para.append(sent)
+ else:
+ para.extend(sent)
+ if self._group_by_para:
+ output.append(para)
+ else:
+ output.extend(para)
+ return output
+
+ def _parse_tag(self, tag_word_tuple):
+ (tag, word) = tag_word_tuple
+ if tag.startswith('w'):
+ tag = ANA.search(tag).group(1)
+ else: # tag.startswith('c')
+ tag = TYPE.search(tag).group(1)
+ return (word, tag)
+
+
+class Pl196xCorpusReader(CategorizedCorpusReader, XMLCorpusReader):
+
+ headLen = 2770
+
+ def __init__(self, *args, **kwargs):
+ if 'textid_file' in kwargs: self._textids = kwargs['textid_file']
+ else: self._textids = None
+
+ XMLCorpusReader.__init__(self, *args)
+ CategorizedCorpusReader.__init__(self, kwargs)
+
+ self._init_textids()
+
+ def _init_textids(self):
+ self._f2t = defaultdict(list)
+ self._t2f = defaultdict(list)
+ if self._textids is not None:
+ for line in self.open(self._textids).readlines():
+ line = line.strip()
+ file_id, text_ids = line.split(' ', 1)
+ if file_id not in self.fileids():
+ raise ValueError('In text_id mapping file %s: %s '
+ 'not found' % (catfile, file_id))
+ for text_id in text_ids.split(self._delimiter):
+ self._add_textids(file_id, text_id)
+
+ def _add_textids(self, file_id, text_id):
+ self._f2t[file_id].append(text_id)
+ self._t2f[text_id].append(file_id)
+
+ def _resolve(self, fileids, categories, textids=None):
+ tmp = None
+ if fileids is not None:
+ if not tmp:
+ tmp = fileids, None
+ else:
+ raise ValueError('Specify only fileids, categories or textids')
+ if categories is not None:
+ if not tmp:
+ tmp = self.fileids(categories), None
+ else:
+ raise ValueError('Specify only fileids, categories or textids')
+ if textids is not None:
+ if not tmp:
+ if isinstance(textids, compat.string_types): textids = [textids]
+ files = sum((self._t2f[t] for t in textids), [])
+ tdict = dict()
+ for f in files:
+ tdict[f] = (set(self._f2t[f]) & set(textids))
+ tmp = files, tdict
+ else:
+ raise ValueError('Specify only fileids, categories or textids')
+ return None, None
+
+ def decode_tag(self, tag):
+ # to be implemented
+ return tag
+
+ def textids(self, fileids=None, categories=None):
+ """
+ In the pl196x corpus each category is stored in single
+ file and thus both methods provide identical functionality. In order
+ to accommodate finer granularity, a non-standard textids() method was
+ implemented. All the main functions can be supplied with a list
+ of required chunks---giving much more control to the user.
+ """
+ fileids, _ = self._resolve(fileids, categories)
+ if fileids is None: return sorted(self._t2f)
+
+ if isinstance(fileids, compat.string_types):
+ fileids = [fileids]
+ return sorted(sum((self._f2t[d] for d in fileids), []))
+
+ def words(self, fileids=None, categories=None, textids=None):
+ fileids, textids = self._resolve(fileids, categories, textids)
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+
+ if textids:
+ return concat([TEICorpusView(self.abspath(fileid),
+ False, False, False,
+ headLen=self.headLen,
+ textids=textids[fileid])
+ for fileid in fileids])
+ else:
+ return concat([TEICorpusView(self.abspath(fileid),
+ False, False, False,
+ headLen=self.headLen)
+ for fileid in fileids])
+
+ def sents(self, fileids=None, categories=None, textids=None):
+ fileids, textids = self._resolve(fileids, categories, textids)
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+
+ if textids:
+ return concat([TEICorpusView(self.abspath(fileid),
+ False, True, False,
+ headLen=self.headLen,
+ textids=textids[fileid])
+ for fileid in fileids])
+ else:
+ return concat([TEICorpusView(self.abspath(fileid),
+ False, True, False,
+ headLen=self.headLen)
+ for fileid in fileids])
+
+ def paras(self, fileids=None, categories=None, textids=None):
+ fileids, textids = self._resolve(fileids, categories, textids)
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+
+ if textids:
+ return concat([TEICorpusView(self.abspath(fileid),
+ False, True, True,
+ headLen=self.headLen,
+ textids=textids[fileid])
+ for fileid in fileids])
+ else:
+ return concat([TEICorpusView(self.abspath(fileid),
+ False, True, True,
+ headLen=self.headLen)
+ for fileid in fileids])
+
+ def tagged_words(self, fileids=None, categories=None, textids=None):
+ fileids, textids = self._resolve(fileids, categories, textids)
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+
+ if textids:
+ return concat([TEICorpusView(self.abspath(fileid),
+ True, False, False,
+ headLen=self.headLen,
+ textids=textids[fileid])
+ for fileid in fileids])
+ else:
+ return concat([TEICorpusView(self.abspath(fileid),
+ True, False, False,
+ headLen=self.headLen)
+ for fileid in fileids])
+
+ def tagged_sents(self, fileids=None, categories=None, textids=None):
+ fileids, textids = self._resolve(fileids, categories, textids)
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+
+ if textids:
+ return concat([TEICorpusView(self.abspath(fileid),
+ True, True, False,
+ headLen=self.headLen,
+ textids=textids[fileid])
+ for fileid in fileids])
+ else:
+ return concat([TEICorpusView(self.abspath(fileid),
+ True, True, False,
+ headLen=self.headLen)
+ for fileid in fileids])
+
+ def tagged_paras(self, fileids=None, categories=None, textids=None):
+ fileids, textids = self._resolve(fileids, categories, textids)
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+
+ if textids:
+ return concat([TEICorpusView(self.abspath(fileid),
+ True, True, True,
+ headLen=self.headLen,
+ textids=textids[fileid])
+ for fileid in fileids])
+ else:
+ return concat([TEICorpusView(self.abspath(fileid),
+ True, True, True,
+ headLen=self.headLen)
+ for fileid in fileids])
+
+ def xml(self, fileids=None, categories=None):
+ fileids, _ = self._resolve(fileids, categories)
+ if len(fileids) == 1: return XMLCorpusReader.xml(self, fileids[0])
+ else: raise TypeError('Expected a single file')
+
+ def raw(self, fileids=None, categories=None):
+ fileids, _ = self._resolve(fileids, categories)
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
diff --git a/nltk/corpus/reader/plaintext.py b/nltk/corpus/reader/plaintext.py
new file mode 100644
index 0000000..a456912
--- /dev/null
+++ b/nltk/corpus/reader/plaintext.py
@@ -0,0 +1,228 @@
+# Natural Language Toolkit: Plaintext Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# Nitin Madnani <nmadnani at umiacs.umd.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A reader for corpora that consist of plaintext documents.
+"""
+
+import codecs
+
+import nltk.data
+from nltk.compat import string_types
+from nltk.tokenize import *
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class PlaintextCorpusReader(CorpusReader):
+ """
+ Reader for corpora that consist of plaintext documents. Paragraphs
+ are assumed to be split using blank lines. Sentences and words can
+ be tokenized using the default tokenizers, or by custom tokenizers
+ specificed as parameters to the constructor.
+
+ This corpus reader can be customized (e.g., to skip preface
+ sections of specific document formats) by creating a subclass and
+ overriding the ``CorpusView`` class variable.
+ """
+
+ CorpusView = StreamBackedCorpusView
+ """The corpus view class used by this reader. Subclasses of
+ ``PlaintextCorpusReader`` may specify alternative corpus view
+ classes (e.g., to skip the preface sections of documents.)"""
+
+ def __init__(self, root, fileids,
+ word_tokenizer=WordPunctTokenizer(),
+ sent_tokenizer=nltk.data.LazyLoader(
+ 'tokenizers/punkt/english.pickle'),
+ para_block_reader=read_blankline_block,
+ encoding='utf8'):
+ """
+ Construct a new plaintext corpus reader for a set of documents
+ located at the given root directory. Example usage:
+
+ >>> root = '/usr/local/share/nltk_data/corpora/webtext/'
+ >>> reader = PlaintextCorpusReader(root, '.*\.txt') # doctest: +SKIP
+
+ :param root: The root directory for this corpus.
+ :param fileids: A list or regexp specifying the fileids in this corpus.
+ :param word_tokenizer: Tokenizer for breaking sentences or
+ paragraphs into words.
+ :param sent_tokenizer: Tokenizer for breaking paragraphs
+ into words.
+ :param para_block_reader: The block reader used to divide the
+ corpus into paragraph blocks.
+ """
+ CorpusReader.__init__(self, root, fileids, encoding)
+ self._word_tokenizer = word_tokenizer
+ self._sent_tokenizer = sent_tokenizer
+ self._para_block_reader = para_block_reader
+
+ def raw(self, fileids=None):
+ """
+ :return: the given file(s) as a single string.
+ :rtype: str
+ """
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def words(self, fileids=None):
+ """
+ :return: the given file(s) as a list of words
+ and punctuation symbols.
+ :rtype: list(str)
+ """
+ return concat([self.CorpusView(path, self._read_word_block, encoding=enc)
+ for (path, enc, fileid)
+ in self.abspaths(fileids, True, True)])
+
+ def sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ sentences or utterances, each encoded as a list of word
+ strings.
+ :rtype: list(list(str))
+ """
+ if self._sent_tokenizer is None:
+ raise ValueError('No sentence tokenizer for this corpus')
+
+ return concat([self.CorpusView(path, self._read_sent_block, encoding=enc)
+ for (path, enc, fileid)
+ in self.abspaths(fileids, True, True)])
+
+ def paras(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ paragraphs, each encoded as a list of sentences, which are
+ in turn encoded as lists of word strings.
+ :rtype: list(list(list(str)))
+ """
+ if self._sent_tokenizer is None:
+ raise ValueError('No sentence tokenizer for this corpus')
+
+ return concat([self.CorpusView(path, self._read_para_block, encoding=enc)
+ for (path, enc, fileid)
+ in self.abspaths(fileids, True, True)])
+
+ def _read_word_block(self, stream):
+ words = []
+ for i in range(20): # Read 20 lines at a time.
+ words.extend(self._word_tokenizer.tokenize(stream.readline()))
+ return words
+
+ def _read_sent_block(self, stream):
+ sents = []
+ for para in self._para_block_reader(stream):
+ sents.extend([self._word_tokenizer.tokenize(sent)
+ for sent in self._sent_tokenizer.tokenize(para)])
+ return sents
+
+ def _read_para_block(self, stream):
+ paras = []
+ for para in self._para_block_reader(stream):
+ paras.append([self._word_tokenizer.tokenize(sent)
+ for sent in self._sent_tokenizer.tokenize(para)])
+ return paras
+
+
+class CategorizedPlaintextCorpusReader(CategorizedCorpusReader,
+ PlaintextCorpusReader):
+ """
+ A reader for plaintext corpora whose documents are divided into
+ categories based on their file identifiers.
+ """
+ def __init__(self, *args, **kwargs):
+ """
+ Initialize the corpus reader. Categorization arguments
+ (``cat_pattern``, ``cat_map``, and ``cat_file``) are passed to
+ the ``CategorizedCorpusReader`` constructor. The remaining arguments
+ are passed to the ``PlaintextCorpusReader`` constructor.
+ """
+ CategorizedCorpusReader.__init__(self, kwargs)
+ PlaintextCorpusReader.__init__(self, *args, **kwargs)
+
+ def _resolve(self, fileids, categories):
+ if fileids is not None and categories is not None:
+ raise ValueError('Specify fileids or categories, not both')
+ if categories is not None:
+ return self.fileids(categories)
+ else:
+ return fileids
+ def raw(self, fileids=None, categories=None):
+ return PlaintextCorpusReader.raw(
+ self, self._resolve(fileids, categories))
+ def words(self, fileids=None, categories=None):
+ return PlaintextCorpusReader.words(
+ self, self._resolve(fileids, categories))
+ def sents(self, fileids=None, categories=None):
+ return PlaintextCorpusReader.sents(
+ self, self._resolve(fileids, categories))
+ def paras(self, fileids=None, categories=None):
+ return PlaintextCorpusReader.paras(
+ self, self._resolve(fileids, categories))
+
+# is there a better way?
+class PortugueseCategorizedPlaintextCorpusReader(CategorizedPlaintextCorpusReader):
+ def __init__(self, *args, **kwargs):
+ CategorizedCorpusReader.__init__(self, kwargs)
+ kwargs['sent_tokenizer'] = nltk.data.LazyLoader('tokenizers/punkt/portuguese.pickle')
+ PlaintextCorpusReader.__init__(self, *args, **kwargs)
+
+class EuroparlCorpusReader(PlaintextCorpusReader):
+
+ """
+ Reader for Europarl corpora that consist of plaintext documents.
+ Documents are divided into chapters instead of paragraphs as
+ for regular plaintext documents. Chapters are separated using blank
+ lines. Everything is inherited from ``PlaintextCorpusReader`` except
+ that:
+ - Since the corpus is pre-processed and pre-tokenized, the
+ word tokenizer should just split the line at whitespaces.
+ - For the same reason, the sentence tokenizer should just
+ split the paragraph at line breaks.
+ - There is a new 'chapters()' method that returns chapters instead
+ instead of paragraphs.
+ - The 'paras()' method inherited from PlaintextCorpusReader is
+ made non-functional to remove any confusion between chapters
+ and paragraphs for Europarl.
+ """
+
+ def _read_word_block(self, stream):
+ words = []
+ for i in range(20): # Read 20 lines at a time.
+ words.extend(stream.readline().split())
+ return words
+
+ def _read_sent_block(self, stream):
+ sents = []
+ for para in self._para_block_reader(stream):
+ sents.extend([sent.split() for sent in para.splitlines()])
+ return sents
+
+ def _read_para_block(self, stream):
+ paras = []
+ for para in self._para_block_reader(stream):
+ paras.append([sent.split() for sent in para.splitlines()])
+ return paras
+
+ def chapters(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ chapters, each encoded as a list of sentences, which are
+ in turn encoded as lists of word strings.
+ :rtype: list(list(list(str)))
+ """
+ return concat([self.CorpusView(fileid, self._read_para_block,
+ encoding=enc)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def paras(self, fileids=None):
+ raise NotImplementedError('The Europarl corpus reader does not support paragraphs. Please use chapters() instead.')
+
diff --git a/nltk/corpus/reader/ppattach.py b/nltk/corpus/reader/ppattach.py
new file mode 100644
index 0000000..9462308
--- /dev/null
+++ b/nltk/corpus/reader/ppattach.py
@@ -0,0 +1,95 @@
+# Natural Language Toolkit: PP Attachment Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Read lines from the Prepositional Phrase Attachment Corpus.
+
+The PP Attachment Corpus contains several files having the format:
+
+sentence_id verb noun1 preposition noun2 attachment
+
+For example:
+
+42960 gives authority to administration V
+46742 gives inventors of microchip N
+
+The PP attachment is to the verb phrase (V) or noun phrase (N), i.e.:
+
+(VP gives (NP authority) (PP to administration))
+(VP gives (NP inventors (PP of microchip)))
+
+The corpus contains the following files:
+
+training: training set
+devset: development test set, used for algorithm development.
+test: test set, used to report results
+bitstrings: word classes derived from Mutual Information Clustering for the Wall Street Journal.
+
+Ratnaparkhi, Adwait (1994). A Maximum Entropy Model for Prepositional
+Phrase Attachment. Proceedings of the ARPA Human Language Technology
+Conference. [http://www.cis.upenn.edu/~adwait/papers/hlt94.ps]
+
+The PP Attachment Corpus is distributed with NLTK with the permission
+of the author.
+"""
+from __future__ import unicode_literals
+
+from nltk import compat
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+
+ at compat.python_2_unicode_compatible
+class PPAttachment(object):
+ def __init__(self, sent, verb, noun1, prep, noun2, attachment):
+ self.sent = sent
+ self.verb = verb
+ self.noun1 = noun1
+ self.prep = prep
+ self.noun2 = noun2
+ self.attachment = attachment
+
+ def __repr__(self):
+ return ('PPAttachment(sent=%r, verb=%r, noun1=%r, prep=%r, '
+ 'noun2=%r, attachment=%r)' %
+ (self.sent, self.verb, self.noun1, self.prep,
+ self.noun2, self.attachment))
+
+class PPAttachmentCorpusReader(CorpusReader):
+ """
+ sentence_id verb noun1 preposition noun2 attachment
+ """
+ def attachments(self, fileids):
+ return concat([StreamBackedCorpusView(fileid, self._read_obj_block,
+ encoding=enc)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tuples(self, fileids):
+ return concat([StreamBackedCorpusView(fileid, self._read_tuple_block,
+ encoding=enc)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def raw(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def _read_tuple_block(self, stream):
+ line = stream.readline()
+ if line:
+ return [tuple(line.split())]
+ else:
+ return []
+
+ def _read_obj_block(self, stream):
+ line = stream.readline()
+ if line:
+ return [PPAttachment(*line.split())]
+ else:
+ return []
+
diff --git a/nltk/corpus/reader/propbank.py b/nltk/corpus/reader/propbank.py
new file mode 100644
index 0000000..49bc361
--- /dev/null
+++ b/nltk/corpus/reader/propbank.py
@@ -0,0 +1,481 @@
+# Natural Language Toolkit: PropBank Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import unicode_literals
+import re
+from xml.etree import ElementTree
+
+from nltk import compat
+from nltk.tree import Tree
+from nltk.internals import raise_unorderable_types
+from nltk.compat import total_ordering
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class PropbankCorpusReader(CorpusReader):
+ """
+ Corpus reader for the propbank corpus, which augments the Penn
+ Treebank with information about the predicate argument structure
+ of every verb instance. The corpus consists of two parts: the
+ predicate-argument annotations themselves, and a set of "frameset
+ files" which define the argument labels used by the annotations,
+ on a per-verb basis. Each "frameset file" contains one or more
+ predicates, such as ``'turn'`` or ``'turn_on'``, each of which is
+ divided into coarse-grained word senses called "rolesets". For
+ each "roleset", the frameset file provides descriptions of the
+ argument roles, along with examples.
+ """
+ def __init__(self, root, propfile, framefiles='',
+ verbsfile=None, parse_fileid_xform=None,
+ parse_corpus=None, encoding='utf8'):
+ """
+ :param root: The root directory for this corpus.
+ :param propfile: The name of the file containing the predicate-
+ argument annotations (relative to ``root``).
+ :param framefiles: A list or regexp specifying the frameset
+ fileids for this corpus.
+ :param parse_fileid_xform: A transform that should be applied
+ to the fileids in this corpus. This should be a function
+ of one argument (a fileid) that returns a string (the new
+ fileid).
+ :param parse_corpus: The corpus containing the parse trees
+ corresponding to this corpus. These parse trees are
+ necessary to resolve the tree pointers used by propbank.
+ """
+ # If framefiles is specified as a regexp, expand it.
+ if isinstance(framefiles, compat.string_types):
+ framefiles = find_corpus_fileids(root, framefiles)
+ framefiles = list(framefiles)
+ # Initialze the corpus reader.
+ CorpusReader.__init__(self, root, [propfile, verbsfile] + framefiles,
+ encoding)
+
+ # Record our frame fileids & prop file.
+ self._propfile = propfile
+ self._framefiles = framefiles
+ self._verbsfile = verbsfile
+ self._parse_fileid_xform = parse_fileid_xform
+ self._parse_corpus = parse_corpus
+
+ def raw(self, fileids=None):
+ """
+ :return: the text contents of the given fileids, as a single string.
+ """
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def instances(self, baseform=None):
+ """
+ :return: a corpus view that acts as a list of
+ ``PropBankInstance`` objects, one for each noun in the corpus.
+ """
+ kwargs = {}
+ if baseform is not None:
+ kwargs['instance_filter'] = lambda inst: inst.baseform==baseform
+ return StreamBackedCorpusView(self.abspath(self._propfile),
+ lambda stream: self._read_instance_block(stream, **kwargs),
+ encoding=self.encoding(self._propfile))
+
+ def lines(self):
+ """
+ :return: a corpus view that acts as a list of strings, one for
+ each line in the predicate-argument annotation file.
+ """
+ return StreamBackedCorpusView(self.abspath(self._propfile),
+ read_line_block,
+ encoding=self.encoding(self._propfile))
+
+ def roleset(self, roleset_id):
+ """
+ :return: the xml description for the given roleset.
+ """
+ baseform = roleset_id.split('.')[0]
+ framefile = 'frames/%s.xml' % baseform
+ if framefile not in self._framefiles:
+ raise ValueError('Frameset file for %s not found' %
+ roleset_id)
+
+ # n.b.: The encoding for XML fileids is specified by the file
+ # itself; so we ignore self._encoding here.
+ etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
+ for roleset in etree.findall('predicate/roleset'):
+ if roleset.attrib['id'] == roleset_id:
+ return roleset
+ else:
+ raise ValueError('Roleset %s not found in %s' %
+ (roleset_id, framefile))
+
+ def rolesets(self, baseform=None):
+ """
+ :return: list of xml descriptions for rolesets.
+ """
+ if baseform is not None:
+ framefile = 'frames/%s.xml' % baseform
+ if framefile not in self._framefiles:
+ raise ValueError('Frameset file for %s not found' %
+ baseform)
+ framefiles = [framefile]
+ else:
+ framefiles = self._framefiles
+
+ rsets = []
+ for framefile in framefiles:
+ # n.b.: The encoding for XML fileids is specified by the file
+ # itself; so we ignore self._encoding here.
+ etree = ElementTree.parse(self.abspath(framefile).open()).getroot()
+ rsets.append(etree.findall('predicate/roleset'))
+ return LazyConcatenation(rsets)
+
+ def verbs(self):
+ """
+ :return: a corpus view that acts as a list of all verb lemmas
+ in this corpus (from the verbs.txt file).
+ """
+ return StreamBackedCorpusView(self.abspath(self._verbsfile),
+ read_line_block,
+ encoding=self.encoding(self._verbsfile))
+
+ def _read_instance_block(self, stream, instance_filter=lambda inst: True):
+ block = []
+
+ # Read 100 at a time.
+ for i in range(100):
+ line = stream.readline().strip()
+ if line:
+ inst = PropbankInstance.parse(
+ line, self._parse_fileid_xform,
+ self._parse_corpus)
+ if instance_filter(inst):
+ block.append(inst)
+
+ return block
+
+######################################################################
+#{ Propbank Instance & related datatypes
+######################################################################
+
+ at compat.python_2_unicode_compatible
+class PropbankInstance(object):
+
+ def __init__(self, fileid, sentnum, wordnum, tagger, roleset,
+ inflection, predicate, arguments, parse_corpus=None):
+
+ self.fileid = fileid
+ """The name of the file containing the parse tree for this
+ instance's sentence."""
+
+ self.sentnum = sentnum
+ """The sentence number of this sentence within ``fileid``.
+ Indexing starts from zero."""
+
+ self.wordnum = wordnum
+ """The word number of this instance's predicate within its
+ containing sentence. Word numbers are indexed starting from
+ zero, and include traces and other empty parse elements."""
+
+ self.tagger = tagger
+ """An identifier for the tagger who tagged this instance; or
+ ``'gold'`` if this is an adjuticated instance."""
+
+ self.roleset = roleset
+ """The name of the roleset used by this instance's predicate.
+ Use ``propbank.roleset() <PropbankCorpusReader.roleset>`` to
+ look up information about the roleset."""
+
+ self.inflection = inflection
+ """A ``PropbankInflection`` object describing the inflection of
+ this instance's predicate."""
+
+ self.predicate = predicate
+ """A ``PropbankTreePointer`` indicating the position of this
+ instance's predicate within its containing sentence."""
+
+ self.arguments = tuple(arguments)
+ """A list of tuples (argloc, argid), specifying the location
+ and identifier for each of the predicate's argument in the
+ containing sentence. Argument identifiers are strings such as
+ ``'ARG0'`` or ``'ARGM-TMP'``. This list does *not* contain
+ the predicate."""
+
+ self.parse_corpus = parse_corpus
+ """A corpus reader for the parse trees corresponding to the
+ instances in this propbank corpus."""
+
+ @property
+ def baseform(self):
+ """The baseform of the predicate."""
+ return self.roleset.split('.')[0]
+
+ @property
+ def sensenumber(self):
+ """The sense number of the predicate."""
+ return self.roleset.split('.')[1]
+
+ @property
+ def predid(self):
+ """Identifier of the predicate."""
+ return 'rel'
+
+ def __repr__(self):
+ return ('<PropbankInstance: %s, sent %s, word %s>' %
+ (self.fileid, self.sentnum, self.wordnum))
+
+ def __str__(self):
+ s = '%s %s %s %s %s %s' % (self.fileid, self.sentnum, self.wordnum,
+ self.tagger, self.roleset, self.inflection)
+ items = self.arguments + ((self.predicate, 'rel'),)
+ for (argloc, argid) in sorted(items):
+ s += ' %s-%s' % (argloc, argid)
+ return s
+
+ def _get_tree(self):
+ if self.parse_corpus is None: return None
+ if self.fileid not in self.parse_corpus.fileids(): return None
+ return self.parse_corpus.parsed_sents(self.fileid)[self.sentnum]
+ tree = property(_get_tree, doc="""
+ The parse tree corresponding to this instance, or None if
+ the corresponding tree is not available.""")
+
+ @staticmethod
+ def parse(s, parse_fileid_xform=None, parse_corpus=None):
+ pieces = s.split()
+ if len(pieces) < 7:
+ raise ValueError('Badly formatted propbank line: %r' % s)
+
+ # Divide the line into its basic pieces.
+ (fileid, sentnum, wordnum,
+ tagger, roleset, inflection) = pieces[:6]
+ rel = [p for p in pieces[6:] if p.endswith('-rel')]
+ args = [p for p in pieces[6:] if not p.endswith('-rel')]
+ if len(rel) != 1:
+ raise ValueError('Badly formatted propbank line: %r' % s)
+
+ # Apply the fileid selector, if any.
+ if parse_fileid_xform is not None:
+ fileid = parse_fileid_xform(fileid)
+
+ # Convert sentence & word numbers to ints.
+ sentnum = int(sentnum)
+ wordnum = int(wordnum)
+
+ # Parse the inflection
+ inflection = PropbankInflection.parse(inflection)
+
+ # Parse the predicate location.
+ predicate = PropbankTreePointer.parse(rel[0][:-4])
+
+ # Parse the arguments.
+ arguments = []
+ for arg in args:
+ argloc, argid = arg.split('-', 1)
+ arguments.append( (PropbankTreePointer.parse(argloc), argid) )
+
+ # Put it all together.
+ return PropbankInstance(fileid, sentnum, wordnum, tagger,
+ roleset, inflection, predicate,
+ arguments, parse_corpus)
+
+class PropbankPointer(object):
+ """
+ A pointer used by propbank to identify one or more constituents in
+ a parse tree. ``PropbankPointer`` is an abstract base class with
+ three concrete subclasses:
+
+ - ``PropbankTreePointer`` is used to point to single constituents.
+ - ``PropbankSplitTreePointer`` is used to point to 'split'
+ constituents, which consist of a sequence of two or more
+ ``PropbankTreePointer`` pointers.
+ - ``PropbankChainTreePointer`` is used to point to entire trace
+ chains in a tree. It consists of a sequence of pieces, which
+ can be ``PropbankTreePointer`` or ``PropbankSplitTreePointer`` pointers.
+ """
+ def __init__(self):
+ if self.__class__ == PropbankPointer:
+ raise NotImplementedError()
+
+ at compat.python_2_unicode_compatible
+class PropbankChainTreePointer(PropbankPointer):
+ def __init__(self, pieces):
+ self.pieces = pieces
+ """A list of the pieces that make up this chain. Elements may
+ be either ``PropbankSplitTreePointer`` or
+ ``PropbankTreePointer`` pointers."""
+
+ def __str__(self):
+ return '*'.join('%s' % p for p in self.pieces)
+ def __repr__(self):
+ return '<PropbankChainTreePointer: %s>' % self
+ def select(self, tree):
+ if tree is None: raise ValueError('Parse tree not avaialable')
+ return Tree('*CHAIN*', [p.select(tree) for p in self.pieces])
+
+
+ at compat.python_2_unicode_compatible
+class PropbankSplitTreePointer(PropbankPointer):
+ def __init__(self, pieces):
+ self.pieces = pieces
+ """A list of the pieces that make up this chain. Elements are
+ all ``PropbankTreePointer`` pointers."""
+
+ def __str__(self):
+ return ','.join('%s' % p for p in self.pieces)
+ def __repr__(self):
+ return '<PropbankSplitTreePointer: %s>' % self
+ def select(self, tree):
+ if tree is None: raise ValueError('Parse tree not avaialable')
+ return Tree('*SPLIT*', [p.select(tree) for p in self.pieces])
+
+
+ at total_ordering
+ at compat.python_2_unicode_compatible
+class PropbankTreePointer(PropbankPointer):
+ """
+ wordnum:height*wordnum:height*...
+ wordnum:height,
+
+ """
+ def __init__(self, wordnum, height):
+ self.wordnum = wordnum
+ self.height = height
+
+ @staticmethod
+ def parse(s):
+ # Deal with chains (xx*yy*zz)
+ pieces = s.split('*')
+ if len(pieces) > 1:
+ return PropbankChainTreePointer([PropbankTreePointer.parse(elt)
+ for elt in pieces])
+
+ # Deal with split args (xx,yy,zz)
+ pieces = s.split(',')
+ if len(pieces) > 1:
+ return PropbankSplitTreePointer([PropbankTreePointer.parse(elt)
+ for elt in pieces])
+
+ # Deal with normal pointers.
+ pieces = s.split(':')
+ if len(pieces) != 2: raise ValueError('bad propbank pointer %r' % s)
+ return PropbankTreePointer(int(pieces[0]), int(pieces[1]))
+
+ def __str__(self):
+ return '%s:%s' % (self.wordnum, self.height)
+
+ def __repr__(self):
+ return 'PropbankTreePointer(%d, %d)' % (self.wordnum, self.height)
+
+ def __eq__(self, other):
+ while isinstance(other, (PropbankChainTreePointer,
+ PropbankSplitTreePointer)):
+ other = other.pieces[0]
+
+ if not isinstance(other, PropbankTreePointer):
+ return self is other
+
+ return (self.wordnum == other.wordnum and self.height == other.height)
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ while isinstance(other, (PropbankChainTreePointer,
+ PropbankSplitTreePointer)):
+ other = other.pieces[0]
+
+ if not isinstance(other, PropbankTreePointer):
+ return id(self) < id(other)
+
+ return (self.wordnum, -self.height) < (other.wordnum, -other.height)
+
+ def select(self, tree):
+ if tree is None: raise ValueError('Parse tree not avaialable')
+ return tree[self.treepos(tree)]
+
+ def treepos(self, tree):
+ """
+ Convert this pointer to a standard 'tree position' pointer,
+ given that it points to the given tree.
+ """
+ if tree is None: raise ValueError('Parse tree not avaialable')
+ stack = [tree]
+ treepos = []
+
+ wordnum = 0
+ while True:
+ #print treepos
+ #print stack[-1]
+ # tree node:
+ if isinstance(stack[-1], Tree):
+ # Select the next child.
+ if len(treepos) < len(stack):
+ treepos.append(0)
+ else:
+ treepos[-1] += 1
+ # Update the stack.
+ if treepos[-1] < len(stack[-1]):
+ stack.append(stack[-1][treepos[-1]])
+ else:
+ # End of node's child list: pop up a level.
+ stack.pop()
+ treepos.pop()
+ # word node:
+ else:
+ if wordnum == self.wordnum:
+ return tuple(treepos[:len(treepos)-self.height-1])
+ else:
+ wordnum += 1
+ stack.pop()
+
+ at compat.python_2_unicode_compatible
+class PropbankInflection(object):
+ #{ Inflection Form
+ INFINITIVE = 'i'
+ GERUND = 'g'
+ PARTICIPLE = 'p'
+ FINITE = 'v'
+ #{ Inflection Tense
+ FUTURE = 'f'
+ PAST = 'p'
+ PRESENT = 'n'
+ #{ Inflection Aspect
+ PERFECT = 'p'
+ PROGRESSIVE = 'o'
+ PERFECT_AND_PROGRESSIVE = 'b'
+ #{ Inflection Person
+ THIRD_PERSON = '3'
+ #{ Inflection Voice
+ ACTIVE = 'a'
+ PASSIVE = 'p'
+ #{ Inflection
+ NONE = '-'
+ #}
+
+ def __init__(self, form='-', tense='-', aspect='-', person='-', voice='-'):
+ self.form = form
+ self.tense = tense
+ self.aspect = aspect
+ self.person = person
+ self.voice = voice
+
+ def __str__(self):
+ return self.form+self.tense+self.aspect+self.person+self.voice
+
+ def __repr__(self):
+ return '<PropbankInflection: %s>' % self
+
+ _VALIDATE = re.compile(r'[igpv\-][fpn\-][pob\-][3\-][ap\-]$')
+
+ @staticmethod
+ def parse(s):
+ if not isinstance(s, compat.string_types):
+ raise TypeError('expected a string')
+ if (len(s) != 5 or
+ not PropbankInflection._VALIDATE.match(s)):
+ raise ValueError('Bad propbank inflection string %r' % s)
+ return PropbankInflection(*s)
+
diff --git a/nltk/corpus/reader/rte.py b/nltk/corpus/reader/rte.py
new file mode 100644
index 0000000..83e88a5
--- /dev/null
+++ b/nltk/corpus/reader/rte.py
@@ -0,0 +1,146 @@
+# Natural Language Toolkit: RTE Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Corpus reader for the Recognizing Textual Entailment (RTE) Challenge Corpora.
+
+The files were taken from the RTE1, RTE2 and RTE3 datasets and the files
+were regularized.
+
+Filenames are of the form rte*_dev.xml and rte*_test.xml. The latter are the
+gold standard annotated files.
+
+Each entailment corpus is a list of 'text'/'hypothesis' pairs. The following
+example is taken from RTE3::
+
+ <pair id="1" entailment="YES" task="IE" length="short" >
+
+ <t>The sale was made to pay Yukos' US$ 27.5 billion tax bill,
+ Yuganskneftegaz was originally sold for US$ 9.4 billion to a little known
+ company Baikalfinansgroup which was later bought by the Russian
+ state-owned oil company Rosneft .</t>
+
+ <h>Baikalfinansgroup was sold to Rosneft.</h>
+ </pair>
+
+In order to provide globally unique IDs for each pair, a new attribute
+``challenge`` has been added to the root element ``entailment-corpus`` of each
+file, taking values 1, 2 or 3. The GID is formatted 'm-n', where 'm' is the
+challenge number and 'n' is the pair ID.
+"""
+from __future__ import unicode_literals
+from nltk import compat
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.xmldocs import *
+
+
+def norm(value_string):
+ """
+ Normalize the string value in an RTE pair's ``value`` or ``entailment``
+ attribute as an integer (1, 0).
+
+ :param value_string: the label used to classify a text/hypothesis pair
+ :type value_string: str
+ :rtype: int
+ """
+
+ valdict = {"TRUE": 1,
+ "FALSE": 0,
+ "YES": 1,
+ "NO": 0}
+ return valdict[value_string.upper()]
+
+ at compat.python_2_unicode_compatible
+class RTEPair(object):
+ """
+ Container for RTE text-hypothesis pairs.
+
+ The entailment relation is signalled by the ``value`` attribute in RTE1, and by
+ ``entailment`` in RTE2 and RTE3. These both get mapped on to the ``entailment``
+ attribute of this class.
+ """
+ def __init__(self, pair, challenge=None, id=None, text=None, hyp=None,
+ value=None, task=None, length=None):
+ """
+ :param challenge: version of the RTE challenge (i.e., RTE1, RTE2 or RTE3)
+ :param id: identifier for the pair
+ :param text: the text component of the pair
+ :param hyp: the hypothesis component of the pair
+ :param value: classification label for the pair
+ :param task: attribute for the particular NLP task that the data was drawn from
+ :param length: attribute for the length of the text of the pair
+ """
+ self.challenge = challenge
+ self.id = pair.attrib["id"]
+ self.gid = "%s-%s" % (self.challenge, self.id)
+ self.text = pair[0].text
+ self.hyp = pair[1].text
+
+ if "value" in pair.attrib:
+ self.value = norm(pair.attrib["value"])
+ elif "entailment" in pair.attrib:
+ self.value = norm(pair.attrib["entailment"])
+ else:
+ self.value = value
+ if "task" in pair.attrib:
+ self.task = pair.attrib["task"]
+ else:
+ self.task = task
+ if "length" in pair.attrib:
+ self.length = pair.attrib["length"]
+ else:
+ self.length = length
+
+ def __repr__(self):
+ if self.challenge:
+ return '<RTEPair: gid=%s-%s>' % (self.challenge, self.id)
+ else:
+ return '<RTEPair: id=%s>' % self.id
+
+
+class RTECorpusReader(XMLCorpusReader):
+ """
+ Corpus reader for corpora in RTE challenges.
+
+ This is just a wrapper around the XMLCorpusReader. See module docstring above for the expected
+ structure of input documents.
+ """
+
+ def _read_etree(self, doc):
+ """
+ Map the XML input into an RTEPair.
+
+ This uses the ``getiterator()`` method from the ElementTree package to
+ find all the ``<pair>`` elements.
+
+ :param doc: a parsed XML document
+ :rtype: list(RTEPair)
+ """
+ try:
+ challenge = doc.attrib['challenge']
+ except KeyError:
+ challenge = None
+ return [RTEPair(pair, challenge=challenge)
+ for pair in doc.getiterator("pair")]
+
+
+ def pairs(self, fileids):
+ """
+ Build a list of RTEPairs from a RTE corpus.
+
+ :param fileids: a list of RTE corpus fileids
+ :type: list
+ :rtype: list(RTEPair)
+ """
+ if isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self._read_etree(self.xml(fileid)) for fileid in fileids])
+
+
+
+
+
diff --git a/nltk/corpus/reader/semcor.py b/nltk/corpus/reader/semcor.py
new file mode 100644
index 0000000..15dc544
--- /dev/null
+++ b/nltk/corpus/reader/semcor.py
@@ -0,0 +1,256 @@
+# Natural Language Toolkit: SemCor Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Nathan Schneider <nschneid at cs.cmu.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Corpus reader for the SemCor Corpus.
+"""
+from __future__ import absolute_import, unicode_literals
+__docformat__ = 'epytext en'
+
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.xmldocs import XMLCorpusReader, XMLCorpusView
+from nltk.tree import Tree
+
+class SemcorCorpusReader(XMLCorpusReader):
+ """
+ Corpus reader for the SemCor Corpus.
+ For access to the complete XML data structure, use the ``xml()``
+ method. For access to simple word lists and tagged word lists, use
+ ``words()``, ``sents()``, ``tagged_words()``, and ``tagged_sents()``.
+ """
+ def __init__(self, root, fileids, wordnet, lazy=True):
+ XMLCorpusReader.__init__(self, root, fileids)
+ self._lazy = lazy
+ self._wordnet = wordnet
+
+ def words(self, fileids=None):
+ """
+ :return: the given file(s) as a list of words and punctuation symbols.
+ :rtype: list(str)
+ """
+ return self._items(fileids, 'word', False, False, False)
+
+ def chunks(self, fileids=None):
+ """
+ :return: the given file(s) as a list of chunks,
+ each of which is a list of words and punctuation symbols
+ that form a unit.
+ :rtype: list(list(str))
+ """
+ return self._items(fileids, 'chunk', False, False, False)
+
+ def tagged_chunks(self, fileids=None, tag=('pos' or 'sem' or 'both')):
+ """
+ :return: the given file(s) as a list of tagged chunks, represented
+ in tree form.
+ :rtype: list(Tree)
+
+ :param tag: `'pos'` (part of speech), `'sem'` (semantic), or `'both'`
+ to indicate the kind of tags to include. Semantic tags consist of
+ WordNet lemma IDs, plus an `'NE'` node if the chunk is a named entity
+ without a specific entry in WordNet. (Named entities of type 'other'
+ have no lemma. Other chunks not in WordNet have no semantic tag.
+ Punctuation tokens have `None` for their part of speech tag.)
+ """
+ return self._items(fileids, 'chunk', False, tag!='sem', tag!='pos')
+
+ def sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of sentences, each encoded
+ as a list of word strings.
+ :rtype: list(list(str))
+ """
+ return self._items(fileids, 'word', True, False, False)
+
+ def chunk_sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of sentences, each encoded
+ as a list of chunks.
+ :rtype: list(list(list(str)))
+ """
+ return self._items(fileids, 'chunk', True, False, False)
+
+ def tagged_sents(self, fileids=None, tag=('pos' or 'sem' or 'both')):
+ """
+ :return: the given file(s) as a list of sentences. Each sentence
+ is represented as a list of tagged chunks (in tree form).
+ :rtype: list(list(Tree))
+
+ :param tag: `'pos'` (part of speech), `'sem'` (semantic), or `'both'`
+ to indicate the kind of tags to include. Semantic tags consist of
+ WordNet lemma IDs, plus an `'NE'` node if the chunk is a named entity
+ without a specific entry in WordNet. (Named entities of type 'other'
+ have no lemma. Other chunks not in WordNet have no semantic tag.
+ Punctuation tokens have `None` for their part of speech tag.)
+ """
+ return self._items(fileids, 'chunk', True, tag!='sem', tag!='pos')
+
+ def _items(self, fileids, unit, bracket_sent, pos_tag, sem_tag):
+ if unit=='word' and not bracket_sent:
+ # the result of the SemcorWordView may be a multiword unit, so the
+ # LazyConcatenation will make sure the sentence is flattened
+ _ = lambda *args: LazyConcatenation((SemcorWordView if self._lazy else self._words)(*args))
+ else:
+ _ = SemcorWordView if self._lazy else self._words
+ return concat([_(fileid, unit, bracket_sent, pos_tag, sem_tag, self._wordnet)
+ for fileid in self.abspaths(fileids)])
+
+ def _words(self, fileid, unit, bracket_sent, pos_tag, sem_tag):
+ """
+ Helper used to implement the view methods -- returns a list of
+ tokens, (segmented) words, chunks, or sentences. The tokens
+ and chunks may optionally be tagged (with POS and sense
+ information).
+
+ :param fileid: The name of the underlying file.
+ :param unit: One of `'token'`, `'word'`, or `'chunk'`.
+ :param bracket_sent: If true, include sentence bracketing.
+ :param pos_tag: Whether to include part-of-speech tags.
+ :param sem_tag: Whether to include semantic tags, namely WordNet lemma
+ and OOV named entity status.
+ """
+ assert unit in ('token', 'word', 'chunk')
+ result = []
+
+ xmldoc = ElementTree.parse(fileid).getroot()
+ for xmlsent in xmldoc.findall('.//s'):
+ sent = []
+ for xmlword in _all_xmlwords_in(xmlsent):
+ itm = SemcorCorpusReader._word(xmlword, unit, pos_tag, sem_tag, self._wordnet)
+ if unit=='word':
+ sent.extend(itm)
+ else:
+ sent.append(itm)
+
+ if bracket_sent:
+ result.append(SemcorSentence(xmlsent.attrib['snum'], sent))
+ else:
+ result.extend(sent)
+
+ assert None not in result
+ return result
+
+ @staticmethod
+ def _word(xmlword, unit, pos_tag, sem_tag, wordnet):
+ tkn = xmlword.text
+ if not tkn:
+ tkn = "" # fixes issue 337?
+
+ lemma = xmlword.get('lemma', tkn) # lemma or NE class
+ lexsn = xmlword.get('lexsn') # lex_sense (locator for the lemma's sense)
+ if lexsn is not None:
+ sense_key = lemma + '%' + lexsn
+ wnpos = ('n','v','a','r','s')[int(lexsn.split(':')[0])-1] # see http://wordnet.princeton.edu/man/senseidx.5WN.html
+ else:
+ sense_key = wnpos = None
+ redef = xmlword.get('rdf', tkn) # redefinition--this indicates the lookup string
+ # does not exactly match the enclosed string, e.g. due to typographical adjustments
+ # or discontinuity of a multiword expression. If a redefinition has occurred,
+ # the "rdf" attribute holds its inflected form and "lemma" holds its lemma.
+ # For NEs, "rdf", "lemma", and "pn" all hold the same value (the NE class).
+ sensenum = xmlword.get('wnsn') # WordNet sense number
+ isOOVEntity = 'pn' in xmlword.keys() # a "personal name" (NE) not in WordNet
+ pos = xmlword.get('pos') # part of speech for the whole chunk (None for punctuation)
+
+ if unit=='token':
+ if not pos_tag and not sem_tag:
+ itm = tkn
+ else:
+ itm = (tkn,) + ((pos,) if pos_tag else ()) + ((lemma, wnpos, sensenum, isOOVEntity) if sem_tag else ())
+ return itm
+ else:
+ ww = tkn.split('_') # TODO: case where punctuation intervenes in MWE
+ if unit=='word':
+ return ww
+ else:
+ if sensenum is not None:
+ try:
+ sense = wordnet.lemma_from_key(sense_key) # Lemma object
+ except Exception:
+ # cannot retrieve the wordnet.Lemma object. possible reasons:
+ # (a) the wordnet corpus is not downloaded;
+ # (b) a nonexistant sense is annotated: e.g., such.s.00 triggers:
+ # nltk.corpus.reader.wordnet.WordNetError: No synset found for key u'such%5:00:01:specified:00'
+ # solution: just use the lemma name as a string
+ try:
+ sense = '%s.%s.%02d' % (lemma, wnpos, int(sensenum)) # e.g.: reach.v.02
+ except ValueError:
+ sense = lemma+'.'+wnpos+'.'+sensenum # e.g. the sense number may be "2;1"
+
+ bottom = [Tree(pos, ww)] if pos_tag else ww
+
+ if sem_tag and isOOVEntity:
+ if sensenum is not None:
+ return Tree(sense, [Tree('NE', bottom)])
+ else: # 'other' NE
+ return Tree('NE', bottom)
+ elif sem_tag and sensenum is not None:
+ return Tree(sense, bottom)
+ elif pos_tag:
+ return bottom[0]
+ else:
+ return bottom # chunk as a list
+
+def _all_xmlwords_in(elt, result=None):
+ if result is None: result = []
+ for child in elt:
+ if child.tag in ('wf', 'punc'): result.append(child)
+ else: _all_xmlwords_in(child, result)
+ return result
+
+class SemcorSentence(list):
+ """
+ A list of words, augmented by an attribute ``num`` used to record
+ the sentence identifier (the ``n`` attribute from the XML).
+ """
+ def __init__(self, num, items):
+ self.num = num
+ list.__init__(self, items)
+
+class SemcorWordView(XMLCorpusView):
+ """
+ A stream backed corpus view specialized for use with the BNC corpus.
+ """
+ def __init__(self, fileid, unit, bracket_sent, pos_tag, sem_tag, wordnet):
+ """
+ :param fileid: The name of the underlying file.
+ :param unit: One of `'token'`, `'word'`, or `'chunk'`.
+ :param bracket_sent: If true, include sentence bracketing.
+ :param pos_tag: Whether to include part-of-speech tags.
+ :param sem_tag: Whether to include semantic tags, namely WordNet lemma
+ and OOV named entity status.
+ """
+ if bracket_sent: tagspec = '.*/s'
+ else: tagspec = '.*/s/(punc|wf)'
+
+ self._unit = unit
+ self._sent = bracket_sent
+ self._pos_tag = pos_tag
+ self._sem_tag = sem_tag
+ self._wordnet = wordnet
+
+ XMLCorpusView.__init__(self, fileid, tagspec)
+
+ def handle_elt(self, elt, context):
+ if self._sent: return self.handle_sent(elt)
+ else: return self.handle_word(elt)
+
+ def handle_word(self, elt):
+ return SemcorCorpusReader._word(elt, self._unit, self._pos_tag, self._sem_tag, self._wordnet)
+
+ def handle_sent(self, elt):
+ sent = []
+ for child in elt:
+ if child.tag in ('wf','punc'):
+ itm = self.handle_word(child)
+ if self._unit=='word':
+ sent.extend(itm)
+ else:
+ sent.append(itm)
+ else:
+ raise ValueError('Unexpected element %s' % child.tag)
+ return SemcorSentence(elt.attrib['snum'], sent)
diff --git a/nltk/corpus/reader/senseval.py b/nltk/corpus/reader/senseval.py
new file mode 100644
index 0000000..bbbead9
--- /dev/null
+++ b/nltk/corpus/reader/senseval.py
@@ -0,0 +1,201 @@
+# Natural Language Toolkit: Senseval 2 Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# Steven Bird <stevenbird1 at gmail.com> (modifications)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Read from the Senseval 2 Corpus.
+
+SENSEVAL [http://www.senseval.org/]
+Evaluation exercises for Word Sense Disambiguation.
+Organized by ACL-SIGLEX [http://www.siglex.org/]
+
+Prepared by Ted Pedersen <tpederse at umn.edu>, University of Minnesota,
+http://www.d.umn.edu/~tpederse/data.html
+Distributed with permission.
+
+The NLTK version of the Senseval 2 files uses well-formed XML.
+Each instance of the ambiguous words "hard", "interest", "line", and "serve"
+is tagged with a sense identifier, and supplied with context.
+"""
+from __future__ import print_function, unicode_literals
+
+import re
+from xml.etree import ElementTree
+
+from nltk import compat
+from nltk.tokenize import *
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+ at compat.python_2_unicode_compatible
+class SensevalInstance(object):
+ def __init__(self, word, position, context, senses):
+ self.word = word
+ self.senses = tuple(senses)
+ self.position = position
+ self.context = context
+
+ def __repr__(self):
+ return ('SensevalInstance(word=%r, position=%r, '
+ 'context=%r, senses=%r)' %
+ (self.word, self.position, self.context, self.senses))
+
+
+class SensevalCorpusReader(CorpusReader):
+ def instances(self, fileids=None):
+ return concat([SensevalCorpusView(fileid, enc)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def raw(self, fileids=None):
+ """
+ :return: the text contents of the given fileids, as a single string.
+ """
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def _entry(self, tree):
+ elts = []
+ for lexelt in tree.findall('lexelt'):
+ for inst in lexelt.findall('instance'):
+ sense = inst[0].attrib['senseid']
+ context = [(w.text, w.attrib['pos'])
+ for w in inst[1]]
+ elts.append( (sense, context) )
+ return elts
+
+
+class SensevalCorpusView(StreamBackedCorpusView):
+ def __init__(self, fileid, encoding):
+ StreamBackedCorpusView.__init__(self, fileid, encoding=encoding)
+
+ self._word_tokenizer = WhitespaceTokenizer()
+ self._lexelt_starts = [0] # list of streampos
+ self._lexelts = [None] # list of lexelt names
+
+ def read_block(self, stream):
+ # Decide which lexical element we're in.
+ lexelt_num = bisect.bisect_right(self._lexelt_starts, stream.tell())-1
+ lexelt = self._lexelts[lexelt_num]
+
+ instance_lines = []
+ in_instance = False
+ while True:
+ line = stream.readline()
+ if line == '':
+ assert instance_lines == []
+ return []
+
+ # Start of a lexical element?
+ if line.lstrip().startswith('<lexelt'):
+ lexelt_num += 1
+ m = re.search('item=("[^"]+"|\'[^\']+\')', line)
+ assert m is not None # <lexelt> has no 'item=...'
+ lexelt = m.group(1)[1:-1]
+ if lexelt_num < len(self._lexelts):
+ assert lexelt == self._lexelts[lexelt_num]
+ else:
+ self._lexelts.append(lexelt)
+ self._lexelt_starts.append(stream.tell())
+
+ # Start of an instance?
+ if line.lstrip().startswith('<instance'):
+ assert instance_lines == []
+ in_instance = True
+
+ # Body of an instance?
+ if in_instance:
+ instance_lines.append(line)
+
+ # End of an instance?
+ if line.lstrip().startswith('</instance'):
+ xml_block = '\n'.join(instance_lines)
+ xml_block = _fixXML(xml_block)
+ inst = ElementTree.fromstring(xml_block)
+ return [self._parse_instance(inst, lexelt)]
+
+ def _parse_instance(self, instance, lexelt):
+ senses = []
+ context = []
+ position = None
+ for child in instance:
+ if child.tag == 'answer':
+ senses.append(child.attrib['senseid'])
+ elif child.tag == 'context':
+ context += self._word_tokenizer.tokenize(child.text)
+ for cword in child:
+ if cword.tag == 'compound':
+ cword = cword[0] # is this ok to do?
+
+ if cword.tag == 'head':
+ # Some santiy checks:
+ assert position is None, 'head specified twice'
+ assert cword.text.strip() or len(cword)==1
+ assert not (cword.text.strip() and len(cword)==1)
+ # Record the position of the head:
+ position = len(context)
+ # Addd on the head word itself:
+ if cword.text.strip():
+ context.append(cword.text.strip())
+ elif cword[0].tag == 'wf':
+ context.append((cword[0].text,
+ cword[0].attrib['pos']))
+ if cword[0].tail:
+ context += self._word_tokenizer.tokenize(
+ cword[0].tail)
+ else:
+ assert False, 'expected CDATA or wf in <head>'
+ elif cword.tag == 'wf':
+ context.append((cword.text, cword.attrib['pos']))
+ elif cword.tag == 's':
+ pass # Sentence boundary marker.
+
+ else:
+ print('ACK', cword.tag)
+ assert False, 'expected CDATA or <wf> or <head>'
+ if cword.tail:
+ context += self._word_tokenizer.tokenize(cword.tail)
+ else:
+ assert False, 'unexpected tag %s' % child.tag
+ return SensevalInstance(lexelt, position, context, senses)
+
+def _fixXML(text):
+ """
+ Fix the various issues with Senseval pseudo-XML.
+ """
+ # <~> or <^> => ~ or ^
+ text = re.sub(r'<([~\^])>', r'\1', text)
+ # fix lone &
+ text = re.sub(r'(\s+)\&(\s+)', r'\1&\2', text)
+ # fix """
+ text = re.sub(r'"""', '\'"\'', text)
+ # fix <s snum=dd> => <s snum="dd"/>
+ text = re.sub(r'(<[^<]*snum=)([^">]+)>', r'\1"\2"/>', text)
+ # fix foreign word tag
+ text = re.sub(r'<\&frasl>\s*<p[^>]*>', 'FRASL', text)
+ # remove <&I .>
+ text = re.sub(r'<\&I[^>]*>', '', text)
+ # fix <{word}>
+ text = re.sub(r'<{([^}]+)}>', r'\1', text)
+ # remove <@>, <p>, </p>
+ text = re.sub(r'<(@|/?p)>', r'', text)
+ # remove <&M .> and <&T .> and <&Ms .>
+ text = re.sub(r'<&\w+ \.>', r'', text)
+ # remove <!DOCTYPE... > lines
+ text = re.sub(r'<!DOCTYPE[^>]*>', r'', text)
+ # remove <[hi]> and <[/p]> etc
+ text = re.sub(r'<\[\/?[^>]+\]*>', r'', text)
+ # take the thing out of the brackets: <…>
+ text = re.sub(r'<(\&\w+;)>', r'\1', text)
+ # and remove the & for those patterns that aren't regular XML
+ text = re.sub(r'&(?!amp|gt|lt|apos|quot)', r'', text)
+ # fix 'abc <p="foo"/>' style tags - now <wf pos="foo">abc</wf>
+ text = re.sub(r'[ \t]*([^<>\s]+?)[ \t]*<p="([^"]*"?)"/>',
+ r' <wf pos="\2">\1</wf>', text)
+ text = re.sub(r'\s*"\s*<p=\'"\'/>', " <wf pos='\"'>\"</wf>", text)
+ return text
diff --git a/nltk/corpus/reader/sentiwordnet.py b/nltk/corpus/reader/sentiwordnet.py
new file mode 100644
index 0000000..3d357b8
--- /dev/null
+++ b/nltk/corpus/reader/sentiwordnet.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: WordNet
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Christopher Potts <cgpotts at stanford.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+An NLTK interface for SentiWordNet
+
+SentiWordNet is a lexical resource for opinion mining.
+SentiWordNet assigns to each synset of WordNet three
+sentiment scores: positivity, negativity, and objectivity.
+
+For details about SentiWordNet see:
+http://sentiwordnet.isti.cnr.it/
+
+ >>> from nltk.corpus import sentiwordnet as swn
+ >>> print(swn.senti_synset('breakdown.n.03'))
+ <breakdown.n.03: PosScore=0.0 NegScore=0.25>
+ >>> list(swn.senti_synsets('slow'))
+ [SentiSynset('decelerate.v.01'), SentiSynset('slow.v.02'),\
+ SentiSynset('slow.v.03'), SentiSynset('slow.a.01'),\
+ SentiSynset('slow.a.02'), SentiSynset('slow.a.04'),\
+ SentiSynset('slowly.r.01'), SentiSynset('behind.r.03')]
+ >>> happy = swn.senti_synsets('happy', 'a')
+ >>> happy0 = list(happy)[0]
+ >>> happy0.pos_score()
+ 0.875
+ >>> happy0.neg_score()
+ 0.0
+ >>> happy0.obj_score()
+ 0.125
+"""
+
+import re
+from nltk.compat import python_2_unicode_compatible
+from nltk.corpus.reader import CorpusReader
+
+ at python_2_unicode_compatible
+class SentiWordNetCorpusReader(CorpusReader):
+ def __init__(self, root, fileids, encoding='utf-8'):
+ """
+ Construct a new SentiWordNet Corpus Reader, using data from
+ the specified file.
+ """
+ super(SentiWordNetCorpusReader, self).__init__(root, fileids,
+ encoding=encoding)
+ if len(self._fileids) != 1:
+ raise ValueError('Exactly one file must be specified')
+ self._db = {}
+ self._parse_src_file()
+
+ def _parse_src_file(self):
+ lines = self.open(self._fileids[0]).read().splitlines()
+ lines = filter((lambda x : not re.search(r"^\s*#", x)), lines)
+ for i, line in enumerate(lines):
+ fields = [field.strip() for field in re.split(r"\t+", line)]
+ try:
+ pos, offset, pos_score, neg_score, synset_terms, gloss = fields
+ except:
+ raise ValueError('Line %s formatted incorrectly: %s\n' % (i, line))
+ if pos and offset:
+ offset = int(offset)
+ self._db[(pos, offset)] = (float(pos_score), float(neg_score))
+
+ def senti_synset(self, *vals):
+ from nltk.corpus import wordnet as wn
+ if tuple(vals) in self._db:
+ pos_score, neg_score = self._db[tuple(vals)]
+ pos, offset = vals
+ synset = wn._synset_from_pos_and_offset(pos, offset)
+ return SentiSynset(pos_score, neg_score, synset)
+ else:
+ synset = wn.synset(vals[0])
+ pos = synset.pos()
+ offset = synset.offset()
+ if (pos, offset) in self._db:
+ pos_score, neg_score = self._db[(pos, offset)]
+ return SentiSynset(pos_score, neg_score, synset)
+ else:
+ return None
+
+ def senti_synsets(self, string, pos=None):
+ from nltk.corpus import wordnet as wn
+ sentis = []
+ synset_list = wn.synsets(string, pos)
+ for synset in synset_list:
+ sentis.append(self.senti_synset(synset.name()))
+ sentis = filter(lambda x : x, sentis)
+ return sentis
+
+ def all_senti_synsets(self):
+ from nltk.corpus import wordnet as wn
+ for key, fields in self._db.items():
+ pos, offset = key
+ pos_score, neg_score = fields
+ synset = wn._synset_from_pos_and_offset(pos, offset)
+ yield SentiSynset(pos_score, neg_score, synset)
+
+
+ at python_2_unicode_compatible
+class SentiSynset(object):
+ def __init__(self, pos_score, neg_score, synset):
+ self._pos_score = pos_score
+ self._neg_score = neg_score
+ self._obj_score = 1.0 - (self._pos_score + self._neg_score)
+ self.synset = synset
+
+ def pos_score(self):
+ return self._pos_score
+
+ def neg_score(self):
+ return self._neg_score
+
+ def obj_score(self):
+ return self._obj_score
+
+ def __str__(self):
+ """Prints just the Pos/Neg scores for now."""
+ s = "<"
+ s += self.synset.name() + ": "
+ s += "PosScore=%s " % self._pos_score
+ s += "NegScore=%s" % self._neg_score
+ s += ">"
+ return s
+
+ def __repr__(self):
+ return "Senti" + repr(self.synset)
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/corpus/reader/sinica_treebank.py b/nltk/corpus/reader/sinica_treebank.py
new file mode 100644
index 0000000..3d872db
--- /dev/null
+++ b/nltk/corpus/reader/sinica_treebank.py
@@ -0,0 +1,75 @@
+# Natural Language Toolkit: Sinica Treebank Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Sinica Treebank Corpus Sample
+
+http://rocling.iis.sinica.edu.tw/CKIP/engversion/treebank.htm
+
+10,000 parsed sentences, drawn from the Academia Sinica Balanced
+Corpus of Modern Chinese. Parse tree notation is based on
+Information-based Case Grammar. Tagset documentation is available
+at http://www.sinica.edu.tw/SinicaCorpus/modern_e_wordtype.html
+
+Language and Knowledge Processing Group, Institute of Information
+Science, Academia Sinica
+
+It is distributed with the Natural Language Toolkit under the terms of
+the Creative Commons Attribution-NonCommercial-ShareAlike License
+[http://creativecommons.org/licenses/by-nc-sa/2.5/].
+
+References:
+
+Feng-Yi Chen, Pi-Fang Tsai, Keh-Jiann Chen, and Chu-Ren Huang (1999)
+The Construction of Sinica Treebank. Computational Linguistics and
+Chinese Language Processing, 4, pp 87-104.
+
+Huang Chu-Ren, Keh-Jiann Chen, Feng-Yi Chen, Keh-Jiann Chen, Zhao-Ming
+Gao, and Kuang-Yu Chen. 2000. Sinica Treebank: Design Criteria,
+Annotation Guidelines, and On-line Interface. Proceedings of 2nd
+Chinese Language Processing Workshop, Association for Computational
+Linguistics.
+
+Chen Keh-Jiann and Yu-Ming Hsieh (2004) Chinese Treebanks and Grammar
+Extraction, Proceedings of IJCNLP-04, pp560-565.
+"""
+
+import os
+import re
+
+from nltk.tree import sinica_parse
+from nltk.tag import map_tag
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+IDENTIFIER = re.compile(r'^#\S+\s')
+APPENDIX = re.compile(r'(?<=\))#.*$')
+TAGWORD = re.compile(r':([^:()|]+):([^:()|]+)')
+WORD = re.compile(r':[^:()|]+:([^:()|]+)')
+
+class SinicaTreebankCorpusReader(SyntaxCorpusReader):
+ """
+ Reader for the sinica treebank.
+ """
+ def _read_block(self, stream):
+ sent = stream.readline()
+ sent = IDENTIFIER.sub('', sent)
+ sent = APPENDIX.sub('', sent)
+ return [sent]
+
+ def _parse(self, sent):
+ return sinica_parse(sent)
+
+ def _tag(self, sent, tagset=None):
+ tagged_sent = [(w,t) for (t,w) in TAGWORD.findall(sent)]
+ if tagset and tagset != self._tagset:
+ tagged_sent = [(w, map_tag(self._tagset, tagset, t)) for (w,t) in tagged_sent]
+ return tagged_sent
+
+ def _word(self, sent):
+ return WORD.findall(sent)
diff --git a/nltk/corpus/reader/string_category.py b/nltk/corpus/reader/string_category.py
new file mode 100644
index 0000000..98859ac
--- /dev/null
+++ b/nltk/corpus/reader/string_category.py
@@ -0,0 +1,61 @@
+# Natural Language Toolkit: String Category Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Read tuples from a corpus consisting of categorized strings.
+For example, from the question classification corpus:
+
+NUM:dist How far is it from Denver to Aspen ?
+LOC:city What county is Modesto , California in ?
+HUM:desc Who was Galileo ?
+DESC:def What is an atom ?
+NUM:date When did Hawaii become a state ?
+"""
+
+# based on PPAttachmentCorpusReader
+
+import os
+
+from nltk import compat
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+# [xx] Should the order of the tuple be reversed -- in most other places
+# in nltk, we use the form (data, tag) -- e.g., tagged words and
+# labeled texts for classifiers.
+class StringCategoryCorpusReader(CorpusReader):
+ def __init__(self, root, fileids, delimiter=' ', encoding='utf8'):
+ """
+ :param root: The root directory for this corpus.
+ :param fileids: A list or regexp specifying the fileids in this corpus.
+ :param delimiter: Field delimiter
+ """
+ CorpusReader.__init__(self, root, fileids, encoding)
+ self._delimiter = delimiter
+
+ def tuples(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([StreamBackedCorpusView(fileid, self._read_tuple_block,
+ encoding=enc)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def raw(self, fileids=None):
+ """
+ :return: the text contents of the given fileids, as a single string.
+ """
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def _read_tuple_block(self, stream):
+ line = stream.readline().strip()
+ if line:
+ return [tuple(line.split(self._delimiter, 1))]
+ else:
+ return []
diff --git a/nltk/corpus/reader/switchboard.py b/nltk/corpus/reader/switchboard.py
new file mode 100644
index 0000000..460da31
--- /dev/null
+++ b/nltk/corpus/reader/switchboard.py
@@ -0,0 +1,119 @@
+# Natural Language Toolkit: Switchboard Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import unicode_literals
+import re
+
+from nltk.tag import str2tuple, map_tag
+from nltk import compat
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+
+ at compat.python_2_unicode_compatible
+class SwitchboardTurn(list):
+ """
+ A specialized list object used to encode switchboard utterances.
+ The elements of the list are the words in the utterance; and two
+ attributes, ``speaker`` and ``id``, are provided to retrieve the
+ spearker identifier and utterance id. Note that utterance ids
+ are only unique within a given discourse.
+ """
+ def __init__(self, words, speaker, id):
+ list.__init__(self, words)
+ self.speaker = speaker
+ self.id = int(id)
+
+ def __repr__(self):
+ if len(self) == 0:
+ text = ''
+ elif isinstance(self[0], tuple):
+ text = ' '.join('%s/%s' % w for w in self)
+ else:
+ text = ' '.join(self)
+ return '<%s.%s: %r>' % (self.speaker, self.id, text)
+
+
+class SwitchboardCorpusReader(CorpusReader):
+ _FILES = ['tagged']
+ # Use the "tagged" file even for non-tagged data methods, since
+ # it's tokenized.
+
+ def __init__(self, root, tagset=None):
+ CorpusReader.__init__(self, root, self._FILES)
+ self._tagset = tagset
+
+ def words(self):
+ return StreamBackedCorpusView(self.abspath('tagged'),
+ self._words_block_reader)
+
+ def tagged_words(self, tagset=None):
+ def tagged_words_block_reader(stream):
+ return self._tagged_words_block_reader(stream, tagset)
+ return StreamBackedCorpusView(self.abspath('tagged'),
+ tagged_words_block_reader)
+
+ def turns(self):
+ return StreamBackedCorpusView(self.abspath('tagged'),
+ self._turns_block_reader)
+
+ def tagged_turns(self, tagset=None):
+ def tagged_turns_block_reader(stream):
+ return self._tagged_turns_block_reader(stream, tagset)
+ return StreamBackedCorpusView(self.abspath('tagged'),
+ tagged_turns_block_reader)
+
+ def discourses(self):
+ return StreamBackedCorpusView(self.abspath('tagged'),
+ self._discourses_block_reader)
+
+ def tagged_discourses(self, tagset=False):
+ def tagged_discourses_block_reader(stream):
+ return self._tagged_discourses_block_reader(stream, tagset)
+ return StreamBackedCorpusView(self.abspath('tagged'),
+ tagged_discourses_block_reader)
+
+ def _discourses_block_reader(self, stream):
+ # returns at most 1 discourse. (The other methods depend on this.)
+ return [[self._parse_utterance(u, include_tag=False)
+ for b in read_blankline_block(stream)
+ for u in b.split('\n') if u.strip()]]
+
+ def _tagged_discourses_block_reader(self, stream, tagset=None):
+ # returns at most 1 discourse. (The other methods depend on this.)
+ return [[self._parse_utterance(u, include_tag=True,
+ tagset=tagset)
+ for b in read_blankline_block(stream)
+ for u in b.split('\n') if u.strip()]]
+
+ def _turns_block_reader(self, stream):
+ return self._discourses_block_reader(stream)[0]
+
+ def _tagged_turns_block_reader(self, stream, tagset=None):
+ return self._tagged_discourses_block_reader(stream, tagset)[0]
+
+ def _words_block_reader(self, stream):
+ return sum(self._discourses_block_reader(stream)[0], [])
+
+ def _tagged_words_block_reader(self, stream, tagset=None):
+ return sum(self._tagged_discourses_block_reader(stream,
+ tagset)[0], [])
+
+ _UTTERANCE_RE = re.compile('(\w+)\.(\d+)\:\s*(.*)')
+ _SEP = '/'
+ def _parse_utterance(self, utterance, include_tag, tagset=None):
+ m = self._UTTERANCE_RE.match(utterance)
+ if m is None:
+ raise ValueError('Bad utterance %r' % utterance)
+ speaker, id, text = m.groups()
+ words = [str2tuple(s, self._SEP) for s in text.split()]
+ if not include_tag:
+ words = [w for (w,t) in words]
+ elif tagset and tagset != self._tagset:
+ words = [(w, map_tag(self._tagset, tagset, t)) for (w,t) in words]
+ return SwitchboardTurn(words, speaker, id)
+
diff --git a/nltk/corpus/reader/tagged.py b/nltk/corpus/reader/tagged.py
new file mode 100644
index 0000000..1e23774
--- /dev/null
+++ b/nltk/corpus/reader/tagged.py
@@ -0,0 +1,294 @@
+# Natural Language Toolkit: Tagged Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# Jacob Perkins <japerk at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A reader for corpora whose documents contain part-of-speech-tagged words.
+"""
+
+import os
+
+from nltk import compat
+from nltk.tag import str2tuple, map_tag
+from nltk.tokenize import *
+
+from nltk.corpus.reader.api import *
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.timit import read_timit_block
+
+class TaggedCorpusReader(CorpusReader):
+ """
+ Reader for simple part-of-speech tagged corpora. Paragraphs are
+ assumed to be split using blank lines. Sentences and words can be
+ tokenized using the default tokenizers, or by custom tokenizers
+ specified as parameters to the constructor. Words are parsed
+ using ``nltk.tag.str2tuple``. By default, ``'/'`` is used as the
+ separator. I.e., words should have the form::
+
+ word1/tag1 word2/tag2 word3/tag3 ...
+
+ But custom separators may be specified as parameters to the
+ constructor. Part of speech tags are case-normalized to upper
+ case.
+ """
+ def __init__(self, root, fileids,
+ sep='/', word_tokenizer=WhitespaceTokenizer(),
+ sent_tokenizer=RegexpTokenizer('\n', gaps=True),
+ para_block_reader=read_blankline_block,
+ encoding='utf8',
+ tagset=None):
+ """
+ Construct a new Tagged Corpus reader for a set of documents
+ located at the given root directory. Example usage:
+
+ >>> root = '/...path to corpus.../'
+ >>> reader = TaggedCorpusReader(root, '.*', '.txt') # doctest: +SKIP
+
+ :param root: The root directory for this corpus.
+ :param fileids: A list or regexp specifying the fileids in this corpus.
+ """
+ CorpusReader.__init__(self, root, fileids, encoding)
+ self._sep = sep
+ self._word_tokenizer = word_tokenizer
+ self._sent_tokenizer = sent_tokenizer
+ self._para_block_reader = para_block_reader
+ self._tagset = tagset
+
+ def raw(self, fileids=None):
+ """
+ :return: the given file(s) as a single string.
+ :rtype: str
+ """
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+ def words(self, fileids=None):
+ """
+ :return: the given file(s) as a list of words
+ and punctuation symbols.
+ :rtype: list(str)
+ """
+ return concat([TaggedCorpusView(fileid, enc,
+ False, False, False,
+ self._sep, self._word_tokenizer,
+ self._sent_tokenizer,
+ self._para_block_reader,
+ None)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def sents(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ sentences or utterances, each encoded as a list of word
+ strings.
+ :rtype: list(list(str))
+ """
+ return concat([TaggedCorpusView(fileid, enc,
+ False, True, False,
+ self._sep, self._word_tokenizer,
+ self._sent_tokenizer,
+ self._para_block_reader,
+ None)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def paras(self, fileids=None):
+ """
+ :return: the given file(s) as a list of
+ paragraphs, each encoded as a list of sentences, which are
+ in turn encoded as lists of word strings.
+ :rtype: list(list(list(str)))
+ """
+ return concat([TaggedCorpusView(fileid, enc,
+ False, True, True,
+ self._sep, self._word_tokenizer,
+ self._sent_tokenizer,
+ self._para_block_reader,
+ None)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tagged_words(self, fileids=None, tagset=None):
+ """
+ :return: the given file(s) as a list of tagged
+ words and punctuation symbols, encoded as tuples
+ ``(word,tag)``.
+ :rtype: list(tuple(str,str))
+ """
+ if tagset and tagset != self._tagset:
+ tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
+ else:
+ tag_mapping_function = None
+ return concat([TaggedCorpusView(fileid, enc,
+ True, False, False,
+ self._sep, self._word_tokenizer,
+ self._sent_tokenizer,
+ self._para_block_reader,
+ tag_mapping_function)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tagged_sents(self, fileids=None, tagset=None):
+ """
+ :return: the given file(s) as a list of
+ sentences, each encoded as a list of ``(word,tag)`` tuples.
+
+ :rtype: list(list(tuple(str,str)))
+ """
+ if tagset and tagset != self._tagset:
+ tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
+ else:
+ tag_mapping_function = None
+ return concat([TaggedCorpusView(fileid, enc,
+ True, True, False,
+ self._sep, self._word_tokenizer,
+ self._sent_tokenizer,
+ self._para_block_reader,
+ tag_mapping_function)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+ def tagged_paras(self, fileids=None, tagset=None):
+ """
+ :return: the given file(s) as a list of
+ paragraphs, each encoded as a list of sentences, which are
+ in turn encoded as lists of ``(word,tag)`` tuples.
+ :rtype: list(list(list(tuple(str,str))))
+ """
+ if tagset and tagset != self._tagset:
+ tag_mapping_function = lambda t: map_tag(self._tagset, tagset, t)
+ else:
+ tag_mapping_function = None
+ return concat([TaggedCorpusView(fileid, enc,
+ True, True, True,
+ self._sep, self._word_tokenizer,
+ self._sent_tokenizer,
+ self._para_block_reader,
+ tag_mapping_function)
+ for (fileid, enc) in self.abspaths(fileids, True)])
+
+class CategorizedTaggedCorpusReader(CategorizedCorpusReader,
+ TaggedCorpusReader):
+ """
+ A reader for part-of-speech tagged corpora whose documents are
+ divided into categories based on their file identifiers.
+ """
+ def __init__(self, *args, **kwargs):
+ """
+ Initialize the corpus reader. Categorization arguments
+ (``cat_pattern``, ``cat_map``, and ``cat_file``) are passed to
+ the ``CategorizedCorpusReader`` constructor. The remaining arguments
+ are passed to the ``TaggedCorpusReader``.
+ """
+ CategorizedCorpusReader.__init__(self, kwargs)
+ TaggedCorpusReader.__init__(self, *args, **kwargs)
+
+ def _resolve(self, fileids, categories):
+ if fileids is not None and categories is not None:
+ raise ValueError('Specify fileids or categories, not both')
+ if categories is not None:
+ return self.fileids(categories)
+ else:
+ return fileids
+ def raw(self, fileids=None, categories=None):
+ return TaggedCorpusReader.raw(
+ self, self._resolve(fileids, categories))
+ def words(self, fileids=None, categories=None):
+ return TaggedCorpusReader.words(
+ self, self._resolve(fileids, categories))
+ def sents(self, fileids=None, categories=None):
+ return TaggedCorpusReader.sents(
+ self, self._resolve(fileids, categories))
+ def paras(self, fileids=None, categories=None):
+ return TaggedCorpusReader.paras(
+ self, self._resolve(fileids, categories))
+ def tagged_words(self, fileids=None, categories=None, tagset=None):
+ return TaggedCorpusReader.tagged_words(
+ self, self._resolve(fileids, categories), tagset)
+ def tagged_sents(self, fileids=None, categories=None, tagset=None):
+ return TaggedCorpusReader.tagged_sents(
+ self, self._resolve(fileids, categories), tagset)
+ def tagged_paras(self, fileids=None, categories=None, tagset=None):
+ return TaggedCorpusReader.tagged_paras(
+ self, self._resolve(fileids, categories), tagset)
+
+class TaggedCorpusView(StreamBackedCorpusView):
+ """
+ A specialized corpus view for tagged documents. It can be
+ customized via flags to divide the tagged corpus documents up by
+ sentence or paragraph, and to include or omit part of speech tags.
+ ``TaggedCorpusView`` objects are typically created by
+ ``TaggedCorpusReader`` (not directly by nltk users).
+ """
+ def __init__(self, corpus_file, encoding, tagged, group_by_sent,
+ group_by_para, sep, word_tokenizer, sent_tokenizer,
+ para_block_reader, tag_mapping_function=None):
+ self._tagged = tagged
+ self._group_by_sent = group_by_sent
+ self._group_by_para = group_by_para
+ self._sep = sep
+ self._word_tokenizer = word_tokenizer
+ self._sent_tokenizer = sent_tokenizer
+ self._para_block_reader = para_block_reader
+ self._tag_mapping_function = tag_mapping_function
+ StreamBackedCorpusView.__init__(self, corpus_file, encoding=encoding)
+
+ def read_block(self, stream):
+ """Reads one paragraph at a time."""
+ block = []
+ for para_str in self._para_block_reader(stream):
+ para = []
+ for sent_str in self._sent_tokenizer.tokenize(para_str):
+ sent = [str2tuple(s, self._sep) for s in
+ self._word_tokenizer.tokenize(sent_str)]
+ if self._tag_mapping_function:
+ sent = [(w, self._tag_mapping_function(t)) for (w,t) in sent]
+ if not self._tagged:
+ sent = [w for (w,t) in sent]
+ if self._group_by_sent:
+ para.append(sent)
+ else:
+ para.extend(sent)
+ if self._group_by_para:
+ block.append(para)
+ else:
+ block.extend(para)
+ return block
+
+# needs to implement simplified tags
+class MacMorphoCorpusReader(TaggedCorpusReader):
+ """
+ A corpus reader for the MAC_MORPHO corpus. Each line contains a
+ single tagged word, using '_' as a separator. Sentence boundaries
+ are based on the end-sentence tag ('_.'). Paragraph information
+ is not included in the corpus, so each paragraph returned by
+ ``self.paras()`` and ``self.tagged_paras()`` contains a single
+ sentence.
+ """
+ def __init__(self, root, fileids, encoding='utf8', tagset=None):
+ TaggedCorpusReader.__init__(
+ self, root, fileids, sep='_',
+ word_tokenizer=LineTokenizer(),
+ sent_tokenizer=RegexpTokenizer('.*\n'),
+ para_block_reader=self._read_block,
+ encoding=encoding,
+ tagset=tagset)
+
+ def _read_block(self, stream):
+ return read_regexp_block(stream, r'.*', r'.*_\.')
+
+class TimitTaggedCorpusReader(TaggedCorpusReader):
+ """
+ A corpus reader for tagged sentences that are included in the TIMIT corpus.
+ """
+ def __init__(self, *args, **kwargs):
+ TaggedCorpusReader.__init__(
+ self, para_block_reader=read_timit_block, *args, **kwargs)
+
+ def paras(self):
+ raise NotImplementedError('use sents() instead')
+
+ def tagged_paras(self):
+ raise NotImplementedError('use tagged_sents() instead')
diff --git a/nltk/corpus/reader/timit.py b/nltk/corpus/reader/timit.py
new file mode 100644
index 0000000..9d922b9
--- /dev/null
+++ b/nltk/corpus/reader/timit.py
@@ -0,0 +1,450 @@
+# Natural Language Toolkit: TIMIT Corpus Reader
+#
+# Copyright (C) 2001-2007 NLTK Project
+# Author: Haejoong Lee <haejoong at ldc.upenn.edu>
+# Steven Bird <stevenbird1 at gmail.com>
+# Jacob Perkins <japerk at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# [xx] this docstring is out-of-date:
+"""
+Read tokens, phonemes and audio data from the NLTK TIMIT Corpus.
+
+This corpus contains selected portion of the TIMIT corpus.
+
+ - 16 speakers from 8 dialect regions
+ - 1 male and 1 female from each dialect region
+ - total 130 sentences (10 sentences per speaker. Note that some
+ sentences are shared among other speakers, especially sa1 and sa2
+ are spoken by all speakers.)
+ - total 160 recording of sentences (10 recordings per speaker)
+ - audio format: NIST Sphere, single channel, 16kHz sampling,
+ 16 bit sample, PCM encoding
+
+
+Module contents
+===============
+
+The timit corpus reader provides 4 functions and 4 data items.
+
+ - utterances
+
+ List of utterances in the corpus. There are total 160 utterances,
+ each of which corresponds to a unique utterance of a speaker.
+ Here's an example of an utterance identifier in the list::
+
+ dr1-fvmh0/sx206
+ - _---- _---
+ | | | | |
+ | | | | |
+ | | | | `--- sentence number
+ | | | `----- sentence type (a:all, i:shared, x:exclusive)
+ | | `--------- speaker ID
+ | `------------ sex (m:male, f:female)
+ `-------------- dialect region (1..8)
+
+ - speakers
+
+ List of speaker IDs. An example of speaker ID::
+
+ dr1-fvmh0
+
+ Note that if you split an item ID with colon and take the first element of
+ the result, you will get a speaker ID.
+
+ >>> itemid = 'dr1-fvmh0/sx206'
+ >>> spkrid , sentid = itemid.split('/')
+ >>> spkrid
+ 'dr1-fvmh0'
+
+ The second element of the result is a sentence ID.
+
+ - dictionary()
+
+ Phonetic dictionary of words contained in this corpus. This is a Python
+ dictionary from words to phoneme lists.
+
+ - spkrinfo()
+
+ Speaker information table. It's a Python dictionary from speaker IDs to
+ records of 10 fields. Speaker IDs the same as the ones in timie.speakers.
+ Each record is a dictionary from field names to values, and the fields are
+ as follows::
+
+ id speaker ID as defined in the original TIMIT speaker info table
+ sex speaker gender (M:male, F:female)
+ dr speaker dialect region (1:new england, 2:northern,
+ 3:north midland, 4:south midland, 5:southern, 6:new york city,
+ 7:western, 8:army brat (moved around))
+ use corpus type (TRN:training, TST:test)
+ in this sample corpus only TRN is available
+ recdate recording date
+ birthdate speaker birth date
+ ht speaker height
+ race speaker race (WHT:white, BLK:black, AMR:american indian,
+ SPN:spanish-american, ORN:oriental,???:unknown)
+ edu speaker education level (HS:high school, AS:associate degree,
+ BS:bachelor's degree (BS or BA), MS:master's degree (MS or MA),
+ PHD:doctorate degree (PhD,JD,MD), ??:unknown)
+ comments comments by the recorder
+
+The 4 functions are as follows.
+
+ - tokenized(sentences=items, offset=False)
+
+ Given a list of items, returns an iterator of a list of word lists,
+ each of which corresponds to an item (sentence). If offset is set to True,
+ each element of the word list is a tuple of word(string), start offset and
+ end offset, where offset is represented as a number of 16kHz samples.
+
+ - phonetic(sentences=items, offset=False)
+
+ Given a list of items, returns an iterator of a list of phoneme lists,
+ each of which corresponds to an item (sentence). If offset is set to True,
+ each element of the phoneme list is a tuple of word(string), start offset
+ and end offset, where offset is represented as a number of 16kHz samples.
+
+ - audiodata(item, start=0, end=None)
+
+ Given an item, returns a chunk of audio samples formatted into a string.
+ When the fuction is called, if start and end are omitted, the entire
+ samples of the recording will be returned. If only end is omitted,
+ samples from the start offset to the end of the recording will be returned.
+
+ - play(data)
+
+ Play the given audio samples. The audio samples can be obtained from the
+ timit.audiodata function.
+
+"""
+from __future__ import print_function, unicode_literals
+
+import sys
+import os
+import re
+import tempfile
+import time
+
+from nltk import compat
+from nltk.tree import Tree
+from nltk.internals import import_from_stdlib
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class TimitCorpusReader(CorpusReader):
+ """
+ Reader for the TIMIT corpus (or any other corpus with the same
+ file layout and use of file formats). The corpus root directory
+ should contain the following files:
+
+ - timitdic.txt: dictionary of standard transcriptions
+ - spkrinfo.txt: table of speaker information
+
+ In addition, the root directory should contain one subdirectory
+ for each speaker, containing three files for each utterance:
+
+ - <utterance-id>.txt: text content of utterances
+ - <utterance-id>.wrd: tokenized text content of utterances
+ - <utterance-id>.phn: phonetic transcription of utterances
+ - <utterance-id>.wav: utterance sound file
+ """
+
+ _FILE_RE = (r'(\w+-\w+/\w+\.(phn|txt|wav|wrd))|' +
+ r'timitdic\.txt|spkrinfo\.txt')
+ """A regexp matching fileids that are used by this corpus reader."""
+ _UTTERANCE_RE = r'\w+-\w+/\w+\.txt'
+
+ def __init__(self, root, encoding='utf8'):
+ """
+ Construct a new TIMIT corpus reader in the given directory.
+ :param root: The root directory for this corpus.
+ """
+ # Ensure that wave files don't get treated as unicode data:
+ if isinstance(encoding, compat.string_types):
+ encoding = [('.*\.wav', None), ('.*', encoding)]
+
+ CorpusReader.__init__(self, root,
+ find_corpus_fileids(root, self._FILE_RE),
+ encoding=encoding)
+
+ self._utterances = [name[:-4] for name in
+ find_corpus_fileids(root, self._UTTERANCE_RE)]
+ """A list of the utterance identifiers for all utterances in
+ this corpus."""
+
+ self._speakerinfo = None
+ self._root = root
+ self.speakers = sorted(set(u.split('/')[0] for u in self._utterances))
+
+ def fileids(self, filetype=None):
+ """
+ Return a list of file identifiers for the files that make up
+ this corpus.
+
+ :param filetype: If specified, then ``filetype`` indicates that
+ only the files that have the given type should be
+ returned. Accepted values are: ``txt``, ``wrd``, ``phn``,
+ ``wav``, or ``metadata``,
+ """
+ if filetype is None:
+ return CorpusReader.fileids(self)
+ elif filetype in ('txt', 'wrd', 'phn', 'wav'):
+ return ['%s.%s' % (u, filetype) for u in self._utterances]
+ elif filetype == 'metadata':
+ return ['timitdic.txt', 'spkrinfo.txt']
+ else:
+ raise ValueError('Bad value for filetype: %r' % filetype)
+
+ def utteranceids(self, dialect=None, sex=None, spkrid=None,
+ sent_type=None, sentid=None):
+ """
+ :return: A list of the utterance identifiers for all
+ utterances in this corpus, or for the given speaker, dialect
+ region, gender, sentence type, or sentence number, if
+ specified.
+ """
+ if isinstance(dialect, compat.string_types): dialect = [dialect]
+ if isinstance(sex, compat.string_types): sex = [sex]
+ if isinstance(spkrid, compat.string_types): spkrid = [spkrid]
+ if isinstance(sent_type, compat.string_types): sent_type = [sent_type]
+ if isinstance(sentid, compat.string_types): sentid = [sentid]
+
+ utterances = self._utterances[:]
+ if dialect is not None:
+ utterances = [u for u in utterances if u[2] in dialect]
+ if sex is not None:
+ utterances = [u for u in utterances if u[4] in sex]
+ if spkrid is not None:
+ utterances = [u for u in utterances if u[:9] in spkrid]
+ if sent_type is not None:
+ utterances = [u for u in utterances if u[11] in sent_type]
+ if sentid is not None:
+ utterances = [u for u in utterances if u[10:] in spkrid]
+ return utterances
+
+ def transcription_dict(self):
+ """
+ :return: A dictionary giving the 'standard' transcription for
+ each word.
+ """
+ _transcriptions = {}
+ for line in self.open('timitdic.txt'):
+ if not line.strip() or line[0] == ';': continue
+ m = re.match(r'\s*(\S+)\s+/(.*)/\s*$', line)
+ if not m: raise ValueError('Bad line: %r' % line)
+ _transcriptions[m.group(1)] = m.group(2).split()
+ return _transcriptions
+
+ def spkrid(self, utterance):
+ return utterance.split('/')[0]
+
+ def sentid(self, utterance):
+ return utterance.split('/')[1]
+
+ def utterance(self, spkrid, sentid):
+ return '%s/%s' % (spkrid, sentid)
+
+ def spkrutteranceids(self, speaker):
+ """
+ :return: A list of all utterances associated with a given
+ speaker.
+ """
+ return [utterance for utterance in self._utterances
+ if utterance.startswith(speaker+'/')]
+
+ def spkrinfo(self, speaker):
+ """
+ :return: A dictionary mapping .. something.
+ """
+ if speaker in self._utterances:
+ speaker = self.spkrid(speaker)
+
+ if self._speakerinfo is None:
+ self._speakerinfo = {}
+ for line in self.open('spkrinfo.txt'):
+ if not line.strip() or line[0] == ';': continue
+ rec = line.strip().split(None, 9)
+ key = "dr%s-%s%s" % (rec[2],rec[1].lower(),rec[0].lower())
+ self._speakerinfo[key] = SpeakerInfo(*rec)
+
+ return self._speakerinfo[speaker]
+
+ def phones(self, utterances=None):
+ return [line.split()[-1]
+ for fileid in self._utterance_fileids(utterances, '.phn')
+ for line in self.open(fileid) if line.strip()]
+
+ def phone_times(self, utterances=None):
+ """
+ offset is represented as a number of 16kHz samples!
+ """
+ return [(line.split()[2], int(line.split()[0]), int(line.split()[1]))
+ for fileid in self._utterance_fileids(utterances, '.phn')
+ for line in self.open(fileid) if line.strip()]
+
+ def words(self, utterances=None):
+ return [line.split()[-1]
+ for fileid in self._utterance_fileids(utterances, '.wrd')
+ for line in self.open(fileid) if line.strip()]
+
+ def word_times(self, utterances=None):
+ return [(line.split()[2], int(line.split()[0]), int(line.split()[1]))
+ for fileid in self._utterance_fileids(utterances, '.wrd')
+ for line in self.open(fileid) if line.strip()]
+
+ def sents(self, utterances=None):
+ return [[line.split()[-1]
+ for line in self.open(fileid) if line.strip()]
+ for fileid in self._utterance_fileids(utterances, '.wrd')]
+
+ def sent_times(self, utterances=None):
+ return [(line.split(None,2)[-1].strip(),
+ int(line.split()[0]), int(line.split()[1]))
+ for fileid in self._utterance_fileids(utterances, '.txt')
+ for line in self.open(fileid) if line.strip()]
+
+ def phone_trees(self, utterances=None):
+ if utterances is None: utterances = self._utterances
+ if isinstance(utterances, compat.string_types): utterances = [utterances]
+
+ trees = []
+ for utterance in utterances:
+ word_times = self.word_times(utterance)
+ phone_times = self.phone_times(utterance)
+ sent_times = self.sent_times(utterance)
+
+ while sent_times:
+ (sent, sent_start, sent_end) = sent_times.pop(0)
+ trees.append(Tree('S', []))
+ while (word_times and phone_times and
+ phone_times[0][2] <= word_times[0][1]):
+ trees[-1].append(phone_times.pop(0)[0])
+ while word_times and word_times[0][2] <= sent_end:
+ (word, word_start, word_end) = word_times.pop(0)
+ trees[-1].append(Tree(word, []))
+ while phone_times and phone_times[0][2] <= word_end:
+ trees[-1][-1].append(phone_times.pop(0)[0])
+ while phone_times and phone_times[0][2] <= sent_end:
+ trees[-1].append(phone_times.pop(0)[0])
+ return trees
+
+ # [xx] NOTE: This is currently broken -- we're assuming that the
+ # fileids are WAV fileids (aka RIFF), but they're actually NIST SPHERE
+ # fileids.
+ def wav(self, utterance, start=0, end=None):
+ # nltk.chunk conflicts with the stdlib module 'chunk'
+ wave = import_from_stdlib('wave')
+
+ w = wave.open(self.open(utterance+'.wav'), 'rb')
+
+ if end is None:
+ end = w.getnframes()
+
+ # Skip past frames before start, then read the frames we want
+ w.readframes(start)
+ frames = w.readframes(end-start)
+
+ # Open a new temporary file -- the wave module requires
+ # an actual file, and won't work w/ stringio. :(
+ tf = tempfile.TemporaryFile()
+ out = wave.open(tf, 'w')
+
+ # Write the parameters & data to the new file.
+ out.setparams(w.getparams())
+ out.writeframes(frames)
+ out.close()
+
+ # Read the data back from the file, and return it. The
+ # file will automatically be deleted when we return.
+ tf.seek(0)
+ return tf.read()
+
+ def audiodata(self, utterance, start=0, end=None):
+ assert(end is None or end > start)
+ headersize = 44
+ if end is None:
+ data = self.open(utterance+'.wav').read()
+ else:
+ data = self.open(utterance+'.wav').read(headersize+end*2)
+ return data[headersize+start*2:]
+
+ def _utterance_fileids(self, utterances, extension):
+ if utterances is None: utterances = self._utterances
+ if isinstance(utterances, compat.string_types): utterances = [utterances]
+ return ['%s%s' % (u, extension) for u in utterances]
+
+ def play(self, utterance, start=0, end=None):
+ """
+ Play the given audio sample.
+
+ :param utterance: The utterance id of the sample to play
+ """
+ # Method 1: os audio dev.
+ try:
+ import ossaudiodev
+ try:
+ dsp = ossaudiodev.open('w')
+ dsp.setfmt(ossaudiodev.AFMT_S16_LE)
+ dsp.channels(1)
+ dsp.speed(16000)
+ dsp.write(self.audiodata(utterance, start, end))
+ dsp.close()
+ except IOError as e:
+ print(("can't acquire the audio device; please "
+ "activate your audio device."), file=sys.stderr)
+ print("system error message:", str(e), file=sys.stderr)
+ return
+ except ImportError:
+ pass
+
+ # Method 2: pygame
+ try:
+ # FIXME: this won't work under python 3
+ import pygame.mixer, StringIO
+ pygame.mixer.init(16000)
+ f = StringIO.StringIO(self.wav(utterance, start, end))
+ pygame.mixer.Sound(f).play()
+ while pygame.mixer.get_busy():
+ time.sleep(0.01)
+ return
+ except ImportError:
+ pass
+
+ # Method 3: complain. :)
+ print(("you must install pygame or ossaudiodev "
+ "for audio playback."), file=sys.stderr)
+
+
+ at compat.python_2_unicode_compatible
+class SpeakerInfo(object):
+ def __init__(self, id, sex, dr, use, recdate, birthdate,
+ ht, race, edu, comments=None):
+ self.id = id
+ self.sex = sex
+ self.dr = dr
+ self.use = use
+ self.recdate = recdate
+ self.birthdate = birthdate
+ self.ht = ht
+ self.race = race
+ self.edu = edu
+ self.comments = comments
+
+ def __repr__(self):
+ attribs = 'id sex dr use recdate birthdate ht race edu comments'
+ args = ['%s=%r' % (attr, getattr(self, attr))
+ for attr in attribs.split()]
+ return 'SpeakerInfo(%s)' % (', '.join(args))
+
+
+def read_timit_block(stream):
+ """
+ Block reader for timit tagged sentences, which are preceded by a sentence
+ number that will be ignored.
+ """
+ line = stream.readline()
+ if not line: return []
+ n, sent = line.split(' ', 1)
+ return [sent]
diff --git a/nltk/corpus/reader/toolbox.py b/nltk/corpus/reader/toolbox.py
new file mode 100644
index 0000000..cc8dfad
--- /dev/null
+++ b/nltk/corpus/reader/toolbox.py
@@ -0,0 +1,68 @@
+# Natural Language Toolkit: Toolbox Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Greg Aumann <greg_aumann at sil.org>
+# Stuart Robinson <Stuart.Robinson at mpi.nl>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Module for reading, writing and manipulating
+Toolbox databases and settings fileids.
+"""
+
+import os
+import re
+import codecs
+
+from nltk import compat
+from nltk.toolbox import ToolboxData
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class ToolboxCorpusReader(CorpusReader):
+ def xml(self, fileids, key=None):
+ return concat([ToolboxData(path, enc).parse(key=key)
+ for (path, enc) in self.abspaths(fileids, True)])
+
+ def fields(self, fileids, strip=True, unwrap=True, encoding='utf8',
+ errors='strict', unicode_fields=None):
+ return concat([list(ToolboxData(fileid,enc).fields(
+ strip, unwrap, encoding, errors, unicode_fields))
+ for (fileid, enc)
+ in self.abspaths(fileids, include_encoding=True)])
+
+ # should probably be done lazily:
+ def entries(self, fileids, **kwargs):
+ if 'key' in kwargs:
+ key = kwargs['key']
+ del kwargs['key']
+ else:
+ key = 'lx' # the default key in MDF
+ entries = []
+ for marker, contents in self.fields(fileids, **kwargs):
+ if marker == key:
+ entries.append((contents, []))
+ else:
+ try:
+ entries[-1][-1].append((marker, contents))
+ except IndexError:
+ pass
+ return entries
+
+ def words(self, fileids, key='lx'):
+ return [contents for marker, contents in self.fields(fileids) if marker == key]
+
+ def raw(self, fileids):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+
+def demo():
+ pass
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/corpus/reader/udhr.py b/nltk/corpus/reader/udhr.py
new file mode 100644
index 0000000..523c521
--- /dev/null
+++ b/nltk/corpus/reader/udhr.py
@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+"""
+UDHR corpus reader. It mostly deals with encodings.
+"""
+from __future__ import absolute_import, unicode_literals
+
+from nltk.corpus.reader.util import find_corpus_fileids
+from nltk.corpus.reader.plaintext import PlaintextCorpusReader
+
+class UdhrCorpusReader(PlaintextCorpusReader):
+
+ ENCODINGS = [
+ ('.*-Latin1$', 'latin-1'),
+ ('.*-Hebrew$', 'hebrew'),
+ ('.*-Arabic$', 'cp1256'),
+ ('Czech_Cesky-UTF8', 'cp1250'), # yeah
+ ('.*-Cyrillic$', 'cyrillic'),
+ ('.*-SJIS$', 'SJIS'),
+ ('.*-GB2312$', 'GB2312'),
+ ('.*-Latin2$', 'ISO-8859-2'),
+ ('.*-Greek$', 'greek'),
+ ('.*-UTF8$', 'utf-8'),
+
+ ('Hungarian_Magyar-Unicode', 'utf-16-le'),
+ ('Amahuaca', 'latin1'),
+ ('Turkish_Turkce-Turkish', 'latin5'),
+ ('Lithuanian_Lietuviskai-Baltic', 'latin4'),
+ ('Japanese_Nihongo-EUC', 'EUC-JP'),
+ ('Japanese_Nihongo-JIS', 'iso2022_jp'),
+ ('Chinese_Mandarin-HZ', 'hz'),
+ ('Abkhaz\-Cyrillic\+Abkh', 'cp1251'),
+ ]
+
+ SKIP = set([
+ # The following files are not fully decodable because they
+ # were truncated at wrong bytes:
+ 'Burmese_Myanmar-UTF8',
+ 'Japanese_Nihongo-JIS',
+ 'Chinese_Mandarin-HZ',
+ 'Chinese_Mandarin-UTF8',
+ 'Gujarati-UTF8',
+ 'Hungarian_Magyar-Unicode',
+ 'Lao-UTF8',
+ 'Magahi-UTF8',
+ 'Marathi-UTF8',
+ 'Tamil-UTF8',
+
+ # Unfortunately, encodings required for reading
+ # the following files are not supported by Python:
+ 'Vietnamese-VPS',
+ 'Vietnamese-VIQR',
+ 'Vietnamese-TCVN',
+ 'Magahi-Agra',
+ 'Bhojpuri-Agra',
+ 'Esperanto-T61', # latin3 raises an exception
+
+ # The following files are encoded for specific fonts:
+ 'Burmese_Myanmar-WinResearcher',
+ 'Armenian-DallakHelv',
+ 'Tigrinya_Tigrigna-VG2Main',
+ 'Amharic-Afenegus6..60375', # ?
+ 'Navaho_Dine-Navajo-Navaho-font',
+
+ # What are these?
+ 'Azeri_Azerbaijani_Cyrillic-Az.Times.Cyr.Normal0117',
+ 'Azeri_Azerbaijani_Latin-Az.Times.Lat0117',
+
+ # The following files are unintended:
+ 'Czech-Latin2-err',
+ 'Russian_Russky-UTF8~',
+ ])
+
+
+ def __init__(self, root='udhr'):
+ fileids = find_corpus_fileids(root, r'(?!README|\.).*')
+ super(UdhrCorpusReader, self).__init__(
+ root,
+ [fileid for fileid in fileids if fileid not in self.SKIP],
+ encoding=self.ENCODINGS
+ )
diff --git a/nltk/corpus/reader/util.py b/nltk/corpus/reader/util.py
new file mode 100644
index 0000000..7959794
--- /dev/null
+++ b/nltk/corpus/reader/util.py
@@ -0,0 +1,797 @@
+# Natural Language Toolkit: Corpus Reader Utilities
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import os
+import bisect
+import re
+import tempfile
+from functools import reduce
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+# Use the c version of ElementTree, which is faster, if possible:
+try: from xml.etree import cElementTree as ElementTree
+except ImportError: from xml.etree import ElementTree
+
+from nltk import compat
+from nltk.tokenize import wordpunct_tokenize
+from nltk.internals import slice_bounds
+from nltk.data import PathPointer, FileSystemPathPointer, ZipFilePathPointer
+from nltk.data import SeekableUnicodeStreamReader
+from nltk.util import AbstractLazySequence, LazySubsequence, LazyConcatenation, py25
+
+######################################################################
+#{ Corpus View
+######################################################################
+
+class StreamBackedCorpusView(AbstractLazySequence):
+ """
+ A 'view' of a corpus file, which acts like a sequence of tokens:
+ it can be accessed by index, iterated over, etc. However, the
+ tokens are only constructed as-needed -- the entire corpus is
+ never stored in memory at once.
+
+ The constructor to ``StreamBackedCorpusView`` takes two arguments:
+ a corpus fileid (specified as a string or as a ``PathPointer``);
+ and a block reader. A "block reader" is a function that reads
+ zero or more tokens from a stream, and returns them as a list. A
+ very simple example of a block reader is:
+
+ >>> def simple_block_reader(stream):
+ ... return stream.readline().split()
+
+ This simple block reader reads a single line at a time, and
+ returns a single token (consisting of a string) for each
+ whitespace-separated substring on the line.
+
+ When deciding how to define the block reader for a given
+ corpus, careful consideration should be given to the size of
+ blocks handled by the block reader. Smaller block sizes will
+ increase the memory requirements of the corpus view's internal
+ data structures (by 2 integers per block). On the other hand,
+ larger block sizes may decrease performance for random access to
+ the corpus. (But note that larger block sizes will *not*
+ decrease performance for iteration.)
+
+ Internally, ``CorpusView`` maintains a partial mapping from token
+ index to file position, with one entry per block. When a token
+ with a given index *i* is requested, the ``CorpusView`` constructs
+ it as follows:
+
+ 1. First, it searches the toknum/filepos mapping for the token
+ index closest to (but less than or equal to) *i*.
+
+ 2. Then, starting at the file position corresponding to that
+ index, it reads one block at a time using the block reader
+ until it reaches the requested token.
+
+ The toknum/filepos mapping is created lazily: it is initially
+ empty, but every time a new block is read, the block's
+ initial token is added to the mapping. (Thus, the toknum/filepos
+ map has one entry per block.)
+
+ In order to increase efficiency for random access patterns that
+ have high degrees of locality, the corpus view may cache one or
+ more blocks.
+
+ :note: Each ``CorpusView`` object internally maintains an open file
+ object for its underlying corpus file. This file should be
+ automatically closed when the ``CorpusView`` is garbage collected,
+ but if you wish to close it manually, use the ``close()``
+ method. If you access a ``CorpusView``'s items after it has been
+ closed, the file object will be automatically re-opened.
+
+ :warning: If the contents of the file are modified during the
+ lifetime of the ``CorpusView``, then the ``CorpusView``'s behavior
+ is undefined.
+
+ :warning: If a unicode encoding is specified when constructing a
+ ``CorpusView``, then the block reader may only call
+ ``stream.seek()`` with offsets that have been returned by
+ ``stream.tell()``; in particular, calling ``stream.seek()`` with
+ relative offsets, or with offsets based on string lengths, may
+ lead to incorrect behavior.
+
+ :ivar _block_reader: The function used to read
+ a single block from the underlying file stream.
+ :ivar _toknum: A list containing the token index of each block
+ that has been processed. In particular, ``_toknum[i]`` is the
+ token index of the first token in block ``i``. Together
+ with ``_filepos``, this forms a partial mapping between token
+ indices and file positions.
+ :ivar _filepos: A list containing the file position of each block
+ that has been processed. In particular, ``_toknum[i]`` is the
+ file position of the first character in block ``i``. Together
+ with ``_toknum``, this forms a partial mapping between token
+ indices and file positions.
+ :ivar _stream: The stream used to access the underlying corpus file.
+ :ivar _len: The total number of tokens in the corpus, if known;
+ or None, if the number of tokens is not yet known.
+ :ivar _eofpos: The character position of the last character in the
+ file. This is calculated when the corpus view is initialized,
+ and is used to decide when the end of file has been reached.
+ :ivar _cache: A cache of the most recently read block. It
+ is encoded as a tuple (start_toknum, end_toknum, tokens), where
+ start_toknum is the token index of the first token in the block;
+ end_toknum is the token index of the first token not in the
+ block; and tokens is a list of the tokens in the block.
+ """
+ def __init__(self, fileid, block_reader=None, startpos=0,
+ encoding='utf8'):
+ """
+ Create a new corpus view, based on the file ``fileid``, and
+ read with ``block_reader``. See the class documentation
+ for more information.
+
+ :param fileid: The path to the file that is read by this
+ corpus view. ``fileid`` can either be a string or a
+ ``PathPointer``.
+
+ :param startpos: The file position at which the view will
+ start reading. This can be used to skip over preface
+ sections.
+
+ :param encoding: The unicode encoding that should be used to
+ read the file's contents. If no encoding is specified,
+ then the file's contents will be read as a non-unicode
+ string (i.e., a str).
+ """
+ if block_reader:
+ self.read_block = block_reader
+ # Initialize our toknum/filepos mapping.
+ self._toknum = [0]
+ self._filepos = [startpos]
+ self._encoding = encoding
+ # We don't know our length (number of tokens) yet.
+ self._len = None
+
+ self._fileid = fileid
+ self._stream = None
+
+ self._current_toknum = None
+ """This variable is set to the index of the next token that
+ will be read, immediately before ``self.read_block()`` is
+ called. This is provided for the benefit of the block
+ reader, which under rare circumstances may need to know
+ the current token number."""
+
+ self._current_blocknum = None
+ """This variable is set to the index of the next block that
+ will be read, immediately before ``self.read_block()`` is
+ called. This is provided for the benefit of the block
+ reader, which under rare circumstances may need to know
+ the current block number."""
+
+ # Find the length of the file.
+ try:
+ if isinstance(self._fileid, PathPointer):
+ self._eofpos = self._fileid.file_size()
+ else:
+ self._eofpos = os.stat(self._fileid).st_size
+ except Exception as exc:
+ raise ValueError('Unable to open or access %r -- %s' %
+ (fileid, exc))
+
+ # Maintain a cache of the most recently read block, to
+ # increase efficiency of random access.
+ self._cache = (-1, -1, None)
+
+ fileid = property(lambda self: self._fileid, doc="""
+ The fileid of the file that is accessed by this view.
+
+ :type: str or PathPointer""")
+
+ def read_block(self, stream):
+ """
+ Read a block from the input stream.
+
+ :return: a block of tokens from the input stream
+ :rtype: list(any)
+ :param stream: an input stream
+ :type stream: stream
+ """
+ raise NotImplementedError('Abstract Method')
+
+ def _open(self):
+ """
+ Open the file stream associated with this corpus view. This
+ will be called performed if any value is read from the view
+ while its file stream is closed.
+ """
+ if isinstance(self._fileid, PathPointer):
+ self._stream = self._fileid.open(self._encoding)
+ elif self._encoding:
+ self._stream = SeekableUnicodeStreamReader(
+ open(self._fileid, 'rb'), self._encoding)
+ else:
+ self._stream = open(self._fileid, 'rb')
+
+ def close(self):
+ """
+ Close the file stream associated with this corpus view. This
+ can be useful if you are worried about running out of file
+ handles (although the stream should automatically be closed
+ upon garbage collection of the corpus view). If the corpus
+ view is accessed after it is closed, it will be automatically
+ re-opened.
+ """
+ if self._stream is not None:
+ self._stream.close()
+ self._stream = None
+
+ def __len__(self):
+ if self._len is None:
+ # iterate_from() sets self._len when it reaches the end
+ # of the file:
+ for tok in self.iterate_from(self._toknum[-1]): pass
+ return self._len
+
+ def __getitem__(self, i):
+ if isinstance(i, slice):
+ start, stop = slice_bounds(self, i)
+ # Check if it's in the cache.
+ offset = self._cache[0]
+ if offset <= start and stop <= self._cache[1]:
+ return self._cache[2][start-offset:stop-offset]
+ # Construct & return the result.
+ return LazySubsequence(self, start, stop)
+ else:
+ # Handle negative indices
+ if i < 0: i += len(self)
+ if i < 0: raise IndexError('index out of range')
+ # Check if it's in the cache.
+ offset = self._cache[0]
+ if offset <= i < self._cache[1]:
+ return self._cache[2][i-offset]
+ # Use iterate_from to extract it.
+ try:
+ return next(self.iterate_from(i))
+ except StopIteration:
+ raise IndexError('index out of range')
+
+ # If we wanted to be thread-safe, then this method would need to
+ # do some locking.
+ def iterate_from(self, start_tok):
+ # Start by feeding from the cache, if possible.
+ if self._cache[0] <= start_tok < self._cache[1]:
+ for tok in self._cache[2][start_tok-self._cache[0]:]:
+ yield tok
+ start_tok += 1
+
+ # Decide where in the file we should start. If `start` is in
+ # our mapping, then we can jump straight to the correct block;
+ # otherwise, start at the last block we've processed.
+ if start_tok < self._toknum[-1]:
+ block_index = bisect.bisect_right(self._toknum, start_tok)-1
+ toknum = self._toknum[block_index]
+ filepos = self._filepos[block_index]
+ else:
+ block_index = len(self._toknum)-1
+ toknum = self._toknum[-1]
+ filepos = self._filepos[-1]
+
+ # Open the stream, if it's not open already.
+ if self._stream is None:
+ self._open()
+
+ # Each iteration through this loop, we read a single block
+ # from the stream.
+ while filepos < self._eofpos:
+ # Read the next block.
+ self._stream.seek(filepos)
+ self._current_toknum = toknum
+ self._current_blocknum = block_index
+ tokens = self.read_block(self._stream)
+ assert isinstance(tokens, (tuple, list, AbstractLazySequence)), (
+ 'block reader %s() should return list or tuple.' %
+ self.read_block.__name__)
+ num_toks = len(tokens)
+ new_filepos = self._stream.tell()
+ assert new_filepos > filepos, (
+ 'block reader %s() should consume at least 1 byte (filepos=%d)' %
+ (self.read_block.__name__, filepos))
+
+ # Update our cache.
+ self._cache = (toknum, toknum+num_toks, list(tokens))
+
+ # Update our mapping.
+ assert toknum <= self._toknum[-1]
+ if num_toks > 0:
+ block_index += 1
+ if toknum == self._toknum[-1]:
+ assert new_filepos > self._filepos[-1] # monotonic!
+ self._filepos.append(new_filepos)
+ self._toknum.append(toknum+num_toks)
+ else:
+ # Check for consistency:
+ assert new_filepos == self._filepos[block_index], (
+ 'inconsistent block reader (num chars read)')
+ assert toknum+num_toks == self._toknum[block_index], (
+ 'inconsistent block reader (num tokens returned)')
+
+ # If we reached the end of the file, then update self._len
+ if new_filepos == self._eofpos:
+ self._len = toknum + num_toks
+ # Generate the tokens in this block (but skip any tokens
+ # before start_tok). Note that between yields, our state
+ # may be modified.
+ for tok in tokens[max(0, start_tok-toknum):]:
+ yield tok
+ # If we're at the end of the file, then we're done.
+ assert new_filepos <= self._eofpos
+ if new_filepos == self._eofpos:
+ break
+ # Update our indices
+ toknum += num_toks
+ filepos = new_filepos
+
+ # If we reach this point, then we should know our length.
+ assert self._len is not None
+
+ # Use concat for these, so we can use a ConcatenatedCorpusView
+ # when possible.
+ def __add__(self, other):
+ return concat([self, other])
+ def __radd__(self, other):
+ return concat([other, self])
+ def __mul__(self, count):
+ return concat([self] * count)
+ def __rmul__(self, count):
+ return concat([self] * count)
+
+class ConcatenatedCorpusView(AbstractLazySequence):
+ """
+ A 'view' of a corpus file that joins together one or more
+ ``StreamBackedCorpusViews<StreamBackedCorpusView>``. At most
+ one file handle is left open at any time.
+ """
+ def __init__(self, corpus_views):
+ self._pieces = corpus_views
+ """A list of the corpus subviews that make up this
+ concatenation."""
+
+ self._offsets = [0]
+ """A list of offsets, indicating the index at which each
+ subview begins. In particular::
+ offsets[i] = sum([len(p) for p in pieces[:i]])"""
+
+ self._open_piece = None
+ """The most recently accessed corpus subview (or None).
+ Before a new subview is accessed, this subview will be closed."""
+
+ def __len__(self):
+ if len(self._offsets) <= len(self._pieces):
+ # Iterate to the end of the corpus.
+ for tok in self.iterate_from(self._offsets[-1]): pass
+
+ return self._offsets[-1]
+
+ def close(self):
+ for piece in self._pieces:
+ piece.close()
+
+ def iterate_from(self, start_tok):
+ piecenum = bisect.bisect_right(self._offsets, start_tok)-1
+
+ while piecenum < len(self._pieces):
+ offset = self._offsets[piecenum]
+ piece = self._pieces[piecenum]
+
+ # If we've got another piece open, close it first.
+ if self._open_piece is not piece:
+ if self._open_piece is not None:
+ self._open_piece.close()
+ self._open_piece = piece
+
+ # Get everything we can from this piece.
+ for tok in piece.iterate_from(max(0, start_tok-offset)):
+ yield tok
+
+ # Update the offset table.
+ if piecenum+1 == len(self._offsets):
+ self._offsets.append(self._offsets[-1] + len(piece))
+
+ # Move on to the next piece.
+ piecenum += 1
+
+def concat(docs):
+ """
+ Concatenate together the contents of multiple documents from a
+ single corpus, using an appropriate concatenation function. This
+ utility function is used by corpus readers when the user requests
+ more than one document at a time.
+ """
+ if len(docs) == 1:
+ return docs[0]
+ if len(docs) == 0:
+ raise ValueError('concat() expects at least one object!')
+
+ types = set(d.__class__ for d in docs)
+
+ # If they're all strings, use string concatenation.
+ if all(isinstance(doc, compat.string_types) for doc in docs):
+ return ''.join(docs)
+
+ # If they're all corpus views, then use ConcatenatedCorpusView.
+ for typ in types:
+ if not issubclass(typ, (StreamBackedCorpusView,
+ ConcatenatedCorpusView)):
+ break
+ else:
+ return ConcatenatedCorpusView(docs)
+
+ # If they're all lazy sequences, use a lazy concatenation
+ for typ in types:
+ if not issubclass(typ, AbstractLazySequence):
+ break
+ else:
+ return LazyConcatenation(docs)
+
+ # Otherwise, see what we can do:
+ if len(types) == 1:
+ typ = list(types)[0]
+
+ if issubclass(typ, list):
+ return reduce((lambda a,b:a+b), docs, [])
+
+ if issubclass(typ, tuple):
+ return reduce((lambda a,b:a+b), docs, ())
+
+ if ElementTree.iselement(typ):
+ xmltree = ElementTree.Element('documents')
+ for doc in docs: xmltree.append(doc)
+ return xmltree
+
+ # No method found!
+ raise ValueError("Don't know how to concatenate types: %r" % types)
+
+######################################################################
+#{ Corpus View for Pickled Sequences
+######################################################################
+
+class PickleCorpusView(StreamBackedCorpusView):
+ """
+ A stream backed corpus view for corpus files that consist of
+ sequences of serialized Python objects (serialized using
+ ``pickle.dump``). One use case for this class is to store the
+ result of running feature detection on a corpus to disk. This can
+ be useful when performing feature detection is expensive (so we
+ don't want to repeat it); but the corpus is too large to store in
+ memory. The following example illustrates this technique:
+
+ >>> from nltk.corpus.reader.util import PickleCorpusView
+ >>> from nltk.util import LazyMap
+ >>> feature_corpus = LazyMap(detect_features, corpus) # doctest: +SKIP
+ >>> PickleCorpusView.write(feature_corpus, some_fileid) # doctest: +SKIP
+ >>> pcv = PickleCorpusView(some_fileid) # doctest: +SKIP
+ """
+ BLOCK_SIZE = 100
+ PROTOCOL = -1
+
+ def __init__(self, fileid, delete_on_gc=False):
+ """
+ Create a new corpus view that reads the pickle corpus
+ ``fileid``.
+
+ :param delete_on_gc: If true, then ``fileid`` will be deleted
+ whenever this object gets garbage-collected.
+ """
+ self._delete_on_gc = delete_on_gc
+ StreamBackedCorpusView.__init__(self, fileid)
+
+ def read_block(self, stream):
+ result = []
+ for i in range(self.BLOCK_SIZE):
+ try: result.append(pickle.load(stream))
+ except EOFError: break
+ return result
+
+ def __del__(self):
+ """
+ If ``delete_on_gc`` was set to true when this
+ ``PickleCorpusView`` was created, then delete the corpus view's
+ fileid. (This method is called whenever a
+ ``PickledCorpusView`` is garbage-collected.
+ """
+ if getattr(self, '_delete_on_gc'):
+ if os.path.exists(self._fileid):
+ try: os.remove(self._fileid)
+ except (OSError, IOError): pass
+ self.__dict__.clear() # make the garbage collector's job easier
+
+ @classmethod
+ def write(cls, sequence, output_file):
+ if isinstance(output_file, compat.string_types):
+ output_file = open(output_file, 'wb')
+ for item in sequence:
+ pickle.dump(item, output_file, cls.PROTOCOL)
+
+ @classmethod
+ def cache_to_tempfile(cls, sequence, delete_on_gc=True):
+ """
+ Write the given sequence to a temporary file as a pickle
+ corpus; and then return a ``PickleCorpusView`` view for that
+ temporary corpus file.
+
+ :param delete_on_gc: If true, then the temporary file will be
+ deleted whenever this object gets garbage-collected.
+ """
+ try:
+ fd, output_file_name = tempfile.mkstemp('.pcv', 'nltk-')
+ output_file = os.fdopen(fd, 'wb')
+ cls.write(sequence, output_file)
+ output_file.close()
+ return PickleCorpusView(output_file_name, delete_on_gc)
+ except (OSError, IOError) as e:
+ raise ValueError('Error while creating temp file: %s' % e)
+
+
+
+######################################################################
+#{ Block Readers
+######################################################################
+
+def read_whitespace_block(stream):
+ toks = []
+ for i in range(20): # Read 20 lines at a time.
+ toks.extend(stream.readline().split())
+ return toks
+
+def read_wordpunct_block(stream):
+ toks = []
+ for i in range(20): # Read 20 lines at a time.
+ toks.extend(wordpunct_tokenize(stream.readline()))
+ return toks
+
+def read_line_block(stream):
+ toks = []
+ for i in range(20):
+ line = stream.readline()
+ if not line: return toks
+ toks.append(line.rstrip('\n'))
+ return toks
+
+def read_blankline_block(stream):
+ s = ''
+ while True:
+ line = stream.readline()
+ # End of file:
+ if not line:
+ if s: return [s]
+ else: return []
+ # Blank line:
+ elif line and not line.strip():
+ if s: return [s]
+ # Other line:
+ else:
+ s += line
+
+def read_alignedsent_block(stream):
+ s = ''
+ while True:
+ line = stream.readline()
+ if line[0] == '=' or line[0] == '\n' or line[:2] == '\r\n':
+ continue
+ # End of file:
+ if not line:
+ if s: return [s]
+ else: return []
+ # Other line:
+ else:
+ s += line
+ if re.match('^\d+-\d+', line) is not None:
+ return [s]
+
+def read_regexp_block(stream, start_re, end_re=None):
+ """
+ Read a sequence of tokens from a stream, where tokens begin with
+ lines that match ``start_re``. If ``end_re`` is specified, then
+ tokens end with lines that match ``end_re``; otherwise, tokens end
+ whenever the next line matching ``start_re`` or EOF is found.
+ """
+ # Scan until we find a line matching the start regexp.
+ while True:
+ line = stream.readline()
+ if not line: return [] # end of file.
+ if re.match(start_re, line): break
+
+ # Scan until we find another line matching the regexp, or EOF.
+ lines = [line]
+ while True:
+ oldpos = stream.tell()
+ line = stream.readline()
+ # End of file:
+ if not line:
+ return [''.join(lines)]
+ # End of token:
+ if end_re is not None and re.match(end_re, line):
+ return [''.join(lines)]
+ # Start of new token: backup to just before it starts, and
+ # return the token we've already collected.
+ if end_re is None and re.match(start_re, line):
+ stream.seek(oldpos)
+ return [''.join(lines)]
+ # Anything else is part of the token.
+ lines.append(line)
+
+def read_sexpr_block(stream, block_size=16384, comment_char=None):
+ """
+ Read a sequence of s-expressions from the stream, and leave the
+ stream's file position at the end the last complete s-expression
+ read. This function will always return at least one s-expression,
+ unless there are no more s-expressions in the file.
+
+ If the file ends in in the middle of an s-expression, then that
+ incomplete s-expression is returned when the end of the file is
+ reached.
+
+ :param block_size: The default block size for reading. If an
+ s-expression is longer than one block, then more than one
+ block will be read.
+ :param comment_char: A character that marks comments. Any lines
+ that begin with this character will be stripped out.
+ (If spaces or tabs precede the comment character, then the
+ line will not be stripped.)
+ """
+ start = stream.tell()
+ block = stream.read(block_size)
+ encoding = getattr(stream, 'encoding', None)
+ assert encoding is not None or isinstance(block, compat.text_type)
+ if encoding not in (None, 'utf-8'):
+ import warnings
+ warnings.warn('Parsing may fail, depending on the properties '
+ 'of the %s encoding!' % encoding)
+ # (e.g., the utf-16 encoding does not work because it insists
+ # on adding BOMs to the beginning of encoded strings.)
+
+ if comment_char:
+ COMMENT = re.compile('(?m)^%s.*$' % re.escape(comment_char))
+ while True:
+ try:
+ # If we're stripping comments, then make sure our block ends
+ # on a line boundary; and then replace any comments with
+ # space characters. (We can't just strip them out -- that
+ # would make our offset wrong.)
+ if comment_char:
+ block += stream.readline()
+ block = re.sub(COMMENT, _sub_space, block)
+ # Read the block.
+ tokens, offset = _parse_sexpr_block(block)
+ # Skip whitespace
+ offset = re.compile(r'\s*').search(block, offset).end()
+
+ # Move to the end position.
+ if encoding is None:
+ stream.seek(start+offset)
+ else:
+ stream.seek(start+len(block[:offset].encode(encoding)))
+
+ # Return the list of tokens we processed
+ return tokens
+ except ValueError as e:
+ if e.args[0] == 'Block too small':
+ next_block = stream.read(block_size)
+ if next_block:
+ block += next_block
+ continue
+ else:
+ # The file ended mid-sexpr -- return what we got.
+ return [block.strip()]
+ else: raise
+
+def _sub_space(m):
+ """Helper function: given a regexp match, return a string of
+ spaces that's the same length as the matched string."""
+ return ' '*(m.end()-m.start())
+
+def _parse_sexpr_block(block):
+ tokens = []
+ start = end = 0
+
+ while end < len(block):
+ m = re.compile(r'\S').search(block, end)
+ if not m:
+ return tokens, end
+
+ start = m.start()
+
+ # Case 1: sexpr is not parenthesized.
+ if m.group() != '(':
+ m2 = re.compile(r'[\s(]').search(block, start)
+ if m2:
+ end = m2.start()
+ else:
+ if tokens: return tokens, end
+ raise ValueError('Block too small')
+
+ # Case 2: parenthesized sexpr.
+ else:
+ nesting = 0
+ for m in re.compile(r'[()]').finditer(block, start):
+ if m.group()=='(': nesting += 1
+ else: nesting -= 1
+ if nesting == 0:
+ end = m.end()
+ break
+ else:
+ if tokens: return tokens, end
+ raise ValueError('Block too small')
+
+ tokens.append(block[start:end])
+
+ return tokens, end
+
+
+######################################################################
+#{ Finding Corpus Items
+######################################################################
+
+def find_corpus_fileids(root, regexp):
+ if not isinstance(root, PathPointer):
+ raise TypeError('find_corpus_fileids: expected a PathPointer')
+ regexp += '$'
+
+ # Find fileids in a zipfile: scan the zipfile's namelist. Filter
+ # out entries that end in '/' -- they're directories.
+ if isinstance(root, ZipFilePathPointer):
+ fileids = [name[len(root.entry):] for name in root.zipfile.namelist()
+ if not name.endswith('/')]
+ items = [name for name in fileids if re.match(regexp, name)]
+ return sorted(items)
+
+ # Find fileids in a directory: use os.walk to search all (proper
+ # or symlinked) subdirectories, and match paths against the regexp.
+ elif isinstance(root, FileSystemPathPointer):
+ items = []
+ # workaround for py25 which doesn't support followlinks
+ kwargs = {}
+ if not py25():
+ kwargs = {'followlinks': True}
+ for dirname, subdirs, fileids in os.walk(root.path, **kwargs):
+ prefix = ''.join('%s/' % p for p in _path_from(root.path, dirname))
+ items += [prefix+fileid for fileid in fileids
+ if re.match(regexp, prefix+fileid)]
+ # Don't visit svn directories:
+ if '.svn' in subdirs: subdirs.remove('.svn')
+ return sorted(items)
+
+ else:
+ raise AssertionError("Don't know how to handle %r" % root)
+
+def _path_from(parent, child):
+ if os.path.split(parent)[1] == '':
+ parent = os.path.split(parent)[0]
+ path = []
+ while parent != child:
+ child, dirname = os.path.split(child)
+ path.insert(0, dirname)
+ assert os.path.split(child)[0] != child
+ return path
+
+######################################################################
+#{ Paragraph structure in Treebank files
+######################################################################
+
+def tagged_treebank_para_block_reader(stream):
+ # Read the next paragraph.
+ para = ''
+ while True:
+ line = stream.readline()
+ # End of paragraph:
+ if re.match('======+\s*$', line):
+ if para.strip(): return [para]
+ # End of file:
+ elif line == '':
+ if para.strip(): return [para]
+ else: return []
+ # Content line:
+ else:
+ para += line
+
diff --git a/nltk/corpus/reader/verbnet.py b/nltk/corpus/reader/verbnet.py
new file mode 100644
index 0000000..2f9ae93
--- /dev/null
+++ b/nltk/corpus/reader/verbnet.py
@@ -0,0 +1,396 @@
+# Natural Language Toolkit: Verbnet Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+An NLTK interface to the VerbNet verb lexicon
+
+For details about VerbNet see:
+http://verbs.colorado.edu/~mpalmer/projects/verbnet.html
+"""
+from __future__ import unicode_literals
+
+import re
+import textwrap
+from collections import defaultdict
+
+from nltk import compat
+from nltk.corpus.reader.xmldocs import XMLCorpusReader
+
+class VerbnetCorpusReader(XMLCorpusReader):
+
+ # No unicode encoding param, since the data files are all XML.
+ def __init__(self, root, fileids, wrap_etree=False):
+ XMLCorpusReader.__init__(self, root, fileids, wrap_etree)
+
+ self._lemma_to_class = defaultdict(list)
+ """A dictionary mapping from verb lemma strings to lists of
+ verbnet class identifiers."""
+
+ self._wordnet_to_class = defaultdict(list)
+ """A dictionary mapping from wordnet identifier strings to
+ lists of verbnet class identifiers."""
+
+ self._class_to_fileid = {}
+ """A dictionary mapping from class identifiers to
+ corresponding file identifiers. The keys of this dictionary
+ provide a complete list of all classes and subclasses."""
+
+ self._shortid_to_longid = {}
+
+ # Initialize the dictionaries. Use the quick (regexp-based)
+ # method instead of the slow (xml-based) method, because it
+ # runs 2-30 times faster.
+ self._quick_index()
+
+ _LONGID_RE = re.compile(r'([^\-\.]*)-([\d+.\-]+)$')
+ """Regular expression that matches (and decomposes) longids"""
+
+ _SHORTID_RE = re.compile(r'[\d+.\-]+$')
+ """Regular expression that matches shortids"""
+
+ _INDEX_RE = re.compile(r'<MEMBER name="\??([^"]+)" wn="([^"]*)"[^>]+>|'
+ r'<VNSUBCLASS ID="([^"]+)"/?>')
+ """Regular expression used by ``_index()`` to quickly scan the corpus
+ for basic information."""
+
+ def lemmas(self, classid=None):
+ """
+ Return a list of all verb lemmas that appear in any class, or
+ in the ``classid`` if specified.
+ """
+ if classid is None:
+ return sorted(self._lemma_to_class.keys())
+ else:
+ # [xx] should this include subclass members?
+ vnclass = self.vnclass(classid)
+ return [member.get('name') for member in
+ vnclass.findall('MEMBERS/MEMBER')]
+
+ def wordnetids(self, classid=None):
+ """
+ Return a list of all wordnet identifiers that appear in any
+ class, or in ``classid`` if specified.
+ """
+ if classid is None:
+ return sorted(self._wordnet_to_class.keys())
+ else:
+ # [xx] should this include subclass members?
+ vnclass = self.vnclass(classid)
+ return sum([member.get('wn','').split() for member in
+ vnclass.findall('MEMBERS/MEMBER')], [])
+
+ def classids(self, lemma=None, wordnetid=None, fileid=None, classid=None):
+ """
+ Return a list of the verbnet class identifiers. If a file
+ identifier is specified, then return only the verbnet class
+ identifiers for classes (and subclasses) defined by that file.
+ If a lemma is specified, then return only verbnet class
+ identifiers for classes that contain that lemma as a member.
+ If a wordnetid is specified, then return only identifiers for
+ classes that contain that wordnetid as a member. If a classid
+ is specified, then return only identifiers for subclasses of
+ the specified verbnet class.
+ """
+ if len([x for x in [lemma, wordnetid, fileid, classid]
+ if x is not None]) > 1:
+ raise ValueError('Specify at most one of: fileid, wordnetid, '
+ 'fileid, classid')
+ if fileid is not None:
+ return [c for (c,f) in self._class_to_fileid.items()
+ if f == fileid]
+ elif lemma is not None:
+ return self._lemma_to_class[lemma]
+ elif wordnetid is not None:
+ return self._wordnet_to_class[wordnetid]
+ elif classid is not None:
+ xmltree = self.vnclass(classid)
+ return [subclass.get('ID') for subclass in
+ xmltree.findall('SUBCLASSES/VNSUBCLASS')]
+ else:
+ return sorted(self._class_to_fileid.keys())
+
+ def vnclass(self, fileid_or_classid):
+ """
+ Return an ElementTree containing the xml for the specified
+ verbnet class.
+
+ :param fileid_or_classid: An identifier specifying which class
+ should be returned. Can be a file identifier (such as
+ ``'put-9.1.xml'``), or a verbnet class identifier (such as
+ ``'put-9.1'``) or a short verbnet class identifier (such as
+ ``'9.1'``).
+ """
+ # File identifier: just return the xml.
+ if fileid_or_classid in self._fileids:
+ return self.xml(fileid_or_classid)
+
+ # Class identifier: get the xml, and find the right elt.
+ classid = self.longid(fileid_or_classid)
+ if classid in self._class_to_fileid:
+ fileid = self._class_to_fileid[self.longid(classid)]
+ tree = self.xml(fileid)
+ if classid == tree.get('ID'):
+ return tree
+ else:
+ for subclass in tree.findall('.//VNSUBCLASS'):
+ if classid == subclass.get('ID'):
+ return subclass
+ else:
+ assert False # we saw it during _index()!
+
+ else:
+ raise ValueError('Unknown identifier %s' % fileid_or_classid)
+
+ def fileids(self, vnclass_ids=None):
+ """
+ Return a list of fileids that make up this corpus. If
+ ``vnclass_ids`` is specified, then return the fileids that make
+ up the specified verbnet class(es).
+ """
+ if vnclass_ids is None:
+ return self._fileids
+ elif isinstance(vnclass_ids, compat.string_types):
+ return [self._class_to_fileid[self.longid(vnclass_ids)]]
+ else:
+ return [self._class_to_fileid[self.longid(vnclass_id)]
+ for vnclass_id in vnclass_ids]
+
+
+ ######################################################################
+ #{ Index Initialization
+ ######################################################################
+
+ def _index(self):
+ """
+ Initialize the indexes ``_lemma_to_class``,
+ ``_wordnet_to_class``, and ``_class_to_fileid`` by scanning
+ through the corpus fileids. This is fast with cElementTree
+ (<0.1 secs), but quite slow (>10 secs) with the python
+ implementation of ElementTree.
+ """
+ for fileid in self._fileids:
+ self._index_helper(self.xml(fileid), fileid)
+
+ def _index_helper(self, xmltree, fileid):
+ """Helper for ``_index()``"""
+ vnclass = xmltree.get('ID')
+ self._class_to_fileid[vnclass] = fileid
+ self._shortid_to_longid[self.shortid(vnclass)] = vnclass
+ for member in xmltree.findall('MEMBERS/MEMBER'):
+ self._lemma_to_class[member.get('name')].append(vnclass)
+ for wn in member.get('wn', '').split():
+ self._wordnet_to_class[wn].append(vnclass)
+ for subclass in xmltree.findall('SUBCLASSES/VNSUBCLASS'):
+ self._index_helper(subclass, fileid)
+
+ def _quick_index(self):
+ """
+ Initialize the indexes ``_lemma_to_class``,
+ ``_wordnet_to_class``, and ``_class_to_fileid`` by scanning
+ through the corpus fileids. This doesn't do proper xml parsing,
+ but is good enough to find everything in the standard verbnet
+ corpus -- and it runs about 30 times faster than xml parsing
+ (with the python ElementTree; only 2-3 times faster with
+ cElementTree).
+ """
+ # nb: if we got rid of wordnet_to_class, this would run 2-3
+ # times faster.
+ for fileid in self._fileids:
+ vnclass = fileid[:-4] # strip the '.xml'
+ self._class_to_fileid[vnclass] = fileid
+ self._shortid_to_longid[self.shortid(vnclass)] = vnclass
+ for m in self._INDEX_RE.finditer(self.open(fileid).read()):
+ groups = m.groups()
+ if groups[0] is not None:
+ self._lemma_to_class[groups[0]].append(vnclass)
+ for wn in groups[1].split():
+ self._wordnet_to_class[wn].append(vnclass)
+ elif groups[2] is not None:
+ self._class_to_fileid[groups[2]] = fileid
+ vnclass = groups[2] # for <MEMBER> elts.
+ self._shortid_to_longid[self.shortid(vnclass)] = vnclass
+ else:
+ assert False, 'unexpected match condition'
+
+ ######################################################################
+ #{ Identifier conversion
+ ######################################################################
+
+ def longid(self, shortid):
+ """Given a short verbnet class identifier (eg '37.10'), map it
+ to a long id (eg 'confess-37.10'). If ``shortid`` is already a
+ long id, then return it as-is"""
+ if self._LONGID_RE.match(shortid):
+ return shortid # it's already a longid.
+ elif not self._SHORTID_RE.match(shortid):
+ raise ValueError('vnclass identifier %r not found' % shortid)
+ try:
+ return self._shortid_to_longid[shortid]
+ except KeyError:
+ raise ValueError('vnclass identifier %r not found' % shortid)
+
+ def shortid(self, longid):
+ """Given a long verbnet class identifier (eg 'confess-37.10'),
+ map it to a short id (eg '37.10'). If ``longid`` is already a
+ short id, then return it as-is."""
+ if self._SHORTID_RE.match(longid):
+ return longid # it's already a shortid.
+ m = self._LONGID_RE.match(longid)
+ if m:
+ return m.group(2)
+ else:
+ raise ValueError('vnclass identifier %r not found' % longid)
+
+ ######################################################################
+ #{ Pretty Printing
+ ######################################################################
+
+ def pprint(self, vnclass):
+ """
+ Return a string containing a pretty-printed representation of
+ the given verbnet class.
+
+ :param vnclass: A verbnet class identifier; or an ElementTree
+ containing the xml contents of a verbnet class.
+ """
+ if isinstance(vnclass, compat.string_types):
+ vnclass = self.vnclass(vnclass)
+
+ s = vnclass.get('ID') + '\n'
+ s += self.pprint_subclasses(vnclass, indent=' ') + '\n'
+ s += self.pprint_members(vnclass, indent=' ') + '\n'
+ s += ' Thematic roles:\n'
+ s += self.pprint_themroles(vnclass, indent=' ') + '\n'
+ s += ' Frames:\n'
+ s += '\n'.join(self.pprint_frame(vnframe, indent=' ')
+ for vnframe in vnclass.findall('FRAMES/FRAME'))
+ return s
+
+ def pprint_subclasses(self, vnclass, indent=''):
+ """
+ Return a string containing a pretty-printed representation of
+ the given verbnet class's subclasses.
+
+ :param vnclass: A verbnet class identifier; or an ElementTree
+ containing the xml contents of a verbnet class.
+ """
+ if isinstance(vnclass, compat.string_types):
+ vnclass = self.vnclass(vnclass)
+
+ subclasses = [subclass.get('ID') for subclass in
+ vnclass.findall('SUBCLASSES/VNSUBCLASS')]
+ if not subclasses: subclasses = ['(none)']
+ s = 'Subclasses: ' + ' '.join(subclasses)
+ return textwrap.fill(s, 70, initial_indent=indent,
+ subsequent_indent=indent+' ')
+
+ def pprint_members(self, vnclass, indent=''):
+ """
+ Return a string containing a pretty-printed representation of
+ the given verbnet class's member verbs.
+
+ :param vnclass: A verbnet class identifier; or an ElementTree
+ containing the xml contents of a verbnet class.
+ """
+ if isinstance(vnclass, compat.string_types):
+ vnclass = self.vnclass(vnclass)
+
+ members = [member.get('name') for member in
+ vnclass.findall('MEMBERS/MEMBER')]
+ if not members: members = ['(none)']
+ s = 'Members: ' + ' '.join(members)
+ return textwrap.fill(s, 70, initial_indent=indent,
+ subsequent_indent=indent+' ')
+
+ def pprint_themroles(self, vnclass, indent=''):
+ """
+ Return a string containing a pretty-printed representation of
+ the given verbnet class's thematic roles.
+
+ :param vnclass: A verbnet class identifier; or an ElementTree
+ containing the xml contents of a verbnet class.
+ """
+ if isinstance(vnclass, compat.string_types):
+ vnclass = self.vnclass(vnclass)
+
+ pieces = []
+ for themrole in vnclass.findall('THEMROLES/THEMROLE'):
+ piece = indent + '* ' + themrole.get('type')
+ modifiers = ['%(Value)s%(type)s' % restr.attrib
+ for restr in themrole.findall('SELRESTRS/SELRESTR')]
+ if modifiers:
+ piece += '[%s]' % ' '.join(modifiers)
+ pieces.append(piece)
+
+ return '\n'.join(pieces)
+
+ def pprint_frame(self, vnframe, indent=''):
+ """
+ Return a string containing a pretty-printed representation of
+ the given verbnet frame.
+
+ :param vnframe: An ElementTree containing the xml contents of
+ a verbnet frame.
+ """
+ s = self.pprint_description(vnframe, indent) + '\n'
+ s += self.pprint_syntax(vnframe, indent+' Syntax: ') + '\n'
+ s += indent + ' Semantics:\n'
+ s += self.pprint_semantics(vnframe, indent+' ')
+ return s
+
+ def pprint_description(self, vnframe, indent=''):
+ """
+ Return a string containing a pretty-printed representation of
+ the given verbnet frame description.
+
+ :param vnframe: An ElementTree containing the xml contents of
+ a verbnet frame.
+ """
+ descr = vnframe.find('DESCRIPTION')
+ s = indent + descr.attrib['primary']
+ if descr.get('secondary', ''):
+ s += ' (%s)' % descr.get('secondary')
+ return s
+
+ def pprint_syntax(self, vnframe, indent=''):
+ """
+ Return a string containing a pretty-printed representation of
+ the given verbnet frame syntax.
+
+ :param vnframe: An ElementTree containing the xml contents of
+ a verbnet frame.
+ """
+ pieces = []
+ for elt in vnframe.find('SYNTAX'):
+ piece = elt.tag
+ modifiers = []
+ if 'value' in elt.attrib:
+ modifiers.append(elt.get('value'))
+ modifiers += ['%(Value)s%(type)s' % restr.attrib
+ for restr in (elt.findall('SELRESTRS/SELRESTR') +
+ elt.findall('SYNRESTRS/SYNRESTR'))]
+ if modifiers:
+ piece += '[%s]' % ' '.join(modifiers)
+ pieces.append(piece)
+
+ return indent + ' '.join(pieces)
+
+ def pprint_semantics(self, vnframe, indent=''):
+ """
+ Return a string containing a pretty-printed representation of
+ the given verbnet frame semantics.
+
+ :param vnframe: An ElementTree containing the xml contents of
+ a verbnet frame.
+ """
+ pieces = []
+ for pred in vnframe.findall('SEMANTICS/PRED'):
+ args = [arg.get('value') for arg in pred.findall('ARGS/ARG')]
+ pieces.append('%s(%s)' % (pred.get('value'), ', '.join(args)))
+ return '\n'.join('%s* %s' % (indent, piece) for piece in pieces)
+
+
diff --git a/nltk/corpus/reader/wordlist.py b/nltk/corpus/reader/wordlist.py
new file mode 100644
index 0000000..b658697
--- /dev/null
+++ b/nltk/corpus/reader/wordlist.py
@@ -0,0 +1,37 @@
+# Natural Language Toolkit: Word List Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk import compat
+from nltk.tokenize import line_tokenize
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class WordListCorpusReader(CorpusReader):
+ """
+ List of words, one per line. Blank lines are ignored.
+ """
+ def words(self, fileids=None):
+ return line_tokenize(self.raw(fileids))
+
+ def raw(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+
+class SwadeshCorpusReader(WordListCorpusReader):
+ def entries(self, fileids=None):
+ """
+ :return: a tuple of words for the specified fileids.
+ """
+ if not fileids:
+ fileids = self.fileids()
+
+ wordlists = [self.words(f) for f in fileids]
+ return list(zip(*wordlists))
diff --git a/nltk/corpus/reader/wordnet.py b/nltk/corpus/reader/wordnet.py
new file mode 100644
index 0000000..f153a62
--- /dev/null
+++ b/nltk/corpus/reader/wordnet.py
@@ -0,0 +1,1960 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: WordNet
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bethard <Steven.Bethard at colorado.edu>
+# Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# Nitin Madnani <nmadnani at ets.org>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+An NLTK interface for WordNet
+
+WordNet is a lexical database of English.
+Using synsets, helps find conceptual relationships between words
+such as hypernyms, hyponyms, synonyms, antonyms etc.
+
+For details about WordNet see:
+http://wordnet.princeton.edu/
+"""
+
+from __future__ import print_function, unicode_literals
+
+import math
+import re
+from itertools import islice, chain
+from operator import itemgetter, attrgetter
+from collections import defaultdict
+
+from nltk.corpus.reader import CorpusReader
+from nltk.util import binary_search_file as _binary_search_file
+from nltk.probability import FreqDist
+from nltk.compat import xrange, python_2_unicode_compatible, total_ordering
+
+######################################################################
+## Table of Contents
+######################################################################
+## - Constants
+## - Data Classes
+## - WordNetError
+## - Lemma
+## - Synset
+## - WordNet Corpus Reader
+## - WordNet Information Content Corpus Reader
+## - Similarity Metrics
+## - Demo
+
+######################################################################
+## Constants
+######################################################################
+
+#: Positive infinity (for similarity functions)
+_INF = 1e300
+
+#{ Part-of-speech constants
+ADJ, ADJ_SAT, ADV, NOUN, VERB = 'a', 's', 'r', 'n', 'v'
+#}
+
+POS_LIST = [NOUN, VERB, ADJ, ADV]
+
+#: A table of strings that are used to express verb frames.
+VERB_FRAME_STRINGS = (
+ None,
+ "Something %s",
+ "Somebody %s",
+ "It is %sing",
+ "Something is %sing PP",
+ "Something %s something Adjective/Noun",
+ "Something %s Adjective/Noun",
+ "Somebody %s Adjective",
+ "Somebody %s something",
+ "Somebody %s somebody",
+ "Something %s somebody",
+ "Something %s something",
+ "Something %s to somebody",
+ "Somebody %s on something",
+ "Somebody %s somebody something",
+ "Somebody %s something to somebody",
+ "Somebody %s something from somebody",
+ "Somebody %s somebody with something",
+ "Somebody %s somebody of something",
+ "Somebody %s something on somebody",
+ "Somebody %s somebody PP",
+ "Somebody %s something PP",
+ "Somebody %s PP",
+ "Somebody's (body part) %s",
+ "Somebody %s somebody to INFINITIVE",
+ "Somebody %s somebody INFINITIVE",
+ "Somebody %s that CLAUSE",
+ "Somebody %s to somebody",
+ "Somebody %s to INFINITIVE",
+ "Somebody %s whether INFINITIVE",
+ "Somebody %s somebody into V-ing something",
+ "Somebody %s something with something",
+ "Somebody %s INFINITIVE",
+ "Somebody %s VERB-ing",
+ "It %s that CLAUSE",
+ "Something %s INFINITIVE")
+
+SENSENUM_RE = re.compile(r'\.\d\d\.')
+
+######################################################################
+## Data Classes
+######################################################################
+
+class WordNetError(Exception):
+ """An exception class for wordnet-related errors."""
+
+
+ at total_ordering
+class _WordNetObject(object):
+ """A common base class for lemmas and synsets."""
+
+ def hypernyms(self):
+ return self._related('@')
+
+ def instance_hypernyms(self):
+ return self._related('@i')
+
+ def hyponyms(self):
+ return self._related('~')
+
+ def instance_hyponyms(self):
+ return self._related('~i')
+
+ def member_holonyms(self):
+ return self._related('#m')
+
+ def substance_holonyms(self):
+ return self._related('#s')
+
+ def part_holonyms(self):
+ return self._related('#p')
+
+ def member_meronyms(self):
+ return self._related('%m')
+
+ def substance_meronyms(self):
+ return self._related('%s')
+
+ def part_meronyms(self):
+ return self._related('%p')
+
+ def topic_domains(self):
+ return self._related(';c')
+
+ def region_domains(self):
+ return self._related(';r')
+
+ def usage_domains(self):
+ return self._related(';u')
+
+ def attributes(self):
+ return self._related('=')
+
+ def entailments(self):
+ return self._related('*')
+
+ def causes(self):
+ return self._related('>')
+
+ def also_sees(self):
+ return self._related('^')
+
+ def verb_groups(self):
+ return self._related('$')
+
+ def similar_tos(self):
+ return self._related('&')
+
+ def __hash__(self):
+ return hash(self._name)
+
+ def __eq__(self, other):
+ return self._name == other._name
+
+ def __ne__(self, other):
+ return self._name != other._name
+
+ def __lt__(self, other):
+ return self._name < other._name
+
+
+ at python_2_unicode_compatible
+class Lemma(_WordNetObject):
+ """
+ The lexical entry for a single morphological form of a
+ sense-disambiguated word.
+
+ Create a Lemma from a "<word>.<pos>.<number>.<lemma>" string where:
+ <word> is the morphological stem identifying the synset
+ <pos> is one of the module attributes ADJ, ADJ_SAT, ADV, NOUN or VERB
+ <number> is the sense number, counting from 0.
+ <lemma> is the morphological form of interest
+
+ Note that <word> and <lemma> can be different, e.g. the Synset
+ 'salt.n.03' has the Lemmas 'salt.n.03.salt', 'salt.n.03.saltiness' and
+ 'salt.n.03.salinity'.
+
+ Lemma attributes, accessible via methods with the same name::
+
+ - name: The canonical name of this lemma.
+ - synset: The synset that this lemma belongs to.
+ - syntactic_marker: For adjectives, the WordNet string identifying the
+ syntactic position relative modified noun. See:
+ http://wordnet.princeton.edu/man/wninput.5WN.html#sect10
+ For all other parts of speech, this attribute is None.
+ - count: The frequency of this lemma in wordnet.
+
+ Lemma methods:
+
+ Lemmas have the following methods for retrieving related Lemmas. They
+ correspond to the names for the pointer symbols defined here:
+ http://wordnet.princeton.edu/man/wninput.5WN.html#sect3
+ These methods all return lists of Lemmas:
+
+ - antonyms
+ - hypernyms, instance_hypernyms
+ - hyponyms, instance_hyponyms
+ - member_holonyms, substance_holonyms, part_holonyms
+ - member_meronyms, substance_meronyms, part_meronyms
+ - topic_domains, region_domains, usage_domains
+ - attributes
+ - derivationally_related_forms
+ - entailments
+ - causes
+ - also_sees
+ - verb_groups
+ - similar_tos
+ - pertainyms
+ """
+
+ __slots__ = ['_wordnet_corpus_reader', '_name', '_syntactic_marker',
+ '_synset', '_frame_strings', '_frame_ids',
+ '_lexname_index', '_lex_id', '_lang', '_key']
+
+ def __init__(self, wordnet_corpus_reader, synset, name,
+ lexname_index, lex_id, syntactic_marker):
+ self._wordnet_corpus_reader = wordnet_corpus_reader
+ self._name = name
+ self._syntactic_marker = syntactic_marker
+ self._synset = synset
+ self._frame_strings = []
+ self._frame_ids = []
+ self._lexname_index = lexname_index
+ self._lex_id = lex_id
+ self._lang = "en"
+
+ self._key = None # gets set later.
+
+ def name(self):
+ return self._name
+
+ def syntactic_marker(self):
+ return self._syntactic_marker
+
+ def synset(self):
+ return self._synset
+
+ def frame_strings(self):
+ return self._frame_strings
+
+ def frame_ids(self):
+ return self._frame_ids
+
+ def lang(self):
+ return self._lang
+
+ def key(self):
+ return self._key
+
+ def __repr__(self):
+ tup = type(self).__name__, self._synset._name, self._name
+ return "%s('%s.%s')" % tup
+
+ def _related(self, relation_symbol):
+ get_synset = self._wordnet_corpus_reader._synset_from_pos_and_offset
+ return sorted([get_synset(pos, offset)._lemmas[lemma_index]
+ for pos, offset, lemma_index
+ in self._synset._lemma_pointers[self._name, relation_symbol]])
+
+ def count(self):
+ """Return the frequency count for this Lemma"""
+ return self._wordnet_corpus_reader.lemma_count(self)
+
+ def antonyms(self):
+ return self._related('!')
+
+ def derivationally_related_forms(self):
+ return self._related('+')
+
+ def pertainyms(self):
+ return self._related('\\')
+
+
+ at python_2_unicode_compatible
+class Synset(_WordNetObject):
+ """Create a Synset from a "<lemma>.<pos>.<number>" string where:
+ <lemma> is the word's morphological stem
+ <pos> is one of the module attributes ADJ, ADJ_SAT, ADV, NOUN or VERB
+ <number> is the sense number, counting from 0.
+
+ Synset attributes, accessible via methods with the same name:
+
+ - name: The canonical name of this synset, formed using the first lemma
+ of this synset. Note that this may be different from the name
+ passed to the constructor if that string used a different lemma to
+ identify the synset.
+ - pos: The synset's part of speech, matching one of the module level
+ attributes ADJ, ADJ_SAT, ADV, NOUN or VERB.
+ - lemmas: A list of the Lemma objects for this synset.
+ - definition: The definition for this synset.
+ - examples: A list of example strings for this synset.
+ - offset: The offset in the WordNet dict file of this synset.
+ - lexname: The name of the lexicographer file containing this synset.
+
+ Synset methods:
+
+ Synsets have the following methods for retrieving related Synsets.
+ They correspond to the names for the pointer symbols defined here:
+ http://wordnet.princeton.edu/man/wninput.5WN.html#sect3
+ These methods all return lists of Synsets.
+
+ - hypernyms, instance_hypernyms
+ - hyponyms, instance_hyponyms
+ - member_holonyms, substance_holonyms, part_holonyms
+ - member_meronyms, substance_meronyms, part_meronyms
+ - attributes
+ - entailments
+ - causes
+ - also_sees
+ - verb_groups
+ - similar_tos
+
+ Additionally, Synsets support the following methods specific to the
+ hypernym relation:
+
+ - root_hypernyms
+ - common_hypernyms
+ - lowest_common_hypernyms
+
+ Note that Synsets do not support the following relations because
+ these are defined by WordNet as lexical relations:
+
+ - antonyms
+ - derivationally_related_forms
+ - pertainyms
+ """
+
+ __slots__ = ['_pos', '_offset', '_name', '_frame_ids',
+ '_lemmas', '_lemma_names',
+ '_definition', '_examples', '_lexname',
+ '_pointers', '_lemma_pointers', '_max_depth',
+ '_min_depth']
+
+ def __init__(self, wordnet_corpus_reader):
+ self._wordnet_corpus_reader = wordnet_corpus_reader
+ # All of these attributes get initialized by
+ # WordNetCorpusReader._synset_from_pos_and_line()
+
+ self._pos = None
+ self._offset = None
+ self._name = None
+ self._frame_ids = []
+ self._lemmas = []
+ self._lemma_names = []
+ self._definition = None
+ self._examples = []
+ self._lexname = None # lexicographer name
+ self._all_hypernyms = None
+
+ self._pointers = defaultdict(set)
+ self._lemma_pointers = defaultdict(set)
+
+ def pos(self):
+ return self._pos
+
+ def offset(self):
+ return self._offset
+
+ def name(self):
+ return self._name
+
+ def frame_ids(self):
+ return self._frame_ids
+
+ def definition(self):
+ return self._definition
+
+ def examples(self):
+ return self._examples
+
+ def lexname(self):
+ return self._lexname
+
+ def _needs_root(self):
+ if self._pos == NOUN:
+ if self._wordnet_corpus_reader.get_version() == '1.6':
+ return True
+ else:
+ return False
+ elif self._pos == VERB:
+ return True
+
+ def lemma_names(self, lang='en'):
+ '''Return all the lemma_names associated with the synset'''
+ if lang=='en':
+ return self._lemma_names
+ else:
+ self._wordnet_corpus_reader._load_lang_data(lang)
+
+ i = self._wordnet_corpus_reader.ss2of(self)
+ for x in self._wordnet_corpus_reader._lang_data[lang][0].keys():
+ if x == i:
+ return self._wordnet_corpus_reader._lang_data[lang][0][x]
+
+ def lemmas(self, lang='en'):
+ '''Return all the lemma objects associated with the synset'''
+ if lang=='en':
+ return self._lemmas
+ else:
+ self._wordnet_corpus_reader._load_lang_data(lang)
+ lemmark = []
+ lemmy = self.lemma_names(lang)
+ for lem in lemmy:
+ temp= Lemma(self._wordnet_corpus_reader, self, lem, self._wordnet_corpus_reader._lexnames.index(self.lexname()), 0, None)
+ temp._lang=lang
+ lemmark.append(temp)
+ return lemmark
+
+ def root_hypernyms(self):
+ """Get the topmost hypernyms of this synset in WordNet."""
+
+ result = []
+ seen = set()
+ todo = [self]
+ while todo:
+ next_synset = todo.pop()
+ if next_synset not in seen:
+ seen.add(next_synset)
+ next_hypernyms = next_synset.hypernyms() + \
+ next_synset.instance_hypernyms()
+ if not next_hypernyms:
+ result.append(next_synset)
+ else:
+ todo.extend(next_hypernyms)
+ return result
+
+# Simpler implementation which makes incorrect assumption that
+# hypernym hierarchy is acyclic:
+#
+# if not self.hypernyms():
+# return [self]
+# else:
+# return list(set(root for h in self.hypernyms()
+# for root in h.root_hypernyms()))
+ def max_depth(self):
+ """
+ :return: The length of the longest hypernym path from this
+ synset to the root.
+ """
+
+ if "_max_depth" not in self.__dict__:
+ hypernyms = self.hypernyms() + self.instance_hypernyms()
+ if not hypernyms:
+ self._max_depth = 0
+ else:
+ self._max_depth = 1 + max(h.max_depth() for h in hypernyms)
+ return self._max_depth
+
+ def min_depth(self):
+ """
+ :return: The length of the shortest hypernym path from this
+ synset to the root.
+ """
+
+ if "_min_depth" not in self.__dict__:
+ hypernyms = self.hypernyms() + self.instance_hypernyms()
+ if not hypernyms:
+ self._min_depth = 0
+ else:
+ self._min_depth = 1 + min(h.min_depth() for h in hypernyms)
+ return self._min_depth
+
+ def closure(self, rel, depth=-1):
+ """Return the transitive closure of source under the rel
+ relationship, breadth-first
+
+ >>> from nltk.corpus import wordnet as wn
+ >>> dog = wn.synset('dog.n.01')
+ >>> hyp = lambda s:s.hypernyms()
+ >>> list(dog.closure(hyp))
+ [Synset('canine.n.02'), Synset('domestic_animal.n.01'),
+ Synset('carnivore.n.01'), Synset('animal.n.01'),
+ Synset('placental.n.01'), Synset('organism.n.01'),
+ Synset('mammal.n.01'), Synset('living_thing.n.01'),
+ Synset('vertebrate.n.01'), Synset('whole.n.02'),
+ Synset('chordate.n.01'), Synset('object.n.01'),
+ Synset('physical_entity.n.01'), Synset('entity.n.01')]
+
+ """
+ from nltk.util import breadth_first
+ synset_offsets = []
+ for synset in breadth_first(self, rel, depth):
+ if synset._offset != self._offset:
+ if synset._offset not in synset_offsets:
+ synset_offsets.append(synset._offset)
+ yield synset
+
+ def hypernym_paths(self):
+ """
+ Get the path(s) from this synset to the root, where each path is a
+ list of the synset nodes traversed on the way to the root.
+
+ :return: A list of lists, where each list gives the node sequence
+ connecting the initial ``Synset`` node and a root node.
+ """
+ paths = []
+
+ hypernyms = self.hypernyms() + self.instance_hypernyms()
+ if len(hypernyms) == 0:
+ paths = [[self]]
+
+ for hypernym in hypernyms:
+ for ancestor_list in hypernym.hypernym_paths():
+ ancestor_list.append(self)
+ paths.append(ancestor_list)
+ return paths
+
+ def common_hypernyms(self, other):
+ """
+ Find all synsets that are hypernyms of this synset and the
+ other synset.
+
+ :type other: Synset
+ :param other: other input synset.
+ :return: The synsets that are hypernyms of both synsets.
+ """
+ if not self._all_hypernyms:
+ self._all_hypernyms = set(self_synset
+ for self_synsets in self._iter_hypernym_lists()
+ for self_synset in self_synsets)
+ if not other._all_hypernyms:
+ other._all_hypernyms = set(other_synset
+ for other_synsets in other._iter_hypernym_lists()
+ for other_synset in other_synsets)
+ return list(self._all_hypernyms.intersection(other._all_hypernyms))
+
+ def lowest_common_hypernyms(self, other, simulate_root=False, use_min_depth=False):
+ """
+ Get a list of lowest synset(s) that both synsets have as a hypernym.
+ When `use_min_depth == False` this means that the synset which appears as a
+ hypernym of both `self` and `other` with the lowest maximum depth is returned
+ or if there are multiple such synsets at the same depth they are all returned
+
+ However, if `use_min_depth == True` then the synset(s) which has/have the lowest
+ minimum depth and appear(s) in both paths is/are returned.
+
+ By setting the use_min_depth flag to True, the behavior of NLTK2 can be preserved.
+ This was changed in NLTK3 to give more accurate results in a small set of cases,
+ generally with synsets concerning people. (eg: 'chef.n.01', 'fireman.n.01', etc.)
+
+ This method is an implementation of Ted Pedersen's "Lowest Common Subsumer" method
+ from the Perl Wordnet module. It can return either "self" or "other" if they are a
+ hypernym of the other.
+
+ :type other: Synset
+ :param other: other input synset
+ :type simulate_root: bool
+ :param simulate_root: The various verb taxonomies do not
+ share a single root which disallows this metric from working for
+ synsets that are not connected. This flag (False by default)
+ creates a fake root that connects all the taxonomies. Set it
+ to True to enable this behavior. For the noun taxonomy,
+ there is usually a default root except for WordNet version 1.6.
+ If you are using wordnet 1.6, a fake root will need to be added
+ for nouns as well.
+ :type use_min_depth: bool
+ :param use_min_depth: This setting mimics older (v2) behavior of NLTK wordnet
+ If True, will use the min_depth function to calculate the lowest common
+ hypernyms. This is known to give strange results for some synset pairs
+ (eg: 'chef.n.01', 'fireman.n.01') but is retained for backwards compatibility
+ :return: The synsets that are the lowest common hypernyms of both synsets
+ """
+ synsets = self.common_hypernyms(other)
+ if simulate_root:
+ fake_synset = Synset(None)
+ fake_synset._name = '*ROOT*'
+ fake_synset.hypernyms = lambda: []
+ fake_synset.instance_hypernyms = lambda: []
+ synsets.append(fake_synset)
+
+ try:
+ if use_min_depth:
+ max_depth = max(s.min_depth() for s in synsets)
+ unsorted_lch = [s for s in synsets if s.min_depth() == max_depth]
+ else:
+ max_depth = max(s.max_depth() for s in synsets)
+ unsorted_lch = [s for s in synsets if s.max_depth() == max_depth]
+ return sorted(unsorted_lch)
+ except ValueError:
+ return []
+
+ def hypernym_distances(self, distance=0, simulate_root=False):
+ """
+ Get the path(s) from this synset to the root, counting the distance
+ of each node from the initial node on the way. A set of
+ (synset, distance) tuples is returned.
+
+ :type distance: int
+ :param distance: the distance (number of edges) from this hypernym to
+ the original hypernym ``Synset`` on which this method was called.
+ :return: A set of ``(Synset, int)`` tuples where each ``Synset`` is
+ a hypernym of the first ``Synset``.
+ """
+ distances = set([(self, distance)])
+ for hypernym in self.hypernyms() + self.instance_hypernyms():
+ distances |= hypernym.hypernym_distances(distance+1, simulate_root=False)
+ if simulate_root:
+ fake_synset = Synset(None)
+ fake_synset._name = '*ROOT*'
+ fake_synset_distance = max(distances, key=itemgetter(1))[1]
+ distances.add((fake_synset, fake_synset_distance+1))
+ return distances
+
+ def shortest_path_distance(self, other, simulate_root=False):
+ """
+ Returns the distance of the shortest path linking the two synsets (if
+ one exists). For each synset, all the ancestor nodes and their
+ distances are recorded and compared. The ancestor node common to both
+ synsets that can be reached with the minimum number of traversals is
+ used. If no ancestor nodes are common, None is returned. If a node is
+ compared with itself 0 is returned.
+
+ :type other: Synset
+ :param other: The Synset to which the shortest path will be found.
+ :return: The number of edges in the shortest path connecting the two
+ nodes, or None if no path exists.
+ """
+
+ if self == other:
+ return 0
+
+ path_distance = None
+
+ dist_list1 = self.hypernym_distances(simulate_root=simulate_root)
+ dist_dict1 = {}
+
+ dist_list2 = other.hypernym_distances(simulate_root=simulate_root)
+ dist_dict2 = {}
+
+ # Transform each distance list into a dictionary. In cases where
+ # there are duplicate nodes in the list (due to there being multiple
+ # paths to the root) the duplicate with the shortest distance from
+ # the original node is entered.
+
+ for (l, d) in [(dist_list1, dist_dict1), (dist_list2, dist_dict2)]:
+ for (key, value) in l:
+ if key in d:
+ if value < d[key]:
+ d[key] = value
+ else:
+ d[key] = value
+
+ # For each ancestor synset common to both subject synsets, find the
+ # connecting path length. Return the shortest of these.
+
+ for synset1 in dist_dict1.keys():
+ for synset2 in dist_dict2.keys():
+ if synset1 == synset2:
+ new_distance = dist_dict1[synset1] + dist_dict2[synset2]
+ if path_distance is None or path_distance < 0 or new_distance < path_distance:
+ path_distance = new_distance
+
+ return path_distance
+
+ def tree(self, rel, depth=-1, cut_mark=None):
+ """
+ >>> from nltk.corpus import wordnet as wn
+ >>> dog = wn.synset('dog.n.01')
+ >>> hyp = lambda s:s.hypernyms()
+ >>> from pprint import pprint
+ >>> pprint(dog.tree(hyp))
+ [Synset('dog.n.01'),
+ [Synset('canine.n.02'),
+ [Synset('carnivore.n.01'),
+ [Synset('placental.n.01'),
+ [Synset('mammal.n.01'),
+ [Synset('vertebrate.n.01'),
+ [Synset('chordate.n.01'),
+ [Synset('animal.n.01'),
+ [Synset('organism.n.01'),
+ [Synset('living_thing.n.01'),
+ [Synset('whole.n.02'),
+ [Synset('object.n.01'),
+ [Synset('physical_entity.n.01'),
+ [Synset('entity.n.01')]]]]]]]]]]]]],
+ [Synset('domestic_animal.n.01'),
+ [Synset('animal.n.01'),
+ [Synset('organism.n.01'),
+ [Synset('living_thing.n.01'),
+ [Synset('whole.n.02'),
+ [Synset('object.n.01'),
+ [Synset('physical_entity.n.01'), [Synset('entity.n.01')]]]]]]]]]
+ """
+
+ tree = [self]
+ if depth != 0:
+ tree += [x.tree(rel, depth-1, cut_mark) for x in rel(self)]
+ elif cut_mark:
+ tree += [cut_mark]
+ return tree
+
+ # interface to similarity methods
+ def path_similarity(self, other, verbose=False, simulate_root=True):
+ """
+ Path Distance Similarity:
+ Return a score denoting how similar two word senses are, based on the
+ shortest path that connects the senses in the is-a (hypernym/hypnoym)
+ taxonomy. The score is in the range 0 to 1, except in those cases where
+ a path cannot be found (will only be true for verbs as there are many
+ distinct verb taxonomies), in which case None is returned. A score of
+ 1 represents identity i.e. comparing a sense with itself will return 1.
+
+ :type other: Synset
+ :param other: The ``Synset`` that this ``Synset`` is being compared to.
+ :type simulate_root: bool
+ :param simulate_root: The various verb taxonomies do not
+ share a single root which disallows this metric from working for
+ synsets that are not connected. This flag (True by default)
+ creates a fake root that connects all the taxonomies. Set it
+ to false to disable this behavior. For the noun taxonomy,
+ there is usually a default root except for WordNet version 1.6.
+ If you are using wordnet 1.6, a fake root will be added for nouns
+ as well.
+ :return: A score denoting the similarity of the two ``Synset`` objects,
+ normally between 0 and 1. None is returned if no connecting path
+ could be found. 1 is returned if a ``Synset`` is compared with
+ itself.
+ """
+
+ distance = self.shortest_path_distance(other, simulate_root=simulate_root and self._needs_root())
+ if distance is None or distance < 0:
+ return None
+ return 1.0 / (distance + 1)
+
+ def lch_similarity(self, other, verbose=False, simulate_root=True):
+ """
+ Leacock Chodorow Similarity:
+ Return a score denoting how similar two word senses are, based on the
+ shortest path that connects the senses (as above) and the maximum depth
+ of the taxonomy in which the senses occur. The relationship is given as
+ -log(p/2d) where p is the shortest path length and d is the taxonomy
+ depth.
+
+ :type other: Synset
+ :param other: The ``Synset`` that this ``Synset`` is being compared to.
+ :type simulate_root: bool
+ :param simulate_root: The various verb taxonomies do not
+ share a single root which disallows this metric from working for
+ synsets that are not connected. This flag (True by default)
+ creates a fake root that connects all the taxonomies. Set it
+ to false to disable this behavior. For the noun taxonomy,
+ there is usually a default root except for WordNet version 1.6.
+ If you are using wordnet 1.6, a fake root will be added for nouns
+ as well.
+ :return: A score denoting the similarity of the two ``Synset`` objects,
+ normally greater than 0. None is returned if no connecting path
+ could be found. If a ``Synset`` is compared with itself, the
+ maximum score is returned, which varies depending on the taxonomy
+ depth.
+ """
+
+ if self._pos != other._pos:
+ raise WordNetError('Computing the lch similarity requires ' + \
+ '%s and %s to have the same part of speech.' % \
+ (self, other))
+
+ need_root = self._needs_root()
+
+ if self._pos not in self._wordnet_corpus_reader._max_depth:
+ self._wordnet_corpus_reader._compute_max_depth(self._pos, need_root)
+
+ depth = self._wordnet_corpus_reader._max_depth[self._pos]
+
+ distance = self.shortest_path_distance(other, simulate_root=simulate_root and need_root)
+
+ if distance is None or distance < 0 or depth == 0:
+ return None
+ return -math.log((distance + 1) / (2.0 * depth))
+
+ def wup_similarity(self, other, verbose=False, simulate_root=True):
+ """
+ Wu-Palmer Similarity:
+ Return a score denoting how similar two word senses are, based on the
+ depth of the two senses in the taxonomy and that of their Least Common
+ Subsumer (most specific ancestor node). Previously, the scores computed
+ by this implementation did _not_ always agree with those given by
+ Pedersen's Perl implementation of WordNet Similarity. However, with
+ the addition of the simulate_root flag (see below), the score for
+ verbs now almost always agree but not always for nouns.
+
+ The LCS does not necessarily feature in the shortest path connecting
+ the two senses, as it is by definition the common ancestor deepest in
+ the taxonomy, not closest to the two senses. Typically, however, it
+ will so feature. Where multiple candidates for the LCS exist, that
+ whose shortest path to the root node is the longest will be selected.
+ Where the LCS has multiple paths to the root, the longer path is used
+ for the purposes of the calculation.
+
+ :type other: Synset
+ :param other: The ``Synset`` that this ``Synset`` is being compared to.
+ :type simulate_root: bool
+ :param simulate_root: The various verb taxonomies do not
+ share a single root which disallows this metric from working for
+ synsets that are not connected. This flag (True by default)
+ creates a fake root that connects all the taxonomies. Set it
+ to false to disable this behavior. For the noun taxonomy,
+ there is usually a default root except for WordNet version 1.6.
+ If you are using wordnet 1.6, a fake root will be added for nouns
+ as well.
+ :return: A float score denoting the similarity of the two ``Synset`` objects,
+ normally greater than zero. If no connecting path between the two
+ senses can be found, None is returned.
+
+ """
+
+ need_root = self._needs_root()
+ # Note that to preserve behavior from NLTK2 we set use_min_depth=True
+ # It is possible that more accurate results could be obtained by
+ # removing this setting and it should be tested later on
+ subsumers = self.lowest_common_hypernyms(other, simulate_root=simulate_root and need_root, use_min_depth=True)
+
+ # If no LCS was found return None
+ if len(subsumers) == 0:
+ return None
+
+ subsumer = subsumers[0]
+
+ # Get the longest path from the LCS to the root,
+ # including a correction:
+ # - add one because the calculations include both the start and end
+ # nodes
+ depth = subsumer.max_depth() + 1
+
+ # Note: No need for an additional add-one correction for non-nouns
+ # to account for an imaginary root node because that is now automatically
+ # handled by simulate_root
+ # if subsumer._pos != NOUN:
+ # depth += 1
+
+ # Get the shortest path from the LCS to each of the synsets it is
+ # subsuming. Add this to the LCS path length to get the path
+ # length from each synset to the root.
+ len1 = self.shortest_path_distance(subsumer, simulate_root=simulate_root and need_root)
+ len2 = other.shortest_path_distance(subsumer, simulate_root=simulate_root and need_root)
+ if len1 is None or len2 is None:
+ return None
+ len1 += depth
+ len2 += depth
+ return (2.0 * depth) / (len1 + len2)
+
+ def res_similarity(self, other, ic, verbose=False):
+ """
+ Resnik Similarity:
+ Return a score denoting how similar two word senses are, based on the
+ Information Content (IC) of the Least Common Subsumer (most specific
+ ancestor node).
+
+ :type other: Synset
+ :param other: The ``Synset`` that this ``Synset`` is being compared to.
+ :type ic: dict
+ :param ic: an information content object (as returned by ``nltk.corpus.wordnet_ic.ic()``).
+ :return: A float score denoting the similarity of the two ``Synset`` objects.
+ Synsets whose LCS is the root node of the taxonomy will have a
+ score of 0 (e.g. N['dog'][0] and N['table'][0]).
+ """
+
+ ic1, ic2, lcs_ic = _lcs_ic(self, other, ic)
+ return lcs_ic
+
+ def jcn_similarity(self, other, ic, verbose=False):
+ """
+ Jiang-Conrath Similarity:
+ Return a score denoting how similar two word senses are, based on the
+ Information Content (IC) of the Least Common Subsumer (most specific
+ ancestor node) and that of the two input Synsets. The relationship is
+ given by the equation 1 / (IC(s1) + IC(s2) - 2 * IC(lcs)).
+
+ :type other: Synset
+ :param other: The ``Synset`` that this ``Synset`` is being compared to.
+ :type ic: dict
+ :param ic: an information content object (as returned by ``nltk.corpus.wordnet_ic.ic()``).
+ :return: A float score denoting the similarity of the two ``Synset`` objects.
+ """
+
+ if self == other:
+ return _INF
+
+ ic1, ic2, lcs_ic = _lcs_ic(self, other, ic)
+
+ # If either of the input synsets are the root synset, or have a
+ # frequency of 0 (sparse data problem), return 0.
+ if ic1 == 0 or ic2 == 0:
+ return 0
+
+ ic_difference = ic1 + ic2 - 2 * lcs_ic
+
+ if ic_difference == 0:
+ return _INF
+
+ return 1 / ic_difference
+
+ def lin_similarity(self, other, ic, verbose=False):
+ """
+ Lin Similarity:
+ Return a score denoting how similar two word senses are, based on the
+ Information Content (IC) of the Least Common Subsumer (most specific
+ ancestor node) and that of the two input Synsets. The relationship is
+ given by the equation 2 * IC(lcs) / (IC(s1) + IC(s2)).
+
+ :type other: Synset
+ :param other: The ``Synset`` that this ``Synset`` is being compared to.
+ :type ic: dict
+ :param ic: an information content object (as returned by ``nltk.corpus.wordnet_ic.ic()``).
+ :return: A float score denoting the similarity of the two ``Synset`` objects,
+ in the range 0 to 1.
+ """
+
+ ic1, ic2, lcs_ic = _lcs_ic(self, other, ic)
+ return (2.0 * lcs_ic) / (ic1 + ic2)
+
+ def _iter_hypernym_lists(self):
+ """
+ :return: An iterator over ``Synset`` objects that are either proper
+ hypernyms or instance of hypernyms of the synset.
+ """
+ todo = [self]
+ seen = set()
+ while todo:
+ for synset in todo:
+ seen.add(synset)
+ yield todo
+ todo = [hypernym
+ for synset in todo
+ for hypernym in (synset.hypernyms() +
+ synset.instance_hypernyms())
+ if hypernym not in seen]
+
+ def __repr__(self):
+ return "%s('%s')" % (type(self).__name__, self._name)
+
+ def _related(self, relation_symbol):
+ get_synset = self._wordnet_corpus_reader._synset_from_pos_and_offset
+ pointer_tuples = self._pointers[relation_symbol]
+ return sorted([get_synset(pos, offset) for pos, offset in pointer_tuples])
+
+
+######################################################################
+## WordNet Corpus Reader
+######################################################################
+
+class WordNetCorpusReader(CorpusReader):
+ """
+ A corpus reader used to access wordnet or its variants.
+ """
+
+ _ENCODING = 'utf8'
+
+ #{ Part-of-speech constants
+ ADJ, ADJ_SAT, ADV, NOUN, VERB = 'a', 's', 'r', 'n', 'v'
+ #}
+
+ #{ Filename constants
+ _FILEMAP = {ADJ: 'adj', ADV: 'adv', NOUN: 'noun', VERB: 'verb'}
+ #}
+
+ #{ Part of speech constants
+ _pos_numbers = {NOUN: 1, VERB: 2, ADJ: 3, ADV: 4, ADJ_SAT: 5}
+ _pos_names = dict(tup[::-1] for tup in _pos_numbers.items())
+ #}
+
+ #: A list of file identifiers for all the fileids used by this
+ #: corpus reader.
+ _FILES = ('cntlist.rev', 'lexnames', 'index.sense',
+ 'index.adj', 'index.adv', 'index.noun', 'index.verb',
+ 'data.adj', 'data.adv', 'data.noun', 'data.verb',
+ 'adj.exc', 'adv.exc', 'noun.exc', 'verb.exc', )
+
+ def __init__(self, root, omw_reader):
+ """
+ Construct a new wordnet corpus reader, with the given root
+ directory.
+ """
+ super(WordNetCorpusReader, self).__init__(root, self._FILES,
+ encoding=self._ENCODING)
+
+ # A index that provides the file offset
+ # Map from lemma -> pos -> synset_index -> offset
+ self._lemma_pos_offset_map = defaultdict(dict)
+
+ # A cache so we don't have to reconstuct synsets
+ # Map from pos -> offset -> synset
+ self._synset_offset_cache = defaultdict(dict)
+
+ # A lookup for the maximum depth of each part of speech. Useful for
+ # the lch similarity metric.
+ self._max_depth = defaultdict(dict)
+
+ # Corpus reader containing omw data.
+ self._omw_reader = omw_reader
+
+ # A cache to store the wordnet data of multiple languages
+ self._lang_data = defaultdict(list)
+
+ self._data_file_map = {}
+ self._exception_map = {}
+ self._lexnames = []
+ self._key_count_file = None
+ self._key_synset_file = None
+
+ # Load the lexnames
+ for i, line in enumerate(self.open('lexnames')):
+ index, lexname, _ = line.split()
+ assert int(index) == i
+ self._lexnames.append(lexname)
+
+ # Load the indices for lemmas and synset offsets
+ self._load_lemma_pos_offset_map()
+
+ # load the exception file data into memory
+ self._load_exception_map()
+
+# Open Multilingual WordNet functions, contributed by
+# Nasruddin A’aidil Shari, Sim Wei Ying Geraldine, and Soe Lynn
+
+ def of2ss(self, of):
+ ''' take an id and return the synsets '''
+ return self._synset_from_pos_and_offset(of[-1], int(of[:8]))
+
+ def ss2of(self, ss):
+ ''' return the ILI of the synset '''
+ return ( "0"*8 + str(ss.offset()) +"-"+ str(ss.pos()))[-10:]
+
+ def _load_lang_data(self, lang):
+ ''' load the wordnet data of the requested language from the file to the cache, _lang_data '''
+
+ if lang not in self.langs():
+ raise WordNetError("Language is not supported.")
+
+ if lang in self._lang_data.keys():
+ return
+
+ f = self._omw_reader.open('{0:}/wn-data-{0:}.tab'.format(lang))
+
+ self._lang_data[lang].append(defaultdict(list))
+ self._lang_data[lang].append(defaultdict(list))
+
+ for l in f.readlines():
+ l = l.replace('\n', '')
+ l = l.replace(' ', '_')
+ if l[0] != '#':
+ word = l.split('\t')
+ self._lang_data[lang][0][word[0]].append(word[2])
+ self._lang_data[lang][1][word[2]].append(word[0])
+ f.close()
+
+ def langs(self):
+ ''' return a list of languages supported by Multilingual Wordnet '''
+ import os
+ langs = []
+ fileids = self._omw_reader.fileids()
+ for fileid in fileids:
+ file_name, file_extension = os.path.splitext(fileid)
+ if file_extension == '.tab':
+ langs.append(file_name.split('-')[-1])
+
+ return langs
+
+
+ def _load_lemma_pos_offset_map(self):
+ for suffix in self._FILEMAP.values():
+
+ # parse each line of the file (ignoring comment lines)
+ for i, line in enumerate(self.open('index.%s' % suffix)):
+ if line.startswith(' '):
+ continue
+
+ _iter = iter(line.split())
+ _next_token = lambda: next(_iter)
+ try:
+
+ # get the lemma and part-of-speech
+ lemma = _next_token()
+ pos = _next_token()
+
+ # get the number of synsets for this lemma
+ n_synsets = int(_next_token())
+ assert n_synsets > 0
+
+ # get the pointer symbols for all synsets of this lemma
+ n_pointers = int(_next_token())
+ _ = [_next_token() for _ in xrange(n_pointers)]
+
+ # same as number of synsets
+ n_senses = int(_next_token())
+ assert n_synsets == n_senses
+
+ # get number of senses ranked according to frequency
+ _ = int(_next_token())
+
+ # get synset offsets
+ synset_offsets = [int(_next_token()) for _ in xrange(n_synsets)]
+
+ # raise more informative error with file name and line number
+ except (AssertionError, ValueError) as e:
+ tup = ('index.%s' % suffix), (i + 1), e
+ raise WordNetError('file %s, line %i: %s' % tup)
+
+ # map lemmas and parts of speech to synsets
+ self._lemma_pos_offset_map[lemma][pos] = synset_offsets
+ if pos == ADJ:
+ self._lemma_pos_offset_map[lemma][ADJ_SAT] = synset_offsets
+
+ def _load_exception_map(self):
+ # load the exception file data into memory
+ for pos, suffix in self._FILEMAP.items():
+ self._exception_map[pos] = {}
+ for line in self.open('%s.exc' % suffix):
+ terms = line.split()
+ self._exception_map[pos][terms[0]] = terms[1:]
+ self._exception_map[ADJ_SAT] = self._exception_map[ADJ]
+
+ def _compute_max_depth(self, pos, simulate_root):
+ """
+ Compute the max depth for the given part of speech. This is
+ used by the lch similarity metric.
+ """
+ depth = 0
+ for ii in self.all_synsets(pos):
+ try:
+ depth = max(depth, ii.max_depth())
+ except RuntimeError:
+ print(ii)
+ if simulate_root:
+ depth += 1
+ self._max_depth[pos] = depth
+
+ def get_version(self):
+ fh = self._data_file(ADJ)
+ for line in fh:
+ match = re.search(r'WordNet (\d+\.\d+) Copyright', line)
+ if match is not None:
+ version = match.group(1)
+ fh.seek(0)
+ return version
+
+ #////////////////////////////////////////////////////////////
+ # Loading Lemmas
+ #////////////////////////////////////////////////////////////
+
+ def lemma(self, name, lang='en'):
+ '''Return lemma object that matches the name'''
+ synset_name, lemma_name = name.rsplit('.', 1)
+ synset = self.synset(synset_name)
+ for lemma in synset.lemmas(lang):
+ if lemma._name == lemma_name:
+ return lemma
+ raise WordNetError('no lemma %r in %r' % (lemma_name, synset_name))
+
+ def lemma_from_key(self, key):
+ # Keys are case sensitive and always lower-case
+ key = key.lower()
+
+ lemma_name, lex_sense = key.split('%')
+ pos_number, lexname_index, lex_id, _, _ = lex_sense.split(':')
+ pos = self._pos_names[int(pos_number)]
+
+ # open the key -> synset file if necessary
+ if self._key_synset_file is None:
+ self._key_synset_file = self.open('index.sense')
+
+ # Find the synset for the lemma.
+ synset_line = _binary_search_file(self._key_synset_file, key)
+ if not synset_line:
+ raise WordNetError("No synset found for key %r" % key)
+ offset = int(synset_line.split()[1])
+ synset = self._synset_from_pos_and_offset(pos, offset)
+
+ # return the corresponding lemma
+ for lemma in synset._lemmas:
+ if lemma._key == key:
+ return lemma
+ raise WordNetError("No lemma found for for key %r" % key)
+
+ #////////////////////////////////////////////////////////////
+ # Loading Synsets
+ #////////////////////////////////////////////////////////////
+ def synset(self, name):
+ # split name into lemma, part of speech and synset number
+ lemma, pos, synset_index_str = name.lower().rsplit('.', 2)
+ synset_index = int(synset_index_str) - 1
+
+ # get the offset for this synset
+ try:
+ offset = self._lemma_pos_offset_map[lemma][pos][synset_index]
+ except KeyError:
+ message = 'no lemma %r with part of speech %r'
+ raise WordNetError(message % (lemma, pos))
+ except IndexError:
+ n_senses = len(self._lemma_pos_offset_map[lemma][pos])
+ message = "lemma %r with part of speech %r has only %i %s"
+ if n_senses == 1:
+ tup = lemma, pos, n_senses, "sense"
+ else:
+ tup = lemma, pos, n_senses, "senses"
+ raise WordNetError(message % tup)
+
+ # load synset information from the appropriate file
+ synset = self._synset_from_pos_and_offset(pos, offset)
+
+ # some basic sanity checks on loaded attributes
+ if pos == 's' and synset._pos == 'a':
+ message = ('adjective satellite requested but only plain '
+ 'adjective found for lemma %r')
+ raise WordNetError(message % lemma)
+ assert synset._pos == pos or (pos == 'a' and synset._pos == 's')
+
+ # Return the synset object.
+ return synset
+
+ def _data_file(self, pos):
+ """
+ Return an open file pointer for the data file for the given
+ part of speech.
+ """
+ if pos == ADJ_SAT:
+ pos = ADJ
+ if self._data_file_map.get(pos) is None:
+ fileid = 'data.%s' % self._FILEMAP[pos]
+ self._data_file_map[pos] = self.open(fileid)
+ return self._data_file_map[pos]
+
+ def _synset_from_pos_and_offset(self, pos, offset):
+ # Check to see if the synset is in the cache
+ if offset in self._synset_offset_cache[pos]:
+ return self._synset_offset_cache[pos][offset]
+
+ data_file = self._data_file(pos)
+ data_file.seek(offset)
+ data_file_line = data_file.readline()
+ synset = self._synset_from_pos_and_line(pos, data_file_line)
+ assert synset._offset == offset
+ self._synset_offset_cache[pos][offset] = synset
+ return synset
+
+ def _synset_from_pos_and_line(self, pos, data_file_line):
+ # Construct a new (empty) synset.
+ synset = Synset(self)
+
+ # parse the entry for this synset
+ try:
+
+ # parse out the definitions and examples from the gloss
+ columns_str, gloss = data_file_line.split('|')
+ gloss = gloss.strip()
+ definitions = []
+ for gloss_part in gloss.split(';'):
+ gloss_part = gloss_part.strip()
+ if gloss_part.startswith('"'):
+ synset._examples.append(gloss_part.strip('"'))
+ else:
+ definitions.append(gloss_part)
+ synset._definition = '; '.join(definitions)
+
+ # split the other info into fields
+ _iter = iter(columns_str.split())
+ _next_token = lambda: next(_iter)
+
+ # get the offset
+ synset._offset = int(_next_token())
+
+ # determine the lexicographer file name
+ lexname_index = int(_next_token())
+ synset._lexname = self._lexnames[lexname_index]
+
+ # get the part of speech
+ synset._pos = _next_token()
+
+ # create Lemma objects for each lemma
+ n_lemmas = int(_next_token(), 16)
+ for _ in xrange(n_lemmas):
+ # get the lemma name
+ lemma_name = _next_token()
+ # get the lex_id (used for sense_keys)
+ lex_id = int(_next_token(), 16)
+ # If the lemma has a syntactic marker, extract it.
+ m = re.match(r'(.*?)(\(.*\))?$', lemma_name)
+ lemma_name, syn_mark = m.groups()
+ # create the lemma object
+ lemma = Lemma(self, synset, lemma_name, lexname_index,
+ lex_id, syn_mark)
+ synset._lemmas.append(lemma)
+ synset._lemma_names.append(lemma._name)
+
+ # collect the pointer tuples
+ n_pointers = int(_next_token())
+ for _ in xrange(n_pointers):
+ symbol = _next_token()
+ offset = int(_next_token())
+ pos = _next_token()
+ lemma_ids_str = _next_token()
+ if lemma_ids_str == '0000':
+ synset._pointers[symbol].add((pos, offset))
+ else:
+ source_index = int(lemma_ids_str[:2], 16) - 1
+ target_index = int(lemma_ids_str[2:], 16) - 1
+ source_lemma_name = synset._lemmas[source_index]._name
+ lemma_pointers = synset._lemma_pointers
+ tups = lemma_pointers[source_lemma_name, symbol]
+ tups.add((pos, offset, target_index))
+
+ # read the verb frames
+ try:
+ frame_count = int(_next_token())
+ except StopIteration:
+ pass
+ else:
+ for _ in xrange(frame_count):
+ # read the plus sign
+ plus = _next_token()
+ assert plus == '+'
+ # read the frame and lemma number
+ frame_number = int(_next_token())
+ frame_string_fmt = VERB_FRAME_STRINGS[frame_number]
+ lemma_number = int(_next_token(), 16)
+ # lemma number of 00 means all words in the synset
+ if lemma_number == 0:
+ synset._frame_ids.append(frame_number)
+ for lemma in synset._lemmas:
+ lemma._frame_ids.append(frame_number)
+ lemma._frame_strings.append(frame_string_fmt %
+ lemma._name)
+ # only a specific word in the synset
+ else:
+ lemma = synset._lemmas[lemma_number - 1]
+ lemma._frame_ids.append(frame_number)
+ lemma._frame_strings.append(frame_string_fmt %
+ lemma._name)
+
+ # raise a more informative error with line text
+ except ValueError as e:
+ raise WordNetError('line %r: %s' % (data_file_line, e))
+
+ # set sense keys for Lemma objects - note that this has to be
+ # done afterwards so that the relations are available
+ for lemma in synset._lemmas:
+ if synset._pos == ADJ_SAT:
+ head_lemma = synset.similar_tos()[0]._lemmas[0]
+ head_name = head_lemma._name
+ head_id = '%02d' % head_lemma._lex_id
+ else:
+ head_name = head_id = ''
+ tup = (lemma._name, WordNetCorpusReader._pos_numbers[synset._pos],
+ lemma._lexname_index, lemma._lex_id, head_name, head_id)
+ lemma._key = ('%s%%%d:%02d:%02d:%s:%s' % tup).lower()
+
+ # the canonical name is based on the first lemma
+ lemma_name = synset._lemmas[0]._name.lower()
+ offsets = self._lemma_pos_offset_map[lemma_name][synset._pos]
+ sense_index = offsets.index(synset._offset)
+ tup = lemma_name, synset._pos, sense_index + 1
+ synset._name = '%s.%s.%02i' % tup
+
+ return synset
+
+ #////////////////////////////////////////////////////////////
+ # Retrieve synsets and lemmas.
+ #////////////////////////////////////////////////////////////
+
+ def synsets(self, lemma, pos=None, lang='en'):
+ """Load all synsets with a given lemma and part of speech tag.
+ If no pos is specified, all synsets for all parts of speech
+ will be loaded.
+ If lang is specified, all the synsets associated with the lemma name
+ of that language will be returned.
+ """
+ lemma = lemma.lower()
+
+ if lang == 'en':
+ get_synset = self._synset_from_pos_and_offset
+ index = self._lemma_pos_offset_map
+ if pos is None:
+ pos = POS_LIST
+ return [get_synset(p, offset)
+ for p in pos
+ for form in self._morphy(lemma, p)
+ for offset in index[form].get(p, [])]
+
+ else:
+ self._load_lang_data(lang)
+ synset_list = []
+ for l in self._lang_data[lang][1][lemma]:
+ if pos is not None and l[-1] != pos:
+ continue
+ synset_list.append(self.of2ss(l))
+ return synset_list
+
+ def lemmas(self, lemma, pos=None, lang='en'):
+ """Return all Lemma objects with a name matching the specified lemma
+ name and part of speech tag. Matches any part of speech tag if none is
+ specified."""
+
+ if lang == 'en':
+ lemma = lemma.lower()
+ return [lemma_obj
+ for synset in self.synsets(lemma, pos)
+ for lemma_obj in synset.lemmas()
+ if lemma_obj.name().lower() == lemma]
+
+ else:
+ self._load_lang_data(lang)
+ lemmas = []
+ syn = self.synsets(lemma, lang=lang)
+ for s in syn:
+ if pos is not None and s.pos() != pos:
+ continue
+ a = Lemma(self, s, lemma, self._lexnames.index(s.lexname()), 0, None)
+ a._lang = lang
+ lemmas.append(a)
+ return lemmas
+
+ def all_lemma_names(self, pos=None, lang='en'):
+ """Return all lemma names for all synsets for the given
+ part of speech tag and langauge or languages. If pos is not specified, all synsets
+ for all parts of speech will be used."""
+
+ if lang == 'en':
+ if pos is None:
+ return iter(self._lemma_pos_offset_map)
+ else:
+ return (lemma
+ for lemma in self._lemma_pos_offset_map
+ if pos in self._lemma_pos_offset_map[lemma])
+ else:
+ self._load_lang_data(lang)
+ lemma = []
+ for i in self._lang_data[lang][0]:
+ if pos is not None and i[-1] != pos:
+ continue
+ lemma.extend(self._lang_data[lang][0][i])
+
+ lemma = list(set(lemma))
+ return lemma
+
+ def all_synsets(self, pos=None):
+ """Iterate over all synsets with a given part of speech tag.
+ If no pos is specified, all synsets for all parts of speech
+ will be loaded.
+ """
+ if pos is None:
+ pos_tags = self._FILEMAP.keys()
+ else:
+ pos_tags = [pos]
+
+ cache = self._synset_offset_cache
+ from_pos_and_line = self._synset_from_pos_and_line
+
+ # generate all synsets for each part of speech
+ for pos_tag in pos_tags:
+ # Open the file for reading. Note that we can not re-use
+ # the file poitners from self._data_file_map here, because
+ # we're defining an iterator, and those file pointers might
+ # be moved while we're not looking.
+ if pos_tag == ADJ_SAT:
+ pos_tag = ADJ
+ fileid = 'data.%s' % self._FILEMAP[pos_tag]
+ data_file = self.open(fileid)
+
+ try:
+ # generate synsets for each line in the POS file
+ offset = data_file.tell()
+ line = data_file.readline()
+ while line:
+ if not line[0].isspace():
+ if offset in cache[pos_tag]:
+ # See if the synset is cached
+ synset = cache[pos_tag][offset]
+ else:
+ # Otherwise, parse the line
+ synset = from_pos_and_line(pos_tag, line)
+ cache[pos_tag][offset] = synset
+
+ # adjective satellites are in the same file as
+ # adjectives so only yield the synset if it's actually
+ # a satellite
+ if pos_tag == ADJ_SAT:
+ if synset._pos == pos_tag:
+ yield synset
+
+ # for all other POS tags, yield all synsets (this means
+ # that adjectives also include adjective satellites)
+ else:
+ yield synset
+ offset = data_file.tell()
+ line = data_file.readline()
+
+ # close the extra file handle we opened
+ except:
+ data_file.close()
+ raise
+ else:
+ data_file.close()
+
+ #////////////////////////////////////////////////////////////
+ # Misc
+ #////////////////////////////////////////////////////////////
+ def lemma_count(self, lemma):
+ """Return the frequency count for this Lemma"""
+ # open the count file if we haven't already
+ if self._key_count_file is None:
+ self._key_count_file = self.open('cntlist.rev')
+ # find the key in the counts file and return the count
+ line = _binary_search_file(self._key_count_file, lemma._key)
+ if line:
+ return int(line.rsplit(' ', 1)[-1])
+ else:
+ return 0
+
+ def path_similarity(self, synset1, synset2, verbose=False, simulate_root=True):
+ return synset1.path_similarity(synset2, verbose, simulate_root)
+ path_similarity.__doc__ = Synset.path_similarity.__doc__
+
+ def lch_similarity(self, synset1, synset2, verbose=False, simulate_root=True):
+ return synset1.lch_similarity(synset2, verbose, simulate_root)
+ lch_similarity.__doc__ = Synset.lch_similarity.__doc__
+
+ def wup_similarity(self, synset1, synset2, verbose=False, simulate_root=True):
+ return synset1.wup_similarity(synset2, verbose, simulate_root)
+ wup_similarity.__doc__ = Synset.wup_similarity.__doc__
+
+ def res_similarity(self, synset1, synset2, ic, verbose=False):
+ return synset1.res_similarity(synset2, ic, verbose)
+ res_similarity.__doc__ = Synset.res_similarity.__doc__
+
+ def jcn_similarity(self, synset1, synset2, ic, verbose=False):
+ return synset1.jcn_similarity(synset2, ic, verbose)
+ jcn_similarity.__doc__ = Synset.jcn_similarity.__doc__
+
+ def lin_similarity(self, synset1, synset2, ic, verbose=False):
+ return synset1.lin_similarity(synset2, ic, verbose)
+ lin_similarity.__doc__ = Synset.lin_similarity.__doc__
+
+ #////////////////////////////////////////////////////////////
+ # Morphy
+ #////////////////////////////////////////////////////////////
+ # Morphy, adapted from Oliver Steele's pywordnet
+ def morphy(self, form, pos=None):
+ """
+ Find a possible base form for the given form, with the given
+ part of speech, by checking WordNet's list of exceptional
+ forms, and by recursively stripping affixes for this part of
+ speech until a form in WordNet is found.
+
+ >>> from nltk.corpus import wordnet as wn
+ >>> print(wn.morphy('dogs'))
+ dog
+ >>> print(wn.morphy('churches'))
+ church
+ >>> print(wn.morphy('aardwolves'))
+ aardwolf
+ >>> print(wn.morphy('abaci'))
+ abacus
+ >>> wn.morphy('hardrock', wn.ADV)
+ >>> print(wn.morphy('book', wn.NOUN))
+ book
+ >>> wn.morphy('book', wn.ADJ)
+ """
+
+ if pos is None:
+ morphy = self._morphy
+ analyses = chain(a for p in POS_LIST for a in morphy(form, p))
+ else:
+ analyses = self._morphy(form, pos)
+
+ # get the first one we find
+ first = list(islice(analyses, 1))
+ if len(first) == 1:
+ return first[0]
+ else:
+ return None
+
+ MORPHOLOGICAL_SUBSTITUTIONS = {
+ NOUN: [('s', ''), ('ses', 's'), ('ves', 'f'), ('xes', 'x'),
+ ('zes', 'z'), ('ches', 'ch'), ('shes', 'sh'),
+ ('men', 'man'), ('ies', 'y')],
+ VERB: [('s', ''), ('ies', 'y'), ('es', 'e'), ('es', ''),
+ ('ed', 'e'), ('ed', ''), ('ing', 'e'), ('ing', '')],
+ ADJ: [('er', ''), ('est', ''), ('er', 'e'), ('est', 'e')],
+ ADV: []}
+
+ def _morphy(self, form, pos):
+ # from jordanbg:
+ # Given an original string x
+ # 1. Apply rules once to the input to get y1, y2, y3, etc.
+ # 2. Return all that are in the database
+ # 3. If there are no matches, keep applying rules until you either
+ # find a match or you can't go any further
+
+ exceptions = self._exception_map[pos]
+ substitutions = self.MORPHOLOGICAL_SUBSTITUTIONS[pos]
+
+ def apply_rules(forms):
+ return [form[:-len(old)] + new
+ for form in forms
+ for old, new in substitutions
+ if form.endswith(old)]
+
+ def filter_forms(forms):
+ result = []
+ seen = set()
+ for form in forms:
+ if form in self._lemma_pos_offset_map:
+ if pos in self._lemma_pos_offset_map[form]:
+ if form not in seen:
+ result.append(form)
+ seen.add(form)
+ return result
+
+ # 0. Check the exception lists
+ if form in exceptions:
+ return filter_forms([form] + exceptions[form])
+
+ # 1. Apply rules once to the input to get y1, y2, y3, etc.
+ forms = apply_rules([form])
+
+ # 2. Return all that are in the database (and check the original too)
+ results = filter_forms([form] + forms)
+ if results:
+ return results
+
+ # 3. If there are no matches, keep applying rules until we find a match
+ while forms:
+ forms = apply_rules(forms)
+ results = filter_forms(forms)
+ if results:
+ return results
+
+ # Return an empty list if we can't find anything
+ return []
+
+ #////////////////////////////////////////////////////////////
+ # Create information content from corpus
+ #////////////////////////////////////////////////////////////
+ def ic(self, corpus, weight_senses_equally = False, smoothing = 1.0):
+ """
+ Creates an information content lookup dictionary from a corpus.
+
+ :type corpus: CorpusReader
+ :param corpus: The corpus from which we create an information
+ content dictionary.
+ :type weight_senses_equally: bool
+ :param weight_senses_equally: If this is True, gives all
+ possible senses equal weight rather than dividing by the
+ number of possible senses. (If a word has 3 synses, each
+ sense gets 0.3333 per appearance when this is False, 1.0 when
+ it is true.)
+ :param smoothing: How much do we smooth synset counts (default is 1.0)
+ :type smoothing: float
+ :return: An information content dictionary
+ """
+ counts = FreqDist()
+ for ww in corpus.words():
+ counts[ww] += 1
+
+ ic = {}
+ for pp in POS_LIST:
+ ic[pp] = defaultdict(float)
+
+ # Initialize the counts with the smoothing value
+ if smoothing > 0.0:
+ for ss in self.all_synsets():
+ pos = ss._pos
+ if pos == ADJ_SAT:
+ pos = ADJ
+ ic[pos][ss._offset] = smoothing
+
+ for ww in counts:
+ possible_synsets = self.synsets(ww)
+ if len(possible_synsets) == 0:
+ continue
+
+ # Distribute weight among possible synsets
+ weight = float(counts[ww])
+ if not weight_senses_equally:
+ weight /= float(len(possible_synsets))
+
+ for ss in possible_synsets:
+ pos = ss._pos
+ if pos == ADJ_SAT:
+ pos = ADJ
+ for level in ss._iter_hypernym_lists():
+ for hh in level:
+ ic[pos][hh._offset] += weight
+ # Add the weight to the root
+ ic[pos][0] += weight
+ return ic
+
+
+######################################################################
+## WordNet Information Content Corpus Reader
+######################################################################
+
+class WordNetICCorpusReader(CorpusReader):
+ """
+ A corpus reader for the WordNet information content corpus.
+ """
+
+ def __init__(self, root, fileids):
+ CorpusReader.__init__(self, root, fileids, encoding='utf8')
+
+ # this load function would be more efficient if the data was pickled
+ # Note that we can't use NLTK's frequency distributions because
+ # synsets are overlapping (each instance of a synset also counts
+ # as an instance of its hypernyms)
+ def ic(self, icfile):
+ """
+ Load an information content file from the wordnet_ic corpus
+ and return a dictionary. This dictionary has just two keys,
+ NOUN and VERB, whose values are dictionaries that map from
+ synsets to information content values.
+
+ :type icfile: str
+ :param icfile: The name of the wordnet_ic file (e.g. "ic-brown.dat")
+ :return: An information content dictionary
+ """
+ ic = {}
+ ic[NOUN] = defaultdict(float)
+ ic[VERB] = defaultdict(float)
+ for num, line in enumerate(self.open(icfile)):
+ if num == 0: # skip the header
+ continue
+ fields = line.split()
+ offset = int(fields[0][:-1])
+ value = float(fields[1])
+ pos = _get_pos(fields[0])
+ if len(fields) == 3 and fields[2] == "ROOT":
+ # Store root count.
+ ic[pos][0] += value
+ if value != 0:
+ ic[pos][offset] = value
+ return ic
+
+
+######################################################################
+# Similarity metrics
+######################################################################
+
+# TODO: Add in the option to manually add a new root node; this will be
+# useful for verb similarity as there exist multiple verb taxonomies.
+
+# More information about the metrics is available at
+# http://marimba.d.umn.edu/similarity/measures.html
+
+def path_similarity(synset1, synset2, verbose=False, simulate_root=True):
+ return synset1.path_similarity(synset2, verbose, simulate_root)
+path_similarity.__doc__ = Synset.path_similarity.__doc__
+
+
+def lch_similarity(synset1, synset2, verbose=False, simulate_root=True):
+ return synset1.lch_similarity(synset2, verbose, simulate_root)
+lch_similarity.__doc__ = Synset.lch_similarity.__doc__
+
+
+def wup_similarity(synset1, synset2, verbose=False, simulate_root=True):
+ return synset1.wup_similarity(synset2, verbose, simulate_root)
+wup_similarity.__doc__ = Synset.wup_similarity.__doc__
+
+
+def res_similarity(synset1, synset2, ic, verbose=False):
+ return synset1.res_similarity(synset2, verbose)
+res_similarity.__doc__ = Synset.res_similarity.__doc__
+
+
+def jcn_similarity(synset1, synset2, ic, verbose=False):
+ return synset1.jcn_similarity(synset2, verbose)
+jcn_similarity.__doc__ = Synset.jcn_similarity.__doc__
+
+
+def lin_similarity(synset1, synset2, ic, verbose=False):
+ return synset1.lin_similarity(synset2, verbose)
+lin_similarity.__doc__ = Synset.lin_similarity.__doc__
+
+
+def _lcs_ic(synset1, synset2, ic, verbose=False):
+ """
+ Get the information content of the least common subsumer that has
+ the highest information content value. If two nodes have no
+ explicit common subsumer, assume that they share an artificial
+ root node that is the hypernym of all explicit roots.
+
+ :type synset1: Synset
+ :param synset1: First input synset.
+ :type synset2: Synset
+ :param synset2: Second input synset. Must be the same part of
+ speech as the first synset.
+ :type ic: dict
+ :param ic: an information content object (as returned by ``load_ic()``).
+ :return: The information content of the two synsets and their most
+ informative subsumer
+ """
+ if synset1._pos != synset2._pos:
+ raise WordNetError('Computing the least common subsumer requires ' + \
+ '%s and %s to have the same part of speech.' % \
+ (synset1, synset2))
+
+ ic1 = information_content(synset1, ic)
+ ic2 = information_content(synset2, ic)
+ subsumers = synset1.common_hypernyms(synset2)
+ if len(subsumers) == 0:
+ subsumer_ic = 0
+ else:
+ subsumer_ic = max(information_content(s, ic) for s in subsumers)
+
+ if verbose:
+ print("> LCS Subsumer by content:", subsumer_ic)
+
+ return ic1, ic2, subsumer_ic
+
+
+# Utility functions
+
+def information_content(synset, ic):
+ try:
+ icpos = ic[synset._pos]
+ except KeyError:
+ msg = 'Information content file has no entries for part-of-speech: %s'
+ raise WordNetError(msg % synset._pos)
+
+ counts = icpos[synset._offset]
+ if counts == 0:
+ return _INF
+ else:
+ return -math.log(counts / icpos[0])
+
+
+# get the part of speech (NOUN or VERB) from the information content record
+# (each identifier has a 'n' or 'v' suffix)
+
+def _get_pos(field):
+ if field[-1] == 'n':
+ return NOUN
+ elif field[-1] == 'v':
+ return VERB
+ else:
+ msg = "Unidentified part of speech in WordNet Information Content file for field %s" % field
+ raise ValueError(msg)
+
+
+# unload corpus after tests
+def teardown_module(module=None):
+ from nltk.corpus import wordnet
+ wordnet._unload()
+
+
+######################################################################
+# Demo
+######################################################################
+
+def demo():
+ import nltk
+ print('loading wordnet')
+ wn = WordNetCorpusReader(nltk.data.find('corpora/wordnet'), None)
+ print('done loading')
+ S = wn.synset
+ L = wn.lemma
+
+ print('getting a synset for go')
+ move_synset = S('go.v.21')
+ print(move_synset.name(), move_synset.pos(), move_synset.lexname())
+ print(move_synset.lemma_names())
+ print(move_synset.definition())
+ print(move_synset.examples())
+
+ zap_n = ['zap.n.01']
+ zap_v = ['zap.v.01', 'zap.v.02', 'nuke.v.01', 'microwave.v.01']
+
+ def _get_synsets(synset_strings):
+ return [S(synset) for synset in synset_strings]
+
+ zap_n_synsets = _get_synsets(zap_n)
+ zap_v_synsets = _get_synsets(zap_v)
+
+ print(zap_n_synsets)
+ print(zap_v_synsets)
+
+ print("Navigations:")
+ print(S('travel.v.01').hypernyms())
+ print(S('travel.v.02').hypernyms())
+ print(S('travel.v.03').hypernyms())
+
+ print(L('zap.v.03.nuke').derivationally_related_forms())
+ print(L('zap.v.03.atomize').derivationally_related_forms())
+ print(L('zap.v.03.atomise').derivationally_related_forms())
+ print(L('zap.v.03.zap').derivationally_related_forms())
+
+ print(S('dog.n.01').member_holonyms())
+ print(S('dog.n.01').part_meronyms())
+
+ print(S('breakfast.n.1').hypernyms())
+ print(S('meal.n.1').hyponyms())
+ print(S('Austen.n.1').instance_hypernyms())
+ print(S('composer.n.1').instance_hyponyms())
+
+ print(S('faculty.n.2').member_meronyms())
+ print(S('copilot.n.1').member_holonyms())
+
+ print(S('table.n.2').part_meronyms())
+ print(S('course.n.7').part_holonyms())
+
+ print(S('water.n.1').substance_meronyms())
+ print(S('gin.n.1').substance_holonyms())
+
+ print(L('leader.n.1.leader').antonyms())
+ print(L('increase.v.1.increase').antonyms())
+
+ print(S('snore.v.1').entailments())
+ print(S('heavy.a.1').similar_tos())
+ print(S('light.a.1').attributes())
+ print(S('heavy.a.1').attributes())
+
+ print(L('English.a.1.English').pertainyms())
+
+ print(S('person.n.01').root_hypernyms())
+ print(S('sail.v.01').root_hypernyms())
+ print(S('fall.v.12').root_hypernyms())
+
+ print(S('person.n.01').lowest_common_hypernyms(S('dog.n.01')))
+ print(S('woman.n.01').lowest_common_hypernyms(S('girlfriend.n.02')))
+
+ print(S('dog.n.01').path_similarity(S('cat.n.01')))
+ print(S('dog.n.01').lch_similarity(S('cat.n.01')))
+ print(S('dog.n.01').wup_similarity(S('cat.n.01')))
+
+ wnic = WordNetICCorpusReader(nltk.data.find('corpora/wordnet_ic'),
+ '.*\.dat')
+ ic = wnic.ic('ic-brown.dat')
+ print(S('dog.n.01').jcn_similarity(S('cat.n.01'), ic))
+
+ ic = wnic.ic('ic-semcor.dat')
+ print(S('dog.n.01').lin_similarity(S('cat.n.01'), ic))
+
+ print(S('code.n.03').topic_domains())
+ print(S('pukka.a.01').region_domains())
+ print(S('freaky.a.01').usage_domains())
+
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/corpus/reader/xmldocs.py b/nltk/corpus/reader/xmldocs.py
new file mode 100644
index 0000000..d55001b
--- /dev/null
+++ b/nltk/corpus/reader/xmldocs.py
@@ -0,0 +1,387 @@
+# Natural Language Toolkit: XML Corpus Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Corpus reader for corpora whose documents are xml files.
+
+(note -- not named 'xml' to avoid conflicting w/ standard xml package)
+"""
+from __future__ import print_function, unicode_literals
+
+import codecs
+
+# Use the c version of ElementTree, which is faster, if possible:
+try: from xml.etree import cElementTree as ElementTree
+except ImportError: from xml.etree import ElementTree
+
+from nltk import compat
+from nltk.data import SeekableUnicodeStreamReader
+from nltk.tokenize import WordPunctTokenizer
+from nltk.internals import ElementWrapper
+
+from nltk.corpus.reader.api import CorpusReader
+from nltk.corpus.reader.util import *
+
+class XMLCorpusReader(CorpusReader):
+ """
+ Corpus reader for corpora whose documents are xml files.
+
+ Note that the ``XMLCorpusReader`` constructor does not take an
+ ``encoding`` argument, because the unicode encoding is specified by
+ the XML files themselves. See the XML specs for more info.
+ """
+ def __init__(self, root, fileids, wrap_etree=False):
+ self._wrap_etree = wrap_etree
+ CorpusReader.__init__(self, root, fileids)
+
+ def xml(self, fileid=None):
+ # Make sure we have exactly one file -- no concatenating XML.
+ if fileid is None and len(self._fileids) == 1:
+ fileid = self._fileids[0]
+ if not isinstance(fileid, compat.string_types):
+ raise TypeError('Expected a single file identifier string')
+ # Read the XML in using ElementTree.
+ elt = ElementTree.parse(self.abspath(fileid).open()).getroot()
+ # If requested, wrap it.
+ if self._wrap_etree:
+ elt = ElementWrapper(elt)
+ # Return the ElementTree element.
+ return elt
+
+ def words(self, fileid=None):
+ """
+ Returns all of the words and punctuation symbols in the specified file
+ that were in text nodes -- ie, tags are ignored. Like the xml() method,
+ fileid can only specify one file.
+
+ :return: the given file's text nodes as a list of words and punctuation symbols
+ :rtype: list(str)
+ """
+
+ elt = self.xml(fileid)
+ encoding = self.encoding(fileid)
+ word_tokenizer=WordPunctTokenizer()
+ iterator = elt.getiterator()
+ out = []
+
+ for node in iterator:
+ text = node.text
+ if text is not None:
+ if isinstance(text, bytes):
+ text = text.decode(encoding)
+ toks = word_tokenizer.tokenize(text)
+ out.extend(toks)
+ return out
+
+ def raw(self, fileids=None):
+ if fileids is None: fileids = self._fileids
+ elif isinstance(fileids, compat.string_types): fileids = [fileids]
+ return concat([self.open(f).read() for f in fileids])
+
+
+class XMLCorpusView(StreamBackedCorpusView):
+ """
+ A corpus view that selects out specified elements from an XML
+ file, and provides a flat list-like interface for accessing them.
+ (Note: ``XMLCorpusView`` is not used by ``XMLCorpusReader`` itself,
+ but may be used by subclasses of ``XMLCorpusReader``.)
+
+ Every XML corpus view has a "tag specification", indicating what
+ XML elements should be included in the view; and each (non-nested)
+ element that matches this specification corresponds to one item in
+ the view. Tag specifications are regular expressions over tag
+ paths, where a tag path is a list of element tag names, separated
+ by '/', indicating the ancestry of the element. Some examples:
+
+ - ``'foo'``: A top-level element whose tag is ``foo``.
+ - ``'foo/bar'``: An element whose tag is ``bar`` and whose parent
+ is a top-level element whose tag is ``foo``.
+ - ``'.*/foo'``: An element whose tag is ``foo``, appearing anywhere
+ in the xml tree.
+ - ``'.*/(foo|bar)'``: An wlement whose tag is ``foo`` or ``bar``,
+ appearing anywhere in the xml tree.
+
+ The view items are generated from the selected XML elements via
+ the method ``handle_elt()``. By default, this method returns the
+ element as-is (i.e., as an ElementTree object); but it can be
+ overridden, either via subclassing or via the ``elt_handler``
+ constructor parameter.
+ """
+
+ #: If true, then display debugging output to stdout when reading
+ #: blocks.
+ _DEBUG = False
+
+ #: The number of characters read at a time by this corpus reader.
+ _BLOCK_SIZE = 1024
+
+ def __init__(self, fileid, tagspec, elt_handler=None):
+ """
+ Create a new corpus view based on a specified XML file.
+
+ Note that the ``XMLCorpusView`` constructor does not take an
+ ``encoding`` argument, because the unicode encoding is
+ specified by the XML files themselves.
+
+ :type tagspec: str
+ :param tagspec: A tag specification, indicating what XML
+ elements should be included in the view. Each non-nested
+ element that matches this specification corresponds to one
+ item in the view.
+
+ :param elt_handler: A function used to transform each element
+ to a value for the view. If no handler is specified, then
+ ``self.handle_elt()`` is called, which returns the element
+ as an ElementTree object. The signature of elt_handler is::
+
+ elt_handler(elt, tagspec) -> value
+ """
+ if elt_handler: self.handle_elt = elt_handler
+
+ self._tagspec = re.compile(tagspec+r'\Z')
+ """The tag specification for this corpus view."""
+
+ self._tag_context = {0: ()}
+ """A dictionary mapping from file positions (as returned by
+ ``stream.seek()`` to XML contexts. An XML context is a
+ tuple of XML tag names, indicating which tags have not yet
+ been closed."""
+
+ encoding = self._detect_encoding(fileid)
+ StreamBackedCorpusView.__init__(self, fileid, encoding=encoding)
+
+ def _detect_encoding(self, fileid):
+ if isinstance(fileid, PathPointer):
+ s = fileid.open().readline()
+ else:
+ with open(fileid, 'rb') as infile:
+ s = infile.readline()
+ if s.startswith(codecs.BOM_UTF16_BE):
+ return 'utf-16-be'
+ if s.startswith(codecs.BOM_UTF16_LE):
+ return 'utf-16-le'
+ if s.startswith(codecs.BOM_UTF32_BE):
+ return 'utf-32-be'
+ if s.startswith(codecs.BOM_UTF32_LE):
+ return 'utf-32-le'
+ if s.startswith(codecs.BOM_UTF8):
+ return 'utf-8'
+ m = re.match(br'\s*<\?xml\b.*\bencoding="([^"]+)"', s)
+ if m:
+ return m.group(1).decode()
+ m = re.match(br"\s*<\?xml\b.*\bencoding='([^']+)'", s)
+ if m:
+ return m.group(1).decode()
+ # No encoding found -- what should the default be?
+ return 'utf-8'
+
+ def handle_elt(self, elt, context):
+ """
+ Convert an element into an appropriate value for inclusion in
+ the view. Unless overridden by a subclass or by the
+ ``elt_handler`` constructor argument, this method simply
+ returns ``elt``.
+
+ :return: The view value corresponding to ``elt``.
+
+ :type elt: ElementTree
+ :param elt: The element that should be converted.
+
+ :type context: str
+ :param context: A string composed of element tags separated by
+ forward slashes, indicating the XML context of the given
+ element. For example, the string ``'foo/bar/baz'``
+ indicates that the element is a ``baz`` element whose
+ parent is a ``bar`` element and whose grandparent is a
+ top-level ``foo`` element.
+ """
+ return elt
+
+ #: A regular expression that matches XML fragments that do not
+ #: contain any un-closed tags.
+ _VALID_XML_RE = re.compile(r"""
+ [^<]*
+ (
+ ((<!--.*?-->) | # comment
+ (<![CDATA[.*?]]) | # raw character data
+ (<!DOCTYPE\s+[^\[]*(\[[^\]]*])?\s*>) | # doctype decl
+ (<[^!>][^>]*>)) # tag or PI
+ [^<]*)*
+ \Z""",
+ re.DOTALL|re.VERBOSE)
+
+ #: A regular expression used to extract the tag name from a start tag,
+ #: end tag, or empty-elt tag string.
+ _XML_TAG_NAME = re.compile('<\s*/?\s*([^\s>]+)')
+
+ #: A regular expression used to find all start-tags, end-tags, and
+ #: emtpy-elt tags in an XML file. This regexp is more lenient than
+ #: the XML spec -- e.g., it allows spaces in some places where the
+ #: spec does not.
+ _XML_PIECE = re.compile(r"""
+ # Include these so we can skip them:
+ (?P<COMMENT> <!--.*?--> )|
+ (?P<CDATA> <![CDATA[.*?]]> )|
+ (?P<PI> <\?.*?\?> )|
+ (?P<DOCTYPE> <!DOCTYPE\s+[^\[^>]*(\[[^\]]*])?\s*>)|
+ # These are the ones we actually care about:
+ (?P<EMPTY_ELT_TAG> <\s*[^>/\?!\s][^>]*/\s*> )|
+ (?P<START_TAG> <\s*[^>/\?!\s][^>]*> )|
+ (?P<END_TAG> <\s*/[^>/\?!\s][^>]*> )""",
+ re.DOTALL|re.VERBOSE)
+
+ def _read_xml_fragment(self, stream):
+ """
+ Read a string from the given stream that does not contain any
+ un-closed tags. In particular, this function first reads a
+ block from the stream of size ``self._BLOCK_SIZE``. It then
+ checks if that block contains an un-closed tag. If it does,
+ then this function either backtracks to the last '<', or reads
+ another block.
+ """
+ fragment = ''
+
+ if isinstance(stream, SeekableUnicodeStreamReader):
+ startpos = stream.tell()
+ while True:
+ # Read a block and add it to the fragment.
+ xml_block = stream.read(self._BLOCK_SIZE)
+ fragment += xml_block
+
+ # Do we have a well-formed xml fragment?
+ if self._VALID_XML_RE.match(fragment):
+ return fragment
+
+ # Do we have a fragment that will never be well-formed?
+ if re.search('[<>]', fragment).group(0) == '>':
+ pos = stream.tell() - (
+ len(fragment)-re.search('[<>]', fragment).end())
+ raise ValueError('Unexpected ">" near char %s' % pos)
+
+ # End of file?
+ if not xml_block:
+ raise ValueError('Unexpected end of file: tag not closed')
+
+ # If not, then we must be in the middle of a <..tag..>.
+ # If appropriate, backtrack to the most recent '<'
+ # character.
+ last_open_bracket = fragment.rfind('<')
+ if last_open_bracket > 0:
+ if self._VALID_XML_RE.match(fragment[:last_open_bracket]):
+ if isinstance(stream, SeekableUnicodeStreamReader):
+ stream.seek(startpos)
+ stream.char_seek_forward(last_open_bracket)
+ else:
+ stream.seek(-(len(fragment)-last_open_bracket), 1)
+ return fragment[:last_open_bracket]
+
+ # Otherwise, read another block. (i.e., return to the
+ # top of the loop.)
+
+ def read_block(self, stream, tagspec=None, elt_handler=None):
+ """
+ Read from ``stream`` until we find at least one element that
+ matches ``tagspec``, and return the result of applying
+ ``elt_handler`` to each element found.
+ """
+ if tagspec is None: tagspec = self._tagspec
+ if elt_handler is None: elt_handler = self.handle_elt
+
+ # Use a stack of strings to keep track of our context:
+ context = list(self._tag_context.get(stream.tell()))
+ assert context is not None # check this -- could it ever happen?
+
+ elts = []
+
+ elt_start = None # where does the elt start
+ elt_depth = None # what context depth
+ elt_text = ''
+
+ while elts==[] or elt_start is not None:
+ if isinstance(stream, SeekableUnicodeStreamReader):
+ startpos = stream.tell()
+ xml_fragment = self._read_xml_fragment(stream)
+
+ # End of file.
+ if not xml_fragment:
+ if elt_start is None: break
+ else: raise ValueError('Unexpected end of file')
+
+ # Process each <tag> in the xml fragment.
+ for piece in self._XML_PIECE.finditer(xml_fragment):
+ if self._DEBUG:
+ print('%25s %s' % ('/'.join(context)[-20:], piece.group()))
+
+ if piece.group('START_TAG'):
+ name = self._XML_TAG_NAME.match(piece.group()).group(1)
+ # Keep context up-to-date.
+ context.append(name)
+ # Is this one of the elts we're looking for?
+ if elt_start is None:
+ if re.match(tagspec, '/'.join(context)):
+ elt_start = piece.start()
+ elt_depth = len(context)
+
+ elif piece.group('END_TAG'):
+ name = self._XML_TAG_NAME.match(piece.group()).group(1)
+ # sanity checks:
+ if not context:
+ raise ValueError('Unmatched tag </%s>' % name)
+ if name != context[-1]:
+ raise ValueError('Unmatched tag <%s>...</%s>' %
+ (context[-1], name))
+ # Is this the end of an element?
+ if elt_start is not None and elt_depth == len(context):
+ elt_text += xml_fragment[elt_start:piece.end()]
+ elts.append( (elt_text, '/'.join(context)) )
+ elt_start = elt_depth = None
+ elt_text = ''
+ # Keep context up-to-date
+ context.pop()
+
+ elif piece.group('EMPTY_ELT_TAG'):
+ name = self._XML_TAG_NAME.match(piece.group()).group(1)
+ if elt_start is None:
+ if re.match(tagspec, '/'.join(context)+'/'+name):
+ elts.append((piece.group(),
+ '/'.join(context)+'/'+name))
+
+ if elt_start is not None:
+ # If we haven't found any elements yet, then keep
+ # looping until we do.
+ if elts == []:
+ elt_text += xml_fragment[elt_start:]
+ elt_start = 0
+
+ # If we've found at least one element, then try
+ # backtracking to the start of the element that we're
+ # inside of.
+ else:
+ # take back the last start-tag, and return what
+ # we've gotten so far (elts is non-empty).
+ if self._DEBUG:
+ print(' '*36+'(backtrack)')
+ if isinstance(stream, SeekableUnicodeStreamReader):
+ stream.seek(startpos)
+ stream.char_seek_forward(elt_start)
+ else:
+ stream.seek(-(len(xml_fragment)-elt_start), 1)
+ context = context[:elt_depth-1]
+ elt_start = elt_depth = None
+ elt_text = ''
+
+ # Update the _tag_context dict.
+ pos = stream.tell()
+ if pos in self._tag_context:
+ assert tuple(context) == self._tag_context[pos]
+ else:
+ self._tag_context[pos] = tuple(context)
+
+ return [elt_handler(ElementTree.fromstring(
+ elt.encode('ascii', 'xmlcharrefreplace')),
+ context)
+ for (elt, context) in elts]
diff --git a/nltk/corpus/reader/ycoe.py b/nltk/corpus/reader/ycoe.py
new file mode 100644
index 0000000..bd2c022
--- /dev/null
+++ b/nltk/corpus/reader/ycoe.py
@@ -0,0 +1,242 @@
+# -*- coding: iso-8859-1 -*-
+
+# Natural Language Toolkit: York-Toronto-Helsinki Parsed Corpus of Old English Prose (YCOE)
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Selina Dennis <selina at tranzfusion.net>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Corpus reader for the York-Toronto-Helsinki Parsed Corpus of Old
+English Prose (YCOE), a 1.5 million word syntactically-annotated
+corpus of Old English prose texts. The corpus is distributed by the
+Oxford Text Archive: http://www.ota.ahds.ac.uk/ It is not included
+with NLTK.
+
+The YCOE corpus is divided into 100 files, each representing
+an Old English prose text. Tags used within each text complies
+to the YCOE standard: http://www-users.york.ac.uk/~lang22/YCOE/YcoeHome.htm
+"""
+
+import os
+import re
+
+from nltk import compat
+from nltk.tokenize import RegexpTokenizer
+from nltk.corpus.reader.bracket_parse import BracketParseCorpusReader
+from nltk.corpus.reader.tagged import TaggedCorpusReader
+
+from nltk.corpus.reader.util import *
+from nltk.corpus.reader.api import *
+
+class YCOECorpusReader(CorpusReader):
+ """
+ Corpus reader for the York-Toronto-Helsinki Parsed Corpus of Old
+ English Prose (YCOE), a 1.5 million word syntactically-annotated
+ corpus of Old English prose texts.
+ """
+ def __init__(self, root, encoding='utf8'):
+ CorpusReader.__init__(self, root, [], encoding)
+
+ self._psd_reader = YCOEParseCorpusReader(
+ self.root.join('psd'), '.*', '.psd', encoding=encoding)
+ self._pos_reader = YCOETaggedCorpusReader(
+ self.root.join('pos'), '.*', '.pos')
+
+ # Make sure we have a consistent set of items:
+ documents = set(f[:-4] for f in self._psd_reader.fileids())
+ if set(f[:-4] for f in self._pos_reader.fileids()) != documents:
+ raise ValueError('Items in "psd" and "pos" '
+ 'subdirectories do not match.')
+
+ fileids = sorted(['%s.psd' % doc for doc in documents] +
+ ['%s.pos' % doc for doc in documents])
+ CorpusReader.__init__(self, root, fileids, encoding)
+ self._documents = sorted(documents)
+
+ def documents(self, fileids=None):
+ """
+ Return a list of document identifiers for all documents in
+ this corpus, or for the documents with the given file(s) if
+ specified.
+ """
+ if fileids is None:
+ return self._documents
+ if isinstance(fileids, compat.string_types):
+ fileids = [fileids]
+ for f in fileids:
+ if f not in self._fileids:
+ raise KeyError('File id %s not found' % fileids)
+ # Strip off the '.pos' and '.psd' extensions.
+ return sorted(set(f[:-4] for f in fileids))
+
+ def fileids(self, documents=None):
+ """
+ Return a list of file identifiers for the files that make up
+ this corpus, or that store the given document(s) if specified.
+ """
+ if documents is None:
+ return self._fileids
+ elif isinstance(documents, compat.string_types):
+ documents = [documents]
+ return sorted(set(['%s.pos' % doc for doc in documents] +
+ ['%s.psd' % doc for doc in documents]))
+
+ def _getfileids(self, documents, subcorpus):
+ """
+ Helper that selects the appropriate fileids for a given set of
+ documents from a given subcorpus (pos or psd).
+ """
+ if documents is None:
+ documents = self._documents
+ else:
+ if isinstance(documents, compat.string_types):
+ documents = [documents]
+ for document in documents:
+ if document not in self._documents:
+ if document[-4:] in ('.pos', '.psd'):
+ raise ValueError(
+ 'Expected a document identifier, not a file '
+ 'identifier. (Use corpus.documents() to get '
+ 'a list of document identifiers.')
+ else:
+ raise ValueError('Document identifier %s not found'
+ % document)
+ return ['%s.%s' % (d, subcorpus) for d in documents]
+
+ # Delegate to one of our two sub-readers:
+ def words(self, documents=None):
+ return self._pos_reader.words(self._getfileids(documents, 'pos'))
+ def sents(self, documents=None):
+ return self._pos_reader.sents(self._getfileids(documents, 'pos'))
+ def paras(self, documents=None):
+ return self._pos_reader.paras(self._getfileids(documents, 'pos'))
+ def tagged_words(self, documents=None):
+ return self._pos_reader.tagged_words(self._getfileids(documents, 'pos'))
+ def tagged_sents(self, documents=None):
+ return self._pos_reader.tagged_sents(self._getfileids(documents, 'pos'))
+ def tagged_paras(self, documents=None):
+ return self._pos_reader.tagged_paras(self._getfileids(documents, 'pos'))
+ def parsed_sents(self, documents=None):
+ return self._psd_reader.parsed_sents(self._getfileids(documents, 'psd'))
+
+
+class YCOEParseCorpusReader(BracketParseCorpusReader):
+ """Specialized version of the standard bracket parse corpus reader
+ that strips out (CODE ...) and (ID ...) nodes."""
+ def _parse(self, t):
+ t = re.sub(r'(?u)\((CODE|ID)[^\)]*\)', '', t)
+ if re.match(r'\s*\(\s*\)\s*$', t): return None
+ return BracketParseCorpusReader._parse(self, t)
+
+class YCOETaggedCorpusReader(TaggedCorpusReader):
+ def __init__(self, root, items, encoding='utf8'):
+ gaps_re = r'(?u)(?<=/\.)\s+|\s*\S*_CODE\s*|\s*\S*_ID\s*'
+ sent_tokenizer = RegexpTokenizer(gaps_re, gaps=True)
+ TaggedCorpusReader.__init__(self, root, items, sep='_',
+ sent_tokenizer=sent_tokenizer)
+
+#: A list of all documents and their titles in ycoe.
+documents = {
+ 'coadrian.o34': 'Adrian and Ritheus',
+ 'coaelhom.o3': '�lfric, Supplemental Homilies',
+ 'coaelive.o3': '�lfric\'s Lives of Saints',
+ 'coalcuin': 'Alcuin De virtutibus et vitiis',
+ 'coalex.o23': 'Alexander\'s Letter to Aristotle',
+ 'coapollo.o3': 'Apollonius of Tyre',
+ 'coaugust': 'Augustine',
+ 'cobede.o2': 'Bede\'s History of the English Church',
+ 'cobenrul.o3': 'Benedictine Rule',
+ 'coblick.o23': 'Blickling Homilies',
+ 'coboeth.o2': 'Boethius\' Consolation of Philosophy',
+ 'cobyrhtf.o3': 'Byrhtferth\'s Manual',
+ 'cocanedgD': 'Canons of Edgar (D)',
+ 'cocanedgX': 'Canons of Edgar (X)',
+ 'cocathom1.o3': '�lfric\'s Catholic Homilies I',
+ 'cocathom2.o3': '�lfric\'s Catholic Homilies II',
+ 'cochad.o24': 'Saint Chad',
+ 'cochdrul': 'Chrodegang of Metz, Rule',
+ 'cochristoph': 'Saint Christopher',
+ 'cochronA.o23': 'Anglo-Saxon Chronicle A',
+ 'cochronC': 'Anglo-Saxon Chronicle C',
+ 'cochronD': 'Anglo-Saxon Chronicle D',
+ 'cochronE.o34': 'Anglo-Saxon Chronicle E',
+ 'cocura.o2': 'Cura Pastoralis',
+ 'cocuraC': 'Cura Pastoralis (Cotton)',
+ 'codicts.o34': 'Dicts of Cato',
+ 'codocu1.o1': 'Documents 1 (O1)',
+ 'codocu2.o12': 'Documents 2 (O1/O2)',
+ 'codocu2.o2': 'Documents 2 (O2)',
+ 'codocu3.o23': 'Documents 3 (O2/O3)',
+ 'codocu3.o3': 'Documents 3 (O3)',
+ 'codocu4.o24': 'Documents 4 (O2/O4)',
+ 'coeluc1': 'Honorius of Autun, Elucidarium 1',
+ 'coeluc2': 'Honorius of Autun, Elucidarium 1',
+ 'coepigen.o3': '�lfric\'s Epilogue to Genesis',
+ 'coeuphr': 'Saint Euphrosyne',
+ 'coeust': 'Saint Eustace and his companions',
+ 'coexodusP': 'Exodus (P)',
+ 'cogenesiC': 'Genesis (C)',
+ 'cogregdC.o24': 'Gregory\'s Dialogues (C)',
+ 'cogregdH.o23': 'Gregory\'s Dialogues (H)',
+ 'coherbar': 'Pseudo-Apuleius, Herbarium',
+ 'coinspolD.o34': 'Wulfstan\'s Institute of Polity (D)',
+ 'coinspolX': 'Wulfstan\'s Institute of Polity (X)',
+ 'cojames': 'Saint James',
+ 'colacnu.o23': 'Lacnunga',
+ 'colaece.o2': 'Leechdoms',
+ 'colaw1cn.o3': 'Laws, Cnut I',
+ 'colaw2cn.o3': 'Laws, Cnut II',
+ 'colaw5atr.o3': 'Laws, �thelred V',
+ 'colaw6atr.o3': 'Laws, �thelred VI',
+ 'colawaf.o2': 'Laws, Alfred',
+ 'colawafint.o2': 'Alfred\'s Introduction to Laws',
+ 'colawger.o34': 'Laws, Gerefa',
+ 'colawine.ox2': 'Laws, Ine',
+ 'colawnorthu.o3': 'Northumbra Preosta Lagu',
+ 'colawwllad.o4': 'Laws, William I, Lad',
+ 'coleofri.o4': 'Leofric',
+ 'colsigef.o3': '�lfric\'s Letter to Sigefyrth',
+ 'colsigewB': '�lfric\'s Letter to Sigeweard (B)',
+ 'colsigewZ.o34': '�lfric\'s Letter to Sigeweard (Z)',
+ 'colwgeat': '�lfric\'s Letter to Wulfgeat',
+ 'colwsigeT': '�lfric\'s Letter to Wulfsige (T)',
+ 'colwsigeXa.o34': '�lfric\'s Letter to Wulfsige (Xa)',
+ 'colwstan1.o3': '�lfric\'s Letter to Wulfstan I',
+ 'colwstan2.o3': '�lfric\'s Letter to Wulfstan II',
+ 'comargaC.o34': 'Saint Margaret (C)',
+ 'comargaT': 'Saint Margaret (T)',
+ 'comart1': 'Martyrology, I',
+ 'comart2': 'Martyrology, II',
+ 'comart3.o23': 'Martyrology, III',
+ 'comarvel.o23': 'Marvels of the East',
+ 'comary': 'Mary of Egypt',
+ 'coneot': 'Saint Neot',
+ 'conicodA': 'Gospel of Nicodemus (A)',
+ 'conicodC': 'Gospel of Nicodemus (C)',
+ 'conicodD': 'Gospel of Nicodemus (D)',
+ 'conicodE': 'Gospel of Nicodemus (E)',
+ 'coorosiu.o2': 'Orosius',
+ 'cootest.o3': 'Heptateuch',
+ 'coprefcath1.o3': '�lfric\'s Preface to Catholic Homilies I',
+ 'coprefcath2.o3': '�lfric\'s Preface to Catholic Homilies II',
+ 'coprefcura.o2': 'Preface to the Cura Pastoralis',
+ 'coprefgen.o3': '�lfric\'s Preface to Genesis',
+ 'copreflives.o3': '�lfric\'s Preface to Lives of Saints',
+ 'coprefsolilo': 'Preface to Augustine\'s Soliloquies',
+ 'coquadru.o23': 'Pseudo-Apuleius, Medicina de quadrupedibus',
+ 'corood': 'History of the Holy Rood-Tree',
+ 'cosevensl': 'Seven Sleepers',
+ 'cosolilo': 'St. Augustine\'s Soliloquies',
+ 'cosolsat1.o4': 'Solomon and Saturn I',
+ 'cosolsat2': 'Solomon and Saturn II',
+ 'cotempo.o3': '�lfric\'s De Temporibus Anni',
+ 'coverhom': 'Vercelli Homilies',
+ 'coverhomE': 'Vercelli Homilies (E)',
+ 'coverhomL': 'Vercelli Homilies (L)',
+ 'covinceB': 'Saint Vincent (Bodley 343)',
+ 'covinsal': 'Vindicta Salvatoris',
+ 'cowsgosp.o3': 'West-Saxon Gospels',
+ 'cowulf.o34': 'Wulfstan\'s Homilies'
+ }
diff --git a/nltk/corpus/util.py b/nltk/corpus/util.py
new file mode 100644
index 0000000..f7449b8
--- /dev/null
+++ b/nltk/corpus/util.py
@@ -0,0 +1,127 @@
+# Natural Language Toolkit: Corpus Reader Utility Functions
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+######################################################################
+#{ Lazy Corpus Loader
+######################################################################
+
+from __future__ import unicode_literals
+import re
+import gc
+import nltk
+from nltk.compat import python_2_unicode_compatible
+
+TRY_ZIPFILE_FIRST = False
+
+ at python_2_unicode_compatible
+class LazyCorpusLoader(object):
+ """
+ To see the API documentation for this lazily loaded corpus, first
+ run corpus.ensure_loaded(), and then run help(this_corpus).
+
+ LazyCorpusLoader is a proxy object which is used to stand in for a
+ corpus object before the corpus is loaded. This allows NLTK to
+ create an object for each corpus, but defer the costs associated
+ with loading those corpora until the first time that they're
+ actually accessed.
+
+ The first time this object is accessed in any way, it will load
+ the corresponding corpus, and transform itself into that corpus
+ (by modifying its own ``__class__`` and ``__dict__`` attributes).
+
+ If the corpus can not be found, then accessing this object will
+ raise an exception, displaying installation instructions for the
+ NLTK data package. Once they've properly installed the data
+ package (or modified ``nltk.data.path`` to point to its location),
+ they can then use the corpus object without restarting python.
+ """
+ def __init__(self, name, reader_cls, *args, **kwargs):
+ from nltk.corpus.reader.api import CorpusReader
+ assert issubclass(reader_cls, CorpusReader)
+ self.__name = self.__name__ = name
+ self.__reader_cls = reader_cls
+ self.__args = args
+ self.__kwargs = kwargs
+
+ def __load(self):
+ # Find the corpus root directory.
+ zip_name = re.sub(r'(([^/]*)(/.*)?)', r'\2.zip/\1/', self.__name)
+ if TRY_ZIPFILE_FIRST:
+ try:
+ root = nltk.data.find('corpora/%s' % zip_name)
+ except LookupError as e:
+ try: root = nltk.data.find('corpora/%s' % self.__name)
+ except LookupError: raise e
+ else:
+ try:
+ root = nltk.data.find('corpora/%s' % self.__name)
+ except LookupError as e:
+ try: root = nltk.data.find('corpora/%s' % zip_name)
+ except LookupError: raise e
+
+ # Load the corpus.
+ corpus = self.__reader_cls(root, *self.__args, **self.__kwargs)
+
+ # This is where the magic happens! Transform ourselves into
+ # the corpus by modifying our own __dict__ and __class__ to
+ # match that of the corpus.
+
+ args, kwargs = self.__args, self.__kwargs
+ name, reader_cls = self.__name, self.__reader_cls
+
+ self.__dict__ = corpus.__dict__
+ self.__class__ = corpus.__class__
+
+ # _unload support: assign __dict__ and __class__ back, then do GC.
+ # after reassigning __dict__ there shouldn't be any references to
+ # corpus data so the memory should be deallocated after gc.collect()
+ def _unload(self):
+ lazy_reader = LazyCorpusLoader(name, reader_cls, *args, **kwargs)
+ self.__dict__ = lazy_reader.__dict__
+ self.__class__ = lazy_reader.__class__
+ gc.collect()
+
+ self._unload = _make_bound_method(_unload, self)
+
+ def __getattr__(self, attr):
+
+ # Fix for inspect.isclass under Python 2.6
+ # (see http://bugs.python.org/issue1225107).
+ # Without this fix tests may take extra 1.5GB RAM
+ # because all corpora gets loaded during test collection.
+ if attr == '__bases__':
+ raise AttributeError("LazyCorpusLoader object has no attribute '__bases__'")
+
+ self.__load()
+ # This looks circular, but its not, since __load() changes our
+ # __class__ to something new:
+ return getattr(self, attr)
+
+ def __repr__(self):
+ return '<%s in %r (not loaded yet)>' % (
+ self.__reader_cls.__name__, '.../corpora/'+self.__name)
+
+ def _unload(self):
+ # If an exception occures during corpus loading then
+ # '_unload' method may be unattached, so __getattr__ can be called;
+ # we shouldn't trigger corpus loading again in this case.
+ pass
+
+
+def _make_bound_method(func, self):
+ """
+ Magic for creating bound methods (used for _unload).
+ """
+ class Foo(object):
+ def meth(self): pass
+ f = Foo()
+ bound_method = type(f.meth)
+
+ try:
+ return bound_method(func, self, self.__class__)
+ except TypeError: # python3
+ return bound_method(func, self)
diff --git a/nltk/data.py b/nltk/data.py
new file mode 100644
index 0000000..1b80b18
--- /dev/null
+++ b/nltk/data.py
@@ -0,0 +1,1404 @@
+# Natural Language Toolkit: Utility functions
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Functions to find and load NLTK resource files, such as corpora,
+grammars, and saved processing objects. Resource files are identified
+using URLs, such as ``nltk:corpora/abc/rural.txt`` or
+``http://nltk.org/sample/toy.cfg``. The following URL protocols are
+supported:
+
+ - ``file:path``: Specifies the file whose path is *path*.
+ Both relative and absolute paths may be used.
+
+ - ``http://host/path``: Specifies the file stored on the web
+ server *host* at path *path*.
+
+ - ``nltk:path``: Specifies the file stored in the NLTK data
+ package at *path*. NLTK will search for these files in the
+ directories specified by ``nltk.data.path``.
+
+If no protocol is specified, then the default protocol ``nltk:`` will
+be used.
+
+This module provides to functions that can be used to access a
+resource file, given its URL: ``load()`` loads a given resource, and
+adds it to a resource cache; and ``retrieve()`` copies a given resource
+to a local file.
+"""
+from __future__ import print_function, unicode_literals
+from __future__ import division
+
+import sys
+import io
+import os
+import textwrap
+import re
+import zipfile
+import codecs
+
+from gzip import GzipFile, READ as GZ_READ, WRITE as GZ_WRITE
+
+try:
+ from zlib import Z_SYNC_FLUSH as FLUSH
+except ImportError:
+ from zlib import Z_FINISH as FLUSH
+
+try:
+ import cPickle as pickle
+except ImportError:
+ import pickle
+
+# this import should be more specific:
+import nltk
+
+from nltk.compat import py3_data, text_type, string_types, BytesIO, urlopen
+
+######################################################################
+# Search Path
+######################################################################
+
+path = []
+"""A list of directories where the NLTK data package might reside.
+ These directories will be checked in order when looking for a
+ resource in the data package. Note that this allows users to
+ substitute in their own versions of resources, if they have them
+ (e.g., in their home directory under ~/nltk_data)."""
+
+# User-specified locations:
+path += [d for d in os.environ.get('NLTK_DATA', str('')).split(os.pathsep) if d]
+if os.path.expanduser('~/') != '~/':
+ path.append(os.path.expanduser(str('~/nltk_data')))
+
+if sys.platform.startswith('win'):
+ # Common locations on Windows:
+ path += [
+ str(r'C:\nltk_data'), str(r'D:\nltk_data'), str(r'E:\nltk_data'),
+ os.path.join(sys.prefix, str('nltk_data')),
+ os.path.join(sys.prefix, str('lib'), str('nltk_data')),
+ os.path.join(os.environ.get(str('APPDATA'), str('C:\\')), str('nltk_data'))
+ ]
+else:
+ # Common locations on UNIX & OS X:
+ path += [
+ str('/usr/share/nltk_data'),
+ str('/usr/local/share/nltk_data'),
+ str('/usr/lib/nltk_data'),
+ str('/usr/local/lib/nltk_data')
+ ]
+
+
+######################################################################
+# Util Functions
+######################################################################
+
+def gzip_open_unicode(filename, mode="rb", compresslevel=9,
+ encoding='utf-8', fileobj=None, errors=None, newline=None):
+ if fileobj is None:
+ fileobj=GzipFile(filename, mode, compresslevel, fileobj)
+ return io.TextIOWrapper(fileobj, encoding, errors, newline)
+
+def split_resource_url(resource_url):
+ """
+ Splits a resource url into "<protocol>:<path>".
+
+ >>> windows = sys.platform.startswith('win')
+ >>> split_resource_url('nltk:home/nltk')
+ ('nltk', 'home/nltk')
+ >>> split_resource_url('nltk:/home/nltk')
+ ('nltk', '/home/nltk')
+ >>> split_resource_url('file:/home/nltk')
+ ('file', '/home/nltk')
+ >>> split_resource_url('file:///home/nltk')
+ ('file', '/home/nltk')
+ >>> split_resource_url('file:///C:/home/nltk')
+ ('file', '/C:/home/nltk')
+ """
+ protocol, path_ = resource_url.split(':', 1)
+ if protocol == 'nltk':
+ pass
+ elif protocol == 'file':
+ if path_.startswith('/'):
+ path_ = '/' + path_.lstrip('/')
+ else:
+ path_ = re.sub(r'^/{0,2}', '', path_)
+ return protocol, path_
+
+def normalize_resource_url(resource_url):
+ r"""
+ Normalizes a resource url
+
+ >>> windows = sys.platform.startswith('win')
+ >>> os.path.normpath(split_resource_url(normalize_resource_url('file:grammar.fcfg'))[1]) == \
+ ... ('\\' if windows else '') + os.path.abspath(os.path.join(os.curdir, 'grammar.fcfg'))
+ True
+ >>> not windows or normalize_resource_url('file:C:/dir/file') == 'file:///C:/dir/file'
+ True
+ >>> not windows or normalize_resource_url('file:C:\\dir\\file') == 'file:///C:/dir/file'
+ True
+ >>> not windows or normalize_resource_url('file:C:\\dir/file') == 'file:///C:/dir/file'
+ True
+ >>> not windows or normalize_resource_url('file://C:/dir/file') == 'file:///C:/dir/file'
+ True
+ >>> not windows or normalize_resource_url('file:////C:/dir/file') == 'file:///C:/dir/file'
+ True
+ >>> not windows or normalize_resource_url('nltk:C:/dir/file') == 'file:///C:/dir/file'
+ True
+ >>> not windows or normalize_resource_url('nltk:C:\\dir\\file') == 'file:///C:/dir/file'
+ True
+ >>> windows or normalize_resource_url('file:/dir/file/toy.cfg') == 'file:///dir/file/toy.cfg'
+ True
+ >>> normalize_resource_url('nltk:home/nltk')
+ 'nltk:home/nltk'
+ >>> windows or normalize_resource_url('nltk:/home/nltk') == 'file:///home/nltk'
+ True
+ >>> normalize_resource_url('http://example.com/dir/file')
+ 'http://example.com/dir/file'
+ >>> normalize_resource_url('dir/file')
+ 'nltk:dir/file'
+ """
+ try:
+ protocol, name = split_resource_url(resource_url)
+ except ValueError:
+ # the resource url has no protocol, use the nltk protocol by default
+ protocol = 'nltk'
+ name = resource_url
+ # use file protocol if the path is an absolute path
+ if protocol == 'nltk' and os.path.isabs(name):
+ protocol = 'file://'
+ name = normalize_resource_name(name, False, None)
+ elif protocol == 'file':
+ protocol = 'file://'
+ # name is absolute
+ name = normalize_resource_name(name, False, None)
+ elif protocol == 'nltk':
+ protocol = 'nltk:'
+ name = normalize_resource_name(name, True)
+ else:
+ # handled by urllib
+ protocol += '://'
+ return ''.join([protocol, name])
+
+def normalize_resource_name(resource_name, allow_relative=True, relative_path=None):
+ """
+ :type resource_name: str or unicode
+ :param resource_name: The name of the resource to search for.
+ Resource names are posix-style relative path names, such as
+ ``corpora/brown``. Directory names will automatically
+ be converted to a platform-appropriate path separator.
+ Directory trailing slashes are preserved
+
+ >>> windows = sys.platform.startswith('win')
+ >>> normalize_resource_name('.', True)
+ './'
+ >>> normalize_resource_name('./', True)
+ './'
+ >>> windows or normalize_resource_name('dir/file', False, '/') == '/dir/file'
+ True
+ >>> not windows or normalize_resource_name('C:/file', False, '/') == '/C:/file'
+ True
+ >>> windows or normalize_resource_name('/dir/file', False, '/') == '/dir/file'
+ True
+ >>> windows or normalize_resource_name('../dir/file', False, '/') == '/dir/file'
+ True
+ """
+ is_dir = bool(re.search(r'[\\/.]$', resource_name)) or resource_name.endswith(os.path.sep)
+ if sys.platform.startswith('win'):
+ resource_name = resource_name.lstrip('/')
+ else:
+ resource_name = re.sub(r'^/+', '/', resource_name)
+ if allow_relative:
+ resource_name = os.path.normpath(resource_name)
+ else:
+ if relative_path is None:
+ relative_path = os.curdir
+ resource_name = os.path.abspath(os.path.join(relative_path, resource_name))
+ resource_name = resource_name.replace('\\', '/').replace(os.path.sep, '/')
+ if sys.platform.startswith('win') and os.path.isabs(resource_name):
+ resource_name = '/' + resource_name
+ if is_dir and not resource_name.endswith('/'):
+ resource_name += '/'
+ return resource_name
+
+
+######################################################################
+# Path Pointers
+######################################################################
+
+class PathPointer(object):
+ """
+ An abstract base class for 'path pointers,' used by NLTK's data
+ package to identify specific paths. Two subclasses exist:
+ ``FileSystemPathPointer`` identifies a file that can be accessed
+ directly via a given absolute path. ``ZipFilePathPointer``
+ identifies a file contained within a zipfile, that can be accessed
+ by reading that zipfile.
+ """
+ def open(self, encoding=None):
+ """
+ Return a seekable read-only stream that can be used to read
+ the contents of the file identified by this path pointer.
+
+ :raise IOError: If the path specified by this pointer does
+ not contain a readable file.
+ """
+ raise NotImplementedError('abstract base class')
+
+ def file_size(self):
+ """
+ Return the size of the file pointed to by this path pointer,
+ in bytes.
+
+ :raise IOError: If the path specified by this pointer does
+ not contain a readable file.
+ """
+ raise NotImplementedError('abstract base class')
+
+ def join(self, fileid):
+ """
+ Return a new path pointer formed by starting at the path
+ identified by this pointer, and then following the relative
+ path given by ``fileid``. The path components of ``fileid``
+ should be separated by forward slashes, regardless of
+ the underlying file system's path seperator character.
+ """
+ raise NotImplementedError('abstract base class')
+
+
+class FileSystemPathPointer(PathPointer, text_type):
+ """
+ A path pointer that identifies a file which can be accessed
+ directly via a given absolute path.
+ """
+ @py3_data
+ def __init__(self, _path):
+ """
+ Create a new path pointer for the given absolute path.
+
+ :raise IOError: If the given path does not exist.
+ """
+
+ _path = os.path.abspath(_path)
+ if not os.path.exists(_path):
+ raise IOError('No such file or directory: %r' % _path)
+ self._path = _path
+
+ # There's no need to call str.__init__(), since it's a no-op;
+ # str does all of its setup work in __new__.
+
+ @property
+ def path(self):
+ """The absolute path identified by this path pointer."""
+ return self._path
+
+ def open(self, encoding=None):
+ stream = open(self._path, 'rb')
+ if encoding is not None:
+ stream = SeekableUnicodeStreamReader(stream, encoding)
+ return stream
+
+ def file_size(self):
+ return os.stat(self._path).st_size
+
+ def join(self, fileid):
+ _path = os.path.join(self._path, fileid)
+ return FileSystemPathPointer(_path)
+
+ def __repr__(self):
+ # This should be a byte string under Python 2.x;
+ # we don't want transliteration here so
+ # @python_2_unicode_compatible is not used.
+ return str('FileSystemPathPointer(%r)' % self._path)
+
+ def __str__(self):
+ return self._path
+
+
+class BufferedGzipFile(GzipFile):
+ """
+ A ``GzipFile`` subclass that buffers calls to ``read()`` and ``write()``.
+ This allows faster reads and writes of data to and from gzip-compressed
+ files at the cost of using more memory.
+
+ The default buffer size is 2MB.
+
+ ``BufferedGzipFile`` is useful for loading large gzipped pickle objects
+ as well as writing large encoded feature files for classifier training.
+ """
+ SIZE = 2 * 2**20
+
+ @py3_data
+ def __init__(self, filename=None, mode=None, compresslevel=9,
+ fileobj=None, **kwargs):
+ """
+ Return a buffered gzip file object.
+
+ :param filename: a filesystem path
+ :type filename: str
+ :param mode: a file mode which can be any of 'r', 'rb', 'a', 'ab',
+ 'w', or 'wb'
+ :type mode: str
+ :param compresslevel: The compresslevel argument is an integer from 1
+ to 9 controlling the level of compression; 1 is fastest and
+ produces the least compression, and 9 is slowest and produces the
+ most compression. The default is 9.
+ :type compresslevel: int
+ :param fileobj: a BytesIO stream to read from instead of a file.
+ :type fileobj: BytesIO
+ :param size: number of bytes to buffer during calls to read() and write()
+ :type size: int
+ :rtype: BufferedGzipFile
+ """
+ GzipFile.__init__(self, filename, mode, compresslevel, fileobj)
+ self._size = kwargs.get('size', self.SIZE)
+ self._buffer = BytesIO()
+ # cStringIO does not support len.
+ self._len = 0
+
+ def _reset_buffer(self):
+ # For some reason calling BytesIO.truncate() here will lead to
+ # inconsistent writes so just set _buffer to a new BytesIO object.
+ self._buffer = BytesIO()
+ self._len = 0
+
+ def _write_buffer(self, data):
+ # Simply write to the buffer and increment the buffer size.
+ if data is not None:
+ self._buffer.write(data)
+ self._len += len(data)
+
+ def _write_gzip(self, data):
+ # Write the current buffer to the GzipFile.
+ GzipFile.write(self, self._buffer.getvalue())
+ # Then reset the buffer and write the new data to the buffer.
+ self._reset_buffer()
+ self._write_buffer(data)
+
+ def close(self):
+ # GzipFile.close() doesn't actuallly close anything.
+ if self.mode == GZ_WRITE:
+ self._write_gzip(None)
+ self._reset_buffer()
+ return GzipFile.close(self)
+
+ def flush(self, lib_mode=FLUSH):
+ self._buffer.flush()
+ GzipFile.flush(self, lib_mode)
+
+ def read(self, size=None):
+ if not size:
+ size = self._size
+ contents = BytesIO()
+ while True:
+ blocks = GzipFile.read(self, size)
+ if not blocks:
+ contents.flush()
+ break
+ contents.write(blocks)
+ return contents.getvalue()
+ else:
+ return GzipFile.read(self, size)
+
+ def write(self, data, size=-1):
+ """
+ :param data: bytes to write to file or buffer
+ :type data: bytes
+ :param size: buffer at least size bytes before writing to file
+ :type size: int
+ """
+ if not size:
+ size = self._size
+ if self._len + len(data) <= size:
+ self._write_buffer(data)
+ else:
+ self._write_gzip(data)
+
+
+class GzipFileSystemPathPointer(FileSystemPathPointer):
+ """
+ A subclass of ``FileSystemPathPointer`` that identifies a gzip-compressed
+ file located at a given absolute path. ``GzipFileSystemPathPointer`` is
+ appropriate for loading large gzip-compressed pickle objects efficiently.
+ """
+ def open(self, encoding=None):
+ stream = BufferedGzipFile(self._path, 'rb')
+ if encoding:
+ stream = SeekableUnicodeStreamReader(stream, encoding)
+ return stream
+
+
+class ZipFilePathPointer(PathPointer):
+ """
+ A path pointer that identifies a file contained within a zipfile,
+ which can be accessed by reading that zipfile.
+ """
+ @py3_data
+ def __init__(self, zipfile, entry=''):
+ """
+ Create a new path pointer pointing at the specified entry
+ in the given zipfile.
+
+ :raise IOError: If the given zipfile does not exist, or if it
+ does not contain the specified entry.
+ """
+ if isinstance(zipfile, string_types):
+ zipfile = OpenOnDemandZipFile(os.path.abspath(zipfile))
+
+ # Normalize the entry string, it should be absolute:
+ entry = normalize_resource_name(entry, False, '/').lstrip('/')
+
+ # Check that the entry exists:
+ if entry:
+ try:
+ zipfile.getinfo(entry)
+ except Exception:
+ # Sometimes directories aren't explicitly listed in
+ # the zip file. So if `entry` is a directory name,
+ # then check if the zipfile contains any files that
+ # are under the given directory.
+ if (entry.endswith('/') and
+ [n for n in zipfile.namelist() if n.startswith(entry)]):
+ pass # zipfile contains a file in that directory.
+ else:
+ # Otherwise, complain.
+ raise IOError('Zipfile %r does not contain %r' %
+ (zipfile.filename, entry))
+ self._zipfile = zipfile
+ self._entry = entry
+
+ @property
+ def zipfile(self):
+ """
+ The zipfile.ZipFile object used to access the zip file
+ containing the entry identified by this path pointer.
+ """
+ return self._zipfile
+
+ @property
+ def entry(self):
+ """
+ The name of the file within zipfile that this path
+ pointer points to.
+ """
+ return self._entry
+
+ def open(self, encoding=None):
+ data = self._zipfile.read(self._entry)
+ stream = BytesIO(data)
+ if self._entry.endswith('.gz'):
+ stream = BufferedGzipFile(self._entry, fileobj=stream)
+ elif encoding is not None:
+ stream = SeekableUnicodeStreamReader(stream, encoding)
+ return stream
+
+ def file_size(self):
+ return self._zipfile.getinfo(self._entry).file_size
+
+ def join(self, fileid):
+ entry = '%s/%s' % (self._entry, fileid)
+ return ZipFilePathPointer(self._zipfile, entry)
+
+ def __repr__(self):
+ return str('ZipFilePathPointer(%r, %r)') % (
+ self._zipfile.filename, self._entry)
+
+ def __str__(self):
+ return os.path.normpath(os.path.join(self._zipfile.filename, self._entry))
+
+######################################################################
+# Access Functions
+######################################################################
+
+# Don't use a weak dictionary, because in the common case this
+# causes a lot more reloading that necessary.
+_resource_cache = {}
+"""A dictionary used to cache resources so that they won't
+ need to be loaded more than once."""
+
+def find(resource_name, paths=None):
+ """
+ Find the given resource by searching through the directories and
+ zip files in paths, where a None or empty string specifies an absolute path.
+ Returns a corresponding path name. If the given resource is not
+ found, raise a ``LookupError``, whose message gives a pointer to
+ the installation instructions for the NLTK downloader.
+
+ Zip File Handling:
+
+ - If ``resource_name`` contains a component with a ``.zip``
+ extension, then it is assumed to be a zipfile; and the
+ remaining path components are used to look inside the zipfile.
+
+ - If any element of ``nltk.data.path`` has a ``.zip`` extension,
+ then it is assumed to be a zipfile.
+
+ - If a given resource name that does not contain any zipfile
+ component is not found initially, then ``find()`` will make a
+ second attempt to find that resource, by replacing each
+ component *p* in the path with *p.zip/p*. For example, this
+ allows ``find()`` to map the resource name
+ ``corpora/chat80/cities.pl`` to a zip file path pointer to
+ ``corpora/chat80.zip/chat80/cities.pl``.
+
+ - When using ``find()`` to locate a directory contained in a
+ zipfile, the resource name must end with the forward slash
+ character. Otherwise, ``find()`` will not locate the
+ directory.
+
+ :type resource_name: str or unicode
+ :param resource_name: The name of the resource to search for.
+ Resource names are posix-style relative path names, such as
+ ``corpora/brown``. Directory names will be
+ automatically converted to a platform-appropriate path separator.
+ :rtype: str
+ """
+ resource_name = normalize_resource_name(resource_name, True)
+
+ # Resolve default paths at runtime in-case the user overrides nltk.data.path
+ if paths is None:
+ paths=path
+
+ # Check if the resource name includes a zipfile name
+ m = re.match(r'(.*\.zip)/?(.*)$|', resource_name)
+ zipfile, zipentry = m.groups()
+
+ # Check each item in our path
+ for path_ in paths:
+ # Is the path item a zipfile?
+ if path_ and (os.path.isfile(path_) and path_.endswith('.zip')):
+ try:
+ return ZipFilePathPointer(path_, resource_name)
+ except IOError:
+ # resource not in zipfile
+ continue
+
+ # Is the path item a directory or is resource_name an absolute path?
+ elif not path_ or os.path.isdir(path_):
+ if zipfile is None:
+ p = os.path.join(path_, resource_name)
+ if os.path.exists(p):
+ if p.endswith('.gz'):
+ return GzipFileSystemPathPointer(p)
+ else:
+ return FileSystemPathPointer(p)
+ else:
+ p = os.path.join(path_, zipfile)
+ if os.path.exists(p):
+ try:
+ return ZipFilePathPointer(p, zipentry)
+ except IOError:
+ # resource not in zipfile
+ continue
+
+ # Fallback: if the path doesn't include a zip file, then try
+ # again, assuming that one of the path components is inside a
+ # zipfile of the same name.
+ if zipfile is None:
+ pieces = resource_name.split('/')
+ for i in range(len(pieces)):
+ modified_name = '/'.join(pieces[:i]+[pieces[i]+'.zip']+pieces[i:])
+ try:
+ return find(modified_name, paths)
+ except LookupError:
+ pass
+
+ # Display a friendly error message if the resource wasn't found:
+ msg = textwrap.fill(
+ 'Resource %r not found. Please use the NLTK Downloader to '
+ 'obtain the resource: >>> nltk.download()' %
+ (resource_name,), initial_indent=' ', subsequent_indent=' ',
+ width=66)
+ msg += '\n Searched in:' + ''.join('\n - %r' % d for d in paths)
+ sep = '*'*70
+ resource_not_found = '\n%s\n%s\n%s' % (sep, msg, sep)
+ raise LookupError(resource_not_found)
+
+def retrieve(resource_url, filename=None, verbose=True):
+ """
+ Copy the given resource to a local file. If no filename is
+ specified, then use the URL's filename. If there is already a
+ file named ``filename``, then raise a ``ValueError``.
+
+ :type resource_url: str
+ :param resource_url: A URL specifying where the resource should be
+ loaded from. The default protocol is "nltk:", which searches
+ for the file in the the NLTK data package.
+ """
+ resource_url = normalize_resource_url(resource_url)
+ if filename is None:
+ if resource_url.startswith('file:'):
+ filename = os.path.split(resource_url)[-1]
+ else:
+ filename = re.sub(r'(^\w+:)?.*/', '', resource_url)
+ if os.path.exists(filename):
+ filename = os.path.abspath(filename)
+ raise ValueError("File %r already exists!" % filename)
+
+ if verbose:
+ print('Retrieving %r, saving to %r' % (resource_url, filename))
+
+ # Open the input & output streams.
+ infile = _open(resource_url)
+
+ # Copy infile -> outfile, using 64k blocks.
+ with open(filename, "wb") as outfile:
+ while True:
+ s = infile.read(1024*64) # 64k blocks.
+ outfile.write(s)
+ if not s: break
+
+ infile.close()
+
+#: A dictionary describing the formats that are supported by NLTK's
+#: load() method. Keys are format names, and values are format
+#: descriptions.
+FORMATS = {
+ 'pickle': "A serialized python object, stored using the pickle module.",
+ 'json': "A serialized python object, stored using the json module.",
+ 'yaml': "A serialized python object, stored using the yaml module.",
+ 'cfg': "A context free grammar.",
+ 'pcfg': "A probabilistic CFG.",
+ 'fcfg': "A feature CFG.",
+ 'fol': "A list of first order logic expressions, parsed with "
+ "nltk.sem.logic.Expression.fromstring.",
+ 'logic': "A list of first order logic expressions, parsed by "
+ "nltk.sem.logic._LogicParser. Requires an additional logic_parser "
+ "parameter",
+ 'val': "A semantic valuation, parsed by nltk.sem.parse_valuation().",
+ 'raw': "The raw (byte string) contents of a file.",
+ 'text': "The raw (unicode string) contents of a file. "
+}
+
+#: A dictionary mapping from file extensions to format names, used
+#: by load() when format="auto" to decide the format for a
+#: given resource url.
+AUTO_FORMATS = {
+ 'pickle': 'pickle',
+ 'json': 'json',
+ 'yaml': 'yaml',
+ 'cfg': 'cfg',
+ 'pcfg': 'pcfg',
+ 'fcfg': 'fcfg',
+ 'fol': 'fol',
+ 'logic': 'logic',
+ 'val': 'val',
+ 'txt': 'text',
+ 'text': 'text',
+}
+
+def load(resource_url, format='auto', cache=True, verbose=False,
+ logic_parser=None, fstruct_reader=None, encoding=None):
+ """
+ Load a given resource from the NLTK data package. The following
+ resource formats are currently supported:
+
+ - ``pickle``
+ - ``json``
+ - ``yaml``
+ - ``cfg`` (context free grammars)
+ - ``pcfg`` (probabilistic CFGs)
+ - ``fcfg`` (feature-based CFGs)
+ - ``fol`` (formulas of First Order Logic)
+ - ``logic`` (Logical formulas to be parsed by the given logic_parser)
+ - ``val`` (valuation of First Order Logic model)
+ - ``text`` (the file contents as a unicode string)
+ - ``raw`` (the raw file contents as a byte string)
+
+ If no format is specified, ``load()`` will attempt to determine a
+ format based on the resource name's file extension. If that
+ fails, ``load()`` will raise a ``ValueError`` exception.
+
+ For all text formats (everything except ``pickle``, ``json``, ``yaml`` and ``raw``),
+ it tries to decode the raw contents using UTF-8, and if that doesn't
+ work, it tries with ISO-8859-1 (Latin-1), unless the ``encoding``
+ is specified.
+
+ :type resource_url: str
+ :param resource_url: A URL specifying where the resource should be
+ loaded from. The default protocol is "nltk:", which searches
+ for the file in the the NLTK data package.
+ :type cache: bool
+ :param cache: If true, add this resource to a cache. If load()
+ finds a resource in its cache, then it will return it from the
+ cache rather than loading it. The cache uses weak references,
+ so a resource wil automatically be expunged from the cache
+ when no more objects are using it.
+ :type verbose: bool
+ :param verbose: If true, print a message when loading a resource.
+ Messages are not displayed when a resource is retrieved from
+ the cache.
+ :type logic_parser: LogicParser
+ :param logic_parser: The parser that will be used to parse logical
+ expressions.
+ :type fstruct_reader: FeatStructReader
+ :param fstruct_reader: The parser that will be used to parse the
+ feature structure of an fcfg.
+ :type encoding: str
+ :param encoding: the encoding of the input; only used for text formats.
+ """
+ resource_url=normalize_resource_url(resource_url)
+
+ # Determine the format of the resource.
+ if format == 'auto':
+ resource_url_parts = resource_url.split('.')
+ ext = resource_url_parts[-1]
+ if ext == 'gz':
+ ext = resource_url_parts[-2]
+ format = AUTO_FORMATS.get(ext)
+ if format is None:
+ raise ValueError('Could not determine format for %s based '
+ 'on its file\nextension; use the "format" '
+ 'argument to specify the format explicitly.'
+ % resource_url)
+
+ if format not in FORMATS:
+ raise ValueError('Unknown format type: %s!' % (format,))
+
+ # If we've cached the resource, then just return it.
+ if cache:
+ resource_val = _resource_cache.get((resource_url, format))
+ if resource_val is not None:
+ if verbose:
+ print('<<Using cached copy of %s>>' % (resource_url,))
+ return resource_val
+
+ # Let the user know what's going on.
+ if verbose:
+ print('<<Loading %s>>' % (resource_url,))
+
+ # Load the resource.
+ opened_resource = _open(resource_url)
+
+ if format == 'raw':
+ resource_val = opened_resource.read()
+ elif format == 'pickle':
+ resource_val = pickle.load(opened_resource)
+ elif format == 'json':
+ import json
+ from nltk.jsontags import json_tags
+ resource_val = json.load(opened_resource)
+ tag = None
+ if len(resource_val) != 1:
+ tag = next(resource_val.keys())
+ if tag not in json_tags:
+ raise ValueError('Unknown json tag.')
+ elif format == 'yaml':
+ import yaml
+ resource_val = yaml.load(opened_resource)
+ else:
+ # The resource is a text format.
+ binary_data = opened_resource.read()
+ if encoding is not None:
+ string_data = binary_data.decode(encoding)
+ else:
+ try:
+ string_data = binary_data.decode('utf-8')
+ except UnicodeDecodeError:
+ string_data = binary_data.decode('latin-1')
+ if format == 'text':
+ resource_val = string_data
+ elif format == 'cfg':
+ resource_val = nltk.grammar.CFG.fromstring(
+ string_data, encoding=encoding)
+ elif format == 'pcfg':
+ resource_val = nltk.grammar.PCFG.fromstring(
+ string_data, encoding=encoding)
+ elif format == 'fcfg':
+ resource_val = nltk.grammar.FeatureGrammar.fromstring(
+ string_data, logic_parser=logic_parser,
+ fstruct_reader=fstruct_reader, encoding=encoding)
+ elif format == 'fol':
+ resource_val = nltk.sem.parse_logic(
+ string_data, logic_parser=nltk.sem.logic._LogicParser(),
+ encoding=encoding)
+ elif format == 'logic':
+ resource_val = nltk.sem.parse_logic(
+ string_data, logic_parser=logic_parser, encoding=encoding)
+ elif format == 'val':
+ resource_val = nltk.sem.parse_valuation(
+ string_data, encoding=encoding)
+ else:
+ raise AssertionError("Internal NLTK error: Format %s isn't "
+ "handled by nltk.data.load()" % (format,))
+
+ opened_resource.close()
+
+ # If requested, add it to the cache.
+ if cache:
+ try:
+ _resource_cache[(resource_url, format)] = resource_val
+ # TODO: add this line
+ # print('<<Caching a copy of %s>>' % (resource_url,))
+ except TypeError:
+ # We can't create weak references to some object types, like
+ # strings and tuples. For now, just don't cache them.
+ pass
+
+ return resource_val
+
+def show_cfg(resource_url, escape='##'):
+ """
+ Write out a grammar file, ignoring escaped and empty lines.
+
+ :type resource_url: str
+ :param resource_url: A URL specifying where the resource should be
+ loaded from. The default protocol is "nltk:", which searches
+ for the file in the the NLTK data package.
+ :type escape: str
+ :param escape: Prepended string that signals lines to be ignored
+ """
+ resource_url = normalize_resource_url(resource_url)
+ resource_val = load(resource_url, format='text', cache=False)
+ lines = resource_val.splitlines()
+ for l in lines:
+ if l.startswith(escape): continue
+ if re.match('^$', l): continue
+ print(l)
+
+
+def clear_cache():
+ """
+ Remove all objects from the resource cache.
+ :see: load()
+ """
+ _resource_cache.clear()
+
+def _open(resource_url):
+ """
+ Helper function that returns an open file object for a resource,
+ given its resource URL. If the given resource URL uses the "nltk:"
+ protocol, or uses no protocol, then use ``nltk.data.find`` to find
+ its path, and open it with the given mode; if the resource URL
+ uses the 'file' protocol, then open the file with the given mode;
+ otherwise, delegate to ``urllib2.urlopen``.
+
+ :type resource_url: str
+ :param resource_url: A URL specifying where the resource should be
+ loaded from. The default protocol is "nltk:", which searches
+ for the file in the the NLTK data package.
+ """
+ resource_url = normalize_resource_url(resource_url)
+ protocol, _path = split_resource_url(resource_url)
+
+ if protocol is None or protocol.lower() == 'nltk':
+ return find(_path, path + ['']).open()
+ elif protocol.lower() == 'file':
+ # urllib might not use mode='rb', so handle this one ourselves:
+ return find(_path, ['']).open()
+ else:
+ return urlopen(resource_url)
+
+######################################################################
+# Lazy Resource Loader
+######################################################################
+
+# We shouldn't apply @python_2_unicode_compatible
+# decorator to LazyLoader, this is resource.__class__ responsibility.
+
+class LazyLoader(object):
+ @py3_data
+ def __init__(self, _path):
+ self._path = _path
+
+ def __load(self):
+ resource = load(self._path)
+ # This is where the magic happens! Transform ourselves into
+ # the object by modifying our own __dict__ and __class__ to
+ # match that of `resource`.
+ self.__dict__ = resource.__dict__
+ self.__class__ = resource.__class__
+
+ def __getattr__(self, attr):
+ self.__load()
+ # This looks circular, but its not, since __load() changes our
+ # __class__ to something new:
+ return getattr(self, attr)
+
+ def __repr__(self):
+ self.__load()
+ # This looks circular, but its not, since __load() changes our
+ # __class__ to something new:
+ return repr(self)
+
+######################################################################
+# Open-On-Demand ZipFile
+######################################################################
+
+
+class OpenOnDemandZipFile(zipfile.ZipFile):
+ """
+ A subclass of ``zipfile.ZipFile`` that closes its file pointer
+ whenever it is not using it; and re-opens it when it needs to read
+ data from the zipfile. This is useful for reducing the number of
+ open file handles when many zip files are being accessed at once.
+ ``OpenOnDemandZipFile`` must be constructed from a filename, not a
+ file-like object (to allow re-opening). ``OpenOnDemandZipFile`` is
+ read-only (i.e. ``write()`` and ``writestr()`` are disabled.
+ """
+ @py3_data
+ def __init__(self, filename):
+ if not isinstance(filename, string_types):
+ raise TypeError('ReopenableZipFile filename must be a string')
+ zipfile.ZipFile.__init__(self, filename)
+ assert self.filename == filename
+ self.close()
+
+ def read(self, name):
+ assert self.fp is None
+ self.fp = open(self.filename, 'rb')
+ value = zipfile.ZipFile.read(self, name)
+ self.close()
+ return value
+
+ def write(self, *args, **kwargs):
+ """:raise NotImplementedError: OpenOnDemandZipfile is read-only"""
+ raise NotImplementedError('OpenOnDemandZipfile is read-only')
+
+ def writestr(self, *args, **kwargs):
+ """:raise NotImplementedError: OpenOnDemandZipfile is read-only"""
+ raise NotImplementedError('OpenOnDemandZipfile is read-only')
+
+ def __repr__(self):
+ return repr(str('OpenOnDemandZipFile(%r)') % self.filename)
+
+######################################################################
+#{ Seekable Unicode Stream Reader
+######################################################################
+
+class SeekableUnicodeStreamReader(object):
+ """
+ A stream reader that automatically encodes the source byte stream
+ into unicode (like ``codecs.StreamReader``); but still supports the
+ ``seek()`` and ``tell()`` operations correctly. This is in contrast
+ to ``codecs.StreamReader``, which provide *broken* ``seek()`` and
+ ``tell()`` methods.
+
+ This class was motivated by ``StreamBackedCorpusView``, which
+ makes extensive use of ``seek()`` and ``tell()``, and needs to be
+ able to handle unicode-encoded files.
+
+ Note: this class requires stateless decoders. To my knowledge,
+ this shouldn't cause a problem with any of python's builtin
+ unicode encodings.
+ """
+ DEBUG = True #: If true, then perform extra sanity checks.
+
+ @py3_data
+ def __init__(self, stream, encoding, errors='strict'):
+ # Rewind the stream to its beginning.
+ stream.seek(0)
+
+ self.stream = stream
+ """The underlying stream."""
+
+ self.encoding = encoding
+ """The name of the encoding that should be used to encode the
+ underlying stream."""
+
+ self.errors = errors
+ """The error mode that should be used when decoding data from
+ the underlying stream. Can be 'strict', 'ignore', or
+ 'replace'."""
+
+ self.decode = codecs.getdecoder(encoding)
+ """The function that is used to decode byte strings into
+ unicode strings."""
+
+ self.bytebuffer = b''
+ """A buffer to use bytes that have been read but have not yet
+ been decoded. This is only used when the final bytes from
+ a read do not form a complete encoding for a character."""
+
+ self.linebuffer = None
+ """A buffer used by ``readline()`` to hold characters that have
+ been read, but have not yet been returned by ``read()`` or
+ ``readline()``. This buffer consists of a list of unicode
+ strings, where each string corresponds to a single line.
+ The final element of the list may or may not be a complete
+ line. Note that the existence of a linebuffer makes the
+ ``tell()`` operation more complex, because it must backtrack
+ to the beginning of the buffer to determine the correct
+ file position in the underlying byte stream."""
+
+ self._rewind_checkpoint = 0
+ """The file position at which the most recent read on the
+ underlying stream began. This is used, together with
+ ``_rewind_numchars``, to backtrack to the beginning of
+ ``linebuffer`` (which is required by ``tell()``)."""
+
+ self._rewind_numchars = None
+ """The number of characters that have been returned since the
+ read that started at ``_rewind_checkpoint``. This is used,
+ together with ``_rewind_checkpoint``, to backtrack to the
+ beginning of ``linebuffer`` (which is required by ``tell()``)."""
+
+ self._bom = self._check_bom()
+ """The length of the byte order marker at the beginning of
+ the stream (or None for no byte order marker)."""
+
+ #/////////////////////////////////////////////////////////////////
+ # Read methods
+ #/////////////////////////////////////////////////////////////////
+
+ def read(self, size=None):
+ """
+ Read up to ``size`` bytes, decode them using this reader's
+ encoding, and return the resulting unicode string.
+
+ :param size: The maximum number of bytes to read. If not
+ specified, then read as many bytes as possible.
+ :type size: int
+ :rtype: unicode
+ """
+ chars = self._read(size)
+
+ # If linebuffer is not empty, then include it in the result
+ if self.linebuffer:
+ chars = ''.join(self.linebuffer) + chars
+ self.linebuffer = None
+ self._rewind_numchars = None
+
+ return chars
+
+ def readline(self, size=None):
+ """
+ Read a line of text, decode it using this reader's encoding,
+ and return the resulting unicode string.
+
+ :param size: The maximum number of bytes to read. If no
+ newline is encountered before ``size`` bytes have been read,
+ then the returned value may not be a complete line of text.
+ :type size: int
+ """
+ # If we have a non-empty linebuffer, then return the first
+ # line from it. (Note that the last element of linebuffer may
+ # not be a complete line; so let _read() deal with it.)
+ if self.linebuffer and len(self.linebuffer) > 1:
+ line = self.linebuffer.pop(0)
+ self._rewind_numchars += len(line)
+ return line
+
+ readsize = size or 72
+ chars = ''
+
+ # If there's a remaining incomplete line in the buffer, add it.
+ if self.linebuffer:
+ chars += self.linebuffer.pop()
+ self.linebuffer = None
+
+ while True:
+ startpos = self.stream.tell() - len(self.bytebuffer)
+ new_chars = self._read(readsize)
+
+ # If we're at a '\r', then read one extra character, since
+ # it might be a '\n', to get the proper line ending.
+ if new_chars and new_chars.endswith('\r'):
+ new_chars += self._read(1)
+
+ chars += new_chars
+ lines = chars.splitlines(True)
+ if len(lines) > 1:
+ line = lines[0]
+ self.linebuffer = lines[1:]
+ self._rewind_numchars = len(new_chars)-(len(chars)-len(line))
+ self._rewind_checkpoint = startpos
+ break
+ elif len(lines) == 1:
+ line0withend = lines[0]
+ line0withoutend = lines[0].splitlines(False)[0]
+ if line0withend != line0withoutend: # complete line
+ line = line0withend
+ break
+
+ if not new_chars or size is not None:
+ line = chars
+ break
+
+ # Read successively larger blocks of text.
+ if readsize < 8000:
+ readsize *= 2
+
+ return line
+
+ def readlines(self, sizehint=None, keepends=True):
+ """
+ Read this file's contents, decode them using this reader's
+ encoding, and return it as a list of unicode lines.
+
+ :rtype: list(unicode)
+ :param sizehint: Ignored.
+ :param keepends: If false, then strip newlines.
+ """
+ return self.read().splitlines(keepends)
+
+ def next(self):
+ """Return the next decoded line from the underlying stream."""
+ line = self.readline()
+ if line: return line
+ else: raise StopIteration
+
+ def __next__(self):
+ return self.next()
+
+ def __iter__(self):
+ """Return self"""
+ return self
+
+ def xreadlines(self):
+ """Return self"""
+ return self
+
+ #/////////////////////////////////////////////////////////////////
+ # Pass-through methods & properties
+ #/////////////////////////////////////////////////////////////////
+
+ @property
+ def closed(self):
+ """True if the underlying stream is closed."""
+ return self.stream.closed
+
+ @property
+ def name(self):
+ """The name of the underlying stream."""
+ return self.stream.name
+
+ @property
+ def mode(self):
+ """The mode of the underlying stream."""
+ return self.stream.mode
+
+ def close(self):
+ """
+ Close the underlying stream.
+ """
+ self.stream.close()
+
+ #/////////////////////////////////////////////////////////////////
+ # Seek and tell
+ #/////////////////////////////////////////////////////////////////
+
+ def seek(self, offset, whence=0):
+ """
+ Move the stream to a new file position. If the reader is
+ maintaining any buffers, tehn they will be cleared.
+
+ :param offset: A byte count offset.
+ :param whence: If 0, then the offset is from the start of the file
+ (offset should be positive), if 1, then the offset is from the
+ current position (offset may be positive or negative); and if 2,
+ then the offset is from the end of the file (offset should
+ typically be negative).
+ """
+ if whence == 1:
+ raise ValueError('Relative seek is not supported for '
+ 'SeekableUnicodeStreamReader -- consider '
+ 'using char_seek_forward() instead.')
+ self.stream.seek(offset, whence)
+ self.linebuffer = None
+ self.bytebuffer = b''
+ self._rewind_numchars = None
+ self._rewind_checkpoint = self.stream.tell()
+
+ def char_seek_forward(self, offset):
+ """
+ Move the read pointer forward by ``offset`` characters.
+ """
+ if offset < 0:
+ raise ValueError('Negative offsets are not supported')
+ # Clear all buffers.
+ self.seek(self.tell())
+ # Perform the seek operation.
+ self._char_seek_forward(offset)
+
+ def _char_seek_forward(self, offset, est_bytes=None):
+ """
+ Move the file position forward by ``offset`` characters,
+ ignoring all buffers.
+
+ :param est_bytes: A hint, giving an estimate of the number of
+ bytes that will be neded to move forward by ``offset`` chars.
+ Defaults to ``offset``.
+ """
+ if est_bytes is None: est_bytes = offset
+ bytes = b''
+
+ while True:
+ # Read in a block of bytes.
+ newbytes = self.stream.read(est_bytes-len(bytes))
+ bytes += newbytes
+
+ # Decode the bytes to characters.
+ chars, bytes_decoded = self._incr_decode(bytes)
+
+ # If we got the right number of characters, then seek
+ # backwards over any truncated characters, and return.
+ if len(chars) == offset:
+ self.stream.seek(-len(bytes)+bytes_decoded, 1)
+ return
+
+ # If we went too far, then we can back-up until we get it
+ # right, using the bytes we've already read.
+ if len(chars) > offset:
+ while len(chars) > offset:
+ # Assume at least one byte/char.
+ est_bytes += offset-len(chars)
+ chars, bytes_decoded = self._incr_decode(bytes[:est_bytes])
+ self.stream.seek(-len(bytes)+bytes_decoded, 1)
+ return
+
+ # Otherwise, we haven't read enough bytes yet; loop again.
+ est_bytes += offset - len(chars)
+
+ def tell(self):
+ """
+ Return the current file position on the underlying byte
+ stream. If this reader is maintaining any buffers, then the
+ returned file position will be the position of the beginning
+ of those buffers.
+ """
+ # If nothing's buffered, then just return our current filepos:
+ if self.linebuffer is None:
+ return self.stream.tell() - len(self.bytebuffer)
+
+ # Otherwise, we'll need to backtrack the filepos until we
+ # reach the beginning of the buffer.
+
+ # Store our original file position, so we can return here.
+ orig_filepos = self.stream.tell()
+
+ # Calculate an estimate of where we think the newline is.
+ bytes_read = ( (orig_filepos-len(self.bytebuffer)) -
+ self._rewind_checkpoint )
+ buf_size = sum(len(line) for line in self.linebuffer)
+ est_bytes = int((bytes_read * self._rewind_numchars /
+ (self._rewind_numchars + buf_size)))
+
+ self.stream.seek(self._rewind_checkpoint)
+ self._char_seek_forward(self._rewind_numchars, est_bytes)
+ filepos = self.stream.tell()
+
+ # Sanity check
+ if self.DEBUG:
+ self.stream.seek(filepos)
+ check1 = self._incr_decode(self.stream.read(50))[0]
+ check2 = ''.join(self.linebuffer)
+ assert check1.startswith(check2) or check2.startswith(check1)
+
+ # Return to our original filepos (so we don't have to throw
+ # out our buffer.)
+ self.stream.seek(orig_filepos)
+
+ # Return the calculated filepos
+ return filepos
+
+ #/////////////////////////////////////////////////////////////////
+ # Helper methods
+ #/////////////////////////////////////////////////////////////////
+
+ def _read(self, size=None):
+ """
+ Read up to ``size`` bytes from the underlying stream, decode
+ them using this reader's encoding, and return the resulting
+ unicode string. ``linebuffer`` is not included in the result.
+ """
+ if size == 0: return ''
+
+ # Skip past the byte order marker, if present.
+ if self._bom and self.stream.tell() == 0:
+ self.stream.read(self._bom)
+
+ # Read the requested number of bytes.
+ if size is None:
+ new_bytes = self.stream.read()
+ else:
+ new_bytes = self.stream.read(size)
+ bytes = self.bytebuffer + new_bytes
+
+ # Decode the bytes into unicode characters
+ chars, bytes_decoded = self._incr_decode(bytes)
+
+ # If we got bytes but couldn't decode any, then read further.
+ if (size is not None) and (not chars) and (len(new_bytes) > 0):
+ while not chars:
+ new_bytes = self.stream.read(1)
+ if not new_bytes: break # end of file.
+ bytes += new_bytes
+ chars, bytes_decoded = self._incr_decode(bytes)
+
+ # Record any bytes we didn't consume.
+ self.bytebuffer = bytes[bytes_decoded:]
+
+ # Return the result
+ return chars
+
+ def _incr_decode(self, bytes):
+ """
+ Decode the given byte string into a unicode string, using this
+ reader's encoding. If an exception is encountered that
+ appears to be caused by a truncation error, then just decode
+ the byte string without the bytes that cause the trunctaion
+ error.
+
+ Return a tuple ``(chars, num_consumed)``, where ``chars`` is
+ the decoded unicode string, and ``num_consumed`` is the
+ number of bytes that were consumed.
+ """
+ while True:
+ try:
+ return self.decode(bytes, 'strict')
+ except UnicodeDecodeError as exc:
+ # If the exception occurs at the end of the string,
+ # then assume that it's a truncation error.
+ if exc.end == len(bytes):
+ return self.decode(bytes[:exc.start], self.errors)
+
+ # Otherwise, if we're being strict, then raise it.
+ elif self.errors == 'strict':
+ raise
+
+ # If we're not strict, then re-process it with our
+ # errors setting. This *may* raise an exception.
+ else:
+ return self.decode(bytes, self.errors)
+
+ _BOM_TABLE = {
+ 'utf8': [(codecs.BOM_UTF8, None)],
+ 'utf16': [(codecs.BOM_UTF16_LE, 'utf16-le'),
+ (codecs.BOM_UTF16_BE, 'utf16-be')],
+ 'utf16le': [(codecs.BOM_UTF16_LE, None)],
+ 'utf16be': [(codecs.BOM_UTF16_BE, None)],
+ 'utf32': [(codecs.BOM_UTF32_LE, 'utf32-le'),
+ (codecs.BOM_UTF32_BE, 'utf32-be')],
+ 'utf32le': [(codecs.BOM_UTF32_LE, None)],
+ 'utf32be': [(codecs.BOM_UTF32_BE, None)],
+ }
+
+ def _check_bom(self):
+ # Normalize our encoding name
+ enc = re.sub('[ -]', '', self.encoding.lower())
+
+ # Look up our encoding in the BOM table.
+ bom_info = self._BOM_TABLE.get(enc)
+
+ if bom_info:
+ # Read a prefix, to check against the BOM(s)
+ bytes = self.stream.read(16)
+ self.stream.seek(0)
+
+ # Check for each possible BOM.
+ for (bom, new_encoding) in bom_info:
+ if bytes.startswith(bom):
+ if new_encoding: self.encoding = new_encoding
+ return len(bom)
+
+ return None
+
+__all__ = ['path', 'PathPointer', 'FileSystemPathPointer', 'BufferedGzipFile',
+ 'GzipFileSystemPathPointer', 'GzipFileSystemPathPointer',
+ 'find', 'retrieve', 'FORMATS', 'AUTO_FORMATS', 'load',
+ 'show_cfg', 'clear_cache', 'LazyLoader', 'OpenOnDemandZipFile',
+ 'GzipFileSystemPathPointer', 'SeekableUnicodeStreamReader']
diff --git a/nltk/decorators.py b/nltk/decorators.py
new file mode 100644
index 0000000..3e01304
--- /dev/null
+++ b/nltk/decorators.py
@@ -0,0 +1,217 @@
+"""
+Decorator module by Michele Simionato <michelesimionato at libero.it>
+Copyright Michele Simionato, distributed under the terms of the BSD License (see below).
+http://www.phyast.pitt.edu/~micheles/python/documentation.html
+
+Included in NLTK for its support of a nice memoization decorator.
+"""
+from __future__ import print_function
+__docformat__ = 'restructuredtext en'
+
+## The basic trick is to generate the source code for the decorated function
+## with the right signature and to evaluate it.
+## Uncomment the statement 'print >> sys.stderr, func_src' in _decorator
+## to understand what is going on.
+
+__all__ = ["decorator", "new_wrapper", "getinfo"]
+
+import sys
+
+# Hack to keep NLTK's "tokenize" module from colliding with the "tokenize" in
+# the Python standard library.
+old_sys_path = sys.path[:]
+sys.path = [p for p in sys.path if "nltk" not in p]
+import inspect
+sys.path = old_sys_path
+
+try:
+ set
+except NameError:
+ from sets import Set as set
+
+def getinfo(func):
+ """
+ Returns an info dictionary containing:
+ - name (the name of the function : str)
+ - argnames (the names of the arguments : list)
+ - defaults (the values of the default arguments : tuple)
+ - signature (the signature : str)
+ - doc (the docstring : str)
+ - module (the module name : str)
+ - dict (the function __dict__ : str)
+
+ >>> def f(self, x=1, y=2, *args, **kw): pass
+
+ >>> info = getinfo(f)
+
+ >>> info["name"]
+ 'f'
+ >>> info["argnames"]
+ ['self', 'x', 'y', 'args', 'kw']
+
+ >>> info["defaults"]
+ (1, 2)
+
+ >>> info["signature"]
+ 'self, x, y, *args, **kw'
+ """
+ assert inspect.ismethod(func) or inspect.isfunction(func)
+ regargs, varargs, varkwargs, defaults = inspect.getargspec(func)
+ argnames = list(regargs)
+ if varargs:
+ argnames.append(varargs)
+ if varkwargs:
+ argnames.append(varkwargs)
+ signature = inspect.formatargspec(regargs, varargs, varkwargs, defaults,
+ formatvalue=lambda value: "")[1:-1]
+
+ # pypy compatibility
+ if hasattr(func, '__closure__'):
+ _closure = func.__closure__
+ _globals = func.__globals__
+ else:
+ _closure = func.func_closure
+ _globals = func.func_globals
+
+ return dict(name=func.__name__, argnames=argnames, signature=signature,
+ defaults = func.__defaults__, doc=func.__doc__,
+ module=func.__module__, dict=func.__dict__,
+ globals=_globals, closure=_closure)
+
+# akin to functools.update_wrapper
+def update_wrapper(wrapper, model, infodict=None):
+ infodict = infodict or getinfo(model)
+ wrapper.__name__ = infodict['name']
+ wrapper.__doc__ = infodict['doc']
+ wrapper.__module__ = infodict['module']
+ wrapper.__dict__.update(infodict['dict'])
+ wrapper.__defaults__ = infodict['defaults']
+ wrapper.undecorated = model
+ return wrapper
+
+def new_wrapper(wrapper, model):
+ """
+ An improvement over functools.update_wrapper. The wrapper is a generic
+ callable object. It works by generating a copy of the wrapper with the
+ right signature and by updating the copy, not the original.
+ Moreovoer, 'model' can be a dictionary with keys 'name', 'doc', 'module',
+ 'dict', 'defaults'.
+ """
+ if isinstance(model, dict):
+ infodict = model
+ else: # assume model is a function
+ infodict = getinfo(model)
+ assert not '_wrapper_' in infodict["argnames"], (
+ '"_wrapper_" is a reserved argument name!')
+ src = "lambda %(signature)s: _wrapper_(%(signature)s)" % infodict
+ funcopy = eval(src, dict(_wrapper_=wrapper))
+ return update_wrapper(funcopy, model, infodict)
+
+# helper used in decorator_factory
+def __call__(self, func):
+ return new_wrapper(lambda *a, **k : self.call(func, *a, **k), func)
+
+def decorator_factory(cls):
+ """
+ Take a class with a ``.caller`` method and return a callable decorator
+ object. It works by adding a suitable __call__ method to the class;
+ it raises a TypeError if the class already has a nontrivial __call__
+ method.
+ """
+ attrs = set(dir(cls))
+ if '__call__' in attrs:
+ raise TypeError('You cannot decorate a class with a nontrivial '
+ '__call__ method')
+ if 'call' not in attrs:
+ raise TypeError('You cannot decorate a class without a '
+ '.call method')
+ cls.__call__ = __call__
+ return cls
+
+def decorator(caller):
+ """
+ General purpose decorator factory: takes a caller function as
+ input and returns a decorator with the same attributes.
+ A caller function is any function like this::
+
+ def caller(func, *args, **kw):
+ # do something
+ return func(*args, **kw)
+
+ Here is an example of usage:
+
+ >>> @decorator
+ ... def chatty(f, *args, **kw):
+ ... print("Calling %r" % f.__name__)
+ ... return f(*args, **kw)
+
+ >>> chatty.__name__
+ 'chatty'
+
+ >>> @chatty
+ ... def f(): pass
+ ...
+ >>> f()
+ Calling 'f'
+
+ decorator can also take in input a class with a .caller method; in this
+ case it converts the class into a factory of callable decorator objects.
+ See the documentation for an example.
+ """
+ if inspect.isclass(caller):
+ return decorator_factory(caller)
+ def _decorator(func): # the real meat is here
+ infodict = getinfo(func)
+ argnames = infodict['argnames']
+ assert not ('_call_' in argnames or '_func_' in argnames), (
+ 'You cannot use _call_ or _func_ as argument names!')
+ src = "lambda %(signature)s: _call_(_func_, %(signature)s)" % infodict
+ # import sys; print >> sys.stderr, src # for debugging purposes
+ dec_func = eval(src, dict(_func_=func, _call_=caller))
+ return update_wrapper(dec_func, func, infodict)
+ return update_wrapper(_decorator, caller)
+
+def getattr_(obj, name, default_thunk):
+ "Similar to .setdefault in dictionaries."
+ try:
+ return getattr(obj, name)
+ except AttributeError:
+ default = default_thunk()
+ setattr(obj, name, default)
+ return default
+
+ at decorator
+def memoize(func, *args):
+ dic = getattr_(func, "memoize_dic", dict)
+ # memoize_dic is created at the first call
+ if args in dic:
+ return dic[args]
+ else:
+ result = func(*args)
+ dic[args] = result
+ return result
+
+if __name__ == "__main__":
+ import doctest; doctest.testmod()
+
+########################## LEGALESE ###############################
+
+## Redistributions of source code must retain the above copyright
+## notice, this list of conditions and the following disclaimer.
+## Redistributions in bytecode form must reproduce the above copyright
+## notice, this list of conditions and the following disclaimer in
+## the documentation and/or other materials provided with the
+## distribution.
+
+## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+## A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+## HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+## INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+## BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
+## OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+## ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+## TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+## USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
+## DAMAGE.
diff --git a/nltk/downloader.py b/nltk/downloader.py
new file mode 100644
index 0000000..e441006
--- /dev/null
+++ b/nltk/downloader.py
@@ -0,0 +1,2264 @@
+# Natural Language Toolkit: Corpus & Model Downloader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+The NLTK corpus and module downloader. This module defines several
+interfaces which can be used to download corpora, models, and other
+data packages that can be used with NLTK.
+
+Downloading Packages
+====================
+If called with no arguments, ``download()`` will display an interactive
+interface which can be used to download and install new packages.
+If Tkinter is available, then a graphical interface will be shown,
+otherwise a simple text interface will be provided.
+
+Individual packages can be downloaded by calling the ``download()``
+function with a single argument, giving the package identifier for the
+package that should be downloaded:
+
+ >>> download('treebank') # doctest: +SKIP
+ [nltk_data] Downloading package 'treebank'...
+ [nltk_data] Unzipping corpora/treebank.zip.
+
+NLTK also provides a number of \"package collections\", consisting of
+a group of related packages. To download all packages in a
+colleciton, simply call ``download()`` with the collection's
+identifier:
+
+ >>> download('all-corpora') # doctest: +SKIP
+ [nltk_data] Downloading package 'abc'...
+ [nltk_data] Unzipping corpora/abc.zip.
+ [nltk_data] Downloading package 'alpino'...
+ [nltk_data] Unzipping corpora/alpino.zip.
+ ...
+ [nltk_data] Downloading package 'words'...
+ [nltk_data] Unzipping corpora/words.zip.
+
+Download Directory
+==================
+By default, packages are installed in either a system-wide directory
+(if Python has sufficient access to write to it); or in the current
+user's home directory. However, the ``download_dir`` argument may be
+used to specify a different installation target, if desired.
+
+See ``Downloader.default_download_dir()`` for more a detailed
+description of how the default download directory is chosen.
+
+NLTK Download Server
+====================
+Before downloading any packages, the corpus and module downloader
+contacts the NLTK download server, to retrieve an index file
+describing the available packages. By default, this index file is
+loaded from ``http://nltk.googlecode.com/svn/trunk/nltk_data/index.xml``.
+If necessary, it is possible to create a new ``Downloader`` object,
+specifying a different URL for the package index file.
+
+Usage::
+
+ python nltk/downloader.py [-d DATADIR] [-q] [-f] [-k] PACKAGE_IDS
+
+or::
+
+ python -m nltk.downloader [-d DATADIR] [-q] [-f] [-k] PACKAGE_IDS
+"""
+#----------------------------------------------------------------------
+from __future__ import print_function, division, unicode_literals
+
+"""
+
+ 0 1 2 3
+[label][----][label][----]
+[column ][column ]
+
+Notes
+=====
+Handling data files.. Some questions:
+
+* Should the data files be kept zipped or unzipped? I say zipped.
+
+* Should the data files be kept in svn at all? Advantages: history;
+ automatic version numbers; 'svn up' could be used rather than the
+ downloader to update the corpora. Disadvantages: they're big,
+ which makes working from svn a bit of a pain. And we're planning
+ to potentially make them much bigger. I don't think we want
+ people to have to download 400MB corpora just to use nltk from svn.
+
+* Compromise: keep the data files in trunk/data rather than in
+ trunk/nltk. That way you can check them out in svn if you want
+ to; but you don't need to, and you can use the downloader instead.
+
+* Also: keep models in mind. When we change the code, we'd
+ potentially like the models to get updated. This could require a
+ little thought.
+
+* So.. let's assume we have a trunk/data directory, containing a bunch
+ of packages. The packages should be kept as zip files, because we
+ really shouldn't be editing them much (well -- we may edit models
+ more, but they tend to be binary-ish files anyway, where diffs
+ aren't that helpful). So we'll have trunk/data, with a bunch of
+ files like abc.zip and treebank.zip and propbank.zip. For each
+ package we could also have eg treebank.xml and propbank.xml,
+ describing the contents of the package (name, copyright, license,
+ etc). Collections would also have .xml files. Finally, we would
+ pull all these together to form a single index.xml file. Some
+ directory structure wouldn't hurt. So how about::
+
+ /trunk/data/ ....................... root of data svn
+ index.xml ........................ main index file
+ src/ ............................. python scripts
+ packages/ ........................ dir for packages
+ corpora/ ....................... zip & xml files for corpora
+ grammars/ ...................... zip & xml files for grammars
+ taggers/ ....................... zip & xml files for taggers
+ tokenizers/ .................... zip & xml files for tokenizers
+ etc.
+ collections/ ..................... xml files for collections
+
+ Where the root (/trunk/data) would contain a makefile; and src/
+ would contain a script to update the info.xml file. It could also
+ contain scripts to rebuild some of the various model files. The
+ script that builds index.xml should probably check that each zip
+ file expands entirely into a single subdir, whose name matches the
+ package's uid.
+
+Changes I need to make:
+ - in index: change "size" to "filesize" or "compressed-size"
+ - in index: add "unzipped-size"
+ - when checking status: check both compressed & uncompressed size.
+ uncompressed size is important to make sure we detect a problem
+ if something got partially unzipped. define new status values
+ to differentiate stale vs corrupt vs corruptly-uncompressed??
+ (we shouldn't need to re-download the file if the zip file is ok
+ but it didn't get uncompressed fully.)
+ - add other fields to the index: author, license, copyright, contact,
+ etc.
+
+the current grammars/ package would become a single new package (eg
+toy-grammars or book-grammars).
+
+xml file should have:
+ - authorship info
+ - license info
+ - copyright info
+ - contact info
+ - info about what type of data/annotation it contains?
+ - recommended corpus reader?
+
+collections can contain other collections. they can also contain
+multiple package types (corpora & models). Have a single 'basics'
+package that includes everything we talk about in the book?
+
+n.b.: there will have to be a fallback to the punkt tokenizer, in case
+they didn't download that model.
+
+default: unzip or not?
+
+"""
+import time, os, zipfile, sys, textwrap, threading, itertools
+from hashlib import md5
+
+try:
+ TKINTER = True
+ from tkinter import (Tk, Frame, Label, Entry, Button, Canvas, Menu, IntVar,
+ TclError)
+ from tkinter.messagebox import showerror
+ from nltk.draw.table import Table
+ from nltk.draw.util import ShowText
+except:
+ TKINTER = False
+ TclError = ValueError
+
+from xml.etree import ElementTree
+import nltk
+from nltk import compat
+#urllib2 = nltk.internals.import_from_stdlib('urllib2')
+
+
+######################################################################
+# Directory entry objects (from the data server's index file)
+######################################################################
+
+ at compat.python_2_unicode_compatible
+class Package(object):
+ """
+ A directory entry for a downloadable package. These entries are
+ extracted from the XML index file that is downloaded by
+ ``Downloader``. Each package consists of a single file; but if
+ that file is a zip file, then it can be automatically decompressed
+ when the package is installed.
+ """
+ def __init__(self, id, url, name=None, subdir='',
+ size=None, unzipped_size=None,
+ checksum=None, svn_revision=None,
+ copyright='Unknown', contact='Unknown',
+ license='Unknown', author='Unknown',
+ unzip=True,
+ **kw):
+ self.id = id
+ """A unique identifier for this package."""
+
+ self.name = name or id
+ """A string name for this package."""
+
+ self.subdir = subdir
+ """The subdirectory where this package should be installed.
+ E.g., ``'corpora'`` or ``'taggers'``."""
+
+ self.url = url
+ """A URL that can be used to download this package's file."""
+
+ self.size = int(size)
+ """The filesize (in bytes) of the package file."""
+
+ self.unzipped_size = int(unzipped_size)
+ """The total filesize of the files contained in the package's
+ zipfile."""
+
+ self.checksum = checksum
+ """The MD-5 checksum of the package file."""
+
+ self.svn_revision = svn_revision
+ """A subversion revision number for this package."""
+
+ self.copyright = copyright
+ """Copyright holder for this package."""
+
+ self.contact = contact
+ """Name & email of the person who should be contacted with
+ questions about this package."""
+
+ self.license = license
+ """License information for this package."""
+
+ self.author = author
+ """Author of this package."""
+
+ ext = os.path.splitext(url.split('/')[-1])[1]
+ self.filename = os.path.join(subdir, id+ext)
+ """The filename that should be used for this package's file. It
+ is formed by joining ``self.subdir`` with ``self.id``, and
+ using the same extension as ``url``."""
+
+ self.unzip = bool(int(unzip)) # '0' or '1'
+ """A flag indicating whether this corpus should be unzipped by
+ default."""
+
+ # Include any other attributes provided by the XML file.
+ self.__dict__.update(kw)
+
+ @staticmethod
+ def fromxml(xml):
+ if isinstance(xml, compat.string_types):
+ xml = ElementTree.parse(xml)
+ for key in xml.attrib:
+ xml.attrib[key] = compat.text_type(xml.attrib[key])
+ return Package(**xml.attrib)
+
+ def __lt__(self, other):
+ return self.id < other.id
+
+ def __repr__(self):
+ return '<Package %s>' % self.id
+
+ at compat.python_2_unicode_compatible
+class Collection(object):
+ """
+ A directory entry for a collection of downloadable packages.
+ These entries are extracted from the XML index file that is
+ downloaded by ``Downloader``.
+ """
+ def __init__(self, id, children, name=None, **kw):
+ self.id = id
+ """A unique identifier for this collection."""
+
+ self.name = name or id
+ """A string name for this collection."""
+
+ self.children = children
+ """A list of the ``Collections`` or ``Packages`` directly
+ contained by this collection."""
+
+ self.packages = None
+ """A list of ``Packages`` contained by this collection or any
+ collections it recursively contains."""
+
+ # Include any other attributes provided by the XML file.
+ self.__dict__.update(kw)
+
+ @staticmethod
+ def fromxml(xml):
+ if isinstance(xml, compat.string_types):
+ xml = ElementTree.parse(xml)
+ for key in xml.attrib:
+ xml.attrib[key] = compat.text_type(xml.attrib[key])
+ children = [child.get('ref') for child in xml.findall('item')]
+ return Collection(children=children, **xml.attrib)
+
+ def __lt__(self, other):
+ return self.id < other.id
+
+ def __repr__(self):
+ return '<Collection %s>' % self.id
+
+######################################################################
+# Message Passing Objects
+######################################################################
+
+class DownloaderMessage(object):
+ """A status message object, used by ``incr_download`` to
+ communicate its progress."""
+class StartCollectionMessage(DownloaderMessage):
+ """Data server has started working on a collection of packages."""
+ def __init__(self, collection): self.collection = collection
+class FinishCollectionMessage(DownloaderMessage):
+ """Data server has finished working on a collection of packages."""
+ def __init__(self, collection): self.collection = collection
+class StartPackageMessage(DownloaderMessage):
+ """Data server has started working on a package."""
+ def __init__(self, package): self.package = package
+class FinishPackageMessage(DownloaderMessage):
+ """Data server has finished working on a package."""
+ def __init__(self, package): self.package = package
+class StartDownloadMessage(DownloaderMessage):
+ """Data server has started downloading a package."""
+ def __init__(self, package): self.package = package
+class FinishDownloadMessage(DownloaderMessage):
+ """Data server has finished downloading a package."""
+ def __init__(self, package): self.package = package
+class StartUnzipMessage(DownloaderMessage):
+ """Data server has started unzipping a package."""
+ def __init__(self, package): self.package = package
+class FinishUnzipMessage(DownloaderMessage):
+ """Data server has finished unzipping a package."""
+ def __init__(self, package): self.package = package
+class UpToDateMessage(DownloaderMessage):
+ """The package download file is already up-to-date"""
+ def __init__(self, package): self.package = package
+class StaleMessage(DownloaderMessage):
+ """The package download file is out-of-date or corrupt"""
+ def __init__(self, package): self.package = package
+class ErrorMessage(DownloaderMessage):
+ """Data server encountered an error"""
+ def __init__(self, package, message):
+ self.package = package
+ if isinstance(message, Exception):
+ self.message = str(message)
+ else:
+ self.message = message
+
+class ProgressMessage(DownloaderMessage):
+ """Indicates how much progress the data server has made"""
+ def __init__(self, progress): self.progress = progress
+class SelectDownloadDirMessage(DownloaderMessage):
+ """Indicates what download directory the data server is using"""
+ def __init__(self, download_dir): self.download_dir = download_dir
+
+######################################################################
+# NLTK Data Server
+######################################################################
+
+class Downloader(object):
+ """
+ A class used to access the NLTK data server, which can be used to
+ download corpora and other data packages.
+ """
+
+ #/////////////////////////////////////////////////////////////////
+ # Configuration
+ #/////////////////////////////////////////////////////////////////
+
+ INDEX_TIMEOUT = 60*60 # 1 hour
+ """The amount of time after which the cached copy of the data
+ server index will be considered 'stale,' and will be
+ re-downloaded."""
+
+ # DEFAULT_URL = 'http://nltk.googlecode.com/svn/trunk/nltk_data/index.xml'
+ DEFAULT_URL = 'http://nltk.github.com/nltk_data/'
+ """The default URL for the NLTK data server's index. An
+ alternative URL can be specified when creating a new
+ ``Downloader`` object."""
+
+ #/////////////////////////////////////////////////////////////////
+ # Status Constants
+ #/////////////////////////////////////////////////////////////////
+
+ INSTALLED = 'installed'
+ """A status string indicating that a package or collection is
+ installed and up-to-date."""
+ NOT_INSTALLED = 'not installed'
+ """A status string indicating that a package or collection is
+ not installed."""
+ STALE = 'out of date'
+ """A status string indicating that a package or collection is
+ corrupt or out-of-date."""
+ PARTIAL = 'partial'
+ """A status string indicating that a collection is partially
+ installed (i.e., only some of its packages are installed.)"""
+
+ #/////////////////////////////////////////////////////////////////
+ # Cosntructor
+ #/////////////////////////////////////////////////////////////////
+
+ def __init__(self, server_index_url=None, download_dir=None):
+ self._url = server_index_url or self.DEFAULT_URL
+ """The URL for the data server's index file."""
+
+ self._collections = {}
+ """Dictionary from collection identifier to ``Collection``"""
+
+ self._packages = {}
+ """Dictionary from package identifier to ``Package``"""
+
+ self._download_dir = download_dir
+ """The default directory to which packages will be downloaded."""
+
+ self._index = None
+ """The XML index file downloaded from the data server"""
+
+ self._index_timestamp = None
+ """Time at which ``self._index`` was downloaded. If it is more
+ than ``INDEX_TIMEOUT`` seconds old, it will be re-downloaded."""
+
+ self._status_cache = {}
+ """Dictionary from package/collection identifier to status
+ string (``INSTALLED``, ``NOT_INSTALLED``, ``STALE``, or
+ ``PARTIAL``). Cache is used for packages only, not
+ collections."""
+
+ self._errors = None
+ """Flag for telling if all packages got successfully downloaded or not."""
+
+ # decide where we're going to save things to.
+ if self._download_dir is None:
+ self._download_dir = self.default_download_dir()
+
+ #/////////////////////////////////////////////////////////////////
+ # Information
+ #/////////////////////////////////////////////////////////////////
+
+ def list(self, download_dir=None, show_packages=True,
+ show_collections=True, header=True, more_prompt=False,
+ skip_installed=False):
+ lines = 0 # for more_prompt
+ if download_dir is None:
+ download_dir = self._download_dir
+ print('Using default data directory (%s)' % download_dir)
+ if header:
+ print('='*(26+len(self._url)))
+ print(' Data server index for <%s>' % self._url)
+ print('='*(26+len(self._url)))
+ lines += 3 # for more_prompt
+ stale = partial = False
+
+ categories = []
+ if show_packages: categories.append('packages')
+ if show_collections: categories.append('collections')
+ for category in categories:
+ print('%s:' % category.capitalize())
+ lines += 1 # for more_prompt
+ for info in sorted(getattr(self, category)(), key=str):
+ status = self.status(info, download_dir)
+ if status == self.INSTALLED and skip_installed: continue
+ if status == self.STALE: stale = True
+ if status == self.PARTIAL: partial = True
+ prefix = {self.INSTALLED:'*', self.STALE:'-',
+ self.PARTIAL:'P', self.NOT_INSTALLED: ' '}[status]
+ name = textwrap.fill('-'*27 + (info.name or info.id),
+ 75, subsequent_indent=27*' ')[27:]
+ print(' [%s] %s %s' % (prefix, info.id.ljust(20, '.'), name))
+ lines += len(name.split('\n')) # for more_prompt
+ if more_prompt and lines > 20:
+ user_input = compat.raw_input("Hit Enter to continue: ")
+ if (user_input.lower() in ('x', 'q')): return
+ lines = 0
+ print()
+ msg = '([*] marks installed packages'
+ if stale: msg += '; [-] marks out-of-date or corrupt packages'
+ if partial: msg += '; [P] marks partially installed collections'
+ print(textwrap.fill(msg+')', subsequent_indent=' ', width=76))
+
+ def packages(self):
+ self._update_index()
+ return self._packages.values()
+
+ def corpora(self):
+ self._update_index()
+ return [pkg for (id,pkg) in self._packages.items()
+ if pkg.subdir == 'corpora']
+
+ def models(self):
+ self._update_index()
+ return [pkg for (id,pkg) in self._packages.items()
+ if pkg.subdir != 'corpora']
+
+ def collections(self):
+ self._update_index()
+ return self._collections.values()
+
+ #/////////////////////////////////////////////////////////////////
+ # Downloading
+ #/////////////////////////////////////////////////////////////////
+
+ def _info_or_id(self, info_or_id):
+ if isinstance(info_or_id, compat.string_types):
+ return self.info(info_or_id)
+ else:
+ return info_or_id
+
+ # [xx] When during downloading is it 'safe' to abort? Only unsafe
+ # time is *during* an unzip -- we don't want to leave a
+ # partially-unzipped corpus in place because we wouldn't notice
+ # it. But if we had the exact total size of the unzipped corpus,
+ # then that would be fine. Then we could abort anytime we want!
+ # So this is really what we should do. That way the threaded
+ # downloader in the gui can just kill the download thread anytime
+ # it wants.
+
+ def incr_download(self, info_or_id, download_dir=None, force=False):
+ # If they didn't specify a download_dir, then use the default one.
+ if download_dir is None:
+ download_dir = self._download_dir
+ yield SelectDownloadDirMessage(download_dir)
+
+ # If they gave us a list of ids, then download each one.
+ if isinstance(info_or_id, (list,tuple)):
+ for msg in self._download_list(info_or_id, download_dir, force):
+ yield msg
+ return
+
+ # Look up the requested collection or package.
+ try: info = self._info_or_id(info_or_id)
+ except (IOError, ValueError) as e:
+ yield ErrorMessage(None, 'Error loading %s: %s' %
+ (info_or_id, e))
+ return
+
+ # Handle collections.
+ if isinstance(info, Collection):
+ yield StartCollectionMessage(info)
+ for msg in self.incr_download(info.children, download_dir, force):
+ yield msg
+ yield FinishCollectionMessage(info)
+
+ # Handle Packages (delegate to a helper function).
+ else:
+ for msg in self._download_package(info, download_dir, force):
+ yield msg
+
+ def _num_packages(self, item):
+ if isinstance(item, Package): return 1
+ else: return len(item.packages)
+
+ def _download_list(self, items, download_dir, force):
+ # Look up the requested items.
+ for i in range(len(items)):
+ try: items[i] = self._info_or_id(items[i])
+ except (IOError, ValueError) as e:
+ yield ErrorMessage(items[i], e)
+ return
+
+ # Download each item, re-scaling their progress.
+ num_packages = sum(self._num_packages(item) for item in items)
+ progress = 0
+ for i, item in enumerate(items):
+ if isinstance(item, Package):
+ delta = 1./num_packages
+ else:
+ delta = float(len(item.packages))/num_packages
+ for msg in self.incr_download(item, download_dir, force):
+ if isinstance(msg, ProgressMessage):
+ yield ProgressMessage(progress + msg.progress*delta)
+ else:
+ yield msg
+
+ progress += 100*delta
+
+ def _download_package(self, info, download_dir, force):
+ yield StartPackageMessage(info)
+ yield ProgressMessage(0)
+
+ # Do we already have the current version?
+ status = self.status(info, download_dir)
+ if not force and status == self.INSTALLED:
+ yield UpToDateMessage(info)
+ yield ProgressMessage(100)
+ yield FinishPackageMessage(info)
+ return
+
+ # Remove the package from our status cache
+ self._status_cache.pop(info.id, None)
+
+ # Check for (and remove) any old/stale version.
+ filepath = os.path.join(download_dir, info.filename)
+ if os.path.exists(filepath):
+ if status == self.STALE:
+ yield StaleMessage(info)
+ os.remove(filepath)
+
+ # Ensure the download_dir exists
+ if not os.path.exists(download_dir):
+ os.mkdir(download_dir)
+ if not os.path.exists(os.path.join(download_dir, info.subdir)):
+ os.mkdir(os.path.join(download_dir, info.subdir))
+
+ # Download the file. This will raise an IOError if the url
+ # is not found.
+ yield StartDownloadMessage(info)
+ yield ProgressMessage(5)
+ try:
+ infile = compat.urlopen(info.url)
+ with open(filepath, 'wb') as outfile:
+ #print info.size
+ num_blocks = max(1, float(info.size)/(1024*16))
+ for block in itertools.count():
+ s = infile.read(1024*16) # 16k blocks.
+ outfile.write(s)
+ if not s: break
+ if block % 2 == 0: # how often?
+ yield ProgressMessage(min(80, 5+75*(block/num_blocks)))
+ infile.close()
+ except IOError as e:
+ yield ErrorMessage(info, 'Error downloading %r from <%s>:'
+ '\n %s' % (info.id, info.url, e))
+ return
+ yield FinishDownloadMessage(info)
+ yield ProgressMessage(80)
+
+ # If it's a zipfile, uncompress it.
+ if info.filename.endswith('.zip'):
+ zipdir = os.path.join(download_dir, info.subdir)
+ # Unzip if we're unzipping by default; *or* if it's already
+ # been unzipped (presumably a previous version).
+ if info.unzip or os.path.exists(os.path.join(zipdir, info.id)):
+ yield StartUnzipMessage(info)
+ for msg in _unzip_iter(filepath, zipdir, verbose=False):
+ # Somewhat of a hack, but we need a proper package reference
+ msg.package = info
+ yield msg
+ yield FinishUnzipMessage(info)
+
+ yield FinishPackageMessage(info)
+
+ def download(self, info_or_id=None, download_dir=None, quiet=False,
+ force=False, prefix='[nltk_data] ', halt_on_error=True,
+ raise_on_error=False):
+ # If no info or id is given, then use the interactive shell.
+ if info_or_id is None:
+ # [xx] hmm -- changing self._download_dir here seems like
+ # the wrong thing to do. Maybe the _interactive_download
+ # function should make a new copy of self to use?
+ if download_dir is not None: self._download_dir = download_dir
+ self._interactive_download()
+ return True
+
+ else:
+ # Define a helper function for displaying output:
+ def show(s, prefix2=''):
+ print(textwrap.fill(s, initial_indent=prefix+prefix2,
+ subsequent_indent=prefix+prefix2+' '*4))
+
+ for msg in self.incr_download(info_or_id, download_dir, force):
+ # Error messages
+ if isinstance(msg, ErrorMessage):
+ show(msg.message)
+ if raise_on_error:
+ raise ValueError(msg.message)
+ if halt_on_error:
+ return False
+ self._errors = True
+ if not quiet:
+ print("Error installing package. Retry? [n/y/e]")
+ choice = compat.raw_input().strip()
+ if choice in ['y', 'Y']:
+ if not self.download(msg.package.id, download_dir,
+ quiet, force, prefix,
+ halt_on_error, raise_on_error):
+ return False
+ elif choice in ['e', 'E']:
+ return False
+
+ # All other messages
+ if not quiet:
+ # Collection downloading messages:
+ if isinstance(msg, StartCollectionMessage):
+ show('Downloading collection %r' % msg.collection.id)
+ prefix += ' | '
+ print(prefix)
+ elif isinstance(msg, FinishCollectionMessage):
+ print(prefix)
+ prefix = prefix[:-4]
+ if self._errors:
+ show('Downloaded collection %r with errors' %
+ msg.collection.id)
+ else:
+ show('Done downloading collection %s' %
+ msg.collection.id)
+
+ # Package downloading messages:
+ elif isinstance(msg, StartPackageMessage):
+ show('Downloading package %s to %s...' %
+ (msg.package.id, download_dir))
+ elif isinstance(msg, UpToDateMessage):
+ show('Package %s is already up-to-date!' %
+ msg.package.id, ' ')
+ #elif isinstance(msg, StaleMessage):
+ # show('Package %s is out-of-date or corrupt' %
+ # msg.package.id, ' ')
+ elif isinstance(msg, StartUnzipMessage):
+ show('Unzipping %s.' % msg.package.filename, ' ')
+
+ # Data directory message:
+ elif isinstance(msg, SelectDownloadDirMessage):
+ download_dir = msg.download_dir
+ return True
+
+ def is_stale(self, info_or_id, download_dir=None):
+ return self.status(info_or_id, download_dir) == self.STALE
+
+ def is_installed(self, info_or_id, download_dir=None):
+ return self.status(info_or_id, download_dir) == self.INSTALLED
+
+ def clear_status_cache(self, id=None):
+ if id is None:
+ self._status_cache.clear()
+ else:
+ self._status_cache.pop(id, None)
+
+ def status(self, info_or_id, download_dir=None):
+ """
+ Return a constant describing the status of the given package
+ or collection. Status can be one of ``INSTALLED``,
+ ``NOT_INSTALLED``, ``STALE``, or ``PARTIAL``.
+ """
+ if download_dir is None: download_dir = self._download_dir
+ info = self._info_or_id(info_or_id)
+
+ # Handle collections:
+ if isinstance(info, Collection):
+ pkg_status = [self.status(pkg.id) for pkg in info.packages]
+ if self.STALE in pkg_status:
+ return self.STALE
+ elif self.PARTIAL in pkg_status:
+ return self.PARTIAL
+ elif (self.INSTALLED in pkg_status and
+ self.NOT_INSTALLED in pkg_status):
+ return self.PARTIAL
+ elif self.NOT_INSTALLED in pkg_status:
+ return self.NOT_INSTALLED
+ else:
+ return self.INSTALLED
+
+ # Handle packages:
+ else:
+ filepath = os.path.join(download_dir, info.filename)
+ if download_dir != self._download_dir:
+ status = self._pkg_status(info, filepath)
+ else:
+ if info.id not in self._status_cache:
+ self._status_cache[info.id] = self._pkg_status(info,
+ filepath)
+ return self._status_cache[info.id]
+
+ def _pkg_status(self, info, filepath):
+ if not os.path.exists(filepath):
+ return self.NOT_INSTALLED
+
+ # Check if the file has the correct size.
+ try: filestat = os.stat(filepath)
+ except OSError: return self.NOT_INSTALLED
+ if filestat.st_size != int(info.size):
+ return self.STALE
+
+ # Check if the file's checksum matches
+ if md5_hexdigest(filepath) != info.checksum:
+ return self.STALE
+
+ # If it's a zipfile, and it's been at least partially
+ # unzipped, then check if it's been fully unzipped.
+ if filepath.endswith('.zip'):
+ unzipdir = filepath[:-4]
+ if not os.path.exists(unzipdir):
+ return self.INSTALLED # but not unzipped -- ok!
+ if not os.path.isdir(unzipdir):
+ return self.STALE
+
+ unzipped_size = sum(os.stat(os.path.join(d, f)).st_size
+ for d, _, files in os.walk(unzipdir)
+ for f in files)
+ if unzipped_size != info.unzipped_size:
+ return self.STALE
+
+ # Otherwise, everything looks good.
+ return self.INSTALLED
+
+ def update(self, quiet=False, prefix='[nltk_data] '):
+ """
+ Re-download any packages whose status is STALE.
+ """
+ self.clear_status_cache()
+ for pkg in self.packages():
+ if self.status(pkg) == self.STALE:
+ self.download(pkg, quiet=quiet, prefix=prefix)
+
+ #/////////////////////////////////////////////////////////////////
+ # Index
+ #/////////////////////////////////////////////////////////////////
+
+ def _update_index(self, url=None):
+ """A helper function that ensures that self._index is
+ up-to-date. If the index is older than self.INDEX_TIMEOUT,
+ then download it again."""
+ # Check if the index is aleady up-to-date. If so, do nothing.
+ if not (self._index is None or url is not None or
+ time.time()-self._index_timestamp > self.INDEX_TIMEOUT):
+ return
+
+ # If a URL was specified, then update our URL.
+ self._url = url or self._url
+
+ # Download the index file.
+ self._index = nltk.internals.ElementWrapper(
+ ElementTree.parse(compat.urlopen(self._url)).getroot())
+ self._index_timestamp = time.time()
+
+ # Build a dictionary of packages.
+ packages = [Package.fromxml(p) for p in
+ self._index.findall('packages/package')]
+ self._packages = dict((p.id, p) for p in packages)
+
+ # Build a dictionary of collections.
+ collections = [Collection.fromxml(c) for c in
+ self._index.findall('collections/collection')]
+ self._collections = dict((c.id, c) for c in collections)
+
+ # Replace identifiers with actual children in collection.children.
+ for collection in self._collections.values():
+ for i, child_id in enumerate(collection.children):
+ if child_id in self._packages:
+ collection.children[i] = self._packages[child_id]
+ if child_id in self._collections:
+ collection.children[i] = self._collections[child_id]
+
+ # Fill in collection.packages for each collection.
+ for collection in self._collections.values():
+ packages = {}
+ queue = [collection]
+ for child in queue:
+ if isinstance(child, Collection):
+ queue.extend(child.children)
+ else:
+ packages[child.id] = child
+ collection.packages = packages.values()
+
+ # Flush the status cache
+ self._status_cache.clear()
+
+ def index(self):
+ """
+ Return the XML index describing the packages available from
+ the data server. If necessary, this index will be downloaded
+ from the data server.
+ """
+ self._update_index()
+ return self._index
+
+ def info(self, id):
+ """Return the ``Package`` or ``Collection`` record for the
+ given item."""
+ self._update_index()
+ if id in self._packages: return self._packages[id]
+ if id in self._collections: return self._collections[id]
+ raise ValueError('Package %r not found in index' % id)
+
+ def xmlinfo(self, id):
+ """Return the XML info record for the given item"""
+ self._update_index()
+ for package in self._index.findall('packages/package'):
+ if package.get('id') == id:
+ return package
+ for collection in self._index.findall('collections/collection'):
+ if collection.get('id') == id:
+ return collection
+ raise ValueError('Package %r not found in index' % id)
+
+ #/////////////////////////////////////////////////////////////////
+ # URL & Data Directory
+ #/////////////////////////////////////////////////////////////////
+
+ def _get_url(self):
+ """The URL for the data server's index file."""
+ return self._url
+ def _set_url(self, url):
+ """
+ Set a new URL for the data server. If we're unable to contact
+ the given url, then the original url is kept.
+ """
+ original_url = self._url
+ try:
+ self._update_index(url)
+ except:
+ self._url = original_url
+ raise
+ url = property(_get_url, _set_url)
+
+ def default_download_dir(self):
+ """
+ Return the directory to which packages will be downloaded by
+ default. This value can be overridden using the constructor,
+ or on a case-by-case basis using the ``download_dir`` argument when
+ calling ``download()``.
+
+ On Windows, the default download directory is
+ ``PYTHONHOME/lib/nltk``, where *PYTHONHOME* is the
+ directory containing Python, e.g. ``C:\\Python25``.
+
+ On all other platforms, the default directory is the first of
+ the following which exists or which can be created with write
+ permission: ``/usr/share/nltk_data``, ``/usr/local/share/nltk_data``,
+ ``/usr/lib/nltk_data``, ``/usr/local/lib/nltk_data``, ``~/nltk_data``.
+ """
+ # Check if we have sufficient permissions to install in a
+ # variety of system-wide locations.
+ for nltkdir in nltk.data.path:
+ if (os.path.exists(nltkdir) and
+ nltk.internals.is_writable(nltkdir)):
+ return nltkdir
+
+ # On Windows, use %APPDATA%
+ if sys.platform == 'win32' and 'APPDATA' in os.environ:
+ homedir = os.environ['APPDATA']
+
+ # Otherwise, install in the user's home directory.
+ else:
+ homedir = os.path.expanduser('~/')
+ if homedir == '~/':
+ raise ValueError("Could not find a default download directory")
+
+ # append "nltk_data" to the home directory
+ return os.path.join(homedir, 'nltk_data')
+
+ def _get_download_dir(self):
+ """
+ The default directory to which packages will be downloaded.
+ This defaults to the value returned by ``default_download_dir()``.
+ To override this default on a case-by-case basis, use the
+ ``download_dir`` argument when calling ``download()``.
+ """
+ return self._download_dir
+ def _set_download_dir(self, download_dir):
+ self._download_dir = download_dir
+ # Clear the status cache.
+ self._status_cache.clear()
+ download_dir = property(_get_download_dir, _set_download_dir)
+
+ #/////////////////////////////////////////////////////////////////
+ # Interactive Shell
+ #/////////////////////////////////////////////////////////////////
+
+ def _interactive_download(self):
+ # Try the GUI first; if that doesn't work, try the simple
+ # interactive shell.
+ if TKINTER:
+ try:
+ DownloaderGUI(self).mainloop()
+ except TclError:
+ DownloaderShell(self).run()
+ else:
+ DownloaderShell(self).run()
+
+class DownloaderShell(object):
+ def __init__(self, dataserver):
+ self._ds = dataserver
+
+ def _simple_interactive_menu(self, *options):
+ print('-'*75)
+ spc = (68 - sum(len(o) for o in options))//(len(options)-1)*' '
+ print(' ' + spc.join(options))
+ #w = 76/len(options)
+ #fmt = ' ' + ('%-'+str(w)+'s')*(len(options)-1) + '%s'
+ #print fmt % options
+ print('-'*75)
+
+ def run(self):
+ print('NLTK Downloader')
+ while True:
+ self._simple_interactive_menu(
+ 'd) Download', 'l) List', ' u) Update', 'c) Config', 'h) Help', 'q) Quit')
+ user_input = compat.raw_input('Downloader> ').strip()
+ if not user_input: print(); continue
+ command = user_input.lower().split()[0]
+ args = user_input.split()[1:]
+ try:
+ if command == 'l':
+ print()
+ self._ds.list(self._ds.download_dir, header=False,
+ more_prompt=True)
+ elif command == 'h':
+ self._simple_interactive_help()
+ elif command == 'c':
+ self._simple_interactive_config()
+ elif command in ('q', 'x'):
+ return
+ elif command == 'd':
+ self._simple_interactive_download(args)
+ elif command == 'u':
+ self._simple_interactive_update()
+ else:
+ print('Command %r unrecognized' % user_input)
+ except compat.HTTPError as e:
+ print('Error reading from server: %s'%e)
+ except compat.URLError as e:
+ print('Error connecting to server: %s'%e.reason)
+ # try checking if user_input is a package name, &
+ # downloading it?
+ print()
+
+ def _simple_interactive_download(self, args):
+ if args:
+ for arg in args:
+ try: self._ds.download(arg, prefix=' ')
+ except (IOError, ValueError) as e: print(e)
+ else:
+ while True:
+ print()
+ print('Download which package (l=list; x=cancel)?')
+ user_input = compat.raw_input(' Identifier> ')
+ if user_input.lower()=='l':
+ self._ds.list(self._ds.download_dir, header=False,
+ more_prompt=True, skip_installed=True)
+ continue
+ elif user_input.lower() in ('x', 'q', ''):
+ return
+ elif user_input:
+ for id in user_input.split():
+ try: self._ds.download(id, prefix=' ')
+ except (IOError, ValueError) as e: print(e)
+ break
+
+ def _simple_interactive_update(self):
+ while True:
+ stale_packages = []
+ stale = partial = False
+ for info in sorted(getattr(self._ds, 'packages')(), key=str):
+ if self._ds.status(info) == self._ds.STALE:
+ stale_packages.append((info.id, info.name))
+
+ print()
+ if stale_packages:
+ print('Will update following packages (o=ok; x=cancel)')
+ for pid, pname in stale_packages:
+ name = textwrap.fill('-'*27 + (pname),
+ 75, subsequent_indent=27*' ')[27:]
+ print(' [ ] %s %s' % (pid.ljust(20, '.'), name))
+ print()
+
+ user_input = compat.raw_input(' Identifier> ')
+ if user_input.lower()=='o':
+ for pid, pname in stale_packages:
+ try: self._ds.download(pid, prefix=' ')
+ except (IOError, ValueError) as e: print(e)
+ break
+ elif user_input.lower() in ('x', 'q', ''):
+ return
+ else:
+ print('Nothing to update.')
+ return
+
+ def _simple_interactive_help(self):
+ print()
+ print('Commands:')
+ print(' d) Download a package or collection u) Update out of date packages')
+ print(' l) List packages & collections h) Help')
+ print(' c) View & Modify Configuration q) Quit')
+
+ def _show_config(self):
+ print()
+ print('Data Server:')
+ print(' - URL: <%s>' % self._ds.url)
+ print((' - %d Package Collections Available' %
+ len(self._ds.collections())))
+ print((' - %d Individual Packages Available' %
+ len(self._ds.packages())))
+ print()
+ print('Local Machine:')
+ print(' - Data directory: %s' % self._ds.download_dir)
+
+ def _simple_interactive_config(self):
+ self._show_config()
+ while True:
+ print()
+ self._simple_interactive_menu(
+ 's) Show Config', 'u) Set Server URL',
+ 'd) Set Data Dir', 'm) Main Menu')
+ user_input = compat.raw_input('Config> ').strip().lower()
+ if user_input == 's':
+ self._show_config()
+ elif user_input == 'd':
+ new_dl_dir = compat.raw_input(' New Directory> ').strip().lower()
+ if new_dl_dir in ('', 'x', 'q'):
+ print(' Cancelled!')
+ elif os.path.isdir(new_dl_dir):
+ self._ds.download_dir = new_dl_dir
+ else:
+ print(('Directory %r not found! Create it first.' %
+ new_dl_dir))
+ elif user_input == 'u':
+ new_url = compat.raw_input(' New URL> ').strip().lower()
+ if new_url in ('', 'x', 'q'):
+ print(' Cancelled!')
+ else:
+ if not new_url.startswith('http://'):
+ new_url = 'http://'+new_url
+ try: self._ds.url = new_url
+ except Exception as e:
+ print('Error reading <%r>:\n %s' % (new_url, e))
+ elif user_input == 'm':
+ break
+
+class DownloaderGUI(object):
+ """
+ Graphical interface for downloading packages from the NLTK data
+ server.
+ """
+
+ #/////////////////////////////////////////////////////////////////
+ # Column Configuration
+ #/////////////////////////////////////////////////////////////////
+
+ COLUMNS = ['', 'Identifier', 'Name', 'Size', 'Status',
+ 'Unzipped Size',
+ 'Copyright', 'Contact', 'License', 'Author',
+ 'Subdir', 'Checksum']
+ """A list of the names of columns. This controls the order in
+ which the columns will appear. If this is edited, then
+ ``_package_to_columns()`` may need to be edited to match."""
+
+ COLUMN_WEIGHTS = {'': 0, 'Name': 5, 'Size': 0, 'Status': 0}
+ """A dictionary specifying how columns should be resized when the
+ table is resized. Columns with weight 0 will not be resized at
+ all; and columns with high weight will be resized more.
+ Default weight (for columns not explicitly listed) is 1."""
+
+ COLUMN_WIDTHS = {'':1, 'Identifier':20, 'Name':45,
+ 'Size': 10, 'Unzipped Size': 10,
+ 'Status': 12}
+ """A dictionary specifying how wide each column should be, in
+ characters. The default width (for columns not explicitly
+ listed) is specified by ``DEFAULT_COLUMN_WIDTH``."""
+
+ DEFAULT_COLUMN_WIDTH = 30
+ """The default width for columns that are not explicitly listed
+ in ``COLUMN_WIDTHS``."""
+
+ INITIAL_COLUMNS = ['', 'Identifier', 'Name', 'Size', 'Status']
+ """The set of columns that should be displayed by default."""
+
+ # Perform a few import-time sanity checks to make sure that the
+ # column configuration variables are defined consistently:
+ for c in COLUMN_WEIGHTS: assert c in COLUMNS
+ for c in COLUMN_WIDTHS: assert c in COLUMNS
+ for c in INITIAL_COLUMNS: assert c in COLUMNS
+
+ #/////////////////////////////////////////////////////////////////
+ # Color Configuration
+ #/////////////////////////////////////////////////////////////////
+
+ _BACKDROP_COLOR = ('#000', '#ccc')
+
+ _ROW_COLOR = {Downloader.INSTALLED: ('#afa', '#080'),
+ Downloader.PARTIAL: ('#ffa', '#880'),
+ Downloader.STALE: ('#faa', '#800'),
+ Downloader.NOT_INSTALLED: ('#fff', '#888')}
+
+ _MARK_COLOR = ('#000', '#ccc')
+
+ #_FRONT_TAB_COLOR = ('#ccf', '#008')
+ #_BACK_TAB_COLOR = ('#88a', '#448')
+ _FRONT_TAB_COLOR = ('#fff', '#45c')
+ _BACK_TAB_COLOR = ('#aaa', '#67a')
+
+ _PROGRESS_COLOR = ('#f00', '#aaa')
+
+ _TAB_FONT = 'helvetica -16 bold'
+
+ #/////////////////////////////////////////////////////////////////
+ # Constructor
+ #/////////////////////////////////////////////////////////////////
+
+ def __init__(self, dataserver, use_threads=True):
+ self._ds = dataserver
+ self._use_threads = use_threads
+
+ # For the threaded downloader:
+ self._download_lock = threading.Lock()
+ self._download_msg_queue = []
+ self._download_abort_queue = []
+ self._downloading = False
+
+ # For tkinter after callbacks:
+ self._afterid = {}
+
+ # A message log.
+ self._log_messages = []
+ self._log_indent = 0
+ self._log('NLTK Downloader Started!')
+
+ # Create the main window.
+ top = self.top = Tk()
+ top.geometry('+50+50')
+ top.title('NLTK Downloader')
+ top.configure(background=self._BACKDROP_COLOR[1])
+
+ # Set up some bindings now, in case anything goes wrong.
+ top.bind('<Control-q>', self.destroy)
+ top.bind('<Control-x>', self.destroy)
+ self._destroyed = False
+
+ self._column_vars = {}
+
+ # Initialize the GUI.
+ self._init_widgets()
+ self._init_menu()
+ try:
+ self._fill_table()
+ except compat.HTTPError as e:
+ showerror('Error reading from server', e)
+ except compat.URLError as e:
+ showerror('Error connecting to server', e.reason)
+
+ self._show_info()
+ self._select_columns()
+ self._table.select(0)
+
+ # Make sure we get notified when we're destroyed, so we can
+ # cancel any download in progress.
+ self._table.bind('<Destroy>', self._destroy)
+
+ def _log(self, msg):
+ self._log_messages.append('%s %s%s' % (time.ctime(),
+ ' | '*self._log_indent, msg))
+
+ #/////////////////////////////////////////////////////////////////
+ # Internals
+ #/////////////////////////////////////////////////////////////////
+
+ def _init_widgets(self):
+ # Create the top-level frame structures
+ f1 = Frame(self.top, relief='raised', border=2, padx=8, pady=0)
+ f1.pack(sid='top', expand=True, fill='both')
+ f1.grid_rowconfigure(2, weight=1)
+ f1.grid_columnconfigure(0, weight=1)
+ Frame(f1, height=8).grid(column=0, row=0) # spacer
+ tabframe = Frame(f1)
+ tabframe.grid(column=0, row=1, sticky='news')
+ tableframe = Frame(f1)
+ tableframe.grid(column=0, row=2, sticky='news')
+ buttonframe = Frame(f1)
+ buttonframe.grid(column=0, row=3, sticky='news')
+ Frame(f1, height=8).grid(column=0, row=4) # spacer
+ infoframe = Frame(f1)
+ infoframe.grid(column=0, row=5, sticky='news')
+ Frame(f1, height=8).grid(column=0, row=6) # spacer
+ progressframe = Frame(self.top, padx=3, pady=3,
+ background=self._BACKDROP_COLOR[1])
+ progressframe.pack(side='bottom', fill='x')
+ self.top['border'] = 0
+ self.top['highlightthickness'] = 0
+
+ # Create the tabs
+ self._tab_names = ['Collections', 'Corpora',
+ 'Models', 'All Packages',]
+ self._tabs = {}
+ for i, tab in enumerate(self._tab_names):
+ label = Label(tabframe, text=tab, font=self._TAB_FONT)
+ label.pack(side='left', padx=((i+1)%2)*10)
+ label.bind('<Button-1>', self._select_tab)
+ self._tabs[tab.lower()] = label
+
+ # Create the table.
+ column_weights = [self.COLUMN_WEIGHTS.get(column, 1)
+ for column in self.COLUMNS]
+ self._table = Table(tableframe, self.COLUMNS,
+ column_weights=column_weights,
+ highlightthickness=0, listbox_height=16,
+ reprfunc=self._table_reprfunc)
+ self._table.columnconfig(0, foreground=self._MARK_COLOR[0]) # marked
+ for i, column in enumerate(self.COLUMNS):
+ width = self.COLUMN_WIDTHS.get(column, self.DEFAULT_COLUMN_WIDTH)
+ self._table.columnconfig(i, width=width)
+ self._table.pack(expand=True, fill='both')
+ self._table.focus()
+ self._table.bind_to_listboxes('<Double-Button-1>',
+ self._download)
+ self._table.bind('<space>', self._table_mark)
+ self._table.bind('<Return>', self._download)
+ self._table.bind('<Left>', self._prev_tab)
+ self._table.bind('<Right>', self._next_tab)
+ self._table.bind('<Control-a>', self._mark_all)
+
+ # Create entry boxes for URL & download_dir
+ infoframe.grid_columnconfigure(1, weight=1)
+
+ info = [('url', 'Server Index:', self._set_url),
+ ('download_dir','Download Directory:',self._set_download_dir)]
+ self._info = {}
+ for (i, (key, label, callback)) in enumerate(info):
+ Label(infoframe, text=label).grid(column=0, row=i, sticky='e')
+ entry = Entry(infoframe, font='courier', relief='groove',
+ disabledforeground='black')
+ self._info[key] = (entry, callback)
+ entry.bind('<Return>', self._info_save)
+ entry.bind('<Button-1>', lambda e,key=key: self._info_edit(key))
+ entry.grid(column=1, row=i, sticky='ew')
+
+ # If the user edits url or download_dir, and then clicks outside
+ # the entry box, then save their results.
+ self.top.bind('<Button-1>', self._info_save)
+
+ # Create Download & Refresh buttons.
+ self._download_button = Button(
+ buttonframe, text='Download', command=self._download, width=8)
+ self._download_button.pack(side='left')
+ self._refresh_button = Button(
+ buttonframe, text='Refresh', command=self._refresh, width=8)
+ self._refresh_button.pack(side='right')
+
+ # Create Progress bar
+ self._progresslabel = Label(progressframe, text='',
+ foreground=self._BACKDROP_COLOR[0],
+ background=self._BACKDROP_COLOR[1])
+ self._progressbar = Canvas(progressframe, width=200, height=16,
+ background=self._PROGRESS_COLOR[1],
+ relief='sunken', border=1)
+ self._init_progressbar()
+ self._progressbar.pack(side='right')
+ self._progresslabel.pack(side='left')
+
+ def _init_menu(self):
+ menubar = Menu(self.top)
+
+ filemenu = Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Download', underline=0,
+ command=self._download, accelerator='Return')
+ filemenu.add_separator()
+ filemenu.add_command(label='Change Server Index', underline=7,
+ command=lambda: self._info_edit('url'))
+ filemenu.add_command(label='Change Download Directory', underline=0,
+ command=lambda: self._info_edit('download_dir'))
+ filemenu.add_separator()
+ filemenu.add_command(label='Show Log', underline=5,
+ command=self._show_log)
+ filemenu.add_separator()
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-x')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ # Create a menu to control which columns of the table are
+ # shown. n.b.: we never hide the first two columns (mark and
+ # identifier).
+ viewmenu = Menu(menubar, tearoff=0)
+ for column in self._table.column_names[2:]:
+ var = IntVar(self.top)
+ assert column not in self._column_vars
+ self._column_vars[column] = var
+ if column in self.INITIAL_COLUMNS: var.set(1)
+ viewmenu.add_checkbutton(label=column, underline=0, variable=var,
+ command=self._select_columns)
+ menubar.add_cascade(label='View', underline=0, menu=viewmenu)
+
+ # Create a sort menu
+ # [xx] this should be selectbuttons; and it should include
+ # reversed sorts as options.
+ sortmenu = Menu(menubar, tearoff=0)
+ for column in self._table.column_names[1:]:
+ sortmenu.add_command(label='Sort by %s' % column,
+ command=(lambda c=column:
+ self._table.sort_by(c, 'ascending')))
+ sortmenu.add_separator()
+ #sortmenu.add_command(label='Descending Sort:')
+ for column in self._table.column_names[1:]:
+ sortmenu.add_command(label='Reverse sort by %s' % column,
+ command=(lambda c=column:
+ self._table.sort_by(c, 'descending')))
+ menubar.add_cascade(label='Sort', underline=0, menu=sortmenu)
+
+ helpmenu = Menu(menubar, tearoff=0)
+ helpmenu.add_command(label='About', underline=0,
+ command=self.about)
+ helpmenu.add_command(label='Instructions', underline=0,
+ command=self.help, accelerator='F1')
+ menubar.add_cascade(label='Help', underline=0, menu=helpmenu)
+ self.top.bind('<F1>', self.help)
+
+ self.top.config(menu=menubar)
+
+ def _select_columns(self):
+ for (column, var) in self._column_vars.items():
+ if var.get():
+ self._table.show_column(column)
+ else:
+ self._table.hide_column(column)
+
+ def _refresh(self):
+ self._ds.clear_status_cache()
+ try:
+ self._fill_table()
+ except compat.HTTPError as e:
+ showerror('Error reading from server', e)
+ except compat.URLError as e:
+ showerror('Error connecting to server', e.reason)
+ self._table.select(0)
+
+ def _info_edit(self, info_key):
+ self._info_save() # just in case.
+ (entry, callback) = self._info[info_key]
+ entry['state'] = 'normal'
+ entry['relief'] = 'sunken'
+ entry.focus()
+
+ def _info_save(self, e=None):
+ focus = self._table
+ for entry, callback in self._info.values():
+ if entry['state'] == 'disabled': continue
+ if e is not None and e.widget is entry and e.keysym != 'Return':
+ focus = entry
+ else:
+ entry['state'] = 'disabled'
+ entry['relief'] = 'groove'
+ callback(entry.get())
+ focus.focus()
+
+ def _table_reprfunc(self, row, col, val):
+ if self._table.column_names[col].endswith('Size'):
+ if isinstance(val, compat.string_types): return ' %s' % val
+ elif val < 1024**2: return ' %.1f KB' % (val/1024.**1)
+ elif val < 1024**3: return ' %.1f MB' % (val/1024.**2)
+ else: return ' %.1f GB' % (val/1024.**3)
+
+ if col in (0, ''): return str(val)
+ else: return ' %s' % val
+
+ def _set_url(self, url):
+ if url == self._ds.url: return
+ try:
+ self._ds.url = url
+ self._fill_table()
+ except IOError as e:
+ showerror('Error Setting Server Index', str(e))
+ self._show_info()
+
+
+ def _set_download_dir(self, download_dir):
+ if self._ds.download_dir == download_dir: return
+ # check if the dir exists, and if not, ask if we should create it?
+
+ # Clear our status cache, & re-check what's installed
+ self._ds.download_dir = download_dir
+ try:
+ self._fill_table()
+ except compat.HTTPError as e:
+ showerror('Error reading from server', e)
+ except compat.URLError as e:
+ showerror('Error connecting to server', e.reason)
+ self._show_info()
+
+ def _show_info(self):
+ print('showing info', self._ds.url)
+ for entry,cb in self._info.values():
+ entry['state'] = 'normal'
+ entry.delete(0, 'end')
+ self._info['url'][0].insert(0, self._ds.url)
+ self._info['download_dir'][0].insert(0, self._ds.download_dir)
+ for entry,cb in self._info.values():
+ entry['state'] = 'disabled'
+
+ def _prev_tab(self, *e):
+ for i, tab in enumerate(self._tab_names):
+ if tab.lower() == self._tab and i > 0:
+ self._tab = self._tab_names[i-1].lower()
+ try:
+ return self._fill_table()
+ except compat.HTTPError as e:
+ showerror('Error reading from server', e)
+ except compat.URLError as e:
+ showerror('Error connecting to server', e.reason)
+
+ def _next_tab(self, *e):
+ for i, tab in enumerate(self._tab_names):
+ if tab.lower() == self._tab and i < (len(self._tabs)-1):
+ self._tab = self._tab_names[i+1].lower()
+ try:
+ return self._fill_table()
+ except compat.HTTPError as e:
+ showerror('Error reading from server', e)
+ except compat.URLError as e:
+ showerror('Error connecting to server', e.reason)
+
+ def _select_tab(self, event):
+ self._tab = event.widget['text'].lower()
+ try:
+ self._fill_table()
+ except compat.HTTPError as e:
+ showerror('Error reading from server', e)
+ except compat.URLError as e:
+ showerror('Error connecting to server', e.reason)
+
+ _tab = 'collections'
+ #_tab = 'corpora'
+ _rows = None
+ def _fill_table(self):
+ selected_row = self._table.selected_row()
+ self._table.clear()
+ if self._tab == 'all packages':
+ items = self._ds.packages()
+ elif self._tab == 'corpora':
+ items = self._ds.corpora()
+ elif self._tab == 'models':
+ items = self._ds.models()
+ elif self._tab == 'collections':
+ items = self._ds.collections()
+ else:
+ assert 0, 'bad tab value %r' % self._tab
+ rows = [self._package_to_columns(item) for item in items]
+ self._table.extend(rows)
+
+ # Highlight the active tab.
+ for tab, label in self._tabs.items():
+ if tab == self._tab:
+ label.configure(foreground=self._FRONT_TAB_COLOR[0],
+ background=self._FRONT_TAB_COLOR[1])
+ else:
+ label.configure(foreground=self._BACK_TAB_COLOR[0],
+ background=self._BACK_TAB_COLOR[1])
+
+ self._table.sort_by('Identifier', order='ascending')
+ self._color_table()
+ self._table.select(selected_row)
+
+ # This is a hack, because the scrollbar isn't updating its
+ # position right -- I'm not sure what the underlying cause is
+ # though. (This is on OS X w/ python 2.5) The length of
+ # delay that's necessary seems to depend on how fast the
+ # comptuer is. :-/
+ self.top.after(150, self._table._scrollbar.set,
+ *self._table._mlb.yview())
+ self.top.after(300, self._table._scrollbar.set,
+ *self._table._mlb.yview())
+
+ def _update_table_status(self):
+ for row_num in range(len(self._table)):
+ status = self._ds.status(self._table[row_num, 'Identifier'])
+ self._table[row_num, 'Status'] = status
+ self._color_table()
+
+ def _download(self, *e):
+ # If we're using threads, then delegate to the threaded
+ # downloader instead.
+ if self._use_threads:
+ return self._download_threaded(*e)
+
+ marked = [self._table[row, 'Identifier']
+ for row in range(len(self._table))
+ if self._table[row, 0] != '']
+ selection = self._table.selected_row()
+ if not marked and selection is not None:
+ marked = [self._table[selection, 'Identifier']]
+
+ download_iter = self._ds.incr_download(marked, self._ds.download_dir)
+ self._log_indent = 0
+ self._download_cb(download_iter, marked)
+
+ _DL_DELAY=10
+ def _download_cb(self, download_iter, ids):
+ try: msg = next(download_iter)
+ except StopIteration:
+ #self._fill_table(sort=False)
+ self._update_table_status()
+ afterid = self.top.after(10, self._show_progress, 0)
+ self._afterid['_download_cb'] = afterid
+ return
+
+ def show(s):
+ self._progresslabel['text'] = s
+ self._log(s)
+ if isinstance(msg, ProgressMessage):
+ self._show_progress(msg.progress)
+ elif isinstance(msg, ErrorMessage):
+ show(msg.message)
+ if msg.package is not None:
+ self._select(msg.package.id)
+ self._show_progress(None)
+ return # halt progress.
+ elif isinstance(msg, StartCollectionMessage):
+ show('Downloading collection %s' % msg.collection.id)
+ self._log_indent += 1
+ elif isinstance(msg, StartPackageMessage):
+ show('Downloading package %s' % msg.package.id)
+ elif isinstance(msg, UpToDateMessage):
+ show('Package %s is up-to-date!' % msg.package.id)
+ #elif isinstance(msg, StaleMessage):
+ # show('Package %s is out-of-date or corrupt' % msg.package.id)
+ elif isinstance(msg, FinishDownloadMessage):
+ show('Finished downloading %r.' % msg.package.id)
+ elif isinstance(msg, StartUnzipMessage):
+ show('Unzipping %s' % msg.package.filename)
+ elif isinstance(msg, FinishCollectionMessage):
+ self._log_indent -= 1
+ show('Finished downloading collection %r.' % msg.collection.id)
+ self._clear_mark(msg.collection.id)
+ elif isinstance(msg, FinishPackageMessage):
+ self._clear_mark(msg.package.id)
+ afterid = self.top.after(self._DL_DELAY, self._download_cb,
+ download_iter, ids)
+ self._afterid['_download_cb'] = afterid
+
+ def _select(self, id):
+ for row in range(len(self._table)):
+ if self._table[row, 'Identifier'] == id:
+ self._table.select(row)
+ return
+
+ def _color_table(self):
+ # Color rows according to status.
+ for row in range(len(self._table)):
+ bg, sbg = self._ROW_COLOR[self._table[row, 'Status']]
+ fg, sfg = ('black', 'white')
+ self._table.rowconfig(row, foreground=fg, selectforeground=sfg,
+ background=bg, selectbackground=sbg)
+ # Color the marked column
+ self._table.itemconfigure(row, 0,
+ foreground=self._MARK_COLOR[0],
+ background=self._MARK_COLOR[1])
+
+
+ def _clear_mark(self, id):
+ for row in range(len(self._table)):
+ if self._table[row, 'Identifier'] == id:
+ self._table[row, 0] = ''
+
+ def _mark_all(self, *e):
+ for row in range(len(self._table)):
+ self._table[row,0] = 'X'
+
+ def _table_mark(self, *e):
+ selection = self._table.selected_row()
+ if selection >= 0:
+ if self._table[selection][0] != '':
+ self._table[selection,0] = ''
+ else:
+ self._table[selection,0] = 'X'
+ self._table.select(delta=1)
+
+ def _show_log(self):
+ text = '\n'.join(self._log_messages)
+ ShowText(self.top, 'NLTK Downloader Log', text)
+
+ def _package_to_columns(self, pkg):
+ """
+ Given a package, return a list of values describing that
+ package, one for each column in ``self.COLUMNS``.
+ """
+ row = []
+ for column_index, column_name in enumerate(self.COLUMNS):
+ if column_index == 0: # Mark:
+ row.append('')
+ elif column_name == 'Identifier':
+ row.append(pkg.id)
+ elif column_name == 'Status':
+ row.append(self._ds.status(pkg))
+ else:
+ attr = column_name.lower().replace(' ', '_')
+ row.append(getattr(pkg, attr, 'n/a'))
+ return row
+
+ #/////////////////////////////////////////////////////////////////
+ # External Interface
+ #/////////////////////////////////////////////////////////////////
+
+ def destroy(self, *e):
+ if self._destroyed: return
+ self.top.destroy()
+ self._destroyed = True
+
+ def _destroy(self, *e):
+ if self.top is not None:
+ for afterid in self._afterid.values():
+ self.top.after_cancel(afterid)
+
+ # Abort any download in progress.
+ if self._downloading and self._use_threads:
+ self._abort_download()
+
+ # Make sure the garbage collector destroys these now;
+ # otherwise, they may get destroyed when we're not in the main
+ # thread, which would make Tkinter unhappy.
+ self._column_vars.clear()
+
+ def mainloop(self, *args, **kwargs):
+ self.top.mainloop(*args, **kwargs)
+
+ #/////////////////////////////////////////////////////////////////
+ # HELP
+ #/////////////////////////////////////////////////////////////////
+
+ HELP = textwrap.dedent("""\
+ This tool can be used to download a variety of corpora and models
+ that can be used with NLTK. Each corpus or model is distributed
+ in a single zip file, known as a \"package file.\" You can
+ download packages individually, or you can download pre-defined
+ collections of packages.
+
+ When you download a package, it will be saved to the \"download
+ directory.\" A default download directory is chosen when you run
+
+ the downloader; but you may also select a different download
+ directory. On Windows, the default download directory is
+
+
+ \"package.\"
+
+ The NLTK downloader can be used to download a variety of corpora,
+ models, and other data packages.
+
+ Keyboard shortcuts::
+ [return]\t Download
+ [up]\t Select previous package
+ [down]\t Select next package
+ [left]\t Select previous tab
+ [right]\t Select next tab
+ """)
+
+ def help(self, *e):
+ # The default font's not very legible; try using 'fixed' instead.
+ try:
+ ShowText(self.top, 'Help: NLTK Dowloader',
+ self.HELP.strip(), width=75, font='fixed')
+ except:
+ ShowText(self.top, 'Help: NLTK Downloader',
+ self.HELP.strip(), width=75)
+
+ def about(self, *e):
+ ABOUT = ("NLTK Downloader\n"+
+ "Written by Edward Loper")
+ TITLE = 'About: NLTK Downloader'
+ try:
+ from tkMessageBox import Message
+ Message(message=ABOUT, title=TITLE).show()
+ except ImportError:
+ ShowText(self._top, TITLE, ABOUT)
+
+ #/////////////////////////////////////////////////////////////////
+ # Progress Bar
+ #/////////////////////////////////////////////////////////////////
+
+ _gradient_width = 5
+ def _init_progressbar(self):
+ c = self._progressbar
+ width, height = int(c['width']), int(c['height'])
+ for i in range(0, (int(c['width'])*2)//self._gradient_width):
+ c.create_line(i*self._gradient_width+20, -20,
+ i*self._gradient_width-height-20, height+20,
+ width=self._gradient_width,
+ fill='#%02x0000' % (80 + abs(i%6-3)*12))
+ c.addtag_all('gradient')
+ c.itemconfig('gradient', state='hidden')
+
+ # This is used to display progress
+ c.addtag_withtag('redbox', c.create_rectangle(
+ 0, 0, 0, 0, fill=self._PROGRESS_COLOR[0]))
+
+ def _show_progress(self, percent):
+ c = self._progressbar
+ if percent is None:
+ c.coords('redbox', 0, 0, 0, 0)
+ c.itemconfig('gradient', state='hidden')
+ else:
+ width, height = int(c['width']), int(c['height'])
+ x = percent * int(width) // 100 + 1
+ c.coords('redbox', 0, 0, x, height+1)
+
+ def _progress_alive(self):
+ c = self._progressbar
+ if not self._downloading:
+ c.itemconfig('gradient', state='hidden')
+ else:
+ c.itemconfig('gradient', state='normal')
+ x1, y1, x2, y2 = c.bbox('gradient')
+ if x1 <= -100:
+ c.move('gradient', (self._gradient_width*6)-4, 0)
+ else:
+ c.move('gradient', -4, 0)
+ afterid = self.top.after(200, self._progress_alive)
+ self._afterid['_progress_alive'] = afterid
+
+ #/////////////////////////////////////////////////////////////////
+ # Threaded downloader
+ #/////////////////////////////////////////////////////////////////
+
+ def _download_threaded(self, *e):
+ # If the user tries to start a new download while we're already
+ # downloading something, then abort the current download instead.
+ if self._downloading:
+ self._abort_download()
+ return
+
+ # Change the 'download' button to an 'abort' button.
+ self._download_button['text'] = 'Cancel'
+
+ marked = [self._table[row, 'Identifier']
+ for row in range(len(self._table))
+ if self._table[row, 0] != '']
+ selection = self._table.selected_row()
+ if not marked and selection is not None:
+ marked = [self._table[selection, 'Identifier']]
+
+ # Create a new data server object for the download operation,
+ # just in case the user modifies our data server during the
+ # download (e.g., clicking 'refresh' or editing the index url).
+ ds = Downloader(self._ds.url, self._ds.download_dir)
+
+ # Start downloading in a separate thread.
+ assert self._download_msg_queue == []
+ assert self._download_abort_queue == []
+ self._DownloadThread(ds, marked, self._download_lock,
+ self._download_msg_queue,
+ self._download_abort_queue).start()
+
+ # Monitor the download message queue & display its progress.
+ self._log_indent = 0
+ self._downloading = True
+ self._monitor_message_queue()
+
+ # Display an indication that we're still alive and well by
+ # cycling the progress bar.
+ self._progress_alive()
+
+ def _abort_download(self):
+ if self._downloading:
+ self._download_lock.acquire()
+ self._download_abort_queue.append('abort')
+ self._download_lock.release()
+
+ class _DownloadThread(threading.Thread):
+ def __init__(self, data_server, items, lock, message_queue, abort):
+ self.data_server = data_server
+ self.items = items
+ self.lock = lock
+ self.message_queue = message_queue
+ self.abort = abort
+ threading.Thread.__init__(self)
+
+ def run (self):
+ for msg in self.data_server.incr_download(self.items):
+ self.lock.acquire()
+ self.message_queue.append(msg)
+ # Check if we've been told to kill ourselves:
+ if self.abort:
+ self.message_queue.append('aborted')
+ self.lock.release()
+ return
+ self.lock.release()
+ self.lock.acquire()
+ self.message_queue.append('finished')
+ self.lock.release()
+
+ _MONITOR_QUEUE_DELAY=100
+ def _monitor_message_queue(self):
+ def show(s):
+ self._progresslabel['text'] = s
+ self._log(s)
+
+ # Try to acquire the lock; if it's busy, then just try again later.
+ if not self._download_lock.acquire():
+ return
+ for msg in self._download_msg_queue:
+
+ # Done downloading?
+ if msg == 'finished' or msg == 'aborted':
+ #self._fill_table(sort=False)
+ self._update_table_status()
+ self._downloading = False
+ self._download_button['text'] = 'Download'
+ del self._download_msg_queue[:]
+ del self._download_abort_queue[:]
+ self._download_lock.release()
+ if msg == 'aborted':
+ show('Download aborted!')
+ self._show_progress(None)
+ else:
+ afterid = self.top.after(100, self._show_progress, None)
+ self._afterid['_monitor_message_queue'] = afterid
+ return
+
+ # All other messages
+ elif isinstance(msg, ProgressMessage):
+ self._show_progress(msg.progress)
+ elif isinstance(msg, ErrorMessage):
+ show(msg.message)
+ if msg.package is not None:
+ self._select(msg.package.id)
+ self._show_progress(None)
+ self._downloading = False
+ return # halt progress.
+ elif isinstance(msg, StartCollectionMessage):
+ show('Downloading collection %r' % msg.collection.id)
+ self._log_indent += 1
+ elif isinstance(msg, StartPackageMessage):
+ self._ds.clear_status_cache(msg.package.id)
+ show('Downloading package %r' % msg.package.id)
+ elif isinstance(msg, UpToDateMessage):
+ show('Package %s is up-to-date!' % msg.package.id)
+ #elif isinstance(msg, StaleMessage):
+ # show('Package %s is out-of-date or corrupt; updating it' %
+ # msg.package.id)
+ elif isinstance(msg, FinishDownloadMessage):
+ show('Finished downloading %r.' % msg.package.id)
+ elif isinstance(msg, StartUnzipMessage):
+ show('Unzipping %s' % msg.package.filename)
+ elif isinstance(msg, FinishUnzipMessage):
+ show('Finished installing %s' % msg.package.id)
+ elif isinstance(msg, FinishCollectionMessage):
+ self._log_indent -= 1
+ show('Finished downloading collection %r.' % msg.collection.id)
+ self._clear_mark(msg.collection.id)
+ elif isinstance(msg, FinishPackageMessage):
+ self._update_table_status()
+ self._clear_mark(msg.package.id)
+
+ # Let the user know when we're aborting a download (but
+ # waiting for a good point to abort it, so we don't end up
+ # with a partially unzipped package or anything like that).
+ if self._download_abort_queue:
+ self._progresslabel['text'] = 'Aborting download...'
+
+ # Clear the message queue and then release the lock
+ del self._download_msg_queue[:]
+ self._download_lock.release()
+
+ # Check the queue again after MONITOR_QUEUE_DELAY msec.
+ afterid = self.top.after(self._MONITOR_QUEUE_DELAY,
+ self._monitor_message_queue)
+ self._afterid['_monitor_message_queue'] = afterid
+
+######################################################################
+# Helper Functions
+######################################################################
+# [xx] It may make sense to move these to nltk.internals.
+
+def md5_hexdigest(file):
+ """
+ Calculate and return the MD5 checksum for a given file.
+ ``file`` may either be a filename or an open stream.
+ """
+ if isinstance(file, compat.string_types):
+ with open(file, 'rb') as infile:
+ return _md5_hexdigest(infile)
+ return _md5_hexdigest(file)
+
+def _md5_hexdigest(fp):
+ md5_digest = md5()
+ while True:
+ block = fp.read(1024*16) # 16k blocks
+ if not block: break
+ md5_digest.update(block)
+ return md5_digest.hexdigest()
+
+
+# change this to periodically yield progress messages?
+# [xx] get rid of topdir parameter -- we should be checking
+# this when we build the index, anyway.
+def unzip(filename, root, verbose=True):
+ """
+ Extract the contents of the zip file ``filename`` into the
+ directory ``root``.
+ """
+ for message in _unzip_iter(filename, root, verbose):
+ if isinstance(message, ErrorMessage):
+ raise Exception(message)
+
+def _unzip_iter(filename, root, verbose=True):
+ if verbose:
+ sys.stdout.write('Unzipping %s' % os.path.split(filename)[1])
+ sys.stdout.flush()
+
+ try: zf = zipfile.ZipFile(filename)
+ except zipfile.error as e:
+ yield ErrorMessage(filename, 'Error with downloaded zip file')
+ return
+ except Exception as e:
+ yield ErrorMessage(filename, e)
+ return
+
+ # Get lists of directories & files
+ namelist = zf.namelist()
+ dirlist = set()
+ for x in namelist:
+ if x.endswith('/'):
+ dirlist.add(x)
+ else:
+ dirlist.add(x.rsplit('/',1)[0] + '/')
+ filelist = [x for x in namelist if not x.endswith('/')]
+
+ # Create the target directory if it doesn't exist
+ if not os.path.exists(root):
+ os.mkdir(root)
+
+ # Create the directory structure
+ for dirname in sorted(dirlist):
+ pieces = dirname[:-1].split('/')
+ for i in range(len(pieces)):
+ dirpath = os.path.join(root, *pieces[:i+1])
+ if not os.path.exists(dirpath):
+ os.mkdir(dirpath)
+
+ # Extract files.
+ for i, filename in enumerate(filelist):
+ filepath = os.path.join(root, *filename.split('/'))
+
+ with open(filepath, 'wb') as outfile:
+ try:
+ contents = zf.read(filename)
+ except Exception as e:
+ yield ErrorMessage(filename, e)
+ return
+ outfile.write(contents)
+
+ if verbose and (i*10/len(filelist) > (i-1)*10/len(filelist)):
+ sys.stdout.write('.')
+ sys.stdout.flush()
+ if verbose:
+ print()
+
+######################################################################
+# Index Builder
+######################################################################
+# This may move to a different file sometime.
+import subprocess, zipfile
+
+def build_index(root, base_url):
+ """
+ Create a new data.xml index file, by combining the xml description
+ files for various packages and collections. ``root`` should be the
+ path to a directory containing the package xml and zip files; and
+ the collection xml files. The ``root`` directory is expected to
+ have the following subdirectories::
+
+ root/
+ packages/ .................. subdirectory for packages
+ corpora/ ................. zip & xml files for corpora
+ grammars/ ................ zip & xml files for grammars
+ taggers/ ................. zip & xml files for taggers
+ tokenizers/ .............. zip & xml files for tokenizers
+ etc.
+ collections/ ............... xml files for collections
+
+ For each package, there should be two files: ``package.zip``
+ (where *package* is the package name)
+ which contains the package itself as a compressed zip file; and
+ ``package.xml``, which is an xml description of the package. The
+ zipfile ``package.zip`` should expand to a single subdirectory
+ named ``package/``. The base filename ``package`` must match
+ the identifier given in the package's xml file.
+
+ For each collection, there should be a single file ``collection.zip``
+ describing the collection, where *collection* is the name of the collection.
+
+ All identifiers (for both packages and collections) must be unique.
+ """
+ # Find all packages.
+ packages = []
+ for pkg_xml, zf, subdir in _find_packages(os.path.join(root, 'packages')):
+ zipstat = os.stat(zf.filename)
+ url = '%s/%s/%s' % (base_url, subdir, os.path.split(zf.filename)[1])
+ unzipped_size = sum(zf_info.file_size for zf_info in zf.infolist())
+
+ # Fill in several fields of the package xml with calculated values.
+ pkg_xml.set('unzipped_size', '%s' % unzipped_size)
+ pkg_xml.set('size', '%s' % zipstat.st_size)
+ pkg_xml.set('checksum', '%s' % md5_hexdigest(zf.filename))
+ pkg_xml.set('subdir', subdir)
+ #pkg_xml.set('svn_revision', _svn_revision(zf.filename))
+ pkg_xml.set('url', url)
+
+ # Record the package.
+ packages.append(pkg_xml)
+
+ # Find all collections
+ collections = list(_find_collections(os.path.join(root, 'collections')))
+
+ # Check that all UIDs are unique
+ uids = set()
+ for item in packages+collections:
+ if item.get('id') in uids:
+ raise ValueError('Duplicate UID: %s' % item.get('id'))
+ uids.add(item.get('id'))
+
+ # Put it all together
+ top_elt = ElementTree.Element('nltk_data')
+ top_elt.append(ElementTree.Element('packages'))
+ for package in packages: top_elt[0].append(package)
+ top_elt.append(ElementTree.Element('collections'))
+ for collection in collections: top_elt[1].append(collection)
+
+ _indent_xml(top_elt)
+ return top_elt
+
+def _indent_xml(xml, prefix=''):
+ """
+ Helper for ``build_index()``: Given an XML ``ElementTree``, modify it
+ (and its descendents) ``text`` and ``tail`` attributes to generate
+ an indented tree, where each nested element is indented by 2
+ spaces with respect to its parent.
+ """
+ if len(xml) > 0:
+ xml.text = (xml.text or '').strip() + '\n' + prefix + ' '
+ for child in xml:
+ _indent_xml(child, prefix+' ')
+ for child in xml[:-1]:
+ child.tail = (child.tail or '').strip() + '\n' + prefix + ' '
+ xml[-1].tail = (xml[-1].tail or '').strip() + '\n' + prefix
+
+def _check_package(pkg_xml, zipfilename, zf):
+ """
+ Helper for ``build_index()``: Perform some checks to make sure that
+ the given package is consistent.
+ """
+ # The filename must patch the id given in the XML file.
+ uid = os.path.splitext(os.path.split(zipfilename)[1])[0]
+ if pkg_xml.get('id') != uid:
+ raise ValueError('package identifier mismatch (%s vs %s)' %
+ (pkg_xml.get('id'), uid))
+
+ # Zip file must expand to a subdir whose name matches uid.
+ if sum( (name!=uid and not name.startswith(uid+'/'))
+ for name in zf.namelist() ):
+ raise ValueError('Zipfile %s.zip does not expand to a single '
+ 'subdirectory %s/' % (uid, uid))
+
+# update for git?
+def _svn_revision(filename):
+ """
+ Helper for ``build_index()``: Calculate the subversion revision
+ number for a given file (by using ``subprocess`` to run ``svn``).
+ """
+ p = subprocess.Popen(['svn', 'status', '-v', filename],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ (stdout, stderr) = p.communicate()
+ if p.returncode != 0 or stderr or not stdout:
+ raise ValueError('Error determining svn_revision for %s: %s' %
+ (os.path.split(filename)[1], textwrap.fill(stderr)))
+ return stdout.split()[2]
+
+def _find_collections(root):
+ """
+ Helper for ``build_index()``: Yield a list of ElementTree.Element
+ objects, each holding the xml for a single package collection.
+ """
+ packages = []
+ for dirname, subdirs, files in os.walk(root):
+ for filename in files:
+ if filename.endswith('.xml'):
+ xmlfile = os.path.join(dirname, filename)
+ yield ElementTree.parse(xmlfile).getroot()
+
+def _find_packages(root):
+ """
+ Helper for ``build_index()``: Yield a list of tuples
+ ``(pkg_xml, zf, subdir)``, where:
+ - ``pkg_xml`` is an ``ElementTree.Element`` holding the xml for a
+ package
+ - ``zf`` is a ``zipfile.ZipFile`` for the package's contents.
+ - ``subdir`` is the subdirectory (relative to ``root``) where
+ the package was found (e.g. 'corpora' or 'grammars').
+ """
+ from nltk.corpus.reader.util import _path_from
+ # Find all packages.
+ packages = []
+ for dirname, subdirs, files in os.walk(root):
+ relpath = '/'.join(_path_from(root, dirname))
+ for filename in files:
+ if filename.endswith('.xml'):
+ xmlfilename = os.path.join(dirname, filename)
+ zipfilename = xmlfilename[:-4]+'.zip'
+ try: zf = zipfile.ZipFile(zipfilename)
+ except Exception as e:
+ raise ValueError('Error reading file %r!\n%s' %
+ (zipfilename, e))
+ try: pkg_xml = ElementTree.parse(xmlfilename).getroot()
+ except Exception as e:
+ raise ValueError('Error reading file %r!\n%s' %
+ (xmlfilename, e))
+
+ # Check that the UID matches the filename
+ uid = os.path.split(xmlfilename[:-4])[1]
+ if pkg_xml.get('id') != uid:
+ raise ValueError('package identifier mismatch (%s '
+ 'vs %s)' % (pkg_xml.get('id'), uid))
+
+ # Check that the zipfile expands to a subdir whose
+ # name matches the uid.
+ if sum( (name!=uid and not name.startswith(uid+'/'))
+ for name in zf.namelist() ):
+ raise ValueError('Zipfile %s.zip does not expand to a '
+ 'single subdirectory %s/' % (uid, uid))
+
+ yield pkg_xml, zf, relpath
+ # Don't recurse into svn subdirectories:
+ try: subdirs.remove('.svn')
+ except ValueError: pass
+
+######################################################################
+# Main:
+######################################################################
+
+# There should be a command-line interface
+
+# Aliases
+_downloader = Downloader()
+download = _downloader.download
+
+def download_shell():
+ DownloaderShell(_downloader).run()
+
+def download_gui():
+ DownloaderGUI(_downloader).mainloop()
+
+def update():
+ _downloader.update()
+
+if __name__ == '__main__':
+ from optparse import OptionParser
+ parser = OptionParser()
+ parser.add_option("-d", "--dir", dest="dir",
+ help="download package to directory DIR", metavar="DIR")
+ parser.add_option("-q", "--quiet", dest="quiet", action="store_true",
+ default=False, help="work quietly")
+ parser.add_option("-f", "--force", dest="force", action="store_true",
+ default=False, help="download even if already installed")
+ parser.add_option("-e", "--exit-on-error", dest="halt_on_error", action="store_true",
+ default=False, help="exit if an error occurs")
+ parser.add_option("-u", "--url", dest="server_index_url",
+ default=None, help="download server index url")
+
+ (options, args) = parser.parse_args()
+
+ downloader = Downloader(server_index_url = options.server_index_url)
+
+ if args:
+ for pkg_id in args:
+ rv = downloader.download(info_or_id=pkg_id, download_dir=options.dir,
+ quiet=options.quiet, force=options.force,
+ halt_on_error=options.halt_on_error)
+ if rv==False and options.halt_on_error:
+ break
+ else:
+ downloader.download(download_dir=options.dir,
+ quiet=options.quiet, force=options.force,
+ halt_on_error=options.halt_on_error)
+
diff --git a/nltk/draw/__init__.py b/nltk/draw/__init__.py
new file mode 100644
index 0000000..ad6dec0
--- /dev/null
+++ b/nltk/draw/__init__.py
@@ -0,0 +1,27 @@
+# Natural Language Toolkit: graphical representations package
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Import Tkinter-based modules if Tkinter is installed
+import nltk.compat
+try:
+ import tkinter
+except ImportError:
+ import warnings
+ warnings.warn("nltk.draw package not loaded "
+ "(please install Tkinter library).")
+else:
+ from nltk.draw.cfg import ProductionList, CFGEditor, CFGDemo
+ from nltk.draw.tree import (TreeSegmentWidget, tree_to_treesegment,
+ TreeWidget, TreeView, draw_trees)
+ from nltk.draw.dispersion import dispersion_plot
+ from nltk.draw.table import Table
+
+# skip doctests from this package
+def setup_module(module):
+ from nose import SkipTest
+ raise SkipTest("nltk.draw examples are not doctests")
diff --git a/nltk/draw/cfg.py b/nltk/draw/cfg.py
new file mode 100644
index 0000000..1458e63
--- /dev/null
+++ b/nltk/draw/cfg.py
@@ -0,0 +1,777 @@
+# Natural Language Toolkit: CFG visualization
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Visualization tools for CFGs.
+"""
+
+# Idea for a nice demo:
+# - 3 panes: grammar, treelet, working area
+# - grammar is a list of productions
+# - when you select a production, the treelet that it licenses appears
+# in the treelet area
+# - the working area has the text on the bottom, and S at top. When
+# you select a production, it shows (ghosted) the locations where
+# that production's treelet could be attached to either the text
+# or the tree rooted at S.
+# - the user can drag the treelet onto one of those (or click on them?)
+# - the user can delete pieces of the tree from the working area
+# (right click?)
+# - connecting top to bottom? drag one NP onto another?
+#
+# +-------------------------------------------------------------+
+# | S -> NP VP | S |
+# |[NP -> Det N ]| / \ |
+# | ... | NP VP |
+# | N -> 'dog' | |
+# | N -> 'cat' | |
+# | ... | |
+# +--------------+ |
+# | NP | Det N |
+# | / \ | | | |
+# | Det N | the cat saw the dog |
+# | | |
+# +--------------+----------------------------------------------+
+#
+# Operations:
+# - connect a new treelet -- drag or click shadow
+# - delete a treelet -- right click
+# - if only connected to top, delete everything below
+# - if only connected to bottom, delete everything above
+# - connect top & bottom -- drag a leaf to a root or a root to a leaf
+# - disconnect top & bottom -- right click
+# - if connected to top & bottom, then disconnect
+
+
+
+import nltk.compat
+import re
+
+from tkinter import (Button, Canvas, Entry, Frame, IntVar, Label,
+ Scrollbar, Text, Tk, Toplevel)
+
+from nltk.grammar import (CFG, _read_cfg_production,
+ Nonterminal, nonterminals)
+from nltk.tree import Tree
+from nltk.draw.tree import TreeSegmentWidget, tree_to_treesegment
+from nltk.draw.util import (CanvasFrame, ColorizedList, ShowText,
+ SymbolWidget, TextWidget)
+from nltk import compat
+
+######################################################################
+# Production List
+######################################################################
+
+class ProductionList(ColorizedList):
+ ARROW = SymbolWidget.SYMBOLS['rightarrow']
+
+ def _init_colortags(self, textwidget, options):
+ textwidget.tag_config('terminal', foreground='#006000')
+ textwidget.tag_config('arrow', font='symbol', underline='0')
+ textwidget.tag_config('nonterminal', foreground='blue',
+ font=('helvetica', -12, 'bold'))
+
+ def _item_repr(self, item):
+ contents = []
+ contents.append(('%s\t' % item.lhs(), 'nonterminal'))
+ contents.append((self.ARROW, 'arrow'))
+ for elt in item.rhs():
+ if isinstance(elt, Nonterminal):
+ contents.append((' %s' % elt.symbol(), 'nonterminal'))
+ else:
+ contents.append((' %r' % elt, 'terminal'))
+ return contents
+
+######################################################################
+# CFG Editor
+######################################################################
+
+_CFGEditor_HELP = """
+
+The CFG Editor can be used to create or modify context free grammars.
+A context free grammar consists of a start symbol and a list of
+productions. The start symbol is specified by the text entry field in
+the upper right hand corner of the editor; and the list of productions
+are specified in the main text editing box.
+
+Every non-blank line specifies a single production. Each production
+has the form "LHS -> RHS," where LHS is a single nonterminal, and RHS
+is a list of nonterminals and terminals.
+
+Nonterminals must be a single word, such as S or NP or NP_subj.
+Currently, nonterminals must consists of alphanumeric characters and
+underscores (_). Nonterminals are colored blue. If you place the
+mouse over any nonterminal, then all occurrences of that nonterminal
+will be highlighted.
+
+Termianals must be surrounded by single quotes (') or double
+quotes(\"). For example, "dog" and "New York" are terminals.
+Currently, the string within the quotes must consist of alphanumeric
+characters, underscores, and spaces.
+
+To enter a new production, go to a blank line, and type a nonterminal,
+followed by an arrow (->), followed by a sequence of terminals and
+nonterminals. Note that "->" (dash + greater-than) is automatically
+converted to an arrow symbol. When you move your cursor to a
+different line, your production will automatically be colorized. If
+there are any errors, they will be highlighted in red.
+
+Note that the order of the productions is significant for some
+algorithms. To re-order the productions, use cut and paste to move
+them.
+
+Use the buttons at the bottom of the window when you are done editing
+the CFG:
+ - Ok: apply the new CFG, and exit the editor.
+ - Apply: apply the new CFG, and do not exit the editor.
+ - Reset: revert to the original CFG, and do not exit the editor.
+ - Cancel: revert to the original CFG, and exit the editor.
+
+"""
+
+class CFGEditor(object):
+ """
+ A dialog window for creating and editing context free grammars.
+ ``CFGEditor`` imposes the following restrictions:
+
+ - All nonterminals must be strings consisting of word
+ characters.
+ - All terminals must be strings consisting of word characters
+ and space characters.
+ """
+ # Regular expressions used by _analyze_line. Precompile them, so
+ # we can process the text faster.
+ ARROW = SymbolWidget.SYMBOLS['rightarrow']
+ _LHS_RE = re.compile(r"(^\s*\w+\s*)(->|("+ARROW+"))")
+ _ARROW_RE = re.compile("\s*(->|("+ARROW+"))\s*")
+ _PRODUCTION_RE = re.compile(r"(^\s*\w+\s*)" + # LHS
+ "(->|("+ARROW+"))\s*" + # arrow
+ r"((\w+|'[\w ]*'|\"[\w ]*\"|\|)\s*)*$") # RHS
+ _TOKEN_RE = re.compile("\\w+|->|'[\\w ]+'|\"[\\w ]+\"|("+ARROW+")")
+ _BOLD = ('helvetica', -12, 'bold')
+
+ def __init__(self, parent, cfg=None, set_cfg_callback=None):
+ self._parent = parent
+ if cfg is not None: self._cfg = cfg
+ else: self._cfg = CFG(Nonterminal('S'), [])
+ self._set_cfg_callback = set_cfg_callback
+
+ self._highlight_matching_nonterminals = 1
+
+ # Create the top-level window.
+ self._top = Toplevel(parent)
+ self._init_bindings()
+
+ self._init_startframe()
+ self._startframe.pack(side='top', fill='x', expand=0)
+ self._init_prodframe()
+ self._prodframe.pack(side='top', fill='both', expand=1)
+ self._init_buttons()
+ self._buttonframe.pack(side='bottom', fill='x', expand=0)
+
+ self._textwidget.focus()
+
+ def _init_startframe(self):
+ frame = self._startframe = Frame(self._top)
+ self._start = Entry(frame)
+ self._start.pack(side='right')
+ Label(frame, text='Start Symbol:').pack(side='right')
+ Label(frame, text='Productions:').pack(side='left')
+ self._start.insert(0, self._cfg.start().symbol())
+
+ def _init_buttons(self):
+ frame = self._buttonframe = Frame(self._top)
+ Button(frame, text='Ok', command=self._ok,
+ underline=0, takefocus=0).pack(side='left')
+ Button(frame, text='Apply', command=self._apply,
+ underline=0, takefocus=0).pack(side='left')
+ Button(frame, text='Reset', command=self._reset,
+ underline=0, takefocus=0,).pack(side='left')
+ Button(frame, text='Cancel', command=self._cancel,
+ underline=0, takefocus=0).pack(side='left')
+ Button(frame, text='Help', command=self._help,
+ underline=0, takefocus=0).pack(side='right')
+
+ def _init_bindings(self):
+ self._top.title('CFG Editor')
+ self._top.bind('<Control-q>', self._cancel)
+ self._top.bind('<Alt-q>', self._cancel)
+ self._top.bind('<Control-d>', self._cancel)
+ #self._top.bind('<Control-x>', self._cancel)
+ self._top.bind('<Alt-x>', self._cancel)
+ self._top.bind('<Escape>', self._cancel)
+ #self._top.bind('<Control-c>', self._cancel)
+ self._top.bind('<Alt-c>', self._cancel)
+
+ self._top.bind('<Control-o>', self._ok)
+ self._top.bind('<Alt-o>', self._ok)
+ self._top.bind('<Control-a>', self._apply)
+ self._top.bind('<Alt-a>', self._apply)
+ self._top.bind('<Control-r>', self._reset)
+ self._top.bind('<Alt-r>', self._reset)
+ self._top.bind('<Control-h>', self._help)
+ self._top.bind('<Alt-h>', self._help)
+ self._top.bind('<F1>', self._help)
+
+ def _init_prodframe(self):
+ self._prodframe = Frame(self._top)
+
+ # Create the basic Text widget & scrollbar.
+ self._textwidget = Text(self._prodframe, background='#e0e0e0',
+ exportselection=1)
+ self._textscroll = Scrollbar(self._prodframe, takefocus=0,
+ orient='vertical')
+ self._textwidget.config(yscrollcommand = self._textscroll.set)
+ self._textscroll.config(command=self._textwidget.yview)
+ self._textscroll.pack(side='right', fill='y')
+ self._textwidget.pack(expand=1, fill='both', side='left')
+
+ # Initialize the colorization tags. Each nonterminal gets its
+ # own tag, so they aren't listed here.
+ self._textwidget.tag_config('terminal', foreground='#006000')
+ self._textwidget.tag_config('arrow', font='symbol')
+ self._textwidget.tag_config('error', background='red')
+
+ # Keep track of what line they're on. We use that to remember
+ # to re-analyze a line whenever they leave it.
+ self._linenum = 0
+
+ # Expand "->" to an arrow.
+ self._top.bind('>', self._replace_arrows)
+
+ # Re-colorize lines when appropriate.
+ self._top.bind('<<Paste>>', self._analyze)
+ self._top.bind('<KeyPress>', self._check_analyze)
+ self._top.bind('<ButtonPress>', self._check_analyze)
+
+ # Tab cycles focus. (why doesn't this work??)
+ def cycle(e, textwidget=self._textwidget):
+ textwidget.tk_focusNext().focus()
+ self._textwidget.bind('<Tab>', cycle)
+
+ prod_tuples = [(p.lhs(),[p.rhs()]) for p in self._cfg.productions()]
+ for i in range(len(prod_tuples)-1,0,-1):
+ if (prod_tuples[i][0] == prod_tuples[i-1][0]):
+ if () in prod_tuples[i][1]: continue
+ if () in prod_tuples[i-1][1]: continue
+ print(prod_tuples[i-1][1])
+ print(prod_tuples[i][1])
+ prod_tuples[i-1][1].extend(prod_tuples[i][1])
+ del prod_tuples[i]
+
+ for lhs, rhss in prod_tuples:
+ print(lhs, rhss)
+ s = '%s ->' % lhs
+ for rhs in rhss:
+ for elt in rhs:
+ if isinstance(elt, Nonterminal): s += ' %s' % elt
+ else: s += ' %r' % elt
+ s += ' |'
+ s = s[:-2] + '\n'
+ self._textwidget.insert('end', s)
+
+ self._analyze()
+
+# # Add the producitons to the text widget, and colorize them.
+# prod_by_lhs = {}
+# for prod in self._cfg.productions():
+# if len(prod.rhs()) > 0:
+# prod_by_lhs.setdefault(prod.lhs(),[]).append(prod)
+# for (lhs, prods) in prod_by_lhs.items():
+# self._textwidget.insert('end', '%s ->' % lhs)
+# self._textwidget.insert('end', self._rhs(prods[0]))
+# for prod in prods[1:]:
+# print '\t|'+self._rhs(prod),
+# self._textwidget.insert('end', '\t|'+self._rhs(prod))
+# print
+# self._textwidget.insert('end', '\n')
+# for prod in self._cfg.productions():
+# if len(prod.rhs()) == 0:
+# self._textwidget.insert('end', '%s' % prod)
+# self._analyze()
+
+# def _rhs(self, prod):
+# s = ''
+# for elt in prod.rhs():
+# if isinstance(elt, Nonterminal): s += ' %s' % elt.symbol()
+# else: s += ' %r' % elt
+# return s
+
+ def _clear_tags(self, linenum):
+ """
+ Remove all tags (except ``arrow`` and ``sel``) from the given
+ line of the text widget used for editing the productions.
+ """
+ start = '%d.0'%linenum
+ end = '%d.end'%linenum
+ for tag in self._textwidget.tag_names():
+ if tag not in ('arrow', 'sel'):
+ self._textwidget.tag_remove(tag, start, end)
+
+ def _check_analyze(self, *e):
+ """
+ Check if we've moved to a new line. If we have, then remove
+ all colorization from the line we moved to, and re-colorize
+ the line that we moved from.
+ """
+ linenum = int(self._textwidget.index('insert').split('.')[0])
+ if linenum != self._linenum:
+ self._clear_tags(linenum)
+ self._analyze_line(self._linenum)
+ self._linenum = linenum
+
+ def _replace_arrows(self, *e):
+ """
+ Replace any ``'->'`` text strings with arrows (char \\256, in
+ symbol font). This searches the whole buffer, but is fast
+ enough to be done anytime they press '>'.
+ """
+ arrow = '1.0'
+ while True:
+ arrow = self._textwidget.search('->', arrow, 'end+1char')
+ if arrow == '': break
+ self._textwidget.delete(arrow, arrow+'+2char')
+ self._textwidget.insert(arrow, self.ARROW, 'arrow')
+ self._textwidget.insert(arrow, '\t')
+
+ arrow = '1.0'
+ while True:
+ arrow = self._textwidget.search(self.ARROW, arrow+'+1char',
+ 'end+1char')
+ if arrow == '': break
+ self._textwidget.tag_add('arrow', arrow, arrow+'+1char')
+
+ def _analyze_token(self, match, linenum):
+ """
+ Given a line number and a regexp match for a token on that
+ line, colorize the token. Note that the regexp match gives us
+ the token's text, start index (on the line), and end index (on
+ the line).
+ """
+ # What type of token is it?
+ if match.group()[0] in "'\"": tag = 'terminal'
+ elif match.group() in ('->', self.ARROW): tag = 'arrow'
+ else:
+ # If it's a nonterminal, then set up new bindings, so we
+ # can highlight all instances of that nonterminal when we
+ # put the mouse over it.
+ tag = 'nonterminal_'+match.group()
+ if tag not in self._textwidget.tag_names():
+ self._init_nonterminal_tag(tag)
+
+ start = '%d.%d' % (linenum, match.start())
+ end = '%d.%d' % (linenum, match.end())
+ self._textwidget.tag_add(tag, start, end)
+
+ def _init_nonterminal_tag(self, tag, foreground='blue'):
+ self._textwidget.tag_config(tag, foreground=foreground,
+ font=CFGEditor._BOLD)
+ if not self._highlight_matching_nonterminals:
+ return
+ def enter(e, textwidget=self._textwidget, tag=tag):
+ textwidget.tag_config(tag, background='#80ff80')
+ def leave(e, textwidget=self._textwidget, tag=tag):
+ textwidget.tag_config(tag, background='')
+ self._textwidget.tag_bind(tag, '<Enter>', enter)
+ self._textwidget.tag_bind(tag, '<Leave>', leave)
+
+ def _analyze_line(self, linenum):
+ """
+ Colorize a given line.
+ """
+ # Get rid of any tags that were previously on the line.
+ self._clear_tags(linenum)
+
+ # Get the line line's text string.
+ line = self._textwidget.get(repr(linenum)+'.0', repr(linenum)+'.end')
+
+ # If it's a valid production, then colorize each token.
+ if CFGEditor._PRODUCTION_RE.match(line):
+ # It's valid; Use _TOKEN_RE to tokenize the production,
+ # and call analyze_token on each token.
+ def analyze_token(match, self=self, linenum=linenum):
+ self._analyze_token(match, linenum)
+ return ''
+ CFGEditor._TOKEN_RE.sub(analyze_token, line)
+ elif line.strip() != '':
+ # It's invalid; show the user where the error is.
+ self._mark_error(linenum, line)
+
+ def _mark_error(self, linenum, line):
+ """
+ Mark the location of an error in a line.
+ """
+ arrowmatch = CFGEditor._ARROW_RE.search(line)
+ if not arrowmatch:
+ # If there's no arrow at all, highlight the whole line.
+ start = '%d.0' % linenum
+ end = '%d.end' % linenum
+ elif not CFGEditor._LHS_RE.match(line):
+ # Otherwise, if the LHS is bad, highlight it.
+ start = '%d.0' % linenum
+ end = '%d.%d' % (linenum, arrowmatch.start())
+ else:
+ # Otherwise, highlight the RHS.
+ start = '%d.%d' % (linenum, arrowmatch.end())
+ end = '%d.end' % linenum
+
+ # If we're highlighting 0 chars, highlight the whole line.
+ if self._textwidget.compare(start, '==', end):
+ start = '%d.0' % linenum
+ end = '%d.end' % linenum
+ self._textwidget.tag_add('error', start, end)
+
+ def _analyze(self, *e):
+ """
+ Replace ``->`` with arrows, and colorize the entire buffer.
+ """
+ self._replace_arrows()
+ numlines = int(self._textwidget.index('end').split('.')[0])
+ for linenum in range(1, numlines+1): # line numbers start at 1.
+ self._analyze_line(linenum)
+
+ def _parse_productions(self):
+ """
+ Parse the current contents of the textwidget buffer, to create
+ a list of productions.
+ """
+ productions = []
+
+ # Get the text, normalize it, and split it into lines.
+ text = self._textwidget.get('1.0', 'end')
+ text = re.sub(self.ARROW, '->', text)
+ text = re.sub('\t', ' ', text)
+ lines = text.split('\n')
+
+ # Convert each line to a CFG production
+ for line in lines:
+ line = line.strip()
+ if line=='': continue
+ productions += _read_cfg_production(line)
+ #if line.strip() == '': continue
+ #if not CFGEditor._PRODUCTION_RE.match(line):
+ # raise ValueError('Bad production string %r' % line)
+ #
+ #(lhs_str, rhs_str) = line.split('->')
+ #lhs = Nonterminal(lhs_str.strip())
+ #rhs = []
+ #def parse_token(match, rhs=rhs):
+ # token = match.group()
+ # if token[0] in "'\"": rhs.append(token[1:-1])
+ # else: rhs.append(Nonterminal(token))
+ # return ''
+ #CFGEditor._TOKEN_RE.sub(parse_token, rhs_str)
+ #
+ #productions.append(Production(lhs, *rhs))
+
+ return productions
+
+ def _destroy(self, *e):
+ if self._top is None: return
+ self._top.destroy()
+ self._top = None
+
+ def _ok(self, *e):
+ self._apply()
+ self._destroy()
+
+ def _apply(self, *e):
+ productions = self._parse_productions()
+ start = Nonterminal(self._start.get())
+ cfg = CFG(start, productions)
+ if self._set_cfg_callback is not None:
+ self._set_cfg_callback(cfg)
+
+ def _reset(self, *e):
+ self._textwidget.delete('1.0', 'end')
+ for production in self._cfg.productions():
+ self._textwidget.insert('end', '%s\n' % production)
+ self._analyze()
+ if self._set_cfg_callback is not None:
+ self._set_cfg_callback(self._cfg)
+
+ def _cancel(self, *e):
+ try: self._reset()
+ except: pass
+ self._destroy()
+
+ def _help(self, *e):
+ # The default font's not very legible; try using 'fixed' instead.
+ try:
+ ShowText(self._parent, 'Help: Chart Parser Demo',
+ (_CFGEditor_HELP).strip(), width=75, font='fixed')
+ except:
+ ShowText(self._parent, 'Help: Chart Parser Demo',
+ (_CFGEditor_HELP).strip(), width=75)
+
+######################################################################
+# New Demo (built tree based on cfg)
+######################################################################
+
+class CFGDemo(object):
+ def __init__(self, grammar, text):
+ self._grammar = grammar
+ self._text = text
+
+ # Set up the main window.
+ self._top = Tk()
+ self._top.title('Context Free Grammar Demo')
+
+ # Base font size
+ self._size = IntVar(self._top)
+ self._size.set(12) # = medium
+
+ # Set up the key bindings
+ self._init_bindings(self._top)
+
+ # Create the basic frames
+ frame1 = Frame(self._top)
+ frame1.pack(side='left', fill='y', expand=0)
+ self._init_menubar(self._top)
+ self._init_buttons(self._top)
+ self._init_grammar(frame1)
+ self._init_treelet(frame1)
+ self._init_workspace(self._top)
+
+ #//////////////////////////////////////////////////
+ # Initialization
+ #//////////////////////////////////////////////////
+
+ def _init_bindings(self, top):
+ top.bind('<Control-q>', self.destroy)
+
+ def _init_menubar(self, parent): pass
+
+ def _init_buttons(self, parent): pass
+
+ def _init_grammar(self, parent):
+ self._prodlist = ProductionList(parent, self._grammar, width=20)
+ self._prodlist.pack(side='top', fill='both', expand=1)
+ self._prodlist.focus()
+ self._prodlist.add_callback('select', self._selectprod_cb)
+ self._prodlist.add_callback('move', self._selectprod_cb)
+
+ def _init_treelet(self, parent):
+ self._treelet_canvas = Canvas(parent, background='white')
+ self._treelet_canvas.pack(side='bottom', fill='x')
+ self._treelet = None
+
+ def _init_workspace(self, parent):
+ self._workspace = CanvasFrame(parent, background='white')
+ self._workspace.pack(side='right', fill='both', expand=1)
+ self._tree = None
+ self.reset_workspace()
+
+ #//////////////////////////////////////////////////
+ # Workspace
+ #//////////////////////////////////////////////////
+
+ def reset_workspace(self):
+ c = self._workspace.canvas()
+ fontsize = int(self._size.get())
+ node_font = ('helvetica', -(fontsize+4), 'bold')
+ leaf_font = ('helvetica', -(fontsize+2))
+
+ # Remove the old tree
+ if self._tree is not None:
+ self._workspace.remove_widget(self._tree)
+
+ # The root of the tree.
+ start = self._grammar.start().symbol()
+ rootnode = TextWidget(c, start, font=node_font, draggable=1)
+
+ # The leaves of the tree.
+ leaves = []
+ for word in self._text:
+ leaves.append(TextWidget(c, word, font=leaf_font, draggable=1))
+
+ # Put it all together into one tree
+ self._tree = TreeSegmentWidget(c, rootnode, leaves,
+ color='white')
+
+ # Add it to the workspace.
+ self._workspace.add_widget(self._tree)
+
+ # Move the leaves to the bottom of the workspace.
+ for leaf in leaves: leaf.move(0,100)
+
+ #self._nodes = {start:1}
+ #self._leaves = dict([(l,1) for l in leaves])
+
+ def workspace_markprod(self, production):
+ pass
+
+ def _markproduction(self, prod, tree=None):
+ if tree is None: tree = self._tree
+ for i in range(len(tree.subtrees())-len(prod.rhs())):
+ if tree['color', i] == 'white':
+ self._markproduction
+
+ for j, node in enumerate(prod.rhs()):
+ widget = tree.subtrees()[i+j]
+ if (isinstance(node, Nonterminal) and
+ isinstance(widget, TreeSegmentWidget) and
+ node.symbol == widget.label().text()):
+ pass # matching nonterminal
+ elif (isinstance(node, compat.string_types) and
+ isinstance(widget, TextWidget) and
+ node == widget.text()):
+ pass # matching nonterminal
+ else: break
+ else:
+ # Everything matched!
+ print('MATCH AT', i)
+
+ #//////////////////////////////////////////////////
+ # Grammar
+ #//////////////////////////////////////////////////
+
+ def _selectprod_cb(self, production):
+ canvas = self._treelet_canvas
+
+ self._prodlist.highlight(production)
+ if self._treelet is not None: self._treelet.destroy()
+
+ # Convert the production to a tree.
+ rhs = production.rhs()
+ for (i, elt) in enumerate(rhs):
+ if isinstance(elt, Nonterminal): elt = Tree(elt)
+ tree = Tree(production.lhs().symbol(), *rhs)
+
+ # Draw the tree in the treelet area.
+ fontsize = int(self._size.get())
+ node_font = ('helvetica', -(fontsize+4), 'bold')
+ leaf_font = ('helvetica', -(fontsize+2))
+ self._treelet = tree_to_treesegment(canvas, tree,
+ node_font=node_font,
+ leaf_font=leaf_font)
+ self._treelet['draggable'] = 1
+
+ # Center the treelet.
+ (x1, y1, x2, y2) = self._treelet.bbox()
+ w, h = int(canvas['width']), int(canvas['height'])
+ self._treelet.move((w-x1-x2)/2, (h-y1-y2)/2)
+
+ # Mark the places where we can add it to the workspace.
+ self._markproduction(production)
+
+ def destroy(self, *args):
+ self._top.destroy()
+
+ def mainloop(self, *args, **kwargs):
+ self._top.mainloop(*args, **kwargs)
+
+def demo2():
+ from nltk import Nonterminal, Production, CFG
+ nonterminals = 'S VP NP PP P N Name V Det'
+ (S, VP, NP, PP, P, N, Name, V, Det) = [Nonterminal(s)
+ for s in nonterminals.split()]
+ productions = (
+ # Syntactic Productions
+ Production(S, [NP, VP]),
+ Production(NP, [Det, N]),
+ Production(NP, [NP, PP]),
+ Production(VP, [VP, PP]),
+ Production(VP, [V, NP, PP]),
+ Production(VP, [V, NP]),
+ Production(PP, [P, NP]),
+ Production(PP, []),
+
+ Production(PP, ['up', 'over', NP]),
+
+ # Lexical Productions
+ Production(NP, ['I']), Production(Det, ['the']),
+ Production(Det, ['a']), Production(N, ['man']),
+ Production(V, ['saw']), Production(P, ['in']),
+ Production(P, ['with']), Production(N, ['park']),
+ Production(N, ['dog']), Production(N, ['statue']),
+ Production(Det, ['my']),
+ )
+ grammar = CFG(S, productions)
+
+ text = 'I saw a man in the park'.split()
+ d=CFGDemo(grammar, text)
+ d.mainloop()
+
+######################################################################
+# Old Demo
+######################################################################
+
+def demo():
+ from nltk import Nonterminal, CFG
+ nonterminals = 'S VP NP PP P N Name V Det'
+ (S, VP, NP, PP, P, N, Name, V, Det) = [Nonterminal(s)
+ for s in nonterminals.split()]
+
+ grammar = CFG.fromstring("""
+ S -> NP VP
+ PP -> P NP
+ NP -> Det N
+ NP -> NP PP
+ VP -> V NP
+ VP -> VP PP
+ Det -> 'a'
+ Det -> 'the'
+ Det -> 'my'
+ NP -> 'I'
+ N -> 'dog'
+ N -> 'man'
+ N -> 'park'
+ N -> 'statue'
+ V -> 'saw'
+ P -> 'in'
+ P -> 'up'
+ P -> 'over'
+ P -> 'with'
+ """)
+
+ def cb(grammar): print(grammar)
+ top = Tk()
+ editor = CFGEditor(top, grammar, cb)
+ Label(top, text='\nTesting CFG Editor\n').pack()
+ Button(top, text='Quit', command=top.destroy).pack()
+ top.mainloop()
+
+def demo3():
+ from nltk import Production
+ (S, VP, NP, PP, P, N, Name, V, Det) = \
+ nonterminals('S, VP, NP, PP, P, N, Name, V, Det')
+
+ productions = (
+ # Syntactic Productions
+ Production(S, [NP, VP]),
+ Production(NP, [Det, N]),
+ Production(NP, [NP, PP]),
+ Production(VP, [VP, PP]),
+ Production(VP, [V, NP, PP]),
+ Production(VP, [V, NP]),
+ Production(PP, [P, NP]),
+ Production(PP, []),
+
+ Production(PP, ['up', 'over', NP]),
+
+ # Lexical Productions
+ Production(NP, ['I']), Production(Det, ['the']),
+ Production(Det, ['a']), Production(N, ['man']),
+ Production(V, ['saw']), Production(P, ['in']),
+ Production(P, ['with']), Production(N, ['park']),
+ Production(N, ['dog']), Production(N, ['statue']),
+ Production(Det, ['my']),
+ )
+
+ t = Tk()
+ def destroy(e, t=t): t.destroy()
+ t.bind('q', destroy)
+ p = ProductionList(t, productions)
+ p.pack(expand=1, fill='both')
+ p.add_callback('select', p.markonly)
+ p.add_callback('move', p.markonly)
+ p.focus()
+ p.mark(productions[2])
+ p.mark(productions[8])
+
+if __name__ == '__main__': demo()
diff --git a/nltk/draw/dispersion.py b/nltk/draw/dispersion.py
new file mode 100644
index 0000000..170f556
--- /dev/null
+++ b/nltk/draw/dispersion.py
@@ -0,0 +1,58 @@
+# Natural Language Toolkit: Dispersion Plots
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A utility for displaying lexical dispersion.
+"""
+
+def dispersion_plot(text, words, ignore_case=False):
+ """
+ Generate a lexical dispersion plot.
+
+ :param text: The source text
+ :type text: list(str) or enum(str)
+ :param words: The target words
+ :type words: list of str
+ :param ignore_case: flag to set if case should be ignored when searching text
+ :type ignore_case: bool
+ """
+
+ try:
+ import pylab
+ except ImportError:
+ raise ValueError('The plot function requires the matplotlib package (aka pylab).'
+ 'See http://matplotlib.sourceforge.net/')
+
+ text = list(text)
+ words.reverse()
+
+ if ignore_case:
+ words_to_comp = list(map(str.lower, words))
+ text_to_comp = list(map(str.lower, text))
+ else:
+ words_to_comp = words
+ text_to_comp = text
+
+ points = [(x,y) for x in range(len(text_to_comp))
+ for y in range(len(words_to_comp))
+ if text_to_comp[x] == words_to_comp[y]]
+ if points:
+ x, y = list(zip(*points))
+ else:
+ x = y = ()
+ pylab.plot(x, y, "b|", scalex=.1)
+ pylab.yticks(list(range(len(words))), words, color="b")
+ pylab.ylim(-1, len(words))
+ pylab.title("Lexical Dispersion Plot")
+ pylab.xlabel("Word Offset")
+ pylab.show()
+
+if __name__ == '__main__':
+ import nltk.compat
+ from nltk.corpus import gutenberg
+ words = ['Elinor', 'Marianne', 'Edward', 'Willoughby']
+ dispersion_plot(gutenberg.words('austen-sense.txt'), words)
diff --git a/nltk/draw/table.py b/nltk/draw/table.py
new file mode 100644
index 0000000..a36c208
--- /dev/null
+++ b/nltk/draw/table.py
@@ -0,0 +1,1090 @@
+# Natural Language Toolkit: Table widget
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Tkinter widgets for displaying multi-column listboxes and tables.
+"""
+
+import nltk.compat
+import operator
+
+from tkinter import (Frame, Label, Listbox, Scrollbar, Tk)
+
+
+######################################################################
+# Multi-Column Listbox
+######################################################################
+
+class MultiListbox(Frame):
+ """
+ A multi-column listbox, where the current selection applies to an
+ entire row. Based on the MultiListbox Tkinter widget
+ recipe from the Python Cookbook (http://code.activestate.com/recipes/52266/)
+
+ For the most part, ``MultiListbox`` methods delegate to its
+ contained listboxes. For any methods that do not have docstrings,
+ see ``Tkinter.Listbox`` for a description of what that method does.
+ """
+ #/////////////////////////////////////////////////////////////////
+ # Configuration
+ #/////////////////////////////////////////////////////////////////
+
+ #: Default configuration values for the frame.
+ FRAME_CONFIG = dict(background='#888',
+ takefocus=True,
+ highlightthickness=1)
+
+ #: Default configurations for the column labels.
+ LABEL_CONFIG = dict(borderwidth=1, relief='raised',
+ font='helvetica -16 bold',
+ background='#444', foreground='white')
+
+ #: Default configuration for the column listboxes.
+ LISTBOX_CONFIG = dict(borderwidth=1,
+ selectborderwidth=0,
+ highlightthickness=0,
+ exportselection=False,
+ selectbackground='#888',
+ activestyle='none',
+ takefocus=False)
+
+ #/////////////////////////////////////////////////////////////////
+ # Constructor
+ #/////////////////////////////////////////////////////////////////
+
+ def __init__(self, master, columns, column_weights=None, cnf={}, **kw):
+ """
+ Construct a new multi-column listbox widget.
+
+ :param master: The widget that should contain the new
+ multi-column listbox.
+
+ :param columns: Specifies what columns should be included in
+ the new multi-column listbox. If ``columns`` is an integer,
+ the it is the number of columns to include. If it is
+ a list, then its length indicates the number of columns
+ to include; and each element of the list will be used as
+ a label for the corresponding column.
+
+ :param cnf, kw: Configuration parameters for this widget.
+ Use ``label_*`` to configure all labels; and ``listbox_*``
+ to configure all listboxes. E.g.:
+
+ >>> mlb = MultiListbox(master, 5, label_foreground='red')
+ """
+ # If columns was specified as an int, convert it to a list.
+ if isinstance(columns, int):
+ columns = list(range(columns))
+ include_labels = False
+ else:
+ include_labels = True
+
+ if len(columns) == 0:
+ raise ValueError("Expected at least one column")
+
+ # Instance variables
+ self._column_names = tuple(columns)
+ self._listboxes = []
+ self._labels = []
+
+ # Pick a default value for column_weights, if none was specified.
+ if column_weights is None:
+ column_weights = [1] * len(columns)
+ elif len(column_weights) != len(columns):
+ raise ValueError('Expected one column_weight for each column')
+ self._column_weights = column_weights
+
+ # Configure our widgets.
+ Frame.__init__(self, master, **self.FRAME_CONFIG)
+ self.grid_rowconfigure(1, weight=1)
+ for i, label in enumerate(self._column_names):
+ self.grid_columnconfigure(i, weight=column_weights[i])
+
+ # Create a label for the column
+ if include_labels:
+ l = Label(self, text=label, **self.LABEL_CONFIG)
+ self._labels.append(l)
+ l.grid(column=i, row=0, sticky='news', padx=0, pady=0)
+ l.column_index = i
+
+ # Create a listbox for the column
+ lb = Listbox(self, **self.LISTBOX_CONFIG)
+ self._listboxes.append(lb)
+ lb.grid(column=i, row=1, sticky='news', padx=0, pady=0)
+ lb.column_index = i
+
+ # Clicking or dragging selects:
+ lb.bind('<Button-1>', self._select)
+ lb.bind('<B1-Motion>', self._select)
+ # Scroll whell scrolls:
+ lb.bind('<Button-4>', lambda e: self._scroll(-1))
+ lb.bind('<Button-5>', lambda e: self._scroll(+1))
+ lb.bind('<MouseWheel>', lambda e: self._scroll(e.delta))
+ # Button 2 can be used to scan:
+ lb.bind('<Button-2>', lambda e: self.scan_mark(e.x, e.y))
+ lb.bind('<B2-Motion>', lambda e: self.scan_dragto(e.x, e.y))
+ # Dragging outside the window has no effect (diable
+ # the default listbox behavior, which scrolls):
+ lb.bind('<B1-Leave>', lambda e: 'break')
+ # Columns can be resized by dragging them:
+ l.bind('<Button-1>', self._resize_column)
+
+ # Columns can be resized by dragging them. (This binding is
+ # used if they click on the grid between columns:)
+ self.bind('<Button-1>', self._resize_column)
+
+ # Set up key bindings for the widget:
+ self.bind('<Up>', lambda e: self.select(delta=-1))
+ self.bind('<Down>', lambda e: self.select(delta=1))
+ self.bind('<Prior>', lambda e: self.select(delta=-self._pagesize()))
+ self.bind('<Next>', lambda e: self.select(delta=self._pagesize()))
+
+ # Configuration customizations
+ self.configure(cnf, **kw)
+
+ #/////////////////////////////////////////////////////////////////
+ # Column Resizing
+ #/////////////////////////////////////////////////////////////////
+
+ def _resize_column(self, event):
+ """
+ Callback used to resize a column of the table. Return ``True``
+ if the column is actually getting resized (if the user clicked
+ on the far left or far right 5 pixels of a label); and
+ ``False`` otherwies.
+ """
+ # If we're already waiting for a button release, then ignore
+ # the new button press.
+ if event.widget.bind('<ButtonRelease>'):
+ return False
+
+ # Decide which column (if any) to resize.
+ self._resize_column_index = None
+ if event.widget is self:
+ for i, lb in enumerate(self._listboxes):
+ if abs(event.x-(lb.winfo_x()+lb.winfo_width())) < 10:
+ self._resize_column_index = i
+ elif event.x > (event.widget.winfo_width()-5):
+ self._resize_column_index = event.widget.column_index
+ elif event.x < 5 and event.widget.column_index != 0:
+ self._resize_column_index = event.widget.column_index-1
+
+ # Bind callbacks that are used to resize it.
+ if self._resize_column_index is not None:
+ event.widget.bind('<Motion>', self._resize_column_motion_cb)
+ event.widget.bind('<ButtonRelease-%d>' % event.num,
+ self._resize_column_buttonrelease_cb)
+ return True
+ else:
+ return False
+
+ def _resize_column_motion_cb(self, event):
+ lb = self._listboxes[self._resize_column_index]
+ charwidth = lb.winfo_width() / float(lb['width'])
+
+ x1 = event.x + event.widget.winfo_x()
+ x2 = lb.winfo_x() + lb.winfo_width()
+
+ lb['width'] = max(3, lb['width'] + int((x1-x2)/charwidth))
+
+ def _resize_column_buttonrelease_cb(self, event):
+ event.widget.unbind('<ButtonRelease-%d>' % event.num)
+ event.widget.unbind('<Motion>')
+
+ #/////////////////////////////////////////////////////////////////
+ # Properties
+ #/////////////////////////////////////////////////////////////////
+
+ @property
+ def column_names(self):
+ """
+ A tuple containing the names of the columns used by this
+ multi-column listbox.
+ """
+ return self._column_names
+
+ @property
+ def column_labels(self):
+ """
+ A tuple containing the ``Tkinter.Label`` widgets used to
+ display the label of each column. If this multi-column
+ listbox was created without labels, then this will be an empty
+ tuple. These widgets will all be augmented with a
+ ``column_index`` attribute, which can be used to determine
+ which column they correspond to. This can be convenient,
+ e.g., when defining callbacks for bound events.
+ """
+ return tuple(self._labels)
+
+ @property
+ def listboxes(self):
+ """
+ A tuple containing the ``Tkinter.Listbox`` widgets used to
+ display individual columns. These widgets will all be
+ augmented with a ``column_index`` attribute, which can be used
+ to determine which column they correspond to. This can be
+ convenient, e.g., when defining callbacks for bound events.
+ """
+ return tuple(self._listboxes)
+
+ #/////////////////////////////////////////////////////////////////
+ # Mouse & Keyboard Callback Functions
+ #/////////////////////////////////////////////////////////////////
+
+ def _select(self, e):
+ i = e.widget.nearest(e.y)
+ self.selection_clear(0, 'end')
+ self.selection_set(i)
+ self.activate(i)
+ self.focus()
+
+ def _scroll(self, delta):
+ for lb in self._listboxes:
+ lb.yview_scroll(delta, 'unit')
+ return 'break'
+
+ def _pagesize(self):
+ """:return: The number of rows that makes up one page"""
+ return int(self.index('@0,1000000')) - int(self.index('@0,0'))
+
+ #/////////////////////////////////////////////////////////////////
+ # Row selection
+ #/////////////////////////////////////////////////////////////////
+
+ def select(self, index=None, delta=None, see=True):
+ """
+ Set the selected row. If ``index`` is specified, then select
+ row ``index``. Otherwise, if ``delta`` is specified, then move
+ the current selection by ``delta`` (negative numbers for up,
+ positive numbers for down). This will not move the selection
+ past the top or the bottom of the list.
+
+ :param see: If true, then call ``self.see()`` with the newly
+ selected index, to ensure that it is visible.
+ """
+ if (index is not None) and (delta is not None):
+ raise ValueError('specify index or delta, but not both')
+
+ # If delta was given, then calculate index.
+ if delta is not None:
+ if len(self.curselection()) == 0:
+ index = -1 + delta
+ else:
+ index = int(self.curselection()[0]) + delta
+
+ # Clear all selected rows.
+ self.selection_clear(0, 'end')
+
+ # Select the specified index
+ if index is not None:
+ index = min(max(index, 0), self.size()-1)
+ #self.activate(index)
+ self.selection_set(index)
+ if see: self.see(index)
+
+ #/////////////////////////////////////////////////////////////////
+ # Configuration
+ #/////////////////////////////////////////////////////////////////
+
+ def configure(self, cnf={}, **kw):
+ """
+ Configure this widget. Use ``label_*`` to configure all
+ labels; and ``listbox_*`` to configure all listboxes. E.g.:
+
+ >>> mlb = MultiListbox(master, 5)
+ >>> mlb.configure(label_foreground='red')
+ >>> mlb.configure(listbox_foreground='red')
+ """
+ cnf = dict(list(cnf.items()) + list(kw.items()))
+ for (key, val) in list(cnf.items()):
+ if key.startswith('label_') or key.startswith('label-'):
+ for label in self._labels:
+ label.configure({key[6:]: val})
+ elif key.startswith('listbox_') or key.startswith('listbox-'):
+ for listbox in self._listboxes:
+ listbox.configure({key[8:]: val})
+ else:
+ Frame.configure(self, {key:val})
+
+ def __setitem__(self, key, val):
+ """
+ Configure this widget. This is equivalent to
+ ``self.configure({key,val``)}. See ``configure()``.
+ """
+ self.configure({key:val})
+
+ def rowconfigure(self, row_index, cnf={}, **kw):
+ """
+ Configure all table cells in the given row. Valid keyword
+ arguments are: ``background``, ``bg``, ``foreground``, ``fg``,
+ ``selectbackground``, ``selectforeground``.
+ """
+ for lb in self._listboxes: lb.itemconfigure(row_index, cnf, **kw)
+
+ def columnconfigure(self, col_index, cnf={}, **kw):
+ """
+ Configure all table cells in the given column. Valid keyword
+ arguments are: ``background``, ``bg``, ``foreground``, ``fg``,
+ ``selectbackground``, ``selectforeground``.
+ """
+ lb = self._listboxes[col_index]
+
+ cnf = dict(list(cnf.items()) + list(kw.items()))
+ for (key, val) in list(cnf.items()):
+ if key in ('background', 'bg', 'foreground', 'fg',
+ 'selectbackground', 'selectforeground'):
+ for i in range(lb.size()): lb.itemconfigure(i, {key:val})
+ else:
+ lb.configure({key:val})
+
+ def itemconfigure(self, row_index, col_index, cnf=None, **kw):
+ """
+ Configure the table cell at the given row and column. Valid
+ keyword arguments are: ``background``, ``bg``, ``foreground``,
+ ``fg``, ``selectbackground``, ``selectforeground``.
+ """
+ lb = self._listboxes[col_index]
+ return lb.itemconfigure(row_index, cnf, **kw)
+
+ #/////////////////////////////////////////////////////////////////
+ # Value Access
+ #/////////////////////////////////////////////////////////////////
+
+ def insert(self, index, *rows):
+ """
+ Insert the given row or rows into the table, at the given
+ index. Each row value should be a tuple of cell values, one
+ for each column in the row. Index may be an integer or any of
+ the special strings (such as ``'end'``) accepted by
+ ``Tkinter.Listbox``.
+ """
+ for elt in rows:
+ if len(elt) != len(self._column_names):
+ raise ValueError('rows should be tuples whose length '
+ 'is equal to the number of columns')
+ for (lb,elts) in zip(self._listboxes, list(zip(*rows))):
+ lb.insert(index, *elts)
+
+ def get(self, first, last=None):
+ """
+ Return the value(s) of the specified row(s). If ``last`` is
+ not specified, then return a single row value; otherwise,
+ return a list of row values. Each row value is a tuple of
+ cell values, one for each column in the row.
+ """
+ values = [lb.get(first, last) for lb in self._listboxes]
+ if last:
+ return [tuple(row) for row in zip(*values)]
+ else:
+ return tuple(values)
+
+ def bbox(self, row, col):
+ """
+ Return the bounding box for the given table cell, relative to
+ this widget's top-left corner. The bounding box is a tuple
+ of integers ``(left, top, width, height)``.
+ """
+ dx, dy, _, _ = self.grid_bbox(row=0, column=col)
+ x, y, w, h = self._listboxes[col].bbox(row)
+ return int(x)+int(dx), int(y)+int(dy), int(w), int(h)
+
+ #/////////////////////////////////////////////////////////////////
+ # Hide/Show Columns
+ #/////////////////////////////////////////////////////////////////
+
+ def hide_column(self, col_index):
+ """
+ Hide the given column. The column's state is still
+ maintained: its values will still be returned by ``get()``, and
+ you must supply its values when calling ``insert()``. It is
+ safe to call this on a column that is already hidden.
+
+ :see: ``show_column()``
+ """
+ if self._labels:
+ self._labels[col_index].grid_forget()
+ self.listboxes[col_index].grid_forget()
+ self.grid_columnconfigure(col_index, weight=0)
+
+ def show_column(self, col_index):
+ """
+ Display a column that has been hidden using ``hide_column()``.
+ It is safe to call this on a column that is not hidden.
+ """
+ weight = self._column_weights[col_index]
+ if self._labels:
+ self._labels[col_index].grid(column=col_index, row=0,
+ sticky='news', padx=0, pady=0)
+ self._listboxes[col_index].grid(column=col_index, row=1,
+ sticky='news', padx=0, pady=0)
+ self.grid_columnconfigure(col_index, weight=weight)
+
+ #/////////////////////////////////////////////////////////////////
+ # Binding Methods
+ #/////////////////////////////////////////////////////////////////
+
+ def bind_to_labels(self, sequence=None, func=None, add=None):
+ """
+ Add a binding to each ``Tkinter.Label`` widget in this
+ mult-column listbox that will call ``func`` in response to the
+ event sequence.
+
+ :return: A list of the identifiers of replaced binding
+ functions (if any), allowing for their deletion (to
+ prevent a memory leak).
+ """
+ return [label.bind(sequence, func, add)
+ for label in self.column_labels]
+
+ def bind_to_listboxes(self, sequence=None, func=None, add=None):
+ """
+ Add a binding to each ``Tkinter.Listbox`` widget in this
+ mult-column listbox that will call ``func`` in response to the
+ event sequence.
+
+ :return: A list of the identifiers of replaced binding
+ functions (if any), allowing for their deletion (to
+ prevent a memory leak).
+ """
+ for listbox in self.listboxes:
+ listbox.bind(sequence, func, add)
+
+ def bind_to_columns(self, sequence=None, func=None, add=None):
+ """
+ Add a binding to each ``Tkinter.Label`` and ``Tkinter.Listbox``
+ widget in this mult-column listbox that will call ``func`` in
+ response to the event sequence.
+
+ :return: A list of the identifiers of replaced binding
+ functions (if any), allowing for their deletion (to
+ prevent a memory leak).
+ """
+ return (self.bind_to_labels(sequence, func, add) +
+ self.bind_to_listboxes(sequence, func, add))
+
+ #/////////////////////////////////////////////////////////////////
+ # Simple Delegation
+ #/////////////////////////////////////////////////////////////////
+
+ # These methods delegate to the first listbox:
+ def curselection(self, *args, **kwargs):
+ return self._listboxes[0].curselection(*args, **kwargs)
+ def selection_includes(self, *args, **kwargs):
+ return self._listboxes[0].selection_includes(*args, **kwargs)
+ def itemcget(self, *args, **kwargs):
+ return self._listboxes[0].itemcget(*args, **kwargs)
+ def size(self, *args, **kwargs):
+ return self._listboxes[0].size(*args, **kwargs)
+ def index(self, *args, **kwargs):
+ return self._listboxes[0].index(*args, **kwargs)
+ def nearest(self, *args, **kwargs):
+ return self._listboxes[0].nearest(*args, **kwargs)
+
+ # These methods delegate to each listbox (and return None):
+ def activate(self, *args, **kwargs):
+ for lb in self._listboxes: lb.activate(*args, **kwargs)
+ def delete(self, *args, **kwargs):
+ for lb in self._listboxes: lb.delete(*args, **kwargs)
+ def scan_mark(self, *args, **kwargs):
+ for lb in self._listboxes: lb.scan_mark(*args, **kwargs)
+ def scan_dragto(self, *args, **kwargs):
+ for lb in self._listboxes: lb.scan_dragto(*args, **kwargs)
+ def see(self, *args, **kwargs):
+ for lb in self._listboxes: lb.see(*args, **kwargs)
+ def selection_anchor(self, *args, **kwargs):
+ for lb in self._listboxes: lb.selection_anchor(*args, **kwargs)
+ def selection_clear(self, *args, **kwargs):
+ for lb in self._listboxes: lb.selection_clear(*args, **kwargs)
+ def selection_set(self, *args, **kwargs):
+ for lb in self._listboxes: lb.selection_set(*args, **kwargs)
+ def yview(self, *args, **kwargs):
+ for lb in self._listboxes: v = lb.yview(*args, **kwargs)
+ return v # if called with no arguments
+ def yview_moveto(self, *args, **kwargs):
+ for lb in self._listboxes: lb.yview_moveto(*args, **kwargs)
+ def yview_scroll(self, *args, **kwargs):
+ for lb in self._listboxes: lb.yview_scroll(*args, **kwargs)
+
+ #/////////////////////////////////////////////////////////////////
+ # Aliases
+ #/////////////////////////////////////////////////////////////////
+
+ itemconfig = itemconfigure
+ rowconfig = rowconfigure
+ columnconfig = columnconfigure
+ select_anchor = selection_anchor
+ select_clear = selection_clear
+ select_includes = selection_includes
+ select_set = selection_set
+
+ #/////////////////////////////////////////////////////////////////
+ # These listbox methods are not defined for multi-listbox
+ #/////////////////////////////////////////////////////////////////
+ # def xview(self, *what): pass
+ # def xview_moveto(self, fraction): pass
+ # def xview_scroll(self, number, what): pass
+
+######################################################################
+# Table
+######################################################################
+
+class Table(object):
+ """
+ A display widget for a table of values, based on a ``MultiListbox``
+ widget. For many purposes, ``Table`` can be treated as a
+ list-of-lists. E.g., table[i] is a list of the values for row i;
+ and table.append(row) adds a new row with the given lits of
+ values. Individual cells can be accessed using table[i,j], which
+ refers to the j-th column of the i-th row. This can be used to
+ both read and write values from the table. E.g.:
+
+ >>> table[i,j] = 'hello'
+
+ The column (j) can be given either as an index number, or as a
+ column name. E.g., the following prints the value in the 3rd row
+ for the 'First Name' column:
+
+ >>> print(table[3, 'First Name'])
+ John
+
+ You can configure the colors for individual rows, columns, or
+ cells using ``rowconfig()``, ``columnconfig()``, and ``itemconfig()``.
+ The color configuration for each row will be preserved if the
+ table is modified; however, when new rows are added, any color
+ configurations that have been made for *columns* will not be
+ applied to the new row.
+
+ Note: Although ``Table`` acts like a widget in some ways (e.g., it
+ defines ``grid()``, ``pack()``, and ``bind()``), it is not itself a
+ widget; it just contains one. This is because widgets need to
+ define ``__getitem__()``, ``__setitem__()``, and ``__nonzero__()`` in
+ a way that's incompatible with the fact that ``Table`` behaves as a
+ list-of-lists.
+
+ :ivar _mlb: The multi-column listbox used to display this table's data.
+ :ivar _rows: A list-of-lists used to hold the cell values of this
+ table. Each element of _rows is a row value, i.e., a list of
+ cell values, one for each column in the row.
+ """
+ def __init__(self, master, column_names, rows=None,
+ column_weights=None,
+ scrollbar=True, click_to_sort=True,
+ reprfunc=None, cnf={}, **kw):
+ """
+ Construct a new Table widget.
+
+ :type master: Tkinter.Widget
+ :param master: The widget that should contain the new table.
+ :type column_names: list(str)
+ :param column_names: A list of names for the columns; these
+ names will be used to create labels for each column;
+ and can be used as an index when reading or writing
+ cell values from the table.
+ :type rows: list(list)
+ :param rows: A list of row values used to initialze the table.
+ Each row value should be a tuple of cell values, one for
+ each column in the row.
+ :type scrollbar: bool
+ :param scrollbar: If true, then create a scrollbar for the
+ new table widget.
+ :type click_to_sort: bool
+ :param click_to_sort: If true, then create bindings that will
+ sort the table's rows by a given column's values if the
+ user clicks on that colum's label.
+ :type reprfunc: function
+ :param reprfunc: If specified, then use this function to
+ convert each table cell value to a string suitable for
+ display. ``reprfunc`` has the following signature:
+ reprfunc(row_index, col_index, cell_value) -> str
+ (Note that the column is specified by index, not by name.)
+ :param cnf, kw: Configuration parameters for this widget's
+ contained ``MultiListbox``. See ``MultiListbox.__init__()``
+ for details.
+ """
+ self._num_columns = len(column_names)
+ self._reprfunc = reprfunc
+ self._frame = Frame(master)
+
+ self._column_name_to_index = dict((c,i) for (i,c) in
+ enumerate(column_names))
+
+ # Make a copy of the rows & check that it's valid.
+ if rows is None: self._rows = []
+ else: self._rows = [[v for v in row] for row in rows]
+ for row in self._rows: self._checkrow(row)
+
+ # Create our multi-list box.
+ self._mlb = MultiListbox(self._frame, column_names,
+ column_weights, cnf, **kw)
+ self._mlb.pack(side='left', expand=True, fill='both')
+
+ # Optional scrollbar
+ if scrollbar:
+ sb = Scrollbar(self._frame, orient='vertical',
+ command=self._mlb.yview)
+ self._mlb.listboxes[0]['yscrollcommand'] = sb.set
+ #for listbox in self._mlb.listboxes:
+ # listbox['yscrollcommand'] = sb.set
+ sb.pack(side='right', fill='y')
+ self._scrollbar = sb
+
+ # Set up sorting
+ self._sortkey = None
+ if click_to_sort:
+ for i, l in enumerate(self._mlb.column_labels):
+ l.bind('<Button-1>', self._sort)
+
+ # Fill in our multi-list box.
+ self._fill_table()
+
+ #/////////////////////////////////////////////////////////////////
+ #{ Widget-like Methods
+ #/////////////////////////////////////////////////////////////////
+ # These all just delegate to either our frame or our MLB.
+
+ def pack(self, *args, **kwargs):
+ """Position this table's main frame widget in its parent
+ widget. See ``Tkinter.Frame.pack()`` for more info."""
+ self._frame.pack(*args, **kwargs)
+
+ def grid(self, *args, **kwargs):
+ """Position this table's main frame widget in its parent
+ widget. See ``Tkinter.Frame.grid()`` for more info."""
+ self._frame.grid(*args, **kwargs)
+
+ def focus(self):
+ """Direct (keyboard) input foxus to this widget."""
+ self._mlb.focus()
+
+ def bind(self, sequence=None, func=None, add=None):
+ """Add a binding to this table's main frame that will call
+ ``func`` in response to the event sequence."""
+ self._mlb.bind(sequence, func, add)
+
+ def rowconfigure(self, row_index, cnf={}, **kw):
+ """:see: ``MultiListbox.rowconfigure()``"""
+ self._mlb.rowconfigure(row_index, cnf, **kw)
+
+ def columnconfigure(self, col_index, cnf={}, **kw):
+ """:see: ``MultiListbox.columnconfigure()``"""
+ col_index = self.column_index(col_index)
+ self._mlb.columnconfigure(col_index, cnf, **kw)
+
+ def itemconfigure(self, row_index, col_index, cnf=None, **kw):
+ """:see: ``MultiListbox.itemconfigure()``"""
+ col_index = self.column_index(col_index)
+ return self._mlb.itemconfigure(row_index, col_index, cnf, **kw)
+
+ def bind_to_labels(self, sequence=None, func=None, add=None):
+ """:see: ``MultiListbox.bind_to_labels()``"""
+ return self._mlb.bind_to_labels(sequence, func, add)
+
+ def bind_to_listboxes(self, sequence=None, func=None, add=None):
+ """:see: ``MultiListbox.bind_to_listboxes()``"""
+ return self._mlb.bind_to_listboxes(sequence, func, add)
+
+ def bind_to_columns(self, sequence=None, func=None, add=None):
+ """:see: ``MultiListbox.bind_to_columns()``"""
+ return self._mlb.bind_to_columns(sequence, func, add)
+
+ rowconfig = rowconfigure
+ columnconfig = columnconfigure
+ itemconfig = itemconfigure
+
+ #/////////////////////////////////////////////////////////////////
+ #{ Table as list-of-lists
+ #/////////////////////////////////////////////////////////////////
+
+ def insert(self, row_index, rowvalue):
+ """
+ Insert a new row into the table, so that its row index will be
+ ``row_index``. If the table contains any rows whose row index
+ is greater than or equal to ``row_index``, then they will be
+ shifted down.
+
+ :param rowvalue: A tuple of cell values, one for each column
+ in the new row.
+ """
+ self._checkrow(rowvalue)
+ self._rows.insert(row_index, rowvalue)
+ if self._reprfunc is not None:
+ rowvalue = [self._reprfunc(row_index,j,v)
+ for (j,v) in enumerate(rowvalue)]
+ self._mlb.insert(row_index, rowvalue)
+ if self._DEBUG: self._check_table_vs_mlb()
+
+ def extend(self, rowvalues):
+ """
+ Add new rows at the end of the table.
+
+ :param rowvalues: A list of row values used to initialze the
+ table. Each row value should be a tuple of cell values,
+ one for each column in the row.
+ """
+ for rowvalue in rowvalues: self.append(rowvalue)
+ if self._DEBUG: self._check_table_vs_mlb()
+
+ def append(self, rowvalue):
+ """
+ Add a new row to the end of the table.
+
+ :param rowvalue: A tuple of cell values, one for each column
+ in the new row.
+ """
+ self.insert(len(self._rows), rowvalue)
+ if self._DEBUG: self._check_table_vs_mlb()
+
+ def clear(self):
+ """
+ Delete all rows in this table.
+ """
+ self._rows = []
+ self._mlb.delete(0, 'end')
+ if self._DEBUG: self._check_table_vs_mlb()
+
+ def __getitem__(self, index):
+ """
+ Return the value of a row or a cell in this table. If
+ ``index`` is an integer, then the row value for the ``index``th
+ row. This row value consists of a tuple of cell values, one
+ for each column in the row. If ``index`` is a tuple of two
+ integers, ``(i,j)``, then return the value of the cell in the
+ ``i``th row and the ``j``th column.
+ """
+ if isinstance(index, slice):
+ raise ValueError('Slicing not supported')
+ elif isinstance(index, tuple) and len(index)==2:
+ return self._rows[index[0]][self.column_index(index[1])]
+ else:
+ return tuple(self._rows[index])
+
+ def __setitem__(self, index, val):
+ """
+ Replace the value of a row or a cell in this table with
+ ``val``.
+
+ If ``index`` is an integer, then ``val`` should be a row value
+ (i.e., a tuple of cell values, one for each column). In this
+ case, the values of the ``index``th row of the table will be
+ replaced with the values in ``val``.
+
+ If ``index`` is a tuple of integers, ``(i,j)``, then replace the
+ value of the cell in the ``i``th row and ``j``th column with
+ ``val``.
+ """
+ if isinstance(index, slice):
+ raise ValueError('Slicing not supported')
+
+
+ # table[i,j] = val
+ elif isinstance(index, tuple) and len(index)==2:
+ i, j = index[0], self.column_index(index[1])
+ config_cookie = self._save_config_info([i])
+ self._rows[i][j] = val
+ if self._reprfunc is not None:
+ val = self._reprfunc(i, j, val)
+ self._mlb.listboxes[j].insert(i, val)
+ self._mlb.listboxes[j].delete(i+1)
+ self._restore_config_info(config_cookie)
+
+ # table[i] = val
+ else:
+ config_cookie = self._save_config_info([index])
+ self._checkrow(val)
+ self._rows[index] = list(val)
+ if self._reprfunc is not None:
+ val = [self._reprfunc(index,j,v) for (j,v) in enumerate(val)]
+ self._mlb.insert(index, val)
+ self._mlb.delete(index+1)
+ self._restore_config_info(config_cookie)
+
+ def __delitem__(self, row_index):
+ """
+ Delete the ``row_index``th row from this table.
+ """
+ if isinstance(index, slice):
+ raise ValueError('Slicing not supported')
+ if isinstance(row_index, tuple) and len(row_index)==2:
+ raise ValueError('Cannot delete a single cell!')
+ del self._rows[row_index]
+ self._mlb.delete(row_index)
+ if self._DEBUG: self._check_table_vs_mlb()
+
+ def __len__(self):
+ """
+ :return: the number of rows in this table.
+ """
+ return len(self._rows)
+
+ def _checkrow(self, rowvalue):
+ """
+ Helper function: check that a given row value has the correct
+ number of elements; and if not, raise an exception.
+ """
+ if len(rowvalue) != self._num_columns:
+ raise ValueError('Row %r has %d columns; expected %d' %
+ (rowvalue, len(rowvalue), self._num_columns))
+
+ #/////////////////////////////////////////////////////////////////
+ # Columns
+ #/////////////////////////////////////////////////////////////////
+
+ @property
+ def column_names(self):
+ """A list of the names of the columns in this table."""
+ return self._mlb.column_names
+
+ def column_index(self, i):
+ """
+ If ``i`` is a valid column index integer, then return it as is.
+ Otherwise, check if ``i`` is used as the name for any column;
+ if so, return that column's index. Otherwise, raise a
+ ``KeyError`` exception.
+ """
+ if isinstance(i, int) and 0 <= i < self._num_columns:
+ return i
+ else:
+ # This raises a key error if the column is not found.
+ return self._column_name_to_index[i]
+
+ def hide_column(self, column_index):
+ """:see: ``MultiListbox.hide_column()``"""
+ self._mlb.hide_column(self.column_index(column_index))
+
+ def show_column(self, column_index):
+ """:see: ``MultiListbox.show_column()``"""
+ self._mlb.show_column(self.column_index(column_index))
+
+ #/////////////////////////////////////////////////////////////////
+ # Selection
+ #/////////////////////////////////////////////////////////////////
+
+ def selected_row(self):
+ """
+ Return the index of the currently selected row, or None if
+ no row is selected. To get the row value itself, use
+ ``table[table.selected_row()]``.
+ """
+ sel = self._mlb.curselection()
+ if sel: return int(sel[0])
+ else: return None
+
+ def select(self, index=None, delta=None, see=True):
+ """:see: ``MultiListbox.select()``"""
+ self._mlb.select(index, delta, see)
+
+ #/////////////////////////////////////////////////////////////////
+ # Sorting
+ #/////////////////////////////////////////////////////////////////
+
+ def sort_by(self, column_index, order='toggle'):
+ """
+ Sort the rows in this table, using the specified column's
+ values as a sort key.
+
+ :param column_index: Specifies which column to sort, using
+ either a column index (int) or a column's label name
+ (str).
+
+ :param order: Specifies whether to sort the values in
+ ascending or descending order:
+
+ - ``'ascending'``: Sort from least to greatest.
+ - ``'descending'``: Sort from greatest to least.
+ - ``'toggle'``: If the most recent call to ``sort_by()``
+ sorted the table by the same column (``column_index``),
+ then reverse the rows; otherwise sort in ascending
+ order.
+ """
+ if order not in ('ascending', 'descending', 'toggle'):
+ raise ValueError('sort_by(): order should be "ascending", '
+ '"descending", or "toggle".')
+ column_index = self.column_index(column_index)
+ config_cookie = self._save_config_info(index_by_id=True)
+
+ # Sort the rows.
+ if order == 'toggle' and column_index == self._sortkey:
+ self._rows.reverse()
+ else:
+ self._rows.sort(key=operator.itemgetter(column_index),
+ reverse=(order=='descending'))
+ self._sortkey = column_index
+
+ # Redraw the table.
+ self._fill_table()
+ self._restore_config_info(config_cookie, index_by_id=True, see=True)
+ if self._DEBUG: self._check_table_vs_mlb()
+
+ def _sort(self, event):
+ """Event handler for clicking on a column label -- sort by
+ that column."""
+ column_index = event.widget.column_index
+
+ # If they click on the far-left of far-right of a column's
+ # label, then resize rather than sorting.
+ if self._mlb._resize_column(event):
+ return 'continue'
+
+ # Otherwise, sort.
+ else:
+ self.sort_by(column_index)
+ return 'continue'
+
+ #/////////////////////////////////////////////////////////////////
+ #{ Table Drawing Helpers
+ #/////////////////////////////////////////////////////////////////
+
+ def _fill_table(self, save_config=True):
+ """
+ Re-draw the table from scratch, by clearing out the table's
+ multi-column listbox; and then filling it in with values from
+ ``self._rows``. Note that any cell-, row-, or column-specific
+ color configuration that has been done will be lost. The
+ selection will also be lost -- i.e., no row will be selected
+ after this call completes.
+ """
+ self._mlb.delete(0, 'end')
+ for i, row in enumerate(self._rows):
+ if self._reprfunc is not None:
+ row = [self._reprfunc(i,j,v) for (j,v) in enumerate(row)]
+ self._mlb.insert('end', row)
+
+ def _get_itemconfig(self, r, c):
+ return dict( (k, self._mlb.itemconfig(r, c, k)[-1])
+ for k in ('foreground', 'selectforeground',
+ 'background', 'selectbackground') )
+
+ def _save_config_info(self, row_indices=None, index_by_id=False):
+ """
+ Return a 'cookie' containing information about which row is
+ selected, and what color configurations have been applied.
+ this information can the be re-applied to the table (after
+ making modifications) using ``_restore_config_info()``. Color
+ configuration information will be saved for any rows in
+ ``row_indices``, or in the entire table, if
+ ``row_indices=None``. If ``index_by_id=True``, the the cookie
+ will associate rows with their configuration information based
+ on the rows' python id. This is useful when performing
+ operations that re-arrange the rows (e.g. ``sort``). If
+ ``index_by_id=False``, then it is assumed that all rows will be
+ in the same order when ``_restore_config_info()`` is called.
+ """
+ # Default value for row_indices is all rows.
+ if row_indices is None:
+ row_indices = list(range(len(self._rows)))
+
+ # Look up our current selection.
+ selection = self.selected_row()
+ if index_by_id and selection is not None:
+ selection = id(self._rows[selection])
+
+ # Look up the color configuration info for each row.
+ if index_by_id:
+ config = dict((id(self._rows[r]), [self._get_itemconfig(r, c)
+ for c in range(self._num_columns)])
+ for r in row_indices)
+ else:
+ config = dict((r, [self._get_itemconfig(r, c)
+ for c in range(self._num_columns)])
+ for r in row_indices)
+
+
+ return selection, config
+
+ def _restore_config_info(self, cookie, index_by_id=False, see=False):
+ """
+ Restore selection & color configuration information that was
+ saved using ``_save_config_info``.
+ """
+ selection, config = cookie
+
+ # Clear the selection.
+ if selection is None:
+ self._mlb.selection_clear(0, 'end')
+
+ # Restore selection & color config
+ if index_by_id:
+ for r, row in enumerate(self._rows):
+ if id(row) in config:
+ for c in range(self._num_columns):
+ self._mlb.itemconfigure(r, c, config[id(row)][c])
+ if id(row) == selection:
+ self._mlb.select(r, see=see)
+ else:
+ if selection is not None:
+ self._mlb.select(selection, see=see)
+ for r in config:
+ for c in range(self._num_columns):
+ self._mlb.itemconfigure(r, c, config[r][c])
+
+ #/////////////////////////////////////////////////////////////////
+ # Debugging (Invariant Checker)
+ #/////////////////////////////////////////////////////////////////
+
+ _DEBUG = False
+ """If true, then run ``_check_table_vs_mlb()`` after any operation
+ that modifies the table."""
+
+ def _check_table_vs_mlb(self):
+ """
+ Verify that the contents of the table's ``_rows`` variable match
+ the contents of its multi-listbox (``_mlb``). This is just
+ included for debugging purposes, to make sure that the
+ list-modifying operations are working correctly.
+ """
+ for col in self._mlb.listboxes:
+ assert len(self) == col.size()
+ for row in self:
+ assert len(row) == self._num_columns
+ assert self._num_columns == len(self._mlb.column_names)
+ #assert self._column_names == self._mlb.column_names
+ for i, row in enumerate(self):
+ for j, cell in enumerate(row):
+ if self._reprfunc is not None:
+ cell = self._reprfunc(i, j, cell)
+ assert self._mlb.get(i)[j] == cell
+
+######################################################################
+# Demo/Test Function
+######################################################################
+
+# update this to use new WordNet API
+def demo():
+ root = Tk()
+ root.bind('<Control-q>', lambda e: root.destroy())
+
+ table = Table(root, 'Word Synset Hypernym Hyponym'.split(),
+ column_weights=[0, 1, 1, 1],
+ reprfunc=(lambda i,j,s: ' %s' % s))
+ table.pack(expand=True, fill='both')
+
+ from nltk.corpus import wordnet
+ from nltk.corpus import brown
+ for word, pos in sorted(set(brown.tagged_words()[:500])):
+ if pos[0] != 'N': continue
+ word = word.lower()
+ for synset in wordnet.synsets(word):
+ hyper = (synset.hypernyms()+[''])[0]
+ hypo = (synset.hyponyms()+[''])[0]
+ table.append([word,
+ getattr(synset, 'definition', '*none*'),
+ getattr(hyper, 'definition', '*none*'),
+ getattr(hypo, 'definition', '*none*')])
+
+ table.columnconfig('Word', background='#afa')
+ table.columnconfig('Synset', background='#efe')
+ table.columnconfig('Hypernym', background='#fee')
+ table.columnconfig('Hyponym', background='#ffe')
+ for row in range(len(table)):
+ for column in ('Hypernym', 'Hyponym'):
+ if table[row, column] == '*none*':
+ table.itemconfig(row, column, foreground='#666',
+ selectforeground='#666')
+ root.mainloop()
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/draw/tree.py b/nltk/draw/tree.py
new file mode 100644
index 0000000..aea0c83
--- /dev/null
+++ b/nltk/draw/tree.py
@@ -0,0 +1,966 @@
+# Natural Language Toolkit: Graphical Representations for Trees
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Graphically display a Tree.
+"""
+
+import nltk.compat
+import sys
+
+from tkinter import IntVar, Menu, Tk
+
+from nltk.util import in_idle
+from nltk.tree import Tree
+from nltk.draw.util import (CanvasFrame, CanvasWidget, BoxWidget,
+ TextWidget, ParenWidget, OvalWidget)
+
+##//////////////////////////////////////////////////////
+## Tree Segment
+##//////////////////////////////////////////////////////
+
+class TreeSegmentWidget(CanvasWidget):
+ """
+ A canvas widget that displays a single segment of a hierarchical
+ tree. Each ``TreeSegmentWidget`` connects a single "node widget"
+ to a sequence of zero or more "subtree widgets". By default, the
+ bottom of the node is connected to the top of each subtree by a
+ single line. However, if the ``roof`` attribute is set, then a
+ single triangular "roof" will connect the node to all of its
+ children.
+
+ Attributes:
+ - ``roof``: What sort of connection to draw between the node and
+ its subtrees. If ``roof`` is true, draw a single triangular
+ "roof" over the subtrees. If ``roof`` is false, draw a line
+ between each subtree and the node. Default value is false.
+ - ``xspace``: The amount of horizontal space to leave between
+ subtrees when managing this widget. Default value is 10.
+ - ``yspace``: The amount of space to place between the node and
+ its children when managing this widget. Default value is 15.
+ - ``color``: The color of the lines connecting the node to its
+ subtrees; and of the outline of the triangular roof. Default
+ value is ``'#006060'``.
+ - ``fill``: The fill color for the triangular roof. Default
+ value is ``''`` (no fill).
+ - ``width``: The width of the lines connecting the node to its
+ subtrees; and of the outline of the triangular roof. Default
+ value is 1.
+ - ``orientation``: Determines whether the tree branches downwards
+ or rightwards. Possible values are ``'horizontal'`` and
+ ``'vertical'``. The default value is ``'vertical'`` (i.e.,
+ branch downwards).
+ - ``draggable``: whether the widget can be dragged by the user.
+ """
+ def __init__(self, canvas, label, subtrees, **attribs):
+ """
+ :type node:
+ :type subtrees: list(CanvasWidgetI)
+ """
+ self._label = label
+ self._subtrees = subtrees
+
+ # Attributes
+ self._horizontal = 0
+ self._roof = 0
+ self._xspace = 10
+ self._yspace = 15
+ self._ordered = False
+
+ # Create canvas objects.
+ self._lines = [canvas.create_line(0,0,0,0, fill='#006060')
+ for c in subtrees]
+ self._polygon = canvas.create_polygon(0,0, fill='', state='hidden',
+ outline='#006060')
+
+ # Register child widgets (label + subtrees)
+ self._add_child_widget(label)
+ for subtree in subtrees:
+ self._add_child_widget(subtree)
+
+ # Are we currently managing?
+ self._managing = False
+
+ CanvasWidget.__init__(self, canvas, **attribs)
+
+ def __setitem__(self, attr, value):
+ canvas = self.canvas()
+ if attr == 'roof':
+ self._roof = value
+ if self._roof:
+ for l in self._lines: canvas.itemconfig(l, state='hidden')
+ canvas.itemconfig(self._polygon, state='normal')
+ else:
+ for l in self._lines: canvas.itemconfig(l, state='normal')
+ canvas.itemconfig(self._polygon, state='hidden')
+ elif attr == 'orientation':
+ if value == 'horizontal': self._horizontal = 1
+ elif value == 'vertical': self._horizontal = 0
+ else:
+ raise ValueError('orientation must be horizontal or vertical')
+ elif attr == 'color':
+ for l in self._lines: canvas.itemconfig(l, fill=value)
+ canvas.itemconfig(self._polygon, outline=value)
+ elif isinstance(attr, tuple) and attr[0] == 'color':
+ # Set the color of an individual line.
+ l = self._lines[int(attr[1])]
+ canvas.itemconfig(l, fill=value)
+ elif attr == 'fill':
+ canvas.itemconfig(self._polygon, fill=value)
+ elif attr == 'width':
+ canvas.itemconfig(self._polygon, {attr:value})
+ for l in self._lines: canvas.itemconfig(l, {attr:value})
+ elif attr in ('xspace', 'yspace'):
+ if attr == 'xspace': self._xspace = value
+ elif attr == 'yspace': self._yspace = value
+ self.update(self._label)
+ elif attr == 'ordered':
+ self._ordered = value
+ else:
+ CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr == 'roof': return self._roof
+ elif attr == 'width':
+ return self.canvas().itemcget(self._polygon, attr)
+ elif attr == 'color':
+ return self.canvas().itemcget(self._polygon, 'outline')
+ elif isinstance(attr, tuple) and attr[0] == 'color':
+ l = self._lines[int(attr[1])]
+ return self.canvas().itemcget(l, 'fill')
+ elif attr == 'xspace': return self._xspace
+ elif attr == 'yspace': return self._yspace
+ elif attr == 'orientation':
+ if self._horizontal: return 'horizontal'
+ else: return 'vertical'
+ elif attr == 'ordered':
+ return self._ordered
+ else:
+ return CanvasWidget.__getitem__(self, attr)
+
+ def label(self):
+ return self._label
+
+ def subtrees(self):
+ return self._subtrees[:]
+
+ def set_label(self, label):
+ """
+ Set the node label to ``label``.
+ """
+ self._remove_child_widget(self._label)
+ self._add_child_widget(label)
+ self._label = label
+ self.update(self._label)
+
+ def replace_child(self, oldchild, newchild):
+ """
+ Replace the child ``oldchild`` with ``newchild``.
+ """
+ index = self._subtrees.index(oldchild)
+ self._subtrees[index] = newchild
+ self._remove_child_widget(oldchild)
+ self._add_child_widget(newchild)
+ self.update(newchild)
+
+ def remove_child(self, child):
+ index = self._subtrees.index(child)
+ del self._subtrees[index]
+ self._remove_child_widget(child)
+ self.canvas().delete(self._lines.pop())
+ self.update(self._label)
+
+ def insert_child(self, index, child):
+ canvas = self.canvas()
+ self._subtrees.insert(index, child)
+ self._add_child_widget(child)
+ self._lines.append(canvas.create_line(0,0,0,0, fill='#006060'))
+ self.update(self._label)
+
+ # but.. lines???
+
+ def _tags(self):
+ if self._roof:
+ return [self._polygon]
+ else:
+ return self._lines
+
+ def _subtree_top(self, child):
+ if isinstance(child, TreeSegmentWidget):
+ bbox = child.label().bbox()
+ else:
+ bbox = child.bbox()
+ if self._horizontal:
+ return (bbox[0], (bbox[1]+bbox[3])/2.0)
+ else:
+ return ((bbox[0]+bbox[2])/2.0, bbox[1])
+
+ def _node_bottom(self):
+ bbox = self._label.bbox()
+ if self._horizontal:
+ return (bbox[2], (bbox[1]+bbox[3])/2.0)
+ else:
+ return ((bbox[0]+bbox[2])/2.0, bbox[3])
+
+ def _update(self, child):
+ if len(self._subtrees) == 0: return
+ if self._label.bbox() is None: return # [XX] ???
+
+ # Which lines need to be redrawn?
+ if child is self._label: need_update = self._subtrees
+ else: need_update = [child]
+
+ if self._ordered and not self._managing:
+ need_update = self._maintain_order(child)
+
+ # Update the polygon.
+ (nodex, nodey) = self._node_bottom()
+ (xmin, ymin, xmax, ymax) = self._subtrees[0].bbox()
+ for subtree in self._subtrees[1:]:
+ bbox = subtree.bbox()
+ xmin = min(xmin, bbox[0])
+ ymin = min(ymin, bbox[1])
+ xmax = max(xmax, bbox[2])
+ ymax = max(ymax, bbox[3])
+
+ if self._horizontal:
+ self.canvas().coords(self._polygon, nodex, nodey, xmin,
+ ymin, xmin, ymax, nodex, nodey)
+ else:
+ self.canvas().coords(self._polygon, nodex, nodey, xmin,
+ ymin, xmax, ymin, nodex, nodey)
+
+ # Redraw all lines that need it.
+ for subtree in need_update:
+ (nodex, nodey) = self._node_bottom()
+ line = self._lines[self._subtrees.index(subtree)]
+ (subtreex, subtreey) = self._subtree_top(subtree)
+ self.canvas().coords(line, nodex, nodey, subtreex, subtreey)
+
+ def _maintain_order(self, child):
+ if self._horizontal:
+ return self._maintain_order_horizontal(child)
+ else:
+ return self._maintain_order_vertical(child)
+
+ def _maintain_order_vertical(self, child):
+ (left, top, right, bot) = child.bbox()
+
+ if child is self._label:
+ # Check all the leaves
+ for subtree in self._subtrees:
+ (x1, y1, x2, y2) = subtree.bbox()
+ if bot+self._yspace > y1:
+ subtree.move(0,bot+self._yspace-y1)
+
+ return self._subtrees
+ else:
+ moved = [child]
+ index = self._subtrees.index(child)
+
+ # Check leaves to our right.
+ x = right + self._xspace
+ for i in range(index+1, len(self._subtrees)):
+ (x1, y1, x2, y2) = self._subtrees[i].bbox()
+ if x > x1:
+ self._subtrees[i].move(x-x1, 0)
+ x += x2-x1 + self._xspace
+ moved.append(self._subtrees[i])
+
+ # Check leaves to our left.
+ x = left - self._xspace
+ for i in range(index-1, -1, -1):
+ (x1, y1, x2, y2) = self._subtrees[i].bbox()
+ if x < x2:
+ self._subtrees[i].move(x-x2, 0)
+ x -= x2-x1 + self._xspace
+ moved.append(self._subtrees[i])
+
+ # Check the node
+ (x1, y1, x2, y2) = self._label.bbox()
+ if y2 > top-self._yspace:
+ self._label.move(0, top-self._yspace-y2)
+ moved = self._subtrees
+
+ # Return a list of the nodes we moved
+ return moved
+
+ def _maintain_order_horizontal(self, child):
+ (left, top, right, bot) = child.bbox()
+
+ if child is self._label:
+ # Check all the leaves
+ for subtree in self._subtrees:
+ (x1, y1, x2, y2) = subtree.bbox()
+ if right+self._xspace > x1:
+ subtree.move(right+self._xspace-x1)
+
+ return self._subtrees
+ else:
+ moved = [child]
+ index = self._subtrees.index(child)
+
+ # Check leaves below us.
+ y = bot + self._yspace
+ for i in range(index+1, len(self._subtrees)):
+ (x1, y1, x2, y2) = self._subtrees[i].bbox()
+ if y > y1:
+ self._subtrees[i].move(0, y-y1)
+ y += y2-y1 + self._yspace
+ moved.append(self._subtrees[i])
+
+ # Check leaves above us
+ y = top - self._yspace
+ for i in range(index-1, -1, -1):
+ (x1, y1, x2, y2) = self._subtrees[i].bbox()
+ if y < y2:
+ self._subtrees[i].move(0, y-y2)
+ y -= y2-y1 + self._yspace
+ moved.append(self._subtrees[i])
+
+ # Check the node
+ (x1, y1, x2, y2) = self._label.bbox()
+ if x2 > left-self._xspace:
+ self._label.move(left-self._xspace-x2, 0)
+ moved = self._subtrees
+
+ # Return a list of the nodes we moved
+ return moved
+
+ def _manage_horizontal(self):
+ (nodex, nodey) = self._node_bottom()
+
+ # Put the subtrees in a line.
+ y = 20
+ for subtree in self._subtrees:
+ subtree_bbox = subtree.bbox()
+ dx = nodex - subtree_bbox[0] + self._xspace
+ dy = y - subtree_bbox[1]
+ subtree.move(dx, dy)
+ y += subtree_bbox[3] - subtree_bbox[1] + self._yspace
+
+ # Find the center of their tops.
+ center = 0.0
+ for subtree in self._subtrees:
+ center += self._subtree_top(subtree)[1]
+ center /= len(self._subtrees)
+
+ # Center the subtrees with the node.
+ for subtree in self._subtrees:
+ subtree.move(0, nodey-center)
+
+ def _manage_vertical(self):
+ (nodex, nodey) = self._node_bottom()
+
+ # Put the subtrees in a line.
+ x = 0
+ for subtree in self._subtrees:
+ subtree_bbox = subtree.bbox()
+ dy = nodey - subtree_bbox[1] + self._yspace
+ dx = x - subtree_bbox[0]
+ subtree.move(dx, dy)
+ x += subtree_bbox[2] - subtree_bbox[0] + self._xspace
+
+ # Find the center of their tops.
+ center = 0.0
+ for subtree in self._subtrees:
+ center += self._subtree_top(subtree)[0]/len(self._subtrees)
+
+ # Center the subtrees with the node.
+ for subtree in self._subtrees:
+ subtree.move(nodex-center, 0)
+
+ def _manage(self):
+ self._managing = True
+ (nodex, nodey) = self._node_bottom()
+ if len(self._subtrees) == 0: return
+
+ if self._horizontal: self._manage_horizontal()
+ else: self._manage_vertical()
+
+ # Update lines to subtrees.
+ for subtree in self._subtrees:
+ self._update(subtree)
+
+ self._managing = False
+
+ def __repr__(self):
+ return '[TreeSeg %s: %s]' % (self._label, self._subtrees)
+
+def _tree_to_treeseg(canvas, t, make_node, make_leaf,
+ tree_attribs, node_attribs,
+ leaf_attribs, loc_attribs):
+ if isinstance(t, Tree):
+ label = make_node(canvas, t.label(), **node_attribs)
+ subtrees = [_tree_to_treeseg(canvas, child, make_node, make_leaf,
+ tree_attribs, node_attribs,
+ leaf_attribs, loc_attribs)
+ for child in t]
+ return TreeSegmentWidget(canvas, label, subtrees, **tree_attribs)
+ else:
+ return make_leaf(canvas, t, **leaf_attribs)
+
+def tree_to_treesegment(canvas, t, make_node=TextWidget,
+ make_leaf=TextWidget, **attribs):
+ """
+ Convert a Tree into a ``TreeSegmentWidget``.
+
+ :param make_node: A ``CanvasWidget`` constructor or a function that
+ creates ``CanvasWidgets``. ``make_node`` is used to convert
+ the Tree's nodes into ``CanvasWidgets``. If no constructor
+ is specified, then ``TextWidget`` will be used.
+ :param make_leaf: A ``CanvasWidget`` constructor or a function that
+ creates ``CanvasWidgets``. ``make_leaf`` is used to convert
+ the Tree's leafs into ``CanvasWidgets``. If no constructor
+ is specified, then ``TextWidget`` will be used.
+ :param attribs: Attributes for the canvas widgets that make up the
+ returned ``TreeSegmentWidget``. Any attribute beginning with
+ ``'tree_'`` will be passed to all ``TreeSegmentWidgets`` (with
+ the ``'tree_'`` prefix removed. Any attribute beginning with
+ ``'node_'`` will be passed to all nodes. Any attribute
+ beginning with ``'leaf_'`` will be passed to all leaves. And
+ any attribute beginning with ``'loc_'`` will be passed to all
+ text locations (for Trees).
+ """
+ # Process attribs.
+ tree_attribs = {}
+ node_attribs = {}
+ leaf_attribs = {}
+ loc_attribs = {}
+
+ for (key, value) in list(attribs.items()):
+ if key[:5] == 'tree_': tree_attribs[key[5:]] = value
+ elif key[:5] == 'node_': node_attribs[key[5:]] = value
+ elif key[:5] == 'leaf_': leaf_attribs[key[5:]] = value
+ elif key[:4] == 'loc_': loc_attribs[key[4:]] = value
+ else: raise ValueError('Bad attribute: %s' % key)
+ return _tree_to_treeseg(canvas, t, make_node, make_leaf,
+ tree_attribs, node_attribs,
+ leaf_attribs, loc_attribs)
+
+##//////////////////////////////////////////////////////
+## Tree Widget
+##//////////////////////////////////////////////////////
+
+class TreeWidget(CanvasWidget):
+ """
+ A canvas widget that displays a single Tree.
+ ``TreeWidget`` manages a group of ``TreeSegmentWidgets`` that are
+ used to display a Tree.
+
+ Attributes:
+
+ - ``node_attr``: Sets the attribute ``attr`` on all of the
+ node widgets for this ``TreeWidget``.
+ - ``node_attr``: Sets the attribute ``attr`` on all of the
+ leaf widgets for this ``TreeWidget``.
+ - ``loc_attr``: Sets the attribute ``attr`` on all of the
+ location widgets for this ``TreeWidget`` (if it was built from
+ a Tree). Note that a location widget is a ``TextWidget``.
+
+ - ``xspace``: The amount of horizontal space to leave between
+ subtrees when managing this widget. Default value is 10.
+ - ``yspace``: The amount of space to place between the node and
+ its children when managing this widget. Default value is 15.
+
+ - ``line_color``: The color of the lines connecting each expanded
+ node to its subtrees.
+ - ``roof_color``: The color of the outline of the triangular roof
+ for collapsed trees.
+ - ``roof_fill``: The fill color for the triangular roof for
+ collapsed trees.
+ - ``width``
+
+ - ``orientation``: Determines whether the tree branches downwards
+ or rightwards. Possible values are ``'horizontal'`` and
+ ``'vertical'``. The default value is ``'vertical'`` (i.e.,
+ branch downwards).
+
+ - ``shapeable``: whether the subtrees can be independently
+ dragged by the user. THIS property simply sets the
+ ``DRAGGABLE`` property on all of the ``TreeWidget``'s tree
+ segments.
+ - ``draggable``: whether the widget can be dragged by the user.
+ """
+ def __init__(self, canvas, t, make_node=TextWidget,
+ make_leaf=TextWidget, **attribs):
+ # Node & leaf canvas widget constructors
+ self._make_node = make_node
+ self._make_leaf = make_leaf
+ self._tree = t
+
+ # Attributes.
+ self._nodeattribs = {}
+ self._leafattribs = {}
+ self._locattribs = {'color': '#008000'}
+ self._line_color = '#008080'
+ self._line_width = 1
+ self._roof_color = '#008080'
+ self._roof_fill = '#c0c0c0'
+ self._shapeable = False
+ self._xspace = 10
+ self._yspace = 10
+ self._orientation = 'vertical'
+ self._ordered = False
+
+ # Build trees.
+ self._keys = {} # treeseg -> key
+ self._expanded_trees = {}
+ self._collapsed_trees = {}
+ self._nodes = []
+ self._leaves = []
+ #self._locs = []
+ self._make_collapsed_trees(canvas, t, ())
+ self._treeseg = self._make_expanded_tree(canvas, t, ())
+ self._add_child_widget(self._treeseg)
+
+ CanvasWidget.__init__(self, canvas, **attribs)
+
+ def expanded_tree(self, *path_to_tree):
+ """
+ Return the ``TreeSegmentWidget`` for the specified subtree.
+
+ :param path_to_tree: A list of indices i1, i2, ..., in, where
+ the desired widget is the widget corresponding to
+ ``tree.children()[i1].children()[i2]....children()[in]``.
+ For the root, the path is ``()``.
+ """
+ return self._expanded_trees[path_to_tree]
+
+ def collapsed_tree(self, *path_to_tree):
+ """
+ Return the ``TreeSegmentWidget`` for the specified subtree.
+
+ :param path_to_tree: A list of indices i1, i2, ..., in, where
+ the desired widget is the widget corresponding to
+ ``tree.children()[i1].children()[i2]....children()[in]``.
+ For the root, the path is ``()``.
+ """
+ return self._collapsed_trees[path_to_tree]
+
+ def bind_click_trees(self, callback, button=1):
+ """
+ Add a binding to all tree segments.
+ """
+ for tseg in list(self._expanded_trees.values()):
+ tseg.bind_click(callback, button)
+ for tseg in list(self._collapsed_trees.values()):
+ tseg.bind_click(callback, button)
+
+ def bind_drag_trees(self, callback, button=1):
+ """
+ Add a binding to all tree segments.
+ """
+ for tseg in list(self._expanded_trees.values()):
+ tseg.bind_drag(callback, button)
+ for tseg in list(self._collapsed_trees.values()):
+ tseg.bind_drag(callback, button)
+
+ def bind_click_leaves(self, callback, button=1):
+ """
+ Add a binding to all leaves.
+ """
+ for leaf in self._leaves: leaf.bind_click(callback, button)
+ for leaf in self._leaves: leaf.bind_click(callback, button)
+
+ def bind_drag_leaves(self, callback, button=1):
+ """
+ Add a binding to all leaves.
+ """
+ for leaf in self._leaves: leaf.bind_drag(callback, button)
+ for leaf in self._leaves: leaf.bind_drag(callback, button)
+
+ def bind_click_nodes(self, callback, button=1):
+ """
+ Add a binding to all nodes.
+ """
+ for node in self._nodes: node.bind_click(callback, button)
+ for node in self._nodes: node.bind_click(callback, button)
+
+ def bind_drag_nodes(self, callback, button=1):
+ """
+ Add a binding to all nodes.
+ """
+ for node in self._nodes: node.bind_drag(callback, button)
+ for node in self._nodes: node.bind_drag(callback, button)
+
+ def _make_collapsed_trees(self, canvas, t, key):
+ if not isinstance(t, Tree): return
+ make_node = self._make_node
+ make_leaf = self._make_leaf
+
+ node = make_node(canvas, t.label(), **self._nodeattribs)
+ self._nodes.append(node)
+ leaves = [make_leaf(canvas, l, **self._leafattribs)
+ for l in t.leaves()]
+ self._leaves += leaves
+ treeseg = TreeSegmentWidget(canvas, node, leaves, roof=1,
+ color=self._roof_color,
+ fill=self._roof_fill,
+ width=self._line_width)
+
+ self._collapsed_trees[key] = treeseg
+ self._keys[treeseg] = key
+ #self._add_child_widget(treeseg)
+ treeseg.hide()
+
+ # Build trees for children.
+ for i in range(len(t)):
+ child = t[i]
+ self._make_collapsed_trees(canvas, child, key + (i,))
+
+ def _make_expanded_tree(self, canvas, t, key):
+ make_node = self._make_node
+ make_leaf = self._make_leaf
+
+ if isinstance(t, Tree):
+ node = make_node(canvas, t.label(), **self._nodeattribs)
+ self._nodes.append(node)
+ children = t
+ subtrees = [self._make_expanded_tree(canvas, children[i], key+(i,))
+ for i in range(len(children))]
+ treeseg = TreeSegmentWidget(canvas, node, subtrees,
+ color=self._line_color,
+ width=self._line_width)
+ self._expanded_trees[key] = treeseg
+ self._keys[treeseg] = key
+ return treeseg
+ else:
+ leaf = make_leaf(canvas, t, **self._leafattribs)
+ self._leaves.append(leaf)
+ return leaf
+
+ def __setitem__(self, attr, value):
+ if attr[:5] == 'node_':
+ for node in self._nodes: node[attr[5:]] = value
+ elif attr[:5] == 'leaf_':
+ for leaf in self._leaves: leaf[attr[5:]] = value
+ elif attr == 'line_color':
+ self._line_color = value
+ for tseg in list(self._expanded_trees.values()): tseg['color'] = value
+ elif attr == 'line_width':
+ self._line_width = value
+ for tseg in list(self._expanded_trees.values()): tseg['width'] = value
+ for tseg in list(self._collapsed_trees.values()): tseg['width'] = value
+ elif attr == 'roof_color':
+ self._roof_color = value
+ for tseg in list(self._collapsed_trees.values()): tseg['color'] = value
+ elif attr == 'roof_fill':
+ self._roof_fill = value
+ for tseg in list(self._collapsed_trees.values()): tseg['fill'] = value
+ elif attr == 'shapeable':
+ self._shapeable = value
+ for tseg in list(self._expanded_trees.values()):
+ tseg['draggable'] = value
+ for tseg in list(self._collapsed_trees.values()):
+ tseg['draggable'] = value
+ for leaf in self._leaves: leaf['draggable'] = value
+ elif attr == 'xspace':
+ self._xspace = value
+ for tseg in list(self._expanded_trees.values()):
+ tseg['xspace'] = value
+ for tseg in list(self._collapsed_trees.values()):
+ tseg['xspace'] = value
+ self.manage()
+ elif attr == 'yspace':
+ self._yspace = value
+ for tseg in list(self._expanded_trees.values()):
+ tseg['yspace'] = value
+ for tseg in list(self._collapsed_trees.values()):
+ tseg['yspace'] = value
+ self.manage()
+ elif attr == 'orientation':
+ self._orientation = value
+ for tseg in list(self._expanded_trees.values()):
+ tseg['orientation'] = value
+ for tseg in list(self._collapsed_trees.values()):
+ tseg['orientation'] = value
+ self.manage()
+ elif attr == 'ordered':
+ self._ordered = value
+ for tseg in list(self._expanded_trees.values()):
+ tseg['ordered'] = value
+ for tseg in list(self._collapsed_trees.values()):
+ tseg['ordered'] = value
+ else: CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr[:5] == 'node_':
+ return self._nodeattribs.get(attr[5:], None)
+ elif attr[:5] == 'leaf_':
+ return self._leafattribs.get(attr[5:], None)
+ elif attr[:4] == 'loc_':
+ return self._locattribs.get(attr[4:], None)
+ elif attr == 'line_color': return self._line_color
+ elif attr == 'line_width': return self._line_width
+ elif attr == 'roof_color': return self._roof_color
+ elif attr == 'roof_fill': return self._roof_fill
+ elif attr == 'shapeable': return self._shapeable
+ elif attr == 'xspace': return self._xspace
+ elif attr == 'yspace': return self._yspace
+ elif attr == 'orientation': return self._orientation
+ else: return CanvasWidget.__getitem__(self, attr)
+
+ def _tags(self): return []
+
+ def _manage(self):
+ segs = list(self._expanded_trees.values()) + list(self._collapsed_trees.values())
+ for tseg in segs:
+ if tseg.hidden():
+ tseg.show()
+ tseg.manage()
+ tseg.hide()
+
+ def toggle_collapsed(self, treeseg):
+ """
+ Collapse/expand a tree.
+ """
+ old_treeseg = treeseg
+ if old_treeseg['roof']:
+ new_treeseg = self._expanded_trees[self._keys[old_treeseg]]
+ else:
+ new_treeseg = self._collapsed_trees[self._keys[old_treeseg]]
+
+ # Replace the old tree with the new tree.
+ if old_treeseg.parent() is self:
+ self._remove_child_widget(old_treeseg)
+ self._add_child_widget(new_treeseg)
+ self._treeseg = new_treeseg
+ else:
+ old_treeseg.parent().replace_child(old_treeseg, new_treeseg)
+
+ # Move the new tree to where the old tree was. Show it first,
+ # so we can find its bounding box.
+ new_treeseg.show()
+ (newx, newy) = new_treeseg.label().bbox()[:2]
+ (oldx, oldy) = old_treeseg.label().bbox()[:2]
+ new_treeseg.move(oldx-newx, oldy-newy)
+
+ # Hide the old tree
+ old_treeseg.hide()
+
+ # We could do parent.manage() here instead, if we wanted.
+ new_treeseg.parent().update(new_treeseg)
+
+##//////////////////////////////////////////////////////
+## draw_trees
+##//////////////////////////////////////////////////////
+
+class TreeView(object):
+ def __init__(self, *trees):
+ from math import sqrt, ceil
+
+ self._trees = trees
+
+ self._top = Tk()
+ self._top.title('NLTK')
+ self._top.bind('<Control-x>', self.destroy)
+ self._top.bind('<Control-q>', self.destroy)
+
+ cf = self._cframe = CanvasFrame(self._top)
+ self._top.bind('<Control-p>', self._cframe.print_to_file)
+
+ # Size is variable.
+ self._size = IntVar(self._top)
+ self._size.set(12)
+ bold = ('helvetica', -self._size.get(), 'bold')
+ helv = ('helvetica', -self._size.get())
+
+ # Lay the trees out in a square.
+ self._width = int(ceil(sqrt(len(trees))))
+ self._widgets = []
+ for i in range(len(trees)):
+ widget = TreeWidget(cf.canvas(), trees[i], node_font=bold,
+ leaf_color='#008040', node_color='#004080',
+ roof_color='#004040', roof_fill='white',
+ line_color='#004040', draggable=1,
+ leaf_font=helv)
+ widget.bind_click_trees(widget.toggle_collapsed)
+ self._widgets.append(widget)
+ cf.add_widget(widget, 0, 0)
+
+ self._layout()
+ self._cframe.pack(expand=1, fill='both')
+ self._init_menubar()
+
+ def _layout(self):
+ i = x = y = ymax = 0
+ width = self._width
+ for i in range(len(self._widgets)):
+ widget = self._widgets[i]
+ (oldx, oldy) = widget.bbox()[:2]
+ if i % width == 0:
+ y = ymax
+ x = 0
+ widget.move(x-oldx, y-oldy)
+ x = widget.bbox()[2] + 10
+ ymax = max(ymax, widget.bbox()[3] + 10)
+
+ def _init_menubar(self):
+ menubar = Menu(self._top)
+
+ filemenu = Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Print to Postscript', underline=0,
+ command=self._cframe.print_to_file,
+ accelerator='Ctrl-p')
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-x')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ zoommenu = Menu(menubar, tearoff=0)
+ zoommenu.add_radiobutton(label='Tiny', variable=self._size,
+ underline=0, value=10, command=self.resize)
+ zoommenu.add_radiobutton(label='Small', variable=self._size,
+ underline=0, value=12, command=self.resize)
+ zoommenu.add_radiobutton(label='Medium', variable=self._size,
+ underline=0, value=14, command=self.resize)
+ zoommenu.add_radiobutton(label='Large', variable=self._size,
+ underline=0, value=28, command=self.resize)
+ zoommenu.add_radiobutton(label='Huge', variable=self._size,
+ underline=0, value=50, command=self.resize)
+ menubar.add_cascade(label='Zoom', underline=0, menu=zoommenu)
+
+ self._top.config(menu=menubar)
+
+ def resize(self, *e):
+ bold = ('helvetica', -self._size.get(), 'bold')
+ helv = ('helvetica', -self._size.get())
+ xspace = self._size.get()
+ yspace = self._size.get()
+ for widget in self._widgets:
+ widget['node_font'] = bold
+ widget['leaf_font'] = helv
+ widget['xspace'] = xspace
+ widget['yspace'] = yspace
+ if self._size.get() < 20: widget['line_width'] = 1
+ elif self._size.get() < 30: widget['line_width'] = 2
+ else: widget['line_width'] = 3
+ self._layout()
+
+ def destroy(self, *e):
+ if self._top is None: return
+ self._top.destroy()
+ self._top = None
+
+ def mainloop(self, *args, **kwargs):
+ """
+ Enter the Tkinter mainloop. This function must be called if
+ this demo is created from a non-interactive program (e.g.
+ from a secript); otherwise, the demo will close as soon as
+ the script completes.
+ """
+ if in_idle(): return
+ self._top.mainloop(*args, **kwargs)
+
+def draw_trees(*trees):
+ """
+ Open a new window containing a graphical diagram of the given
+ trees.
+
+ :rtype: None
+ """
+ TreeView(*trees).mainloop()
+ return
+
+##//////////////////////////////////////////////////////
+## Demo Code
+##//////////////////////////////////////////////////////
+
+def demo():
+ import random
+ def fill(cw):
+ cw['fill'] = '#%06d' % random.randint(0,999999)
+
+ cf = CanvasFrame(width=550, height=450, closeenough=2)
+
+ t = Tree('''
+ (S (NP the very big cat)
+ (VP (Adv sorta) (V saw) (NP (Det the) (N dog))))''')
+
+ tc = TreeWidget(cf.canvas(), t, draggable=1,
+ node_font=('helvetica', -14, 'bold'),
+ leaf_font=('helvetica', -12, 'italic'),
+ roof_fill='white', roof_color='black',
+ leaf_color='green4', node_color='blue2')
+ cf.add_widget(tc,10,10)
+
+ def boxit(canvas, text):
+ big = ('helvetica', -16, 'bold')
+ return BoxWidget(canvas, TextWidget(canvas, text,
+ font=big), fill='green')
+ def ovalit(canvas, text):
+ return OvalWidget(canvas, TextWidget(canvas, text),
+ fill='cyan')
+
+ treetok = Tree('(S (NP this tree) (VP (V is) (AdjP shapeable)))')
+ tc2 = TreeWidget(cf.canvas(), treetok, boxit, ovalit, shapeable=1)
+
+ def color(node):
+ node['color'] = '#%04d00' % random.randint(0,9999)
+ def color2(treeseg):
+ treeseg.label()['fill'] = '#%06d' % random.randint(0,9999)
+ treeseg.label().child()['color'] = 'white'
+
+ tc.bind_click_trees(tc.toggle_collapsed)
+ tc2.bind_click_trees(tc2.toggle_collapsed)
+ tc.bind_click_nodes(color, 3)
+ tc2.expanded_tree(1).bind_click(color2, 3)
+ tc2.expanded_tree().bind_click(color2, 3)
+
+ paren = ParenWidget(cf.canvas(), tc2)
+ cf.add_widget(paren, tc.bbox()[2]+10, 10)
+
+ tree3 = Tree('''
+ (S (NP this tree) (AUX was)
+ (VP (V built) (PP (P with) (NP (N tree_to_treesegment)))))''')
+ tc3 = tree_to_treesegment(cf.canvas(), tree3, tree_color='green4',
+ tree_xspace=2, tree_width=2)
+ tc3['draggable'] = 1
+ cf.add_widget(tc3, 10, tc.bbox()[3]+10)
+
+ def orientswitch(treewidget):
+ if treewidget['orientation'] == 'horizontal':
+ treewidget.expanded_tree(1,1).subtrees()[0].set_text('vertical')
+ treewidget.collapsed_tree(1,1).subtrees()[0].set_text('vertical')
+ treewidget.collapsed_tree(1).subtrees()[1].set_text('vertical')
+ treewidget.collapsed_tree().subtrees()[3].set_text('vertical')
+ treewidget['orientation'] = 'vertical'
+ else:
+ treewidget.expanded_tree(1,1).subtrees()[0].set_text('horizontal')
+ treewidget.collapsed_tree(1,1).subtrees()[0].set_text('horizontal')
+ treewidget.collapsed_tree(1).subtrees()[1].set_text('horizontal')
+ treewidget.collapsed_tree().subtrees()[3].set_text('horizontal')
+ treewidget['orientation'] = 'horizontal'
+
+ text = """
+Try clicking, right clicking, and dragging
+different elements of each of the trees.
+The top-left tree is a TreeWidget built from
+a Tree. The top-right is a TreeWidget built
+from a Tree, using non-default widget
+constructors for the nodes & leaves (BoxWidget
+and OvalWidget). The bottom-left tree is
+built from tree_to_treesegment."""
+ twidget = TextWidget(cf.canvas(), text.strip())
+ textbox = BoxWidget(cf.canvas(), twidget, fill='white', draggable=1)
+ cf.add_widget(textbox, tc3.bbox()[2]+10, tc2.bbox()[3]+10)
+
+ tree4 = Tree('(S (NP this tree) (VP (V is) (Adj horizontal)))')
+ tc4 = TreeWidget(cf.canvas(), tree4, draggable=1,
+ line_color='brown2', roof_color='brown2',
+ node_font=('helvetica', -12, 'bold'),
+ node_color='brown4', orientation='horizontal')
+ tc4.manage()
+ cf.add_widget(tc4, tc3.bbox()[2]+10, textbox.bbox()[3]+10)
+ tc4.bind_click(orientswitch)
+ tc4.bind_click_trees(tc4.toggle_collapsed, 3)
+
+ # Run mainloop
+ cf.mainloop()
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/draw/util.py b/nltk/draw/util.py
new file mode 100644
index 0000000..760f56d
--- /dev/null
+++ b/nltk/draw/util.py
@@ -0,0 +1,2356 @@
+# Natural Language Toolkit: Drawing utilities
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Tools for graphically displaying and interacting with the objects and
+processing classes defined by the Toolkit. These tools are primarily
+intended to help students visualize the objects that they create.
+
+The graphical tools are typically built using "canvas widgets", each
+of which encapsulates the graphical elements and bindings used to
+display a complex object on a Tkinter ``Canvas``. For example, NLTK
+defines canvas widgets for displaying trees and directed graphs, as
+well as a number of simpler widgets. These canvas widgets make it
+easier to build new graphical tools and demos. See the class
+documentation for ``CanvasWidget`` for more information.
+
+The ``nltk.draw`` module defines the abstract ``CanvasWidget`` base
+class, and a number of simple canvas widgets. The remaining canvas
+widgets are defined by submodules, such as ``nltk.draw.tree``.
+
+The ``nltk.draw`` module also defines ``CanvasFrame``, which
+encapsulates a ``Canvas`` and its scrollbars. It uses a
+``ScrollWatcherWidget`` to ensure that all canvas widgets contained on
+its canvas are within the scroll region.
+
+Acknowledgements: Many of the ideas behind the canvas widget system
+are derived from ``CLIG``, a Tk-based grapher for linguistic data
+structures. For more information, see the CLIG
+homepage (http://www.ags.uni-sb.de/~konrad/clig.html).
+
+"""
+
+
+import nltk.compat
+from tkinter import (Button, Canvas, Entry, Frame, Label, Menu, Menubutton,
+ RAISED, Scrollbar, StringVar, Text, Tk, Toplevel, Widget)
+
+import tkinter.font, tkinter.messagebox, tkinter.filedialog
+
+from nltk.util import in_idle
+
+##//////////////////////////////////////////////////////
+## CanvasWidget
+##//////////////////////////////////////////////////////
+
+class CanvasWidget(object):
+ """
+ A collection of graphical elements and bindings used to display a
+ complex object on a Tkinter ``Canvas``. A canvas widget is
+ responsible for managing the ``Canvas`` tags and callback bindings
+ necessary to display and interact with the object. Canvas widgets
+ are often organized into hierarchies, where parent canvas widgets
+ control aspects of their child widgets.
+
+ Each canvas widget is bound to a single ``Canvas``. This ``Canvas``
+ is specified as the first argument to the ``CanvasWidget``'s
+ constructor.
+
+ Attributes. Each canvas widget can support a variety of
+ "attributes", which control how the canvas widget is displayed.
+ Some typical examples attributes are ``color``, ``font``, and
+ ``radius``. Each attribute has a default value. This default
+ value can be overridden in the constructor, using keyword
+ arguments of the form ``attribute=value``:
+
+ >>> from nltk.draw.util import TextWidget
+ >>> cn = TextWidget(c, 'test', color='red')
+
+ Attribute values can also be changed after a canvas widget has
+ been constructed, using the ``__setitem__`` operator:
+
+ >>> cn['font'] = 'times'
+
+ The current value of an attribute value can be queried using the
+ ``__getitem__`` operator:
+
+ >>> cn['color']
+ red
+
+ For a list of the attributes supported by a type of canvas widget,
+ see its class documentation.
+
+ Interaction. The attribute ``'draggable'`` controls whether the
+ user can drag a canvas widget around the canvas. By default,
+ canvas widgets are not draggable.
+
+ ``CanvasWidget`` provides callback support for two types of user
+ interaction: clicking and dragging. The method ``bind_click``
+ registers a callback function that is called whenever the canvas
+ widget is clicked. The method ``bind_drag`` registers a callback
+ function that is called after the canvas widget is dragged. If
+ the user clicks or drags a canvas widget with no registered
+ callback function, then the interaction event will propagate to
+ its parent. For each canvas widget, only one callback function
+ may be registered for an interaction event. Callback functions
+ can be deregistered with the ``unbind_click`` and ``unbind_drag``
+ methods.
+
+ Subclassing. ``CanvasWidget`` is an abstract class. Subclasses
+ are required to implement the following methods:
+
+ - ``__init__``: Builds a new canvas widget. It must perform the
+ following three tasks (in order):
+ - Create any new graphical elements.
+ - Call ``_add_child_widget`` on each child widget.
+ - Call the ``CanvasWidget`` constructor.
+ - ``_tags``: Returns a list of the canvas tags for all graphical
+ elements managed by this canvas widget, not including
+ graphical elements managed by its child widgets.
+ - ``_manage``: Arranges the child widgets of this canvas widget.
+ This is typically only called when the canvas widget is
+ created.
+ - ``_update``: Update this canvas widget in response to a
+ change in a single child.
+
+ For a ``CanvasWidget`` with no child widgets, the default
+ definitions for ``_manage`` and ``_update`` may be used.
+
+ If a subclass defines any attributes, then it should implement
+ ``__getitem__`` and ``__setitem__``. If either of these methods is
+ called with an unknown attribute, then they should propagate the
+ request to ``CanvasWidget``.
+
+ Most subclasses implement a number of additional methods that
+ modify the ``CanvasWidget`` in some way. These methods must call
+ ``parent.update(self)`` after making any changes to the canvas
+ widget's graphical elements. The canvas widget must also call
+ ``parent.update(self)`` after changing any attribute value that
+ affects the shape or position of the canvas widget's graphical
+ elements.
+
+ :type __canvas: Tkinter.Canvas
+ :ivar __canvas: This ``CanvasWidget``'s canvas.
+
+ :type __parent: CanvasWidget or None
+ :ivar __parent: This ``CanvasWidget``'s hierarchical parent widget.
+ :type __children: list(CanvasWidget)
+ :ivar __children: This ``CanvasWidget``'s hierarchical child widgets.
+
+ :type __updating: bool
+ :ivar __updating: Is this canvas widget currently performing an
+ update? If it is, then it will ignore any new update requests
+ from child widgets.
+
+ :type __draggable: bool
+ :ivar __draggable: Is this canvas widget draggable?
+ :type __press: event
+ :ivar __press: The ButtonPress event that we're currently handling.
+ :type __drag_x: int
+ :ivar __drag_x: Where it's been moved to (to find dx)
+ :type __drag_y: int
+ :ivar __drag_y: Where it's been moved to (to find dy)
+ :type __callbacks: dictionary
+ :ivar __callbacks: Registered callbacks. Currently, four keys are
+ used: ``1``, ``2``, ``3``, and ``'drag'``. The values are
+ callback functions. Each callback function takes a single
+ argument, which is the ``CanvasWidget`` that triggered the
+ callback.
+ """
+ def __init__(self, canvas, parent=None, **attribs):
+ """
+ Create a new canvas widget. This constructor should only be
+ called by subclass constructors; and it should be called only
+ "after" the subclass has constructed all graphical canvas
+ objects and registered all child widgets.
+
+ :param canvas: This canvas widget's canvas.
+ :type canvas: Tkinter.Canvas
+ :param parent: This canvas widget's hierarchical parent.
+ :type parent: CanvasWidget
+ :param attribs: The new canvas widget's attributes.
+ """
+ if self.__class__ == CanvasWidget:
+ raise TypeError('CanvasWidget is an abstract base class')
+
+ if not isinstance(canvas, Canvas):
+ raise TypeError('Expected a canvas!')
+
+ self.__canvas = canvas
+ self.__parent = parent
+
+ # If the subclass constructor called _add_child_widget, then
+ # self.__children will already exist.
+ if not hasattr(self, '_CanvasWidget__children'): self.__children = []
+
+ # Is this widget hidden?
+ self.__hidden = 0
+
+ # Update control (prevents infinite loops)
+ self.__updating = 0
+
+ # Button-press and drag callback handling.
+ self.__press = None
+ self.__drag_x = self.__drag_y = 0
+ self.__callbacks = {}
+ self.__draggable = 0
+
+ # Set up attributes.
+ for (attr, value) in list(attribs.items()): self[attr] = value
+
+ # Manage this canvas widget
+ self._manage()
+
+ # Register any new bindings
+ for tag in self._tags():
+ self.__canvas.tag_bind(tag, '<ButtonPress-1>',
+ self.__press_cb)
+ self.__canvas.tag_bind(tag, '<ButtonPress-2>',
+ self.__press_cb)
+ self.__canvas.tag_bind(tag, '<ButtonPress-3>',
+ self.__press_cb)
+
+ ##//////////////////////////////////////////////////////
+ ## Inherited methods.
+ ##//////////////////////////////////////////////////////
+
+ def bbox(self):
+ """
+ :return: A bounding box for this ``CanvasWidget``. The bounding
+ box is a tuple of four coordinates, *(xmin, ymin, xmax, ymax)*,
+ for a rectangle which encloses all of the canvas
+ widget's graphical elements. Bounding box coordinates are
+ specified with respect to the coordinate space of the ``Canvas``.
+ :rtype: tuple(int, int, int, int)
+ """
+ if self.__hidden: return (0,0,0,0)
+ if len(self.tags()) == 0: raise ValueError('No tags')
+ return self.__canvas.bbox(*self.tags())
+
+ def width(self):
+ """
+ :return: The width of this canvas widget's bounding box, in
+ its ``Canvas``'s coordinate space.
+ :rtype: int
+ """
+ if len(self.tags()) == 0: raise ValueError('No tags')
+ bbox = self.__canvas.bbox(*self.tags())
+ return bbox[2]-bbox[0]
+
+ def height(self):
+ """
+ :return: The height of this canvas widget's bounding box, in
+ its ``Canvas``'s coordinate space.
+ :rtype: int
+ """
+ if len(self.tags()) == 0: raise ValueError('No tags')
+ bbox = self.__canvas.bbox(*self.tags())
+ return bbox[3]-bbox[1]
+
+ def parent(self):
+ """
+ :return: The hierarchical parent of this canvas widget.
+ ``self`` is considered a subpart of its parent for
+ purposes of user interaction.
+ :rtype: CanvasWidget or None
+ """
+ return self.__parent
+
+ def child_widgets(self):
+ """
+ :return: A list of the hierarchical children of this canvas
+ widget. These children are considered part of ``self``
+ for purposes of user interaction.
+ :rtype: list of CanvasWidget
+ """
+ return self.__children
+
+ def canvas(self):
+ """
+ :return: The canvas that this canvas widget is bound to.
+ :rtype: Tkinter.Canvas
+ """
+ return self.__canvas
+
+ def move(self, dx, dy):
+ """
+ Move this canvas widget by a given distance. In particular,
+ shift the canvas widget right by ``dx`` pixels, and down by
+ ``dy`` pixels. Both ``dx`` and ``dy`` may be negative, resulting
+ in leftward or upward movement.
+
+ :type dx: int
+ :param dx: The number of pixels to move this canvas widget
+ rightwards.
+ :type dy: int
+ :param dy: The number of pixels to move this canvas widget
+ downwards.
+ :rtype: None
+ """
+ if dx == dy == 0: return
+ for tag in self.tags():
+ self.__canvas.move(tag, dx, dy)
+ if self.__parent: self.__parent.update(self)
+
+ def moveto(self, x, y, anchor='NW'):
+ """
+ Move this canvas widget to the given location. In particular,
+ shift the canvas widget such that the corner or side of the
+ bounding box specified by ``anchor`` is at location (``x``,
+ ``y``).
+
+ :param x,y: The location that the canvas widget should be moved
+ to.
+ :param anchor: The corner or side of the canvas widget that
+ should be moved to the specified location. ``'N'``
+ specifies the top center; ``'NE'`` specifies the top right
+ corner; etc.
+ """
+ x1,y1,x2,y2 = self.bbox()
+ if anchor == 'NW': self.move(x-x1, y-y1)
+ if anchor == 'N': self.move(x-x1/2-x2/2, y-y1)
+ if anchor == 'NE': self.move(x-x2, y-y1)
+ if anchor == 'E': self.move(x-x2, y-y1/2-y2/2)
+ if anchor == 'SE': self.move(x-x2, y-y2)
+ if anchor == 'S': self.move(x-x1/2-x2/2, y-y2)
+ if anchor == 'SW': self.move(x-x1, y-y2)
+ if anchor == 'W': self.move(x-x1, y-y1/2-y2/2)
+
+ def destroy(self):
+ """
+ Remove this ``CanvasWidget`` from its ``Canvas``. After a
+ ``CanvasWidget`` has been destroyed, it should not be accessed.
+
+ Note that you only need to destroy a top-level
+ ``CanvasWidget``; its child widgets will be destroyed
+ automatically. If you destroy a non-top-level
+ ``CanvasWidget``, then the entire top-level widget will be
+ destroyed.
+
+ :raise ValueError: if this ``CanvasWidget`` has a parent.
+ :rtype: None
+ """
+ if self.__parent is not None:
+ self.__parent.destroy()
+ return
+
+ for tag in self.tags():
+ self.__canvas.tag_unbind(tag, '<ButtonPress-1>')
+ self.__canvas.tag_unbind(tag, '<ButtonPress-2>')
+ self.__canvas.tag_unbind(tag, '<ButtonPress-3>')
+ self.__canvas.delete(*self.tags())
+ self.__canvas = None
+
+ def update(self, child):
+ """
+ Update the graphical display of this canvas widget, and all of
+ its ancestors, in response to a change in one of this canvas
+ widget's children.
+
+ :param child: The child widget that changed.
+ :type child: CanvasWidget
+ """
+ if self.__hidden or child.__hidden: return
+ # If we're already updating, then do nothing. This prevents
+ # infinite loops when _update modifies its children.
+ if self.__updating: return
+ self.__updating = 1
+
+ # Update this CanvasWidget.
+ self._update(child)
+
+ # Propagate update request to the parent.
+ if self.__parent: self.__parent.update(self)
+
+ # We're done updating.
+ self.__updating = 0
+
+ def manage(self):
+ """
+ Arrange this canvas widget and all of its descendants.
+
+ :rtype: None
+ """
+ if self.__hidden: return
+ for child in self.__children: child.manage()
+ self._manage()
+
+ def tags(self):
+ """
+ :return: a list of the canvas tags for all graphical
+ elements managed by this canvas widget, including
+ graphical elements managed by its child widgets.
+ :rtype: list of int
+ """
+ if self.__canvas is None:
+ raise ValueError('Attempt to access a destroyed canvas widget')
+ tags = []
+ tags += self._tags()
+ for child in self.__children:
+ tags += child.tags()
+ return tags
+
+ def __setitem__(self, attr, value):
+ """
+ Set the value of the attribute ``attr`` to ``value``. See the
+ class documentation for a list of attributes supported by this
+ canvas widget.
+
+ :rtype: None
+ """
+ if attr == 'draggable':
+ self.__draggable = value
+ else:
+ raise ValueError('Unknown attribute %r' % attr)
+
+ def __getitem__(self, attr):
+ """
+ :return: the value of the attribute ``attr``. See the class
+ documentation for a list of attributes supported by this
+ canvas widget.
+ :rtype: (any)
+ """
+ if attr == 'draggable':
+ return self.__draggable
+ else:
+ raise ValueError('Unknown attribute %r' % attr)
+
+ def __repr__(self):
+ """
+ :return: a string representation of this canvas widget.
+ :rtype: str
+ """
+ return '<%s>' % self.__class__.__name__
+
+ def hide(self):
+ """
+ Temporarily hide this canvas widget.
+
+ :rtype: None
+ """
+ self.__hidden = 1
+ for tag in self.tags():
+ self.__canvas.itemconfig(tag, state='hidden')
+
+ def show(self):
+ """
+ Show a hidden canvas widget.
+
+ :rtype: None
+ """
+ self.__hidden = 0
+ for tag in self.tags():
+ self.__canvas.itemconfig(tag, state='normal')
+
+ def hidden(self):
+ """
+ :return: True if this canvas widget is hidden.
+ :rtype: bool
+ """
+ return self.__hidden
+
+ ##//////////////////////////////////////////////////////
+ ## Callback interface
+ ##//////////////////////////////////////////////////////
+
+ def bind_click(self, callback, button=1):
+ """
+ Register a new callback that will be called whenever this
+ ``CanvasWidget`` is clicked on.
+
+ :type callback: function
+ :param callback: The callback function that will be called
+ whenever this ``CanvasWidget`` is clicked. This function
+ will be called with this ``CanvasWidget`` as its argument.
+ :type button: int
+ :param button: Which button the user should use to click on
+ this ``CanvasWidget``. Typically, this should be 1 (left
+ button), 3 (right button), or 2 (middle button).
+ """
+ self.__callbacks[button] = callback
+
+ def bind_drag(self, callback):
+ """
+ Register a new callback that will be called after this
+ ``CanvasWidget`` is dragged. This implicitly makes this
+ ``CanvasWidget`` draggable.
+
+ :type callback: function
+ :param callback: The callback function that will be called
+ whenever this ``CanvasWidget`` is clicked. This function
+ will be called with this ``CanvasWidget`` as its argument.
+ """
+ self.__draggable = 1
+ self.__callbacks['drag'] = callback
+
+ def unbind_click(self, button=1):
+ """
+ Remove a callback that was registered with ``bind_click``.
+
+ :type button: int
+ :param button: Which button the user should use to click on
+ this ``CanvasWidget``. Typically, this should be 1 (left
+ button), 3 (right button), or 2 (middle button).
+ """
+ try: del self.__callbacks[button]
+ except: pass
+
+ def unbind_drag(self):
+ """
+ Remove a callback that was registered with ``bind_drag``.
+ """
+ try: del self.__callbacks['drag']
+ except: pass
+
+ ##//////////////////////////////////////////////////////
+ ## Callback internals
+ ##//////////////////////////////////////////////////////
+
+ def __press_cb(self, event):
+ """
+ Handle a button-press event:
+ - record the button press event in ``self.__press``
+ - register a button-release callback.
+ - if this CanvasWidget or any of its ancestors are
+ draggable, then register the appropriate motion callback.
+ """
+ # If we're already waiting for a button release, then ignore
+ # this new button press.
+ if (self.__canvas.bind('<ButtonRelease-1>') or
+ self.__canvas.bind('<ButtonRelease-2>') or
+ self.__canvas.bind('<ButtonRelease-3>')):
+ return
+
+ # Unbind motion (just in case; this shouldn't be necessary)
+ self.__canvas.unbind('<Motion>')
+
+ # Record the button press event.
+ self.__press = event
+
+ # If any ancestor is draggable, set up a motion callback.
+ # (Only if they pressed button number 1)
+ if event.num == 1:
+ widget = self
+ while widget is not None:
+ if widget['draggable']:
+ widget.__start_drag(event)
+ break
+ widget = widget.parent()
+
+ # Set up the button release callback.
+ self.__canvas.bind('<ButtonRelease-%d>' % event.num,
+ self.__release_cb)
+
+ def __start_drag(self, event):
+ """
+ Begin dragging this object:
+ - register a motion callback
+ - record the drag coordinates
+ """
+ self.__canvas.bind('<Motion>', self.__motion_cb)
+ self.__drag_x = event.x
+ self.__drag_y = event.y
+
+ def __motion_cb(self, event):
+ """
+ Handle a motion event:
+ - move this object to the new location
+ - record the new drag coordinates
+ """
+ self.move(event.x-self.__drag_x, event.y-self.__drag_y)
+ self.__drag_x = event.x
+ self.__drag_y = event.y
+
+ def __release_cb(self, event):
+ """
+ Handle a release callback:
+ - unregister motion & button release callbacks.
+ - decide whether they clicked, dragged, or cancelled
+ - call the appropriate handler.
+ """
+ # Unbind the button release & motion callbacks.
+ self.__canvas.unbind('<ButtonRelease-%d>' % event.num)
+ self.__canvas.unbind('<Motion>')
+
+ # Is it a click or a drag?
+ if (event.time - self.__press.time < 100 and
+ abs(event.x-self.__press.x) + abs(event.y-self.__press.y) < 5):
+ # Move it back, if we were dragging.
+ if self.__draggable and event.num == 1:
+ self.move(self.__press.x - self.__drag_x,
+ self.__press.y - self.__drag_y)
+ self.__click(event.num)
+ elif event.num == 1:
+ self.__drag()
+
+ self.__press = None
+
+ def __drag(self):
+ """
+ If this ``CanvasWidget`` has a drag callback, then call it;
+ otherwise, find the closest ancestor with a drag callback, and
+ call it. If no ancestors have a drag callback, do nothing.
+ """
+ if self.__draggable:
+ if 'drag' in self.__callbacks:
+ cb = self.__callbacks['drag']
+ try:
+ cb(self)
+ except:
+ print('Error in drag callback for %r' % self)
+ elif self.__parent is not None:
+ self.__parent.__drag()
+
+ def __click(self, button):
+ """
+ If this ``CanvasWidget`` has a drag callback, then call it;
+ otherwise, find the closest ancestor with a click callback, and
+ call it. If no ancestors have a click callback, do nothing.
+ """
+ if button in self.__callbacks:
+ cb = self.__callbacks[button]
+ #try:
+ cb(self)
+ #except:
+ # print 'Error in click callback for %r' % self
+ # raise
+ elif self.__parent is not None:
+ self.__parent.__click(button)
+
+ ##//////////////////////////////////////////////////////
+ ## Child/parent Handling
+ ##//////////////////////////////////////////////////////
+
+ def _add_child_widget(self, child):
+ """
+ Register a hierarchical child widget. The child will be
+ considered part of this canvas widget for purposes of user
+ interaction. ``_add_child_widget`` has two direct effects:
+ - It sets ``child``'s parent to this canvas widget.
+ - It adds ``child`` to the list of canvas widgets returned by
+ the ``child_widgets`` member function.
+
+ :param child: The new child widget. ``child`` must not already
+ have a parent.
+ :type child: CanvasWidget
+ """
+ if not hasattr(self, '_CanvasWidget__children'): self.__children = []
+ if child.__parent is not None:
+ raise ValueError('%s already has a parent', child)
+ child.__parent = self
+ self.__children.append(child)
+
+ def _remove_child_widget(self, child):
+ """
+ Remove a hierarchical child widget. This child will no longer
+ be considered part of this canvas widget for purposes of user
+ interaction. ``_add_child_widget`` has two direct effects:
+ - It sets ``child``'s parent to None.
+ - It removes ``child`` from the list of canvas widgets
+ returned by the ``child_widgets`` member function.
+
+ :param child: The child widget to remove. ``child`` must be a
+ child of this canvas widget.
+ :type child: CanvasWidget
+ """
+ self.__children.remove(child)
+ child.__parent = None
+
+ ##//////////////////////////////////////////////////////
+ ## Defined by subclass
+ ##//////////////////////////////////////////////////////
+
+ def _tags(self):
+ """
+ :return: a list of canvas tags for all graphical elements
+ managed by this canvas widget, not including graphical
+ elements managed by its child widgets.
+ :rtype: list of int
+ """
+ raise NotImplementedError()
+
+ def _manage(self):
+ """
+ Arrange the child widgets of this canvas widget. This method
+ is called when the canvas widget is initially created. It is
+ also called if the user calls the ``manage`` method on this
+ canvas widget or any of its ancestors.
+
+ :rtype: None
+ """
+ pass
+
+ def _update(self, child):
+ """
+ Update this canvas widget in response to a change in one of
+ its children.
+
+ :param child: The child that changed.
+ :type child: CanvasWidget
+ :rtype: None
+ """
+ pass
+
+##//////////////////////////////////////////////////////
+## Basic widgets.
+##//////////////////////////////////////////////////////
+
+class TextWidget(CanvasWidget):
+ """
+ A canvas widget that displays a single string of text.
+
+ Attributes:
+ - ``color``: the color of the text.
+ - ``font``: the font used to display the text.
+ - ``justify``: justification for multi-line texts. Valid values
+ are ``left``, ``center``, and ``right``.
+ - ``width``: the width of the text. If the text is wider than
+ this width, it will be line-wrapped at whitespace.
+ - ``draggable``: whether the text can be dragged by the user.
+ """
+ def __init__(self, canvas, text, **attribs):
+ """
+ Create a new text widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :type text: str
+ :param text: The string of text to display.
+ :param attribs: The new canvas widget's attributes.
+ """
+ self._text = text
+ self._tag = canvas.create_text(1, 1, text=text)
+ CanvasWidget.__init__(self, canvas, **attribs)
+
+ def __setitem__(self, attr, value):
+ if attr in ('color', 'font', 'justify', 'width'):
+ if attr == 'color': attr = 'fill'
+ self.canvas().itemconfig(self._tag, {attr:value})
+ else:
+ CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr == 'width':
+ return int(self.canvas().itemcget(self._tag, attr))
+ elif attr in ('color', 'font', 'justify'):
+ if attr == 'color': attr = 'fill'
+ return self.canvas().itemcget(self._tag, attr)
+ else:
+ return CanvasWidget.__getitem__(self, attr)
+
+ def _tags(self): return [self._tag]
+
+ def text(self):
+ """
+ :return: The text displayed by this text widget.
+ :rtype: str
+ """
+ return self.canvas().itemcget(self._tag, 'TEXT')
+
+ def set_text(self, text):
+ """
+ Change the text that is displayed by this text widget.
+
+ :type text: str
+ :param text: The string of text to display.
+ :rtype: None
+ """
+ self.canvas().itemconfig(self._tag, text=text)
+ if self.parent() is not None:
+ self.parent().update(self)
+
+ def __repr__(self):
+ return '[Text: %r]' % self._text
+
+class SymbolWidget(TextWidget):
+ """
+ A canvas widget that displays special symbols, such as the
+ negation sign and the exists operator. Symbols are specified by
+ name. Currently, the following symbol names are defined: ``neg``,
+ ``disj``, ``conj``, ``lambda``, ``merge``, ``forall``, ``exists``,
+ ``subseteq``, ``subset``, ``notsubset``, ``emptyset``, ``imp``,
+ ``rightarrow``, ``equal``, ``notequal``, ``epsilon``.
+
+ Attributes:
+
+ - ``color``: the color of the text.
+ - ``draggable``: whether the text can be dragged by the user.
+
+ :cvar SYMBOLS: A dictionary mapping from symbols to the character
+ in the ``symbol`` font used to render them.
+ """
+ SYMBOLS = {'neg':'\330', 'disj':'\332', 'conj': '\331',
+ 'lambda': '\154', 'merge': '\304',
+ 'forall': '\042', 'exists': '\044',
+ 'subseteq': '\315', 'subset': '\314',
+ 'notsubset': '\313', 'emptyset': '\306',
+ 'imp': '\336', 'rightarrow': chr(222), #'\256',
+ 'equal': '\75', 'notequal': '\271',
+ 'intersection': '\307', 'union': '\310',
+ 'epsilon': 'e',
+ }
+
+ def __init__(self, canvas, symbol, **attribs):
+ """
+ Create a new symbol widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :type symbol: str
+ :param symbol: The name of the symbol to display.
+ :param attribs: The new canvas widget's attributes.
+ """
+ attribs['font'] = 'symbol'
+ TextWidget.__init__(self, canvas, '', **attribs)
+ self.set_symbol(symbol)
+
+ def symbol(self):
+ """
+ :return: the name of the symbol that is displayed by this
+ symbol widget.
+ :rtype: str
+ """
+ return self._symbol
+
+ def set_symbol(self, symbol):
+ """
+ Change the symbol that is displayed by this symbol widget.
+
+ :type symbol: str
+ :param symbol: The name of the symbol to display.
+ """
+ if symbol not in SymbolWidget.SYMBOLS:
+ raise ValueError('Unknown symbol: %s' % symbol)
+ self._symbol = symbol
+ self.set_text(SymbolWidget.SYMBOLS[symbol])
+
+ def __repr__(self):
+ return '[Symbol: %r]' % self._symbol
+
+ @staticmethod
+ def symbolsheet(size=20):
+ """
+ Open a new Tkinter window that displays the entire alphabet
+ for the symbol font. This is useful for constructing the
+ ``SymbolWidget.SYMBOLS`` dictionary.
+ """
+ top = Tk()
+ def destroy(e, top=top): top.destroy()
+ top.bind('q', destroy)
+ Button(top, text='Quit', command=top.destroy).pack(side='bottom')
+ text = Text(top, font=('helvetica', -size), width=20, height=30)
+ text.pack(side='left')
+ sb=Scrollbar(top, command=text.yview)
+ text['yscrollcommand']=sb.set
+ sb.pack(side='right', fill='y')
+ text.tag_config('symbol', font=('symbol', -size))
+ for i in range(256):
+ if i in (0,10): continue # null and newline
+ for k,v in list(SymbolWidget.SYMBOLS.items()):
+ if v == chr(i):
+ text.insert('end', '%-10s\t' % k)
+ break
+ else:
+ text.insert('end', '%-10d \t' % i)
+ text.insert('end', '[%s]\n' % chr(i), 'symbol')
+ top.mainloop()
+
+
+class AbstractContainerWidget(CanvasWidget):
+ """
+ An abstract class for canvas widgets that contain a single child,
+ such as ``BoxWidget`` and ``OvalWidget``. Subclasses must define
+ a constructor, which should create any new graphical elements and
+ then call the ``AbstractCanvasContainer`` constructor. Subclasses
+ must also define the ``_update`` method and the ``_tags`` method;
+ and any subclasses that define attributes should define
+ ``__setitem__`` and ``__getitem__``.
+ """
+ def __init__(self, canvas, child, **attribs):
+ """
+ Create a new container widget. This constructor should only
+ be called by subclass constructors.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :param child: The container's child widget. ``child`` must not
+ have a parent.
+ :type child: CanvasWidget
+ :param attribs: The new canvas widget's attributes.
+ """
+ self._child = child
+ self._add_child_widget(child)
+ CanvasWidget.__init__(self, canvas, **attribs)
+
+ def _manage(self):
+ self._update(self._child)
+
+ def child(self):
+ """
+ :return: The child widget contained by this container widget.
+ :rtype: CanvasWidget
+ """
+ return self._child
+
+ def set_child(self, child):
+ """
+ Change the child widget contained by this container widget.
+
+ :param child: The new child widget. ``child`` must not have a
+ parent.
+ :type child: CanvasWidget
+ :rtype: None
+ """
+ self._remove_child_widget(self._child)
+ self._add_child_widget(child)
+ self._child = child
+ self.update(child)
+
+ def __repr__(self):
+ name = self.__class__.__name__
+ if name[-6:] == 'Widget': name = name[:-6]
+ return '[%s: %r]' % (name, self._child)
+
+class BoxWidget(AbstractContainerWidget):
+ """
+ A canvas widget that places a box around a child widget.
+
+ Attributes:
+ - ``fill``: The color used to fill the interior of the box.
+ - ``outline``: The color used to draw the outline of the box.
+ - ``width``: The width of the outline of the box.
+ - ``margin``: The number of pixels space left between the child
+ and the box.
+ - ``draggable``: whether the text can be dragged by the user.
+ """
+ def __init__(self, canvas, child, **attribs):
+ """
+ Create a new box widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :param child: The child widget. ``child`` must not have a
+ parent.
+ :type child: CanvasWidget
+ :param attribs: The new canvas widget's attributes.
+ """
+ self._child = child
+ self._margin = 1
+ self._box = canvas.create_rectangle(1,1,1,1)
+ canvas.tag_lower(self._box)
+ AbstractContainerWidget.__init__(self, canvas, child, **attribs)
+
+ def __setitem__(self, attr, value):
+ if attr == 'margin': self._margin = value
+ elif attr in ('outline', 'fill', 'width'):
+ self.canvas().itemconfig(self._box, {attr:value})
+ else:
+ CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr == 'margin': return self._margin
+ elif attr == 'width':
+ return float(self.canvas().itemcget(self._box, attr))
+ elif attr in ('outline', 'fill', 'width'):
+ return self.canvas().itemcget(self._box, attr)
+ else:
+ return CanvasWidget.__getitem__(self, attr)
+
+ def _update(self, child):
+ (x1, y1, x2, y2) = child.bbox()
+ margin = self._margin + self['width']/2
+ self.canvas().coords(self._box, x1-margin, y1-margin,
+ x2+margin, y2+margin)
+
+ def _tags(self): return [self._box]
+
+class OvalWidget(AbstractContainerWidget):
+ """
+ A canvas widget that places a oval around a child widget.
+
+ Attributes:
+ - ``fill``: The color used to fill the interior of the oval.
+ - ``outline``: The color used to draw the outline of the oval.
+ - ``width``: The width of the outline of the oval.
+ - ``margin``: The number of pixels space left between the child
+ and the oval.
+ - ``draggable``: whether the text can be dragged by the user.
+ - ``double``: If true, then a double-oval is drawn.
+ """
+ def __init__(self, canvas, child, **attribs):
+ """
+ Create a new oval widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :param child: The child widget. ``child`` must not have a
+ parent.
+ :type child: CanvasWidget
+ :param attribs: The new canvas widget's attributes.
+ """
+ self._child = child
+ self._margin = 1
+ self._oval = canvas.create_oval(1,1,1,1)
+ self._circle = attribs.pop('circle', False)
+ self._double = attribs.pop('double', False)
+ if self._double:
+ self._oval2 = canvas.create_oval(1,1,1,1)
+ else:
+ self._oval2 = None
+ canvas.tag_lower(self._oval)
+ AbstractContainerWidget.__init__(self, canvas, child, **attribs)
+
+ def __setitem__(self, attr, value):
+ c = self.canvas()
+ if attr == 'margin': self._margin = value
+ elif attr == 'double':
+ if value==True and self._oval2 is None:
+ # Copy attributes & position from self._oval.
+ x1, y1, x2, y2 = c.bbox(self._oval)
+ w = self['width']*2
+ self._oval2 = c.create_oval(x1-w, y1-w, x2+w, y2+w,
+ outline=c.itemcget(self._oval, 'outline'),
+ width=c.itemcget(self._oval, 'width'))
+ c.tag_lower(self._oval2)
+ if value==False and self._oval2 is not None:
+ c.delete(self._oval2)
+ self._oval2 = None
+ elif attr in ('outline', 'fill', 'width'):
+ c.itemconfig(self._oval, {attr:value})
+ if self._oval2 is not None and attr!='fill':
+ c.itemconfig(self._oval2, {attr:value})
+ if self._oval2 is not None and attr!='fill':
+ self.canvas().itemconfig(self._oval2, {attr:value})
+ else:
+ CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr == 'margin': return self._margin
+ elif attr == 'double': return self._double is not None
+ elif attr == 'width':
+ return float(self.canvas().itemcget(self._oval, attr))
+ elif attr in ('outline', 'fill', 'width'):
+ return self.canvas().itemcget(self._oval, attr)
+ else:
+ return CanvasWidget.__getitem__(self, attr)
+
+ # The ratio between inscribed & circumscribed ovals
+ RATIO = 1.4142135623730949
+
+ def _update(self, child):
+ R = OvalWidget.RATIO
+ (x1, y1, x2, y2) = child.bbox()
+ margin = self._margin
+
+ # If we're a circle, pretend our contents are square.
+ if self._circle:
+ dx, dy = abs(x1-x2), abs(y1-y2)
+ if dx > dy:
+ y = (y1+y2)/2
+ y1, y2 = y-dx/2, y+dx/2
+ elif dy > dx:
+ x = (x1+x2)/2
+ x1, x2 = x-dy/2, x+dy/2
+
+ # Find the four corners.
+ left = int(( x1*(1+R) + x2*(1-R) ) / 2)
+ right = left + int((x2-x1)*R)
+ top = int(( y1*(1+R) + y2*(1-R) ) / 2)
+ bot = top + int((y2-y1)*R)
+ self.canvas().coords(self._oval, left-margin, top-margin,
+ right+margin, bot+margin)
+ if self._oval2 is not None:
+ self.canvas().coords(self._oval2, left-margin+2, top-margin+2,
+ right+margin-2, bot+margin-2)
+
+ def _tags(self):
+ if self._oval2 is None:
+ return [self._oval]
+ else:
+ return [self._oval, self._oval2]
+
+class ParenWidget(AbstractContainerWidget):
+ """
+ A canvas widget that places a pair of parenthases around a child
+ widget.
+
+ Attributes:
+ - ``color``: The color used to draw the parenthases.
+ - ``width``: The width of the parenthases.
+ - ``draggable``: whether the text can be dragged by the user.
+ """
+ def __init__(self, canvas, child, **attribs):
+ """
+ Create a new parenthasis widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :param child: The child widget. ``child`` must not have a
+ parent.
+ :type child: CanvasWidget
+ :param attribs: The new canvas widget's attributes.
+ """
+ self._child = child
+ self._oparen = canvas.create_arc(1,1,1,1, style='arc',
+ start=90, extent=180)
+ self._cparen = canvas.create_arc(1,1,1,1, style='arc',
+ start=-90, extent=180)
+ AbstractContainerWidget.__init__(self, canvas, child, **attribs)
+
+ def __setitem__(self, attr, value):
+ if attr == 'color':
+ self.canvas().itemconfig(self._oparen, outline=value)
+ self.canvas().itemconfig(self._cparen, outline=value)
+ elif attr == 'width':
+ self.canvas().itemconfig(self._oparen, width=value)
+ self.canvas().itemconfig(self._cparen, width=value)
+ else:
+ CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr == 'color':
+ return self.canvas().itemcget(self._oparen, 'outline')
+ elif attr == 'width':
+ return self.canvas().itemcget(self._oparen, 'width')
+ else:
+ return CanvasWidget.__getitem__(self, attr)
+
+ def _update(self, child):
+ (x1, y1, x2, y2) = child.bbox()
+ width = max((y2-y1)/6, 4)
+ self.canvas().coords(self._oparen, x1-width, y1, x1+width, y2)
+ self.canvas().coords(self._cparen, x2-width, y1, x2+width, y2)
+
+ def _tags(self): return [self._oparen, self._cparen]
+
+class BracketWidget(AbstractContainerWidget):
+ """
+ A canvas widget that places a pair of brackets around a child
+ widget.
+
+ Attributes:
+ - ``color``: The color used to draw the brackets.
+ - ``width``: The width of the brackets.
+ - ``draggable``: whether the text can be dragged by the user.
+ """
+ def __init__(self, canvas, child, **attribs):
+ """
+ Create a new bracket widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :param child: The child widget. ``child`` must not have a
+ parent.
+ :type child: CanvasWidget
+ :param attribs: The new canvas widget's attributes.
+ """
+ self._child = child
+ self._obrack = canvas.create_line(1,1,1,1,1,1,1,1)
+ self._cbrack = canvas.create_line(1,1,1,1,1,1,1,1)
+ AbstractContainerWidget.__init__(self, canvas, child, **attribs)
+
+ def __setitem__(self, attr, value):
+ if attr == 'color':
+ self.canvas().itemconfig(self._obrack, fill=value)
+ self.canvas().itemconfig(self._cbrack, fill=value)
+ elif attr == 'width':
+ self.canvas().itemconfig(self._obrack, width=value)
+ self.canvas().itemconfig(self._cbrack, width=value)
+ else:
+ CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr == 'color':
+ return self.canvas().itemcget(self._obrack, 'outline')
+ elif attr == 'width':
+ return self.canvas().itemcget(self._obrack, 'width')
+ else:
+ return CanvasWidget.__getitem__(self, attr)
+
+ def _update(self, child):
+ (x1, y1, x2, y2) = child.bbox()
+ width = max((y2-y1)/8, 2)
+ self.canvas().coords(self._obrack, x1, y1, x1-width, y1,
+ x1-width, y2, x1, y2)
+ self.canvas().coords(self._cbrack, x2, y1, x2+width, y1,
+ x2+width, y2, x2, y2)
+
+ def _tags(self): return [self._obrack, self._cbrack]
+
+class SequenceWidget(CanvasWidget):
+ """
+ A canvas widget that keeps a list of canvas widgets in a
+ horizontal line.
+
+ Attributes:
+ - ``align``: The vertical alignment of the children. Possible
+ values are ``'top'``, ``'center'``, and ``'bottom'``. By
+ default, children are center-aligned.
+ - ``space``: The amount of horizontal space to place between
+ children. By default, one pixel of space is used.
+ - ``ordered``: If true, then keep the children in their
+ original order.
+ """
+ def __init__(self, canvas, *children, **attribs):
+ """
+ Create a new sequence widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :param children: The widgets that should be aligned
+ horizontally. Each child must not have a parent.
+ :type children: list(CanvasWidget)
+ :param attribs: The new canvas widget's attributes.
+ """
+ self._align = 'center'
+ self._space = 1
+ self._ordered = False
+ self._children = list(children)
+ for child in children: self._add_child_widget(child)
+ CanvasWidget.__init__(self, canvas, **attribs)
+
+ def __setitem__(self, attr, value):
+ if attr == 'align':
+ if value not in ('top', 'bottom', 'center'):
+ raise ValueError('Bad alignment: %r' % value)
+ self._align = value
+ elif attr == 'space': self._space = value
+ elif attr == 'ordered': self._ordered = value
+ else: CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr == 'align': return self._align
+ elif attr == 'space': return self._space
+ elif attr == 'ordered': return self._ordered
+ else: return CanvasWidget.__getitem__(self, attr)
+
+ def _tags(self): return []
+
+ def _yalign(self, top, bot):
+ if self._align == 'top': return top
+ if self._align == 'bottom': return bot
+ if self._align == 'center': return (top+bot)/2
+
+ def _update(self, child):
+ # Align all children with child.
+ (left, top, right, bot) = child.bbox()
+ y = self._yalign(top, bot)
+ for c in self._children:
+ (x1, y1, x2, y2) = c.bbox()
+ c.move(0, y-self._yalign(y1,y2))
+
+ if self._ordered and len(self._children) > 1:
+ index = self._children.index(child)
+
+ x = right + self._space
+ for i in range(index+1, len(self._children)):
+ (x1, y1, x2, y2) = self._children[i].bbox()
+ if x > x1:
+ self._children[i].move(x-x1, 0)
+ x += x2-x1 + self._space
+
+ x = left - self._space
+ for i in range(index-1, -1, -1):
+ (x1, y1, x2, y2) = self._children[i].bbox()
+ if x < x2:
+ self._children[i].move(x-x2, 0)
+ x -= x2-x1 + self._space
+
+ def _manage(self):
+ if len(self._children) == 0: return
+ child = self._children[0]
+
+ # Align all children with child.
+ (left, top, right, bot) = child.bbox()
+ y = self._yalign(top, bot)
+
+ index = self._children.index(child)
+
+ # Line up children to the right of child.
+ x = right + self._space
+ for i in range(index+1, len(self._children)):
+ (x1, y1, x2, y2) = self._children[i].bbox()
+ self._children[i].move(x-x1, y-self._yalign(y1,y2))
+ x += x2-x1 + self._space
+
+ # Line up children to the left of child.
+ x = left - self._space
+ for i in range(index-1, -1, -1):
+ (x1, y1, x2, y2) = self._children[i].bbox()
+ self._children[i].move(x-x2, y-self._yalign(y1,y2))
+ x -= x2-x1 + self._space
+
+ def __repr__(self):
+ return '[Sequence: ' + repr(self._children)[1:-1]+']'
+
+ # Provide an alias for the child_widgets() member.
+ children = CanvasWidget.child_widgets
+
+ def replace_child(self, oldchild, newchild):
+ """
+ Replace the child canvas widget ``oldchild`` with ``newchild``.
+ ``newchild`` must not have a parent. ``oldchild``'s parent will
+ be set to None.
+
+ :type oldchild: CanvasWidget
+ :param oldchild: The child canvas widget to remove.
+ :type newchild: CanvasWidget
+ :param newchild: The canvas widget that should replace
+ ``oldchild``.
+ """
+ index = self._children.index(oldchild)
+ self._children[index] = newchild
+ self._remove_child_widget(oldchild)
+ self._add_child_widget(newchild)
+ self.update(newchild)
+
+ def remove_child(self, child):
+ """
+ Remove the given child canvas widget. ``child``'s parent will
+ be set ot None.
+
+ :type child: CanvasWidget
+ :param child: The child canvas widget to remove.
+ """
+ index = self._children.index(child)
+ del self._children[index]
+ self._remove_child_widget(child)
+ if len(self._children) > 0:
+ self.update(self._children[0])
+
+ def insert_child(self, index, child):
+ """
+ Insert a child canvas widget before a given index.
+
+ :type child: CanvasWidget
+ :param child: The canvas widget that should be inserted.
+ :type index: int
+ :param index: The index where the child widget should be
+ inserted. In particular, the index of ``child`` will be
+ ``index``; and the index of any children whose indices were
+ greater than equal to ``index`` before ``child`` was
+ inserted will be incremented by one.
+ """
+ self._children.insert(index, child)
+ self._add_child_widget(child)
+
+class StackWidget(CanvasWidget):
+ """
+ A canvas widget that keeps a list of canvas widgets in a vertical
+ line.
+
+ Attributes:
+ - ``align``: The horizontal alignment of the children. Possible
+ values are ``'left'``, ``'center'``, and ``'right'``. By
+ default, children are center-aligned.
+ - ``space``: The amount of vertical space to place between
+ children. By default, one pixel of space is used.
+ - ``ordered``: If true, then keep the children in their
+ original order.
+ """
+ def __init__(self, canvas, *children, **attribs):
+ """
+ Create a new stack widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :param children: The widgets that should be aligned
+ vertically. Each child must not have a parent.
+ :type children: list(CanvasWidget)
+ :param attribs: The new canvas widget's attributes.
+ """
+ self._align = 'center'
+ self._space = 1
+ self._ordered = False
+ self._children = list(children)
+ for child in children: self._add_child_widget(child)
+ CanvasWidget.__init__(self, canvas, **attribs)
+
+ def __setitem__(self, attr, value):
+ if attr == 'align':
+ if value not in ('left', 'right', 'center'):
+ raise ValueError('Bad alignment: %r' % value)
+ self._align = value
+ elif attr == 'space': self._space = value
+ elif attr == 'ordered': self._ordered = value
+ else: CanvasWidget.__setitem__(self, attr, value)
+
+ def __getitem__(self, attr):
+ if attr == 'align': return self._align
+ elif attr == 'space': return self._space
+ elif attr == 'ordered': return self._ordered
+ else: return CanvasWidget.__getitem__(self, attr)
+
+ def _tags(self): return []
+
+ def _xalign(self, left, right):
+ if self._align == 'left': return left
+ if self._align == 'right': return right
+ if self._align == 'center': return (left+right)/2
+
+ def _update(self, child):
+ # Align all children with child.
+ (left, top, right, bot) = child.bbox()
+ x = self._xalign(left, right)
+ for c in self._children:
+ (x1, y1, x2, y2) = c.bbox()
+ c.move(x-self._xalign(x1,x2), 0)
+
+ if self._ordered and len(self._children) > 1:
+ index = self._children.index(child)
+
+ y = bot + self._space
+ for i in range(index+1, len(self._children)):
+ (x1, y1, x2, y2) = self._children[i].bbox()
+ if y > y1:
+ self._children[i].move(0, y-y1)
+ y += y2-y1 + self._space
+
+ y = top - self._space
+ for i in range(index-1, -1, -1):
+ (x1, y1, x2, y2) = self._children[i].bbox()
+ if y < y2:
+ self._children[i].move(0, y-y2)
+ y -= y2-y1 + self._space
+
+ def _manage(self):
+ if len(self._children) == 0: return
+ child = self._children[0]
+
+ # Align all children with child.
+ (left, top, right, bot) = child.bbox()
+ x = self._xalign(left, right)
+
+ index = self._children.index(child)
+
+ # Line up children below the child.
+ y = bot + self._space
+ for i in range(index+1, len(self._children)):
+ (x1, y1, x2, y2) = self._children[i].bbox()
+ self._children[i].move(x-self._xalign(x1,x2), y-y1)
+ y += y2-y1 + self._space
+
+ # Line up children above the child.
+ y = top - self._space
+ for i in range(index-1, -1, -1):
+ (x1, y1, x2, y2) = self._children[i].bbox()
+ self._children[i].move(x-self._xalign(x1,x2), y-y2)
+ y -= y2-y1 + self._space
+
+ def __repr__(self):
+ return '[Stack: ' + repr(self._children)[1:-1]+']'
+
+ # Provide an alias for the child_widgets() member.
+ children = CanvasWidget.child_widgets
+
+ def replace_child(self, oldchild, newchild):
+ """
+ Replace the child canvas widget ``oldchild`` with ``newchild``.
+ ``newchild`` must not have a parent. ``oldchild``'s parent will
+ be set to None.
+
+ :type oldchild: CanvasWidget
+ :param oldchild: The child canvas widget to remove.
+ :type newchild: CanvasWidget
+ :param newchild: The canvas widget that should replace
+ ``oldchild``.
+ """
+ index = self._children.index(oldchild)
+ self._children[index] = newchild
+ self._remove_child_widget(oldchild)
+ self._add_child_widget(newchild)
+ self.update(newchild)
+
+ def remove_child(self, child):
+ """
+ Remove the given child canvas widget. ``child``'s parent will
+ be set ot None.
+
+ :type child: CanvasWidget
+ :param child: The child canvas widget to remove.
+ """
+ index = self._children.index(child)
+ del self._children[index]
+ self._remove_child_widget(child)
+ if len(self._children) > 0:
+ self.update(self._children[0])
+
+ def insert_child(self, index, child):
+ """
+ Insert a child canvas widget before a given index.
+
+ :type child: CanvasWidget
+ :param child: The canvas widget that should be inserted.
+ :type index: int
+ :param index: The index where the child widget should be
+ inserted. In particular, the index of ``child`` will be
+ ``index``; and the index of any children whose indices were
+ greater than equal to ``index`` before ``child`` was
+ inserted will be incremented by one.
+ """
+ self._children.insert(index, child)
+ self._add_child_widget(child)
+
+class SpaceWidget(CanvasWidget):
+ """
+ A canvas widget that takes up space but does not display
+ anything. A ``SpaceWidget`` can be used to add space between
+ elements. Each space widget is characterized by a width and a
+ height. If you wish to only create horizontal space, then use a
+ height of zero; and if you wish to only create vertical space, use
+ a width of zero.
+ """
+ def __init__(self, canvas, width, height, **attribs):
+ """
+ Create a new space widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :type width: int
+ :param width: The width of the new space widget.
+ :type height: int
+ :param height: The height of the new space widget.
+ :param attribs: The new canvas widget's attributes.
+ """
+ # For some reason,
+ if width > 4: width -= 4
+ if height > 4: height -= 4
+ self._tag = canvas.create_line(1, 1, width, height, fill='')
+ CanvasWidget.__init__(self, canvas, **attribs)
+
+ # note: width() and height() are already defined by CanvasWidget.
+ def set_width(self, width):
+ """
+ Change the width of this space widget.
+
+ :param width: The new width.
+ :type width: int
+ :rtype: None
+ """
+ [x1, y1, x2, y2] = self.bbox()
+ self.canvas().coords(self._tag, x1, y1, x1+width, y2)
+
+ def set_height(self, height):
+ """
+ Change the height of this space widget.
+
+ :param height: The new height.
+ :type height: int
+ :rtype: None
+ """
+ [x1, y1, x2, y2] = self.bbox()
+ self.canvas().coords(self._tag, x1, y1, x2, y1+height)
+
+ def _tags(self): return [self._tag]
+
+ def __repr__(self): return '[Space]'
+
+class ScrollWatcherWidget(CanvasWidget):
+ """
+ A special canvas widget that adjusts its ``Canvas``'s scrollregion
+ to always include the bounding boxes of all of its children. The
+ scroll-watcher widget will only increase the size of the
+ ``Canvas``'s scrollregion; it will never decrease it.
+ """
+ def __init__(self, canvas, *children, **attribs):
+ """
+ Create a new scroll-watcher widget.
+
+ :type canvas: Tkinter.Canvas
+ :param canvas: This canvas widget's canvas.
+ :type children: list(CanvasWidget)
+ :param children: The canvas widgets watched by the
+ scroll-watcher. The scroll-watcher will ensure that these
+ canvas widgets are always contained in their canvas's
+ scrollregion.
+ :param attribs: The new canvas widget's attributes.
+ """
+ for child in children: self._add_child_widget(child)
+ CanvasWidget.__init__(self, canvas, **attribs)
+
+ def add_child(self, canvaswidget):
+ """
+ Add a new canvas widget to the scroll-watcher. The
+ scroll-watcher will ensure that the new canvas widget is
+ always contained in its canvas's scrollregion.
+
+ :param canvaswidget: The new canvas widget.
+ :type canvaswidget: CanvasWidget
+ :rtype: None
+ """
+ self._add_child_widget(canvaswidget)
+ self.update(canvaswidget)
+
+ def remove_child(self, canvaswidget):
+ """
+ Remove a canvas widget from the scroll-watcher. The
+ scroll-watcher will no longer ensure that the new canvas
+ widget is always contained in its canvas's scrollregion.
+
+ :param canvaswidget: The canvas widget to remove.
+ :type canvaswidget: CanvasWidget
+ :rtype: None
+ """
+ self._remove_child_widget(canvaswidget)
+
+ def _tags(self): return []
+
+ def _update(self, child):
+ self._adjust_scrollregion()
+
+ def _adjust_scrollregion(self):
+ """
+ Adjust the scrollregion of this scroll-watcher's ``Canvas`` to
+ include the bounding boxes of all of its children.
+ """
+ bbox = self.bbox()
+ canvas = self.canvas()
+ scrollregion = [int(n) for n in canvas['scrollregion'].split()]
+ if len(scrollregion) != 4: return
+ if (bbox[0] < scrollregion[0] or bbox[1] < scrollregion[1] or
+ bbox[2] > scrollregion[2] or bbox[3] > scrollregion[3]):
+ scrollregion = ('%d %d %d %d' %
+ (min(bbox[0], scrollregion[0]),
+ min(bbox[1], scrollregion[1]),
+ max(bbox[2], scrollregion[2]),
+ max(bbox[3], scrollregion[3])))
+ canvas['scrollregion'] = scrollregion
+
+##//////////////////////////////////////////////////////
+## Canvas Frame
+##//////////////////////////////////////////////////////
+
+class CanvasFrame(object):
+ """
+ A ``Tkinter`` frame containing a canvas and scrollbars.
+ ``CanvasFrame`` uses a ``ScrollWatcherWidget`` to ensure that all of
+ the canvas widgets contained on its canvas are within its
+ scrollregion. In order for ``CanvasFrame`` to make these checks,
+ all canvas widgets must be registered with ``add_widget`` when they
+ are added to the canvas; and destroyed with ``destroy_widget`` when
+ they are no longer needed.
+
+ If a ``CanvasFrame`` is created with no parent, then it will create
+ its own main window, including a "Done" button and a "Print"
+ button.
+ """
+ def __init__(self, parent=None, **kw):
+ """
+ Create a new ``CanvasFrame``.
+
+ :type parent: Tkinter.BaseWidget or Tkinter.Tk
+ :param parent: The parent ``Tkinter`` widget. If no parent is
+ specified, then ``CanvasFrame`` will create a new main
+ window.
+ :param kw: Keyword arguments for the new ``Canvas``. See the
+ documentation for ``Tkinter.Canvas`` for more information.
+ """
+ # If no parent was given, set up a top-level window.
+ if parent is None:
+ self._parent = Tk()
+ self._parent.title('NLTK')
+ self._parent.bind('<Control-p>', lambda e: self.print_to_file())
+ self._parent.bind('<Control-x>', self.destroy)
+ self._parent.bind('<Control-q>', self.destroy)
+ else:
+ self._parent = parent
+
+ # Create a frame for the canvas & scrollbars
+ self._frame = frame = Frame(self._parent)
+ self._canvas = canvas = Canvas(frame, **kw)
+ xscrollbar = Scrollbar(self._frame, orient='horizontal')
+ yscrollbar = Scrollbar(self._frame, orient='vertical')
+ xscrollbar['command'] = canvas.xview
+ yscrollbar['command'] = canvas.yview
+ canvas['xscrollcommand'] = xscrollbar.set
+ canvas['yscrollcommand'] = yscrollbar.set
+ yscrollbar.pack(fill='y', side='right')
+ xscrollbar.pack(fill='x', side='bottom')
+ canvas.pack(expand=1, fill='both', side='left')
+
+ # Set initial scroll region.
+ scrollregion = '0 0 %s %s' % (canvas['width'], canvas['height'])
+ canvas['scrollregion'] = scrollregion
+
+ self._scrollwatcher = ScrollWatcherWidget(canvas)
+
+ # If no parent was given, pack the frame, and add a menu.
+ if parent is None:
+ self.pack(expand=1, fill='both')
+ self._init_menubar()
+
+ def _init_menubar(self):
+ menubar = Menu(self._parent)
+
+ filemenu = Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Print to Postscript', underline=0,
+ command=self.print_to_file, accelerator='Ctrl-p')
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='Ctrl-x')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ self._parent.config(menu=menubar)
+
+ def print_to_file(self, filename=None):
+ """
+ Print the contents of this ``CanvasFrame`` to a postscript
+ file. If no filename is given, then prompt the user for one.
+
+ :param filename: The name of the file to print the tree to.
+ :type filename: str
+ :rtype: None
+ """
+ if filename is None:
+ from tkinter.filedialog import asksaveasfilename
+ ftypes = [('Postscript files', '.ps'),
+ ('All files', '*')]
+ filename = asksaveasfilename(filetypes=ftypes,
+ defaultextension='.ps')
+ if not filename: return
+ (x0, y0, w, h) = self.scrollregion()
+ self._canvas.postscript(file=filename, x=x0, y=y0,
+ width=w+2, height=h+2,
+ pagewidth=w+2, # points = 1/72 inch
+ pageheight=h+2, # points = 1/72 inch
+ pagex=0, pagey=0)
+
+ def scrollregion(self):
+ """
+ :return: The current scroll region for the canvas managed by
+ this ``CanvasFrame``.
+ :rtype: 4-tuple of int
+ """
+ (x1, y1, x2, y2) = self._canvas['scrollregion'].split()
+ return (int(x1), int(y1), int(x2), int(y2))
+
+ def canvas(self):
+ """
+ :return: The canvas managed by this ``CanvasFrame``.
+ :rtype: Tkinter.Canvas
+ """
+ return self._canvas
+
+ def add_widget(self, canvaswidget, x=None, y=None):
+ """
+ Register a canvas widget with this ``CanvasFrame``. The
+ ``CanvasFrame`` will ensure that this canvas widget is always
+ within the ``Canvas``'s scrollregion. If no coordinates are
+ given for the canvas widget, then the ``CanvasFrame`` will
+ attempt to find a clear area of the canvas for it.
+
+ :type canvaswidget: CanvasWidget
+ :param canvaswidget: The new canvas widget. ``canvaswidget``
+ must have been created on this ``CanvasFrame``'s canvas.
+ :type x: int
+ :param x: The initial x coordinate for the upper left hand
+ corner of ``canvaswidget``, in the canvas's coordinate
+ space.
+ :type y: int
+ :param y: The initial y coordinate for the upper left hand
+ corner of ``canvaswidget``, in the canvas's coordinate
+ space.
+ """
+ if x is None or y is None:
+ (x, y) = self._find_room(canvaswidget, x, y)
+
+ # Move to (x,y)
+ (x1,y1,x2,y2) = canvaswidget.bbox()
+ canvaswidget.move(x-x1,y-y1)
+
+ # Register with scrollwatcher.
+ self._scrollwatcher.add_child(canvaswidget)
+
+ def _find_room(self, widget, desired_x, desired_y):
+ """
+ Try to find a space for a given widget.
+ """
+ (left, top, right, bot) = self.scrollregion()
+ w = widget.width()
+ h = widget.height()
+
+ if w >= (right-left): return (0,0)
+ if h >= (bot-top): return (0,0)
+
+ # Move the widget out of the way, for now.
+ (x1,y1,x2,y2) = widget.bbox()
+ widget.move(left-x2-50, top-y2-50)
+
+ if desired_x is not None:
+ x = desired_x
+ for y in range(top, bot-h, int((bot-top-h)/10)):
+ if not self._canvas.find_overlapping(x-5, y-5, x+w+5, y+h+5):
+ return (x,y)
+
+ if desired_y is not None:
+ y = desired_y
+ for x in range(left, right-w, int((right-left-w)/10)):
+ if not self._canvas.find_overlapping(x-5, y-5, x+w+5, y+h+5):
+ return (x,y)
+
+ for y in range(top, bot-h, int((bot-top-h)/10)):
+ for x in range(left, right-w, int((right-left-w)/10)):
+ if not self._canvas.find_overlapping(x-5, y-5, x+w+5, y+h+5):
+ return (x,y)
+ return (0,0)
+
+ def destroy_widget(self, canvaswidget):
+ """
+ Remove a canvas widget from this ``CanvasFrame``. This
+ deregisters the canvas widget, and destroys it.
+ """
+ self.remove_widget(canvaswidget)
+ canvaswidget.destroy()
+
+ def remove_widget(self, canvaswidget):
+ # Deregister with scrollwatcher.
+ self._scrollwatcher.remove_child(canvaswidget)
+
+ def pack(self, cnf={}, **kw):
+ """
+ Pack this ``CanvasFrame``. See the documentation for
+ ``Tkinter.Pack`` for more information.
+ """
+ self._frame.pack(cnf, **kw)
+ # Adjust to be big enough for kids?
+
+ def destroy(self, *e):
+ """
+ Destroy this ``CanvasFrame``. If this ``CanvasFrame`` created a
+ top-level window, then this will close that window.
+ """
+ if self._parent is None: return
+ self._parent.destroy()
+ self._parent = None
+
+ def mainloop(self, *args, **kwargs):
+ """
+ Enter the Tkinter mainloop. This function must be called if
+ this frame is created from a non-interactive program (e.g.
+ from a secript); otherwise, the frame will close as soon as
+ the script completes.
+ """
+ if in_idle(): return
+ self._parent.mainloop(*args, **kwargs)
+
+##//////////////////////////////////////////////////////
+## Text display
+##//////////////////////////////////////////////////////
+
+class ShowText(object):
+ """
+ A ``Tkinter`` window used to display a text. ``ShowText`` is
+ typically used by graphical tools to display help text, or similar
+ information.
+ """
+ def __init__(self, root, title, text, width=None, height=None,
+ **textbox_options):
+ if width is None or height is None:
+ (width, height) = self.find_dimentions(text, width, height)
+
+ # Create the main window.
+ if root is None:
+ self._top = top = Tk()
+ else:
+ self._top = top = Toplevel(root)
+ top.title(title)
+
+ b = Button(top, text='Ok', command=self.destroy)
+ b.pack(side='bottom')
+
+ tbf = Frame(top)
+ tbf.pack(expand=1, fill='both')
+ scrollbar = Scrollbar(tbf, orient='vertical')
+ scrollbar.pack(side='right', fill='y')
+ textbox = Text(tbf, wrap='word', width=width,
+ height=height, **textbox_options)
+ textbox.insert('end', text)
+ textbox['state'] = 'disabled'
+ textbox.pack(side='left', expand=1, fill='both')
+ scrollbar['command'] = textbox.yview
+ textbox['yscrollcommand'] = scrollbar.set
+
+ # Make it easy to close the window.
+ top.bind('q', self.destroy)
+ top.bind('x', self.destroy)
+ top.bind('c', self.destroy)
+ top.bind('<Return>', self.destroy)
+ top.bind('<Escape>', self.destroy)
+
+ # Focus the scrollbar, so they can use up/down, etc.
+ scrollbar.focus()
+
+ def find_dimentions(self, text, width, height):
+ lines = text.split('\n')
+ if width is None:
+ maxwidth = max(len(line) for line in lines)
+ width = min(maxwidth, 80)
+
+ # Now, find height.
+ height = 0
+ for line in lines:
+ while len(line) > width:
+ brk = line[:width].rfind(' ')
+ line = line[brk:]
+ height += 1
+ height += 1
+ height = min(height, 25)
+
+ return (width, height)
+
+ def destroy(self, *e):
+ if self._top is None: return
+ self._top.destroy()
+ self._top = None
+
+ def mainloop(self, *args, **kwargs):
+ """
+ Enter the Tkinter mainloop. This function must be called if
+ this window is created from a non-interactive program (e.g.
+ from a secript); otherwise, the window will close as soon as
+ the script completes.
+ """
+ if in_idle(): return
+ self._top.mainloop(*args, **kwargs)
+
+##//////////////////////////////////////////////////////
+## Entry dialog
+##//////////////////////////////////////////////////////
+
+class EntryDialog(object):
+ """
+ A dialog box for entering
+ """
+ def __init__(self, parent, original_text='', instructions='',
+ set_callback=None, title=None):
+ self._parent = parent
+ self._original_text = original_text
+ self._set_callback = set_callback
+
+ width = max(30, len(original_text)*3/2)
+ self._top = Toplevel(parent)
+
+ if title: self._top.title(title)
+
+ # The text entry box.
+ entryframe = Frame(self._top)
+ entryframe.pack(expand=1, fill='both', padx=5, pady=5,ipady=10)
+ if instructions:
+ l=Label(entryframe, text=instructions)
+ l.pack(side='top', anchor='w', padx=30)
+ self._entry = Entry(entryframe, width=width)
+ self._entry.pack(expand=1, fill='x', padx=30)
+ self._entry.insert(0, original_text)
+
+ # A divider
+ divider = Frame(self._top, borderwidth=1, relief='sunken')
+ divider.pack(fill='x', ipady=1, padx=10)
+
+ # The buttons.
+ buttons = Frame(self._top)
+ buttons.pack(expand=0, fill='x', padx=5, pady=5)
+ b = Button(buttons, text='Cancel', command=self._cancel, width=8)
+ b.pack(side='right', padx=5)
+ b = Button(buttons, text='Ok', command=self._ok,
+ width=8, default='active')
+ b.pack(side='left', padx=5)
+ b = Button(buttons, text='Apply', command=self._apply, width=8)
+ b.pack(side='left')
+
+ self._top.bind('<Return>', self._ok)
+ self._top.bind('<Control-q>', self._cancel)
+ self._top.bind('<Escape>', self._cancel)
+
+ self._entry.focus()
+
+ def _reset(self, *e):
+ self._entry.delete(0,'end')
+ self._entry.insert(0, self._original_text)
+ if self._set_callback:
+ self._set_callback(self._original_text)
+
+ def _cancel(self, *e):
+ try: self._reset()
+ except: pass
+ self._destroy()
+
+ def _ok(self, *e):
+ self._apply()
+ self._destroy()
+
+ def _apply(self, *e):
+ if self._set_callback:
+ self._set_callback(self._entry.get())
+
+ def _destroy(self, *e):
+ if self._top is None: return
+ self._top.destroy()
+ self._top = None
+
+##//////////////////////////////////////////////////////
+## Colorized List
+##//////////////////////////////////////////////////////
+
+class ColorizedList(object):
+ """
+ An abstract base class for displaying a colorized list of items.
+ Subclasses should define:
+ - ``_init_colortags``, which sets up Text color tags that
+ will be used by the list.
+ - ``_item_repr``, which returns a list of (text,colortag)
+ tuples that make up the colorized representation of the
+ item.
+ :note: Typically, you will want to register a callback for
+ ``'select'`` that calls ``mark`` on the given item.
+ """
+ def __init__(self, parent, items=[], **options):
+ """
+ Construct a new list.
+
+ :param parent: The Tk widget that contains the colorized list
+ :param items: The initial contents of the colorized list.
+ :param options:
+ """
+ self._parent = parent
+ self._callbacks = {}
+
+ # Which items are marked?
+ self._marks = {}
+
+ # Initialize the Tkinter frames.
+ self._init_itemframe(options.copy())
+
+ # Set up key & mouse bindings.
+ self._textwidget.bind('<KeyPress>', self._keypress)
+ self._textwidget.bind('<ButtonPress>', self._buttonpress)
+
+ # Fill in the given CFG's items.
+ self._items = None
+ self.set(items)
+
+ #////////////////////////////////////////////////////////////
+ # Abstract methods
+ #////////////////////////////////////////////////////////////
+
+ def _init_colortags(self, textwidget, options):
+ """
+ Set up any colortags that will be used by this colorized list.
+ E.g.:
+ >>> textwidget.tag_config('terminal', foreground='black')
+ """
+ raise NotImplementedError()
+
+ def _item_repr(self, item):
+ """
+ Return a list of (text, colortag) tuples that make up the
+ colorized representation of the item. Colorized
+ representations may not span multiple lines. I.e., the text
+ strings returned may not contain newline characters.
+ """
+ raise NotImplementedError()
+
+ #////////////////////////////////////////////////////////////
+ # Item Access
+ #////////////////////////////////////////////////////////////
+
+ def get(self, index=None):
+ """
+ :return: A list of the items contained by this list.
+ """
+ if index is None:
+ return self._items[:]
+ else:
+ return self._items[index]
+
+ def set(self, items):
+ """
+ Modify the list of items contained by this list.
+ """
+ items = list(items)
+ if self._items == items: return
+ self._items = list(items)
+
+ self._textwidget['state'] = 'normal'
+ self._textwidget.delete('1.0', 'end')
+ for item in items:
+ for (text, colortag) in self._item_repr(item):
+ assert '\n' not in text, 'item repr may not contain newline'
+ self._textwidget.insert('end', text, colortag)
+ self._textwidget.insert('end', '\n')
+ # Remove the final newline
+ self._textwidget.delete('end-1char', 'end')
+ self._textwidget.mark_set('insert', '1.0')
+ self._textwidget['state'] = 'disabled'
+ # Clear all marks
+ self._marks.clear()
+
+ def unmark(self, item=None):
+ """
+ Remove highlighting from the given item; or from every item,
+ if no item is given.
+ :raise ValueError: If ``item`` is not contained in the list.
+ :raise KeyError: If ``item`` is not marked.
+ """
+ if item is None:
+ self._marks.clear()
+ self._textwidget.tag_remove('highlight', '1.0', 'end+1char')
+ else:
+ index = self._items.index(item)
+ del self._marks[item]
+ (start, end) = ('%d.0' % (index+1), '%d.0' % (index+2))
+ self._textwidget.tag_remove('highlight', start, end)
+
+ def mark(self, item):
+ """
+ Highlight the given item.
+ :raise ValueError: If ``item`` is not contained in the list.
+ """
+ self._marks[item] = 1
+ index = self._items.index(item)
+ (start, end) = ('%d.0' % (index+1), '%d.0' % (index+2))
+ self._textwidget.tag_add('highlight', start, end)
+
+ def markonly(self, item):
+ """
+ Remove any current highlighting, and mark the given item.
+ :raise ValueError: If ``item`` is not contained in the list.
+ """
+ self.unmark()
+ self.mark(item)
+
+ def view(self, item):
+ """
+ Adjust the view such that the given item is visible. If
+ the item is already visible, then do nothing.
+ """
+ index = self._items.index(item)
+ self._textwidget.see('%d.0' % (index+1))
+
+ #////////////////////////////////////////////////////////////
+ # Callbacks
+ #////////////////////////////////////////////////////////////
+
+ def add_callback(self, event, func):
+ """
+ Register a callback function with the list. This function
+ will be called whenever the given event occurs.
+
+ :param event: The event that will trigger the callback
+ function. Valid events are: click1, click2, click3,
+ space, return, select, up, down, next, prior, move
+ :param func: The function that should be called when
+ the event occurs. ``func`` will be called with a
+ single item as its argument. (The item selected
+ or the item moved to).
+ """
+ if event == 'select': events = ['click1', 'space', 'return']
+ elif event == 'move': events = ['up', 'down', 'next', 'prior']
+ else: events = [event]
+
+ for e in events:
+ self._callbacks.setdefault(e,{})[func] = 1
+
+ def remove_callback(self, event, func=None):
+ """
+ Deregister a callback function. If ``func`` is none, then
+ all callbacks are removed for the given event.
+ """
+ if event is None: events = list(self._callbacks.keys())
+ elif event == 'select': events = ['click1', 'space', 'return']
+ elif event == 'move': events = ['up', 'down', 'next', 'prior']
+ else: events = [event]
+
+ for e in events:
+ if func is None: del self._callbacks[e]
+ else:
+ try: del self._callbacks[e][func]
+ except: pass
+
+ #////////////////////////////////////////////////////////////
+ # Tkinter Methods
+ #////////////////////////////////////////////////////////////
+
+ def pack(self, cnf={}, **kw):
+# "@include: Tkinter.Pack.pack"
+ self._itemframe.pack(cnf, **kw)
+
+ def grid(self, cnf={}, **kw):
+# "@include: Tkinter.Grid.grid"
+ self._itemframe.grid(cnf, *kw)
+
+ def focus(self):
+# "@include: Tkinter.Widget.focus"
+ self._textwidget.focus()
+
+ #////////////////////////////////////////////////////////////
+ # Internal Methods
+ #////////////////////////////////////////////////////////////
+
+ def _init_itemframe(self, options):
+ self._itemframe = Frame(self._parent)
+
+ # Create the basic Text widget & scrollbar.
+ options.setdefault('background', '#e0e0e0')
+ self._textwidget = Text(self._itemframe, **options)
+ self._textscroll = Scrollbar(self._itemframe, takefocus=0,
+ orient='vertical')
+ self._textwidget.config(yscrollcommand = self._textscroll.set)
+ self._textscroll.config(command=self._textwidget.yview)
+ self._textscroll.pack(side='right', fill='y')
+ self._textwidget.pack(expand=1, fill='both', side='left')
+
+ # Initialize the colorization tags
+ self._textwidget.tag_config('highlight', background='#e0ffff',
+ border='1', relief='raised')
+ self._init_colortags(self._textwidget, options)
+
+ # How do I want to mark keyboard selection?
+ self._textwidget.tag_config('sel', foreground='')
+ self._textwidget.tag_config('sel', foreground='', background='',
+ border='', underline=1)
+ self._textwidget.tag_lower('highlight', 'sel')
+
+ def _fire_callback(self, event, itemnum):
+ if event not in self._callbacks: return
+ if 0 <= itemnum < len(self._items):
+ item = self._items[itemnum]
+ else:
+ item = None
+ for cb_func in list(self._callbacks[event].keys()):
+ cb_func(item)
+
+ def _buttonpress(self, event):
+ clickloc = '@%d,%d' % (event.x,event.y)
+ insert_point = self._textwidget.index(clickloc)
+ itemnum = int(insert_point.split('.')[0])-1
+ self._fire_callback('click%d' % event.num, itemnum)
+
+ def _keypress(self, event):
+ if event.keysym == 'Return' or event.keysym == 'space':
+ insert_point = self._textwidget.index('insert')
+ itemnum = int(insert_point.split('.')[0])-1
+ self._fire_callback(event.keysym.lower(), itemnum)
+ return
+ elif event.keysym == 'Down': delta='+1line'
+ elif event.keysym == 'Up': delta='-1line'
+ elif event.keysym == 'Next': delta='+10lines'
+ elif event.keysym == 'Prior': delta='-10lines'
+ else: return 'continue'
+
+ self._textwidget.mark_set('insert', 'insert'+delta)
+ self._textwidget.see('insert')
+ self._textwidget.tag_remove('sel', '1.0', 'end+1char')
+ self._textwidget.tag_add('sel', 'insert linestart', 'insert lineend')
+
+ insert_point = self._textwidget.index('insert')
+ itemnum = int(insert_point.split('.')[0])-1
+ self._fire_callback(event.keysym.lower(), itemnum)
+
+ return 'break'
+
+##//////////////////////////////////////////////////////
+## Improved OptionMenu
+##//////////////////////////////////////////////////////
+
+class MutableOptionMenu(Menubutton):
+ def __init__(self, master, values, **options):
+ self._callback = options.get('command')
+ if 'command' in options: del options['command']
+
+ # Create a variable
+ self._variable = variable = StringVar()
+ if len(values) > 0:
+ variable.set(values[0])
+
+ kw = {"borderwidth": 2, "textvariable": variable,
+ "indicatoron": 1, "relief": RAISED, "anchor": "c",
+ "highlightthickness": 2}
+ kw.update(options)
+ Widget.__init__(self, master, "menubutton", kw)
+ self.widgetName = 'tk_optionMenu'
+ self._menu = Menu(self, name="menu", tearoff=0,)
+ self.menuname = self._menu._w
+
+ self._values = []
+ for value in values: self.add(value)
+
+ self["menu"] = self._menu
+
+ def add(self, value):
+ if value in self._values: return
+ def set(value=value): self.set(value)
+ self._menu.add_command(label=value, command=set)
+ self._values.append(value)
+
+ def set(self, value):
+ self._variable.set(value)
+ if self._callback:
+ self._callback(value)
+
+ def remove(self, value):
+ # Might raise indexerror: pass to parent.
+ i = self._values.index(value)
+ del self._values[i]
+ self._menu.delete(i, i)
+
+ def __getitem__(self, name):
+ if name == 'menu':
+ return self.__menu
+ return Widget.__getitem__(self, name)
+
+ def destroy(self):
+ """Destroy this widget and the associated menu."""
+ Menubutton.destroy(self)
+ self._menu = None
+
+##//////////////////////////////////////////////////////
+## Test code.
+##//////////////////////////////////////////////////////
+
+def demo():
+ """
+ A simple demonstration showing how to use canvas widgets.
+ """
+ def fill(cw):
+ from random import randint
+ cw['fill'] = '#00%04d' % randint(0,9999)
+ def color(cw):
+ from random import randint
+ cw['color'] = '#ff%04d' % randint(0,9999)
+
+ cf = CanvasFrame(closeenough=10, width=300, height=300)
+ c = cf.canvas()
+ ct3 = TextWidget(c, 'hiya there', draggable=1)
+ ct2 = TextWidget(c, 'o o\n||\n___\n U', draggable=1, justify='center')
+ co = OvalWidget(c, ct2, outline='red')
+ ct = TextWidget(c, 'o o\n||\n\\___/', draggable=1, justify='center')
+ cp = ParenWidget(c, ct, color='red')
+ cb = BoxWidget(c, cp, fill='cyan', draggable=1, width=3, margin=10)
+ equation = SequenceWidget(c,
+ SymbolWidget(c, 'forall'), TextWidget(c, 'x'),
+ SymbolWidget(c, 'exists'), TextWidget(c, 'y: '),
+ TextWidget(c, 'x'), SymbolWidget(c, 'notequal'),
+ TextWidget(c, 'y'))
+ space = SpaceWidget(c, 0, 30)
+ cstack = StackWidget(c, cb, ct3, space, co, equation, align='center')
+ foo = TextWidget(c, 'try clicking\nand dragging',
+ draggable=1, justify='center')
+ cs = SequenceWidget(c, cstack, foo)
+ zz = BracketWidget(c, cs, color='green4', width=3)
+ cf.add_widget(zz, 60, 30)
+
+ cb.bind_click(fill)
+ ct.bind_click(color)
+ co.bind_click(fill)
+ ct2.bind_click(color)
+ ct3.bind_click(color)
+
+ cf.mainloop()
+ #ShowText(None, 'title', ((('this is text'*150)+'\n')*5))
+
+if __name__ == '__main__':
+ demo()
+
diff --git a/nltk/examples/__init__.py b/nltk/examples/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/nltk/examples/pt.py b/nltk/examples/pt.py
new file mode 100644
index 0000000..dc76567
--- /dev/null
+++ b/nltk/examples/pt.py
@@ -0,0 +1,49 @@
+# -*- coding: iso-8859-1 -*-
+
+# Natural Language Toolkit: Some Portuguese texts for exploration in chapter 1 of the book
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+from nltk.corpus import machado, mac_morpho, floresta, genesis
+from nltk.text import Text
+from nltk.probability import FreqDist
+from nltk.util import bigrams
+from nltk.misc import babelize_shell
+
+print("*** Introductory Examples for the NLTK Book ***")
+print("Loading ptext1, ... and psent1, ...")
+print("Type the name of the text or sentence to view it.")
+print("Type: 'texts()' or 'sents()' to list the materials.")
+
+ptext1 = Text(machado.words('romance/marm05.txt'), name="Mem�rias P�stumas de Br�s Cubas (1881)")
+print("ptext1:", ptext1.name)
+
+ptext2 = Text(machado.words('romance/marm08.txt'), name="Dom Casmurro (1899)")
+print("ptext2:", ptext2.name)
+
+ptext3 = Text(genesis.words('portuguese.txt'), name="G�nesis")
+print("ptext3:", ptext3.name)
+
+ptext4 = Text(mac_morpho.words('mu94se01.txt'), name="Folha de Sao Paulo (1994)")
+print("ptext4:", ptext4.name)
+
+def texts():
+ print("ptext1:", ptext1.name)
+ print("ptext2:", ptext2.name)
+ print("ptext3:", ptext3.name)
+ print("ptext4:", ptext4.name)
+
+psent1 = "o amor da gl�ria era a coisa mais verdadeiramente humana que h� no homem , e , conseq�entemente , a sua mais genu�na fei��o .".split()
+psent2 = "N�o consultes dicion�rios .".split()
+psent3 = "No princ�pio, criou Deus os c�us e a terra.".split()
+psent4 = "A C�ritas acredita que outros cubanos devem chegar ao Brasil .".split()
+
+def sents():
+ print("psent1:", " ".join(psent1))
+ print("psent2:", " ".join(psent2))
+ print("psent3:", " ".join(psent3))
+ print("psent4:", " ".join(psent4))
diff --git a/nltk/featstruct.py b/nltk/featstruct.py
new file mode 100644
index 0000000..2585eb8
--- /dev/null
+++ b/nltk/featstruct.py
@@ -0,0 +1,2500 @@
+# Natural Language Toolkit: Feature Structures
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>,
+# Rob Speer,
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+"""
+Basic data classes for representing feature structures, and for
+performing basic operations on those feature structures. A feature
+structure is a mapping from feature identifiers to feature values,
+where each feature value is either a basic value (such as a string or
+an integer), or a nested feature structure. There are two types of
+feature structure, implemented by two subclasses of ``FeatStruct``:
+
+ - feature dictionaries, implemented by ``FeatDict``, act like
+ Python dictionaries. Feature identifiers may be strings or
+ instances of the ``Feature`` class.
+ - feature lists, implemented by ``FeatList``, act like Python
+ lists. Feature identifiers are integers.
+
+Feature structures are typically used to represent partial information
+about objects. A feature identifier that is not mapped to a value
+stands for a feature whose value is unknown (*not* a feature without
+a value). Two feature structures that represent (potentially
+overlapping) information about the same object can be combined by
+unification. When two inconsistent feature structures are unified,
+the unification fails and returns None.
+
+Features can be specified using "feature paths", or tuples of feature
+identifiers that specify path through the nested feature structures to
+a value. Feature structures may contain reentrant feature values. A
+"reentrant feature value" is a single feature value that can be
+accessed via multiple feature paths. Unification preserves the
+reentrance relations imposed by both of the unified feature
+structures. In the feature structure resulting from unification, any
+modifications to a reentrant feature value will be visible using any
+of its feature paths.
+
+Feature structure variables are encoded using the ``nltk.sem.Variable``
+class. The variables' values are tracked using a bindings
+dictionary, which maps variables to their values. When two feature
+structures are unified, a fresh bindings dictionary is created to
+track their values; and before unification completes, all bound
+variables are replaced by their values. Thus, the bindings
+dictionaries are usually strictly internal to the unification process.
+However, it is possible to track the bindings of variables if you
+choose to, by supplying your own initial bindings dictionary to the
+``unify()`` function.
+
+When unbound variables are unified with one another, they become
+aliased. This is encoded by binding one variable to the other.
+
+Lightweight Feature Structures
+==============================
+Many of the functions defined by ``nltk.featstruct`` can be applied
+directly to simple Python dictionaries and lists, rather than to
+full-fledged ``FeatDict`` and ``FeatList`` objects. In other words,
+Python ``dicts`` and ``lists`` can be used as "light-weight" feature
+structures.
+
+ >>> from nltk.featstruct import unify
+ >>> unify(dict(x=1, y=dict()), dict(a='a', y=dict(b='b'))) # doctest: +SKIP
+ {'y': {'b': 'b'}, 'x': 1, 'a': 'a'}
+
+However, you should keep in mind the following caveats:
+
+ - Python dictionaries & lists ignore reentrance when checking for
+ equality between values. But two FeatStructs with different
+ reentrances are considered nonequal, even if all their base
+ values are equal.
+
+ - FeatStructs can be easily frozen, allowing them to be used as
+ keys in hash tables. Python dictionaries and lists can not.
+
+ - FeatStructs display reentrance in their string representations;
+ Python dictionaries and lists do not.
+
+ - FeatStructs may *not* be mixed with Python dictionaries and lists
+ (e.g., when performing unification).
+
+ - FeatStructs provide a number of useful methods, such as ``walk()``
+ and ``cyclic()``, which are not available for Python dicts and lists.
+
+In general, if your feature structures will contain any reentrances,
+or if you plan to use them as dictionary keys, it is strongly
+recommended that you use full-fledged ``FeatStruct`` objects.
+"""
+from __future__ import print_function, unicode_literals, division
+
+import re
+import copy
+
+from nltk.internals import read_str, raise_unorderable_types
+from nltk.sem.logic import (Variable, Expression, SubstituteBindingsI,
+ _LogicParser, LogicalExpressionException)
+from nltk.compat import (string_types, integer_types, total_ordering,
+ python_2_unicode_compatible, unicode_repr)
+
+######################################################################
+# Feature Structure
+######################################################################
+
+ at total_ordering
+class FeatStruct(SubstituteBindingsI):
+ """
+ A mapping from feature identifiers to feature values, where each
+ feature value is either a basic value (such as a string or an
+ integer), or a nested feature structure. There are two types of
+ feature structure:
+
+ - feature dictionaries, implemented by ``FeatDict``, act like
+ Python dictionaries. Feature identifiers may be strings or
+ instances of the ``Feature`` class.
+ - feature lists, implemented by ``FeatList``, act like Python
+ lists. Feature identifiers are integers.
+
+ Feature structures may be indexed using either simple feature
+ identifiers or 'feature paths.' A feature path is a sequence
+ of feature identifiers that stand for a corresponding sequence of
+ indexing operations. In particular, ``fstruct[(f1,f2,...,fn)]`` is
+ equivalent to ``fstruct[f1][f2]...[fn]``.
+
+ Feature structures may contain reentrant feature structures. A
+ "reentrant feature structure" is a single feature structure
+ object that can be accessed via multiple feature paths. Feature
+ structures may also be cyclic. A feature structure is "cyclic"
+ if there is any feature path from the feature structure to itself.
+
+ Two feature structures are considered equal if they assign the
+ same values to all features, and have the same reentrancies.
+
+ By default, feature structures are mutable. They may be made
+ immutable with the ``freeze()`` method. Once they have been
+ frozen, they may be hashed, and thus used as dictionary keys.
+ """
+
+ _frozen = False
+ """:ivar: A flag indicating whether this feature structure is
+ frozen or not. Once this flag is set, it should never be
+ un-set; and no further modification should be made to this
+ feature structue."""
+
+ ##////////////////////////////////////////////////////////////
+ #{ Constructor
+ ##////////////////////////////////////////////////////////////
+
+ def __new__(cls, features=None, **morefeatures):
+ """
+ Construct and return a new feature structure. If this
+ constructor is called directly, then the returned feature
+ structure will be an instance of either the ``FeatDict`` class
+ or the ``FeatList`` class.
+
+ :param features: The initial feature values for this feature
+ structure:
+ - FeatStruct(string) -> FeatStructReader().read(string)
+ - FeatStruct(mapping) -> FeatDict(mapping)
+ - FeatStruct(sequence) -> FeatList(sequence)
+ - FeatStruct() -> FeatDict()
+ :param morefeatures: If ``features`` is a mapping or None,
+ then ``morefeatures`` provides additional features for the
+ ``FeatDict`` constructor.
+ """
+ # If the FeatStruct constructor is called directly, then decide
+ # whether to create a FeatDict or a FeatList, based on the
+ # contents of the `features` argument.
+ if cls is FeatStruct:
+ if features is None:
+ return FeatDict.__new__(FeatDict, **morefeatures)
+ elif _is_mapping(features):
+ return FeatDict.__new__(FeatDict, features, **morefeatures)
+ elif morefeatures:
+ raise TypeError('Keyword arguments may only be specified '
+ 'if features is None or is a mapping.')
+ if isinstance(features, string_types):
+ if FeatStructReader._START_FDICT_RE.match(features):
+ return FeatDict.__new__(FeatDict, features, **morefeatures)
+ else:
+ return FeatList.__new__(FeatList, features, **morefeatures)
+ elif _is_sequence(features):
+ return FeatList.__new__(FeatList, features)
+ else:
+ raise TypeError('Expected string or mapping or sequence')
+
+ # Otherwise, construct the object as normal.
+ else:
+ return super(FeatStruct, cls).__new__(cls, features,
+ **morefeatures)
+
+ ##////////////////////////////////////////////////////////////
+ #{ Uniform Accessor Methods
+ ##////////////////////////////////////////////////////////////
+ # These helper functions allow the methods defined by FeatStruct
+ # to treat all feature structures as mappings, even if they're
+ # really lists. (Lists are treated as mappings from ints to vals)
+
+ def _keys(self):
+ """Return an iterable of the feature identifiers used by this
+ FeatStruct."""
+ raise NotImplementedError() # Implemented by subclasses.
+
+ def _values(self):
+ """Return an iterable of the feature values directly defined
+ by this FeatStruct."""
+ raise NotImplementedError() # Implemented by subclasses.
+
+ def _items(self):
+ """Return an iterable of (fid,fval) pairs, where fid is a
+ feature identifier and fval is the corresponding feature
+ value, for all features defined by this FeatStruct."""
+ raise NotImplementedError() # Implemented by subclasses.
+
+ ##////////////////////////////////////////////////////////////
+ #{ Equality & Hashing
+ ##////////////////////////////////////////////////////////////
+
+ def equal_values(self, other, check_reentrance=False):
+ """
+ Return True if ``self`` and ``other`` assign the same value to
+ to every feature. In particular, return true if
+ ``self[p]==other[p]`` for every feature path *p* such
+ that ``self[p]`` or ``other[p]`` is a base value (i.e.,
+ not a nested feature structure).
+
+ :param check_reentrance: If True, then also return False if
+ there is any difference between the reentrances of ``self``
+ and ``other``.
+ :note: the ``==`` is equivalent to ``equal_values()`` with
+ ``check_reentrance=True``.
+ """
+ return self._equal(other, check_reentrance, set(), set(), set())
+
+ def __eq__(self, other):
+ """
+ Return true if ``self`` and ``other`` are both feature structures,
+ assign the same values to all features, and contain the same
+ reentrances. I.e., return
+ ``self.equal_values(other, check_reentrance=True)``.
+
+ :see: ``equal_values()``
+ """
+ return self._equal(other, True, set(), set(), set())
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, FeatStruct):
+ # raise_unorderable_types("<", self, other)
+ # Sometimes feature values can be pure strings,
+ # so we need to be able to compare with non-featstructs:
+ return self.__class__.__name__ < other.__class__.__name__
+ else:
+ return len(self) < len(other)
+
+ def __hash__(self):
+ """
+ If this feature structure is frozen, return its hash value;
+ otherwise, raise ``TypeError``.
+ """
+ if not self._frozen:
+ raise TypeError('FeatStructs must be frozen before they '
+ 'can be hashed.')
+ try: return self._hash
+ except AttributeError:
+ self._hash = self._calculate_hashvalue(set())
+ return self._hash
+
+ def _equal(self, other, check_reentrance, visited_self,
+ visited_other, visited_pairs):
+ """
+ Return True iff self and other have equal values.
+
+ :param visited_self: A set containing the ids of all ``self``
+ feature structures we've already visited.
+ :param visited_other: A set containing the ids of all ``other``
+ feature structures we've already visited.
+ :param visited_pairs: A set containing ``(selfid, otherid)`` pairs
+ for all pairs of feature structures we've already visited.
+ """
+ # If we're the same object, then we're equal.
+ if self is other: return True
+
+ # If we have different classes, we're definitely not equal.
+ if self.__class__ != other.__class__: return False
+
+ # If we define different features, we're definitely not equal.
+ # (Perform len test first because it's faster -- we should
+ # do profiling to see if this actually helps)
+ if len(self) != len(other): return False
+ if set(self._keys()) != set(other._keys()): return False
+
+ # If we're checking reentrance, then any time we revisit a
+ # structure, make sure that it was paired with the same
+ # feature structure that it is now. Note: if check_reentrance,
+ # then visited_pairs will never contain two pairs whose first
+ # values are equal, or two pairs whose second values are equal.
+ if check_reentrance:
+ if id(self) in visited_self or id(other) in visited_other:
+ return (id(self), id(other)) in visited_pairs
+
+ # If we're not checking reentrance, then we still need to deal
+ # with cycles. If we encounter the same (self, other) pair a
+ # second time, then we won't learn anything more by examining
+ # their children a second time, so just return true.
+ else:
+ if (id(self), id(other)) in visited_pairs:
+ return True
+
+ # Keep track of which nodes we've visited.
+ visited_self.add(id(self))
+ visited_other.add(id(other))
+ visited_pairs.add( (id(self), id(other)) )
+
+ # Now we have to check all values. If any of them don't match,
+ # then return false.
+ for (fname, self_fval) in self._items():
+ other_fval = other[fname]
+ if isinstance(self_fval, FeatStruct):
+ if not self_fval._equal(other_fval, check_reentrance,
+ visited_self, visited_other,
+ visited_pairs):
+ return False
+ else:
+ if self_fval != other_fval: return False
+
+ # Everything matched up; return true.
+ return True
+
+ def _calculate_hashvalue(self, visited):
+ """
+ Return a hash value for this feature structure.
+
+ :require: ``self`` must be frozen.
+ :param visited: A set containing the ids of all feature
+ structures we've already visited while hashing.
+ """
+ if id(self) in visited: return 1
+ visited.add(id(self))
+
+ hashval = 5831
+ for (fname, fval) in sorted(self._items()):
+ hashval *= 37
+ hashval += hash(fname)
+ hashval *= 37
+ if isinstance(fval, FeatStruct):
+ hashval += fval._calculate_hashvalue(visited)
+ else:
+ hashval += hash(fval)
+ # Convert to a 32 bit int.
+ hashval = int(hashval & 0x7fffffff)
+ return hashval
+
+ ##////////////////////////////////////////////////////////////
+ #{ Freezing
+ ##////////////////////////////////////////////////////////////
+
+ #: Error message used by mutating methods when called on a frozen
+ #: feature structure.
+ _FROZEN_ERROR = "Frozen FeatStructs may not be modified."
+
+ def freeze(self):
+ """
+ Make this feature structure, and any feature structures it
+ contains, immutable. Note: this method does not attempt to
+ 'freeze' any feature value that is not a ``FeatStruct``; it
+ is recommended that you use only immutable feature values.
+ """
+ if self._frozen: return
+ self._freeze(set())
+
+ def frozen(self):
+ """
+ Return True if this feature structure is immutable. Feature
+ structures can be made immutable with the ``freeze()`` method.
+ Immutable feature structures may not be made mutable again,
+ but new mutable copies can be produced with the ``copy()`` method.
+ """
+ return self._frozen
+
+ def _freeze(self, visited):
+ """
+ Make this feature structure, and any feature structure it
+ contains, immutable.
+
+ :param visited: A set containing the ids of all feature
+ structures we've already visited while freezing.
+ """
+ if id(self) in visited: return
+ visited.add(id(self))
+ self._frozen = True
+ for (fname, fval) in sorted(self._items()):
+ if isinstance(fval, FeatStruct):
+ fval._freeze(visited)
+
+ ##////////////////////////////////////////////////////////////
+ #{ Copying
+ ##////////////////////////////////////////////////////////////
+
+ def copy(self, deep=True):
+ """
+ Return a new copy of ``self``. The new copy will not be frozen.
+
+ :param deep: If true, create a deep copy; if false, create
+ a shallow copy.
+ """
+ if deep:
+ return copy.deepcopy(self)
+ else:
+ return self.__class__(self)
+
+ # Subclasses should define __deepcopy__ to ensure that the new
+ # copy will not be frozen.
+ def __deepcopy__(self, memo):
+ raise NotImplementedError() # Implemented by subclasses.
+
+ ##////////////////////////////////////////////////////////////
+ #{ Structural Information
+ ##////////////////////////////////////////////////////////////
+
+ def cyclic(self):
+ """
+ Return True if this feature structure contains itself.
+ """
+ return self._find_reentrances({})[id(self)]
+
+ def walk(self):
+ """
+ Return an iterator that generates this feature structure, and
+ each feature structure it contains. Each feature structure will
+ be generated exactly once.
+ """
+ return self._walk(set())
+
+ def _walk(self, visited):
+ """
+ Return an iterator that generates this feature structure, and
+ each feature structure it contains.
+
+ :param visited: A set containing the ids of all feature
+ structures we've already visited while freezing.
+ """
+ raise NotImplementedError() # Implemented by subclasses.
+
+ def _walk(self, visited):
+ if id(self) in visited: return
+ visited.add(id(self))
+ yield self
+ for fval in self._values():
+ if isinstance(fval, FeatStruct):
+ for elt in fval._walk(visited):
+ yield elt
+
+ # Walk through the feature tree. The first time we see a feature
+ # value, map it to False (not reentrant). If we see a feature
+ # value more than once, then map it to True (reentrant).
+ def _find_reentrances(self, reentrances):
+ """
+ Return a dictionary that maps from the ``id`` of each feature
+ structure contained in ``self`` (including ``self``) to a
+ boolean value, indicating whether it is reentrant or not.
+ """
+ if id(self) in reentrances:
+ # We've seen it more than once.
+ reentrances[id(self)] = True
+ else:
+ # This is the first time we've seen it.
+ reentrances[id(self)] = False
+
+ # Recurse to contained feature structures.
+ for fval in self._values():
+ if isinstance(fval, FeatStruct):
+ fval._find_reentrances(reentrances)
+
+ return reentrances
+
+ ##////////////////////////////////////////////////////////////
+ #{ Variables & Bindings
+ ##////////////////////////////////////////////////////////////
+
+ def substitute_bindings(self, bindings):
+ """:see: ``nltk.featstruct.substitute_bindings()``"""
+ return substitute_bindings(self, bindings)
+
+ def retract_bindings(self, bindings):
+ """:see: ``nltk.featstruct.retract_bindings()``"""
+ return retract_bindings(self, bindings)
+
+ def variables(self):
+ """:see: ``nltk.featstruct.find_variables()``"""
+ return find_variables(self)
+
+ def rename_variables(self, vars=None, used_vars=(), new_vars=None):
+ """:see: ``nltk.featstruct.rename_variables()``"""
+ return rename_variables(self, vars, used_vars, new_vars)
+
+ def remove_variables(self):
+ """
+ Return the feature structure that is obtained by deleting
+ any feature whose value is a ``Variable``.
+
+ :rtype: FeatStruct
+ """
+ return remove_variables(self)
+
+ ##////////////////////////////////////////////////////////////
+ #{ Unification
+ ##////////////////////////////////////////////////////////////
+
+ def unify(self, other, bindings=None, trace=False,
+ fail=None, rename_vars=True):
+ return unify(self, other, bindings, trace, fail, rename_vars)
+
+ def subsumes(self, other):
+ """
+ Return True if ``self`` subsumes ``other``. I.e., return true
+ If unifying ``self`` with ``other`` would result in a feature
+ structure equal to ``other``.
+ """
+ return subsumes(self, other)
+
+ ##////////////////////////////////////////////////////////////
+ #{ String Representations
+ ##////////////////////////////////////////////////////////////
+
+ def __repr__(self):
+ """
+ Display a single-line representation of this feature structure,
+ suitable for embedding in other representations.
+ """
+ return self._repr(self._find_reentrances({}), {})
+
+ def _repr(self, reentrances, reentrance_ids):
+ """
+ Return a string representation of this feature structure.
+
+ :param reentrances: A dictionary that maps from the ``id`` of
+ each feature value in self, indicating whether that value
+ is reentrant or not.
+ :param reentrance_ids: A dictionary mapping from each ``id``
+ of a feature value to a unique identifier. This is modified
+ by ``repr``: the first time a reentrant feature value is
+ displayed, an identifier is added to ``reentrance_ids`` for it.
+ """
+ raise NotImplementedError()
+
+# Mutation: disable if frozen.
+_FROZEN_ERROR = "Frozen FeatStructs may not be modified."
+_FROZEN_NOTICE = "\n%sIf self is frozen, raise ValueError."
+def _check_frozen(method, indent=''):
+ """
+ Given a method function, return a new method function that first
+ checks if ``self._frozen`` is true; and if so, raises ``ValueError``
+ with an appropriate message. Otherwise, call the method and return
+ its result.
+ """
+ def wrapped(self, *args, **kwargs):
+ if self._frozen: raise ValueError(_FROZEN_ERROR)
+ else: return method(self, *args, **kwargs)
+ wrapped.__name__ = method.__name__
+ wrapped.__doc__ = (method.__doc__ or '') + (_FROZEN_NOTICE % indent)
+ return wrapped
+
+
+######################################################################
+# Feature Dictionary
+######################################################################
+
+ at python_2_unicode_compatible
+class FeatDict(FeatStruct, dict):
+ """
+ A feature structure that acts like a Python dictionary. I.e., a
+ mapping from feature identifiers to feature values, where a feature
+ identifier can be a string or a ``Feature``; and where a feature value
+ can be either a basic value (such as a string or an integer), or a nested
+ feature structure. A feature identifiers for a ``FeatDict`` is
+ sometimes called a "feature name".
+
+ Two feature dicts are considered equal if they assign the same
+ values to all features, and have the same reentrances.
+
+ :see: ``FeatStruct`` for information about feature paths, reentrance,
+ cyclic feature structures, mutability, freezing, and hashing.
+ """
+ def __init__(self, features=None, **morefeatures):
+ """
+ Create a new feature dictionary, with the specified features.
+
+ :param features: The initial value for this feature
+ dictionary. If ``features`` is a ``FeatStruct``, then its
+ features are copied (shallow copy). If ``features`` is a
+ dict, then a feature is created for each item, mapping its
+ key to its value. If ``features`` is a string, then it is
+ processed using ``FeatStructReader``. If ``features`` is a list of
+ tuples ``(name, val)``, then a feature is created for each tuple.
+ :param morefeatures: Additional features for the new feature
+ dictionary. If a feature is listed under both ``features`` and
+ ``morefeatures``, then the value from ``morefeatures`` will be
+ used.
+ """
+ if isinstance(features, string_types):
+ FeatStructReader().fromstring(features, self)
+ self.update(**morefeatures)
+ else:
+ # update() checks the types of features.
+ self.update(features, **morefeatures)
+
+ #////////////////////////////////////////////////////////////
+ #{ Dict methods
+ #////////////////////////////////////////////////////////////
+ _INDEX_ERROR = str("Expected feature name or path. Got %r.")
+
+ def __getitem__(self, name_or_path):
+ """If the feature with the given name or path exists, return
+ its value; otherwise, raise ``KeyError``."""
+ if isinstance(name_or_path, (string_types, Feature)):
+ return dict.__getitem__(self, name_or_path)
+ elif isinstance(name_or_path, tuple):
+ try:
+ val = self
+ for fid in name_or_path:
+ if not isinstance(val, FeatStruct):
+ raise KeyError # path contains base value
+ val = val[fid]
+ return val
+ except (KeyError, IndexError):
+ raise KeyError(name_or_path)
+ else:
+ raise TypeError(self._INDEX_ERROR % name_or_path)
+
+ def get(self, name_or_path, default=None):
+ """If the feature with the given name or path exists, return its
+ value; otherwise, return ``default``."""
+ try: return self[name_or_path]
+ except KeyError: return default
+
+ def __contains__(self, name_or_path):
+ """Return true if a feature with the given name or path exists."""
+ try: self[name_or_path]; return True
+ except KeyError: return False
+
+ def has_key(self, name_or_path):
+ """Return true if a feature with the given name or path exists."""
+ return name_or_path in self
+
+ def __delitem__(self, name_or_path):
+ """If the feature with the given name or path exists, delete
+ its value; otherwise, raise ``KeyError``."""
+ if self._frozen: raise ValueError(_FROZEN_ERROR)
+ if isinstance(name_or_path, (string_types, Feature)):
+ return dict.__delitem__(self, name_or_path)
+ elif isinstance(name_or_path, tuple):
+ if len(name_or_path) == 0:
+ raise ValueError("The path () can not be set")
+ else:
+ parent = self[name_or_path[:-1]]
+ if not isinstance(parent, FeatStruct):
+ raise KeyError(name_or_path) # path contains base value
+ del parent[name_or_path[-1]]
+ else:
+ raise TypeError(self._INDEX_ERROR % name_or_path)
+
+ def __setitem__(self, name_or_path, value):
+ """Set the value for the feature with the given name or path
+ to ``value``. If ``name_or_path`` is an invalid path, raise
+ ``KeyError``."""
+ if self._frozen: raise ValueError(_FROZEN_ERROR)
+ if isinstance(name_or_path, (string_types, Feature)):
+ return dict.__setitem__(self, name_or_path, value)
+ elif isinstance(name_or_path, tuple):
+ if len(name_or_path) == 0:
+ raise ValueError("The path () can not be set")
+ else:
+ parent = self[name_or_path[:-1]]
+ if not isinstance(parent, FeatStruct):
+ raise KeyError(name_or_path) # path contains base value
+ parent[name_or_path[-1]] = value
+ else:
+ raise TypeError(self._INDEX_ERROR % name_or_path)
+
+ clear = _check_frozen(dict.clear)
+ pop = _check_frozen(dict.pop)
+ popitem = _check_frozen(dict.popitem)
+ setdefault = _check_frozen(dict.setdefault)
+
+ def update(self, features=None, **morefeatures):
+ if self._frozen: raise ValueError(_FROZEN_ERROR)
+ if features is None:
+ items = ()
+ elif hasattr(features, 'items') and callable(features.items):
+ items = features.items()
+ elif hasattr(features, '__iter__'):
+ items = features
+ else:
+ raise ValueError('Expected mapping or list of tuples')
+
+ for key, val in items:
+ if not isinstance(key, (string_types, Feature)):
+ raise TypeError('Feature names must be strings')
+ self[key] = val
+ for key, val in morefeatures.items():
+ if not isinstance(key, (string_types, Feature)):
+ raise TypeError('Feature names must be strings')
+ self[key] = val
+
+ ##////////////////////////////////////////////////////////////
+ #{ Copying
+ ##////////////////////////////////////////////////////////////
+
+ def __deepcopy__(self, memo):
+ memo[id(self)] = selfcopy = self.__class__()
+ for (key, val) in self._items():
+ selfcopy[copy.deepcopy(key,memo)] = copy.deepcopy(val,memo)
+ return selfcopy
+
+ ##////////////////////////////////////////////////////////////
+ #{ Uniform Accessor Methods
+ ##////////////////////////////////////////////////////////////
+
+ def _keys(self): return self.keys()
+ def _values(self): return self.values()
+ def _items(self): return self.items()
+
+ ##////////////////////////////////////////////////////////////
+ #{ String Representations
+ ##////////////////////////////////////////////////////////////
+
+ def __str__(self):
+ """
+ Display a multi-line representation of this feature dictionary
+ as an FVM (feature value matrix).
+ """
+ return '\n'.join(self._str(self._find_reentrances({}), {}))
+
+ def _repr(self, reentrances, reentrance_ids):
+ segments = []
+ prefix = ''
+ suffix = ''
+
+ # If this is the first time we've seen a reentrant structure,
+ # then assign it a unique identifier.
+ if reentrances[id(self)]:
+ assert id(self) not in reentrance_ids
+ reentrance_ids[id(self)] = repr(len(reentrance_ids)+1)
+
+ # sorting note: keys are unique strings, so we'll never fall
+ # through to comparing values.
+ for (fname, fval) in sorted(self.items()):
+ display = getattr(fname, 'display', None)
+ if id(fval) in reentrance_ids:
+ segments.append('%s->(%s)' %
+ (fname, reentrance_ids[id(fval)]))
+ elif (display == 'prefix' and not prefix and
+ isinstance(fval, (Variable, string_types))):
+ prefix = '%s' % fval
+ elif display == 'slash' and not suffix:
+ if isinstance(fval, Variable):
+ suffix = '/%s' % fval.name
+ else:
+ suffix = '/%s' % unicode_repr(fval)
+ elif isinstance(fval, Variable):
+ segments.append('%s=%s' % (fname, fval.name))
+ elif fval is True:
+ segments.append('+%s' % fname)
+ elif fval is False:
+ segments.append('-%s' % fname)
+ elif isinstance(fval, Expression):
+ segments.append('%s=<%s>' % (fname, fval))
+ elif not isinstance(fval, FeatStruct):
+ segments.append('%s=%s' % (fname, unicode_repr(fval)))
+ else:
+ fval_repr = fval._repr(reentrances, reentrance_ids)
+ segments.append('%s=%s' % (fname, fval_repr))
+ # If it's reentrant, then add on an identifier tag.
+ if reentrances[id(self)]:
+ prefix = '(%s)%s' % (reentrance_ids[id(self)], prefix)
+ return '%s[%s]%s' % (prefix, ', '.join(segments), suffix)
+
+ def _str(self, reentrances, reentrance_ids):
+ """
+ :return: A list of lines composing a string representation of
+ this feature dictionary.
+ :param reentrances: A dictionary that maps from the ``id`` of
+ each feature value in self, indicating whether that value
+ is reentrant or not.
+ :param reentrance_ids: A dictionary mapping from each ``id``
+ of a feature value to a unique identifier. This is modified
+ by ``repr``: the first time a reentrant feature value is
+ displayed, an identifier is added to ``reentrance_ids`` for
+ it.
+ """
+ # If this is the first time we've seen a reentrant structure,
+ # then tack on an id string.
+ if reentrances[id(self)]:
+ assert id(self) not in reentrance_ids
+ reentrance_ids[id(self)] = repr(len(reentrance_ids)+1)
+
+ # Special case: empty feature dict.
+ if len(self) == 0:
+ if reentrances[id(self)]:
+ return ['(%s) []' % reentrance_ids[id(self)]]
+ else:
+ return ['[]']
+
+ # What's the longest feature name? Use this to align names.
+ maxfnamelen = max(len("%s" % k) for k in self.keys())
+
+ lines = []
+ # sorting note: keys are unique strings, so we'll never fall
+ # through to comparing values.
+ for (fname, fval) in sorted(self.items()):
+ fname = ("%s" % fname).ljust(maxfnamelen)
+ if isinstance(fval, Variable):
+ lines.append('%s = %s' % (fname,fval.name))
+
+ elif isinstance(fval, Expression):
+ lines.append('%s = <%s>' % (fname, fval))
+
+ elif isinstance(fval, FeatList):
+ fval_repr = fval._repr(reentrances, reentrance_ids)
+ lines.append('%s = %s' % (fname, unicode_repr(fval_repr)))
+
+ elif not isinstance(fval, FeatDict):
+ # It's not a nested feature structure -- just print it.
+ lines.append('%s = %s' % (fname, unicode_repr(fval)))
+
+ elif id(fval) in reentrance_ids:
+ # It's a feature structure we've seen before -- print
+ # the reentrance id.
+ lines.append('%s -> (%s)' % (fname, reentrance_ids[id(fval)]))
+
+ else:
+ # It's a new feature structure. Separate it from
+ # other values by a blank line.
+ if lines and lines[-1] != '': lines.append('')
+
+ # Recursively print the feature's value (fval).
+ fval_lines = fval._str(reentrances, reentrance_ids)
+
+ # Indent each line to make room for fname.
+ fval_lines = [(' '*(maxfnamelen+3))+l for l in fval_lines]
+
+ # Pick which line we'll display fname on, & splice it in.
+ nameline = (len(fval_lines)-1) // 2
+ fval_lines[nameline] = (
+ fname+' ='+fval_lines[nameline][maxfnamelen+2:])
+
+ # Add the feature structure to the output.
+ lines += fval_lines
+
+ # Separate FeatStructs by a blank line.
+ lines.append('')
+
+ # Get rid of any excess blank lines.
+ if lines[-1] == '': lines.pop()
+
+ # Add brackets around everything.
+ maxlen = max(len(line) for line in lines)
+ lines = ['[ %s%s ]' % (line, ' '*(maxlen-len(line))) for line in lines]
+
+ # If it's reentrant, then add on an identifier tag.
+ if reentrances[id(self)]:
+ idstr = '(%s) ' % reentrance_ids[id(self)]
+ lines = [(' '*len(idstr))+l for l in lines]
+ idline = (len(lines)-1) // 2
+ lines[idline] = idstr + lines[idline][len(idstr):]
+
+ return lines
+
+
+######################################################################
+# Feature List
+######################################################################
+
+class FeatList(FeatStruct, list):
+ """
+ A list of feature values, where each feature value is either a
+ basic value (such as a string or an integer), or a nested feature
+ structure.
+
+ Feature lists may contain reentrant feature values. A "reentrant
+ feature value" is a single feature value that can be accessed via
+ multiple feature paths. Feature lists may also be cyclic.
+
+ Two feature lists are considered equal if they assign the same
+ values to all features, and have the same reentrances.
+
+ :see: ``FeatStruct`` for information about feature paths, reentrance,
+ cyclic feature structures, mutability, freezing, and hashing.
+ """
+ def __init__(self, features=()):
+ """
+ Create a new feature list, with the specified features.
+
+ :param features: The initial list of features for this feature
+ list. If ``features`` is a string, then it is paresd using
+ ``FeatStructReader``. Otherwise, it should be a sequence
+ of basic values and nested feature structures.
+ """
+ if isinstance(features, string_types):
+ FeatStructReader().fromstring(features, self)
+ else:
+ list.__init__(self, features)
+
+ #////////////////////////////////////////////////////////////
+ #{ List methods
+ #////////////////////////////////////////////////////////////
+ _INDEX_ERROR = "Expected int or feature path. Got %r."
+
+ def __getitem__(self, name_or_path):
+ if isinstance(name_or_path, integer_types):
+ return list.__getitem__(self, name_or_path)
+ elif isinstance(name_or_path, tuple):
+ try:
+ val = self
+ for fid in name_or_path:
+ if not isinstance(val, FeatStruct):
+ raise KeyError # path contains base value
+ val = val[fid]
+ return val
+ except (KeyError, IndexError):
+ raise KeyError(name_or_path)
+ else:
+ raise TypeError(self._INDEX_ERROR % name_or_path)
+
+ def __delitem__(self, name_or_path):
+ """If the feature with the given name or path exists, delete
+ its value; otherwise, raise ``KeyError``."""
+ if self._frozen: raise ValueError(_FROZEN_ERROR)
+ if isinstance(name_or_path, (integer_types, slice)):
+ return list.__delitem__(self, name_or_path)
+ elif isinstance(name_or_path, tuple):
+ if len(name_or_path) == 0:
+ raise ValueError("The path () can not be set")
+ else:
+ parent = self[name_or_path[:-1]]
+ if not isinstance(parent, FeatStruct):
+ raise KeyError(name_or_path) # path contains base value
+ del parent[name_or_path[-1]]
+ else:
+ raise TypeError(self._INDEX_ERROR % name_or_path)
+
+ def __setitem__(self, name_or_path, value):
+ """Set the value for the feature with the given name or path
+ to ``value``. If ``name_or_path`` is an invalid path, raise
+ ``KeyError``."""
+ if self._frozen: raise ValueError(_FROZEN_ERROR)
+ if isinstance(name_or_path, (integer_types, slice)):
+ return list.__setitem__(self, name_or_path, value)
+ elif isinstance(name_or_path, tuple):
+ if len(name_or_path) == 0:
+ raise ValueError("The path () can not be set")
+ else:
+ parent = self[name_or_path[:-1]]
+ if not isinstance(parent, FeatStruct):
+ raise KeyError(name_or_path) # path contains base value
+ parent[name_or_path[-1]] = value
+ else:
+ raise TypeError(self._INDEX_ERROR % name_or_path)
+
+# __delslice__ = _check_frozen(list.__delslice__, ' ')
+# __setslice__ = _check_frozen(list.__setslice__, ' ')
+ __iadd__ = _check_frozen(list.__iadd__)
+ __imul__ = _check_frozen(list.__imul__)
+ append = _check_frozen(list.append)
+ extend = _check_frozen(list.extend)
+ insert = _check_frozen(list.insert)
+ pop = _check_frozen(list.pop)
+ remove = _check_frozen(list.remove)
+ reverse = _check_frozen(list.reverse)
+ sort = _check_frozen(list.sort)
+
+ ##////////////////////////////////////////////////////////////
+ #{ Copying
+ ##////////////////////////////////////////////////////////////
+
+ def __deepcopy__(self, memo):
+ memo[id(self)] = selfcopy = self.__class__()
+ selfcopy.extend(copy.deepcopy(fval,memo) for fval in self)
+ return selfcopy
+
+ ##////////////////////////////////////////////////////////////
+ #{ Uniform Accessor Methods
+ ##////////////////////////////////////////////////////////////
+
+ def _keys(self): return list(range(len(self)))
+ def _values(self): return self
+ def _items(self): return enumerate(self)
+
+ ##////////////////////////////////////////////////////////////
+ #{ String Representations
+ ##////////////////////////////////////////////////////////////
+
+ # Special handling for: reentrances, variables, expressions.
+ def _repr(self, reentrances, reentrance_ids):
+ # If this is the first time we've seen a reentrant structure,
+ # then assign it a unique identifier.
+ if reentrances[id(self)]:
+ assert id(self) not in reentrance_ids
+ reentrance_ids[id(self)] = repr(len(reentrance_ids)+1)
+ prefix = '(%s)' % reentrance_ids[id(self)]
+ else:
+ prefix = ''
+
+ segments = []
+ for fval in self:
+ if id(fval) in reentrance_ids:
+ segments.append('->(%s)' % reentrance_ids[id(fval)])
+ elif isinstance(fval, Variable):
+ segments.append(fval.name)
+ elif isinstance(fval, Expression):
+ segments.append('%s' % fval)
+ elif isinstance(fval, FeatStruct):
+ segments.append(fval._repr(reentrances, reentrance_ids))
+ else:
+ segments.append('%s' % unicode_repr(fval))
+
+ return '%s[%s]' % (prefix, ', '.join(segments))
+
+######################################################################
+# Variables & Bindings
+######################################################################
+
+def substitute_bindings(fstruct, bindings, fs_class='default'):
+ """
+ Return the feature structure that is obtained by replacing each
+ variable bound by ``bindings`` with its binding. If a variable is
+ aliased to a bound variable, then it will be replaced by that
+ variable's value. If a variable is aliased to an unbound
+ variable, then it will be replaced by that variable.
+
+ :type bindings: dict(Variable -> any)
+ :param bindings: A dictionary mapping from variables to values.
+ """
+ if fs_class == 'default': fs_class = _default_fs_class(fstruct)
+ fstruct = copy.deepcopy(fstruct)
+ _substitute_bindings(fstruct, bindings, fs_class, set())
+ return fstruct
+
+def _substitute_bindings(fstruct, bindings, fs_class, visited):
+ # Visit each node only once:
+ if id(fstruct) in visited: return
+ visited.add(id(fstruct))
+
+ if _is_mapping(fstruct): items = fstruct.items()
+ elif _is_sequence(fstruct): items = enumerate(fstruct)
+ else: raise ValueError('Expected mapping or sequence')
+ for (fname, fval) in items:
+ while (isinstance(fval, Variable) and fval in bindings):
+ fval = fstruct[fname] = bindings[fval]
+ if isinstance(fval, fs_class):
+ _substitute_bindings(fval, bindings, fs_class, visited)
+ elif isinstance(fval, SubstituteBindingsI):
+ fstruct[fname] = fval.substitute_bindings(bindings)
+
+def retract_bindings(fstruct, bindings, fs_class='default'):
+ """
+ Return the feature structure that is obtained by replacing each
+ feature structure value that is bound by ``bindings`` with the
+ variable that binds it. A feature structure value must be
+ identical to a bound value (i.e., have equal id) to be replaced.
+
+ ``bindings`` is modified to point to this new feature structure,
+ rather than the original feature structure. Feature structure
+ values in ``bindings`` may be modified if they are contained in
+ ``fstruct``.
+ """
+ if fs_class == 'default': fs_class = _default_fs_class(fstruct)
+ (fstruct, new_bindings) = copy.deepcopy((fstruct, bindings))
+ bindings.update(new_bindings)
+ inv_bindings = dict((id(val),var) for (var,val) in bindings.items())
+ _retract_bindings(fstruct, inv_bindings, fs_class, set())
+ return fstruct
+
+def _retract_bindings(fstruct, inv_bindings, fs_class, visited):
+ # Visit each node only once:
+ if id(fstruct) in visited: return
+ visited.add(id(fstruct))
+
+ if _is_mapping(fstruct): items = fstruct.items()
+ elif _is_sequence(fstruct): items = enumerate(fstruct)
+ else: raise ValueError('Expected mapping or sequence')
+ for (fname, fval) in items:
+ if isinstance(fval, fs_class):
+ if id(fval) in inv_bindings:
+ fstruct[fname] = inv_bindings[id(fval)]
+ _retract_bindings(fval, inv_bindings, fs_class, visited)
+
+
+def find_variables(fstruct, fs_class='default'):
+ """
+ :return: The set of variables used by this feature structure.
+ :rtype: set(Variable)
+ """
+ if fs_class == 'default': fs_class = _default_fs_class(fstruct)
+ return _variables(fstruct, set(), fs_class, set())
+
+def _variables(fstruct, vars, fs_class, visited):
+ # Visit each node only once:
+ if id(fstruct) in visited: return
+ visited.add(id(fstruct))
+ if _is_mapping(fstruct): items = fstruct.items()
+ elif _is_sequence(fstruct): items = enumerate(fstruct)
+ else: raise ValueError('Expected mapping or sequence')
+ for (fname, fval) in items:
+ if isinstance(fval, Variable):
+ vars.add(fval)
+ elif isinstance(fval, fs_class):
+ _variables(fval, vars, fs_class, visited)
+ elif isinstance(fval, SubstituteBindingsI):
+ vars.update(fval.variables())
+ return vars
+
+def rename_variables(fstruct, vars=None, used_vars=(), new_vars=None,
+ fs_class='default'):
+ """
+ Return the feature structure that is obtained by replacing
+ any of this feature structure's variables that are in ``vars``
+ with new variables. The names for these new variables will be
+ names that are not used by any variable in ``vars``, or in
+ ``used_vars``, or in this feature structure.
+
+ :type vars: set
+ :param vars: The set of variables that should be renamed.
+ If not specified, ``find_variables(fstruct)`` is used; i.e., all
+ variables will be given new names.
+ :type used_vars: set
+ :param used_vars: A set of variables whose names should not be
+ used by the new variables.
+ :type new_vars: dict(Variable -> Variable)
+ :param new_vars: A dictionary that is used to hold the mapping
+ from old variables to new variables. For each variable *v*
+ in this feature structure:
+
+ - If ``new_vars`` maps *v* to *v'*, then *v* will be
+ replaced by *v'*.
+ - If ``new_vars`` does not contain *v*, but ``vars``
+ does contain *v*, then a new entry will be added to
+ ``new_vars``, mapping *v* to the new variable that is used
+ to replace it.
+
+ To consistently rename the variables in a set of feature
+ structures, simply apply rename_variables to each one, using
+ the same dictionary:
+
+ >>> from nltk.featstruct import FeatStruct
+ >>> fstruct1 = FeatStruct('[subj=[agr=[gender=?y]], obj=[agr=[gender=?y]]]')
+ >>> fstruct2 = FeatStruct('[subj=[agr=[number=?z,gender=?y]], obj=[agr=[number=?z,gender=?y]]]')
+ >>> new_vars = {} # Maps old vars to alpha-renamed vars
+ >>> fstruct1.rename_variables(new_vars=new_vars)
+ [obj=[agr=[gender=?y2]], subj=[agr=[gender=?y2]]]
+ >>> fstruct2.rename_variables(new_vars=new_vars)
+ [obj=[agr=[gender=?y2, number=?z2]], subj=[agr=[gender=?y2, number=?z2]]]
+
+ If new_vars is not specified, then an empty dictionary is used.
+ """
+ if fs_class == 'default': fs_class = _default_fs_class(fstruct)
+
+ # Default values:
+ if new_vars is None: new_vars = {}
+ if vars is None: vars = find_variables(fstruct, fs_class)
+ else: vars = set(vars)
+
+ # Add our own variables to used_vars.
+ used_vars = find_variables(fstruct, fs_class).union(used_vars)
+
+ # Copy ourselves, and rename variables in the copy.
+ return _rename_variables(copy.deepcopy(fstruct), vars, used_vars,
+ new_vars, fs_class, set())
+
+def _rename_variables(fstruct, vars, used_vars, new_vars, fs_class, visited):
+ if id(fstruct) in visited: return
+ visited.add(id(fstruct))
+ if _is_mapping(fstruct): items = fstruct.items()
+ elif _is_sequence(fstruct): items = enumerate(fstruct)
+ else: raise ValueError('Expected mapping or sequence')
+ for (fname, fval) in items:
+ if isinstance(fval, Variable):
+ # If it's in new_vars, then rebind it.
+ if fval in new_vars:
+ fstruct[fname] = new_vars[fval]
+ # If it's in vars, pick a new name for it.
+ elif fval in vars:
+ new_vars[fval] = _rename_variable(fval, used_vars)
+ fstruct[fname] = new_vars[fval]
+ used_vars.add(new_vars[fval])
+ elif isinstance(fval, fs_class):
+ _rename_variables(fval, vars, used_vars, new_vars,
+ fs_class, visited)
+ elif isinstance(fval, SubstituteBindingsI):
+ # Pick new names for any variables in `vars`
+ for var in fval.variables():
+ if var in vars and var not in new_vars:
+ new_vars[var] = _rename_variable(var, used_vars)
+ used_vars.add(new_vars[var])
+ # Replace all variables in `new_vars`.
+ fstruct[fname] = fval.substitute_bindings(new_vars)
+ return fstruct
+
+def _rename_variable(var, used_vars):
+ name, n = re.sub('\d+$', '', var.name), 2
+ if not name: name = '?'
+ while Variable('%s%s' % (name, n)) in used_vars: n += 1
+ return Variable('%s%s' % (name, n))
+
+def remove_variables(fstruct, fs_class='default'):
+ """
+ :rtype: FeatStruct
+ :return: The feature structure that is obtained by deleting
+ all features whose values are ``Variables``.
+ """
+ if fs_class == 'default': fs_class = _default_fs_class(fstruct)
+ return _remove_variables(copy.deepcopy(fstruct), fs_class, set())
+
+def _remove_variables(fstruct, fs_class, visited):
+ if id(fstruct) in visited:
+ return
+ visited.add(id(fstruct))
+
+ if _is_mapping(fstruct):
+ items = list(fstruct.items())
+ elif _is_sequence(fstruct):
+ items = list(enumerate(fstruct))
+ else:
+ raise ValueError('Expected mapping or sequence')
+
+ for (fname, fval) in items:
+ if isinstance(fval, Variable):
+ del fstruct[fname]
+ elif isinstance(fval, fs_class):
+ _remove_variables(fval, fs_class, visited)
+ return fstruct
+
+
+######################################################################
+# Unification
+######################################################################
+
+ at python_2_unicode_compatible
+class _UnificationFailure(object):
+ def __repr__(self):
+ return 'nltk.featstruct.UnificationFailure'
+
+UnificationFailure = _UnificationFailure()
+"""A unique value used to indicate unification failure. It can be
+ returned by ``Feature.unify_base_values()`` or by custom ``fail()``
+ functions to indicate that unificaiton should fail."""
+
+# The basic unification algorithm:
+# 1. Make copies of self and other (preserving reentrance)
+# 2. Destructively unify self and other
+# 3. Apply forward pointers, to preserve reentrance.
+# 4. Replace bound variables with their values.
+def unify(fstruct1, fstruct2, bindings=None, trace=False,
+ fail=None, rename_vars=True, fs_class='default'):
+ """
+ Unify ``fstruct1`` with ``fstruct2``, and return the resulting feature
+ structure. This unified feature structure is the minimal
+ feature structure that contains all feature value assignments from both
+ ``fstruct1`` and ``fstruct2``, and that preserves all reentrancies.
+
+ If no such feature structure exists (because ``fstruct1`` and
+ ``fstruct2`` specify incompatible values for some feature), then
+ unification fails, and ``unify`` returns None.
+
+ Bound variables are replaced by their values. Aliased
+ variables are replaced by their representative variable
+ (if unbound) or the value of their representative variable
+ (if bound). I.e., if variable *v* is in ``bindings``,
+ then *v* is replaced by ``bindings[v]``. This will
+ be repeated until the variable is replaced by an unbound
+ variable or a non-variable value.
+
+ Unbound variables are bound when they are unified with
+ values; and aliased when they are unified with variables.
+ I.e., if variable *v* is not in ``bindings``, and is
+ unified with a variable or value *x*, then
+ ``bindings[v]`` is set to *x*.
+
+ If ``bindings`` is unspecified, then all variables are
+ assumed to be unbound. I.e., ``bindings`` defaults to an
+ empty dict.
+
+ >>> from nltk.featstruct import FeatStruct
+ >>> FeatStruct('[a=?x]').unify(FeatStruct('[b=?x]'))
+ [a=?x, b=?x2]
+
+ :type bindings: dict(Variable -> any)
+ :param bindings: A set of variable bindings to be used and
+ updated during unification.
+ :type trace: bool
+ :param trace: If true, generate trace output.
+ :type rename_vars: bool
+ :param rename_vars: If True, then rename any variables in
+ ``fstruct2`` that are also used in ``fstruct1``, in order to
+ avoid collisions on variable names.
+ """
+ # Decide which class(es) will be treated as feature structures,
+ # for the purposes of unification.
+ if fs_class == 'default':
+ fs_class = _default_fs_class(fstruct1)
+ if _default_fs_class(fstruct2) != fs_class:
+ raise ValueError("Mixing FeatStruct objects with Python "
+ "dicts and lists is not supported.")
+ assert isinstance(fstruct1, fs_class)
+ assert isinstance(fstruct2, fs_class)
+
+ # If bindings are unspecified, use an empty set of bindings.
+ user_bindings = (bindings is not None)
+ if bindings is None: bindings = {}
+
+ # Make copies of fstruct1 and fstruct2 (since the unification
+ # algorithm is destructive). Do it all at once, to preserve
+ # reentrance links between fstruct1 and fstruct2. Copy bindings
+ # as well, in case there are any bound vars that contain parts
+ # of fstruct1 or fstruct2.
+ (fstruct1copy, fstruct2copy, bindings_copy) = (
+ copy.deepcopy((fstruct1, fstruct2, bindings)))
+
+ # Copy the bindings back to the original bindings dict.
+ bindings.update(bindings_copy)
+
+ if rename_vars:
+ vars1 = find_variables(fstruct1copy, fs_class)
+ vars2 = find_variables(fstruct2copy, fs_class)
+ _rename_variables(fstruct2copy, vars1, vars2, {}, fs_class, set())
+
+ # Do the actual unification. If it fails, return None.
+ forward = {}
+ if trace: _trace_unify_start((), fstruct1copy, fstruct2copy)
+ try: result = _destructively_unify(fstruct1copy, fstruct2copy, bindings,
+ forward, trace, fail, fs_class, ())
+ except _UnificationFailureError: return None
+
+ # _destructively_unify might return UnificationFailure, e.g. if we
+ # tried to unify a mapping with a sequence.
+ if result is UnificationFailure:
+ if fail is None: return None
+ else: return fail(fstruct1copy, fstruct2copy, ())
+
+ # Replace any feature structure that has a forward pointer
+ # with the target of its forward pointer.
+ result = _apply_forwards(result, forward, fs_class, set())
+ if user_bindings: _apply_forwards_to_bindings(forward, bindings)
+
+ # Replace bound vars with values.
+ _resolve_aliases(bindings)
+ _substitute_bindings(result, bindings, fs_class, set())
+
+ # Return the result.
+ if trace: _trace_unify_succeed((), result)
+ if trace: _trace_bindings((), bindings)
+ return result
+
+class _UnificationFailureError(Exception):
+ """An exception that is used by ``_destructively_unify`` to abort
+ unification when a failure is encountered."""
+
+def _destructively_unify(fstruct1, fstruct2, bindings, forward,
+ trace, fail, fs_class, path):
+ """
+ Attempt to unify ``fstruct1`` and ``fstruct2`` by modifying them
+ in-place. If the unification succeeds, then ``fstruct1`` will
+ contain the unified value, the value of ``fstruct2`` is undefined,
+ and forward[id(fstruct2)] is set to fstruct1. If the unification
+ fails, then a _UnificationFailureError is raised, and the
+ values of ``fstruct1`` and ``fstruct2`` are undefined.
+
+ :param bindings: A dictionary mapping variables to values.
+ :param forward: A dictionary mapping feature structures ids
+ to replacement structures. When two feature structures
+ are merged, a mapping from one to the other will be added
+ to the forward dictionary; and changes will be made only
+ to the target of the forward dictionary.
+ ``_destructively_unify`` will always 'follow' any links
+ in the forward dictionary for fstruct1 and fstruct2 before
+ actually unifying them.
+ :param trace: If true, generate trace output
+ :param path: The feature path that led us to this unification
+ step. Used for trace output.
+ """
+ # If fstruct1 is already identical to fstruct2, we're done.
+ # Note: this, together with the forward pointers, ensures
+ # that unification will terminate even for cyclic structures.
+ if fstruct1 is fstruct2:
+ if trace: _trace_unify_identity(path, fstruct1)
+ return fstruct1
+
+ # Set fstruct2's forward pointer to point to fstruct1; this makes
+ # fstruct1 the canonical copy for fstruct2. Note that we need to
+ # do this before we recurse into any child structures, in case
+ # they're cyclic.
+ forward[id(fstruct2)] = fstruct1
+
+ # Unifying two mappings:
+ if _is_mapping(fstruct1) and _is_mapping(fstruct2):
+ for fname in fstruct1:
+ if getattr(fname, 'default', None) is not None:
+ fstruct2.setdefault(fname, fname.default)
+ for fname in fstruct2:
+ if getattr(fname, 'default', None) is not None:
+ fstruct1.setdefault(fname, fname.default)
+
+ # Unify any values that are defined in both fstruct1 and
+ # fstruct2. Copy any values that are defined in fstruct2 but
+ # not in fstruct1 to fstruct1. Note: sorting fstruct2's
+ # features isn't actually necessary; but we do it to give
+ # deterministic behavior, e.g. for tracing.
+ for fname, fval2 in sorted(fstruct2.items()):
+ if fname in fstruct1:
+ fstruct1[fname] = _unify_feature_values(
+ fname, fstruct1[fname], fval2, bindings,
+ forward, trace, fail, fs_class, path+(fname,))
+ else:
+ fstruct1[fname] = fval2
+
+ return fstruct1 # Contains the unified value.
+
+ # Unifying two sequences:
+ elif _is_sequence(fstruct1) and _is_sequence(fstruct2):
+ # If the lengths don't match, fail.
+ if len(fstruct1) != len(fstruct2):
+ return UnificationFailure
+
+ # Unify corresponding values in fstruct1 and fstruct2.
+ for findex in range(len(fstruct1)):
+ fstruct1[findex] = _unify_feature_values(
+ findex, fstruct1[findex], fstruct2[findex], bindings,
+ forward, trace, fail, fs_class, path+(findex,))
+
+ return fstruct1 # Contains the unified value.
+
+ # Unifying sequence & mapping: fail. The failure function
+ # doesn't get a chance to recover in this case.
+ elif ((_is_sequence(fstruct1) or _is_mapping(fstruct1)) and
+ (_is_sequence(fstruct2) or _is_mapping(fstruct2))):
+ return UnificationFailure
+
+ # Unifying anything else: not allowed!
+ raise TypeError('Expected mappings or sequences')
+
+def _unify_feature_values(fname, fval1, fval2, bindings, forward,
+ trace, fail, fs_class, fpath):
+ """
+ Attempt to unify ``fval1`` and and ``fval2``, and return the
+ resulting unified value. The method of unification will depend on
+ the types of ``fval1`` and ``fval2``:
+
+ 1. If they're both feature structures, then destructively
+ unify them (see ``_destructively_unify()``.
+ 2. If they're both unbound variables, then alias one variable
+ to the other (by setting bindings[v2]=v1).
+ 3. If one is an unbound variable, and the other is a value,
+ then bind the unbound variable to the value.
+ 4. If one is a feature structure, and the other is a base value,
+ then fail.
+ 5. If they're both base values, then unify them. By default,
+ this will succeed if they are equal, and fail otherwise.
+ """
+ if trace: _trace_unify_start(fpath, fval1, fval2)
+
+ # Look up the "canonical" copy of fval1 and fval2
+ while id(fval1) in forward: fval1 = forward[id(fval1)]
+ while id(fval2) in forward: fval2 = forward[id(fval2)]
+
+ # If fval1 or fval2 is a bound variable, then
+ # replace it by the variable's bound value. This
+ # includes aliased variables, which are encoded as
+ # variables bound to other variables.
+ fvar1 = fvar2 = None
+ while isinstance(fval1, Variable) and fval1 in bindings:
+ fvar1 = fval1
+ fval1 = bindings[fval1]
+ while isinstance(fval2, Variable) and fval2 in bindings:
+ fvar2 = fval2
+ fval2 = bindings[fval2]
+
+ # Case 1: Two feature structures (recursive case)
+ if isinstance(fval1, fs_class) and isinstance(fval2, fs_class):
+ result = _destructively_unify(fval1, fval2, bindings, forward,
+ trace, fail, fs_class, fpath)
+
+ # Case 2: Two unbound variables (create alias)
+ elif (isinstance(fval1, Variable) and
+ isinstance(fval2, Variable)):
+ if fval1 != fval2: bindings[fval2] = fval1
+ result = fval1
+
+ # Case 3: An unbound variable and a value (bind)
+ elif isinstance(fval1, Variable):
+ bindings[fval1] = fval2
+ result = fval1
+ elif isinstance(fval2, Variable):
+ bindings[fval2] = fval1
+ result = fval2
+
+ # Case 4: A feature structure & a base value (fail)
+ elif isinstance(fval1, fs_class) or isinstance(fval2, fs_class):
+ result = UnificationFailure
+
+ # Case 5: Two base values
+ else:
+ # Case 5a: Feature defines a custom unification method for base values
+ if isinstance(fname, Feature):
+ result = fname.unify_base_values(fval1, fval2, bindings)
+ # Case 5b: Feature value defines custom unification method
+ elif isinstance(fval1, CustomFeatureValue):
+ result = fval1.unify(fval2)
+ # Sanity check: unify value should be symmetric
+ if (isinstance(fval2, CustomFeatureValue) and
+ result != fval2.unify(fval1)):
+ raise AssertionError(
+ 'CustomFeatureValue objects %r and %r disagree '
+ 'about unification value: %r vs. %r' %
+ (fval1, fval2, result, fval2.unify(fval1)))
+ elif isinstance(fval2, CustomFeatureValue):
+ result = fval2.unify(fval1)
+ # Case 5c: Simple values -- check if they're equal.
+ else:
+ if fval1 == fval2:
+ result = fval1
+ else:
+ result = UnificationFailure
+
+ # If either value was a bound variable, then update the
+ # bindings. (This is really only necessary if fname is a
+ # Feature or if either value is a CustomFeatureValue.)
+ if result is not UnificationFailure:
+ if fvar1 is not None:
+ bindings[fvar1] = result
+ result = fvar1
+ if fvar2 is not None and fvar2 != fvar1:
+ bindings[fvar2] = result
+ result = fvar2
+
+ # If we unification failed, call the failure function; it
+ # might decide to continue anyway.
+ if result is UnificationFailure:
+ if fail is not None: result = fail(fval1, fval2, fpath)
+ if trace: _trace_unify_fail(fpath[:-1], result)
+ if result is UnificationFailure:
+ raise _UnificationFailureError
+
+ # Normalize the result.
+ if isinstance(result, fs_class):
+ result = _apply_forwards(result, forward, fs_class, set())
+
+ if trace: _trace_unify_succeed(fpath, result)
+ if trace and isinstance(result, fs_class):
+ _trace_bindings(fpath, bindings)
+
+ return result
+
+def _apply_forwards_to_bindings(forward, bindings):
+ """
+ Replace any feature structure that has a forward pointer with
+ the target of its forward pointer (to preserve reentrancy).
+ """
+ for (var, value) in bindings.items():
+ while id(value) in forward:
+ value = forward[id(value)]
+ bindings[var] = value
+
+def _apply_forwards(fstruct, forward, fs_class, visited):
+ """
+ Replace any feature structure that has a forward pointer with
+ the target of its forward pointer (to preserve reentrancy).
+ """
+ # Follow our own forwards pointers (if any)
+ while id(fstruct) in forward: fstruct = forward[id(fstruct)]
+
+ # Visit each node only once:
+ if id(fstruct) in visited: return
+ visited.add(id(fstruct))
+
+ if _is_mapping(fstruct): items = fstruct.items()
+ elif _is_sequence(fstruct): items = enumerate(fstruct)
+ else: raise ValueError('Expected mapping or sequence')
+ for fname, fval in items:
+ if isinstance(fval, fs_class):
+ # Replace w/ forwarded value.
+ while id(fval) in forward:
+ fval = forward[id(fval)]
+ fstruct[fname] = fval
+ # Recurse to child.
+ _apply_forwards(fval, forward, fs_class, visited)
+
+ return fstruct
+
+def _resolve_aliases(bindings):
+ """
+ Replace any bound aliased vars with their binding; and replace
+ any unbound aliased vars with their representative var.
+ """
+ for (var, value) in bindings.items():
+ while isinstance(value, Variable) and value in bindings:
+ value = bindings[var] = bindings[value]
+
+def _trace_unify_start(path, fval1, fval2):
+ if path == ():
+ print('\nUnification trace:')
+ else:
+ fullname = '.'.join("%s" % n for n in path)
+ print(' '+'| '*(len(path)-1)+'|')
+ print(' '+'| '*(len(path)-1)+'| Unify feature: %s' % fullname)
+ print(' '+'| '*len(path)+' / '+_trace_valrepr(fval1))
+ print(' '+'| '*len(path)+'|\\ '+_trace_valrepr(fval2))
+def _trace_unify_identity(path, fval1):
+ print(' '+'| '*len(path)+'|')
+ print(' '+'| '*len(path)+'| (identical objects)')
+ print(' '+'| '*len(path)+'|')
+ print(' '+'| '*len(path)+'+-->'+unicode_repr(fval1))
+def _trace_unify_fail(path, result):
+ if result is UnificationFailure: resume = ''
+ else: resume = ' (nonfatal)'
+ print(' '+'| '*len(path)+'| |')
+ print(' '+'X '*len(path)+'X X <-- FAIL'+resume)
+def _trace_unify_succeed(path, fval1):
+ # Print the result.
+ print(' '+'| '*len(path)+'|')
+ print(' '+'| '*len(path)+'+-->'+unicode_repr(fval1))
+def _trace_bindings(path, bindings):
+ # Print the bindings (if any).
+ if len(bindings) > 0:
+ binditems = sorted(bindings.items(), key=lambda v:v[0].name)
+ bindstr = '{%s}' % ', '.join(
+ '%s: %s' % (var, _trace_valrepr(val))
+ for (var, val) in binditems)
+ print(' '+'| '*len(path)+' Bindings: '+bindstr)
+def _trace_valrepr(val):
+ if isinstance(val, Variable):
+ return '%s' % val
+ else:
+ return '%s' % unicode_repr(val)
+
+def subsumes(fstruct1, fstruct2):
+ """
+ Return True if ``fstruct1`` subsumes ``fstruct2``. I.e., return
+ true if unifying ``fstruct1`` with ``fstruct2`` would result in a
+ feature structure equal to ``fstruct2.``
+
+ :rtype: bool
+ """
+ return fstruct2 == unify(fstruct1, fstruct2)
+
+def conflicts(fstruct1, fstruct2, trace=0):
+ """
+ Return a list of the feature paths of all features which are
+ assigned incompatible values by ``fstruct1`` and ``fstruct2``.
+
+ :rtype: list(tuple)
+ """
+ conflict_list = []
+ def add_conflict(fval1, fval2, path):
+ conflict_list.append(path)
+ return fval1
+ unify(fstruct1, fstruct2, fail=add_conflict, trace=trace)
+ return conflict_list
+
+######################################################################
+# Helper Functions
+######################################################################
+
+def _is_mapping(v):
+ return hasattr(v, '__contains__') and hasattr(v, 'keys')
+
+def _is_sequence(v):
+ return (hasattr(v, '__iter__') and hasattr(v, '__len__') and
+ not isinstance(v, string_types))
+
+def _default_fs_class(obj):
+ if isinstance(obj, FeatStruct): return FeatStruct
+ if isinstance(obj, (dict, list)): return (dict, list)
+ else:
+ raise ValueError('To unify objects of type %s, you must specify '
+ 'fs_class explicitly.' % obj.__class__.__name__)
+######################################################################
+# FeatureValueSet & FeatureValueTuple
+######################################################################
+
+class SubstituteBindingsSequence(SubstituteBindingsI):
+ """
+ A mixin class for sequence clases that distributes variables() and
+ substitute_bindings() over the object's elements.
+ """
+ def variables(self):
+ return ([elt for elt in self if isinstance(elt, Variable)] +
+ sum([list(elt.variables()) for elt in self
+ if isinstance(elt, SubstituteBindingsI)], []))
+
+ def substitute_bindings(self, bindings):
+ return self.__class__([self.subst(v, bindings) for v in self])
+
+ def subst(self, v, bindings):
+ if isinstance(v, SubstituteBindingsI):
+ return v.substitute_bindings(bindings)
+ else:
+ return bindings.get(v, v)
+
+ at python_2_unicode_compatible
+class FeatureValueTuple(SubstituteBindingsSequence, tuple):
+ """
+ A base feature value that is a tuple of other base feature values.
+ FeatureValueTuple implements ``SubstituteBindingsI``, so it any
+ variable substitutions will be propagated to the elements
+ contained by the set. A ``FeatureValueTuple`` is immutable.
+ """
+ def __repr__(self): # [xx] really use %s here?
+ if len(self) == 0: return '()'
+ return '(%s)' % ', '.join('%s' % (b,) for b in self)
+
+
+ at python_2_unicode_compatible
+class FeatureValueSet(SubstituteBindingsSequence, frozenset):
+ """
+ A base feature value that is a set of other base feature values.
+ FeatureValueSet implements ``SubstituteBindingsI``, so it any
+ variable substitutions will be propagated to the elements
+ contained by the set. A ``FeatureValueSet`` is immutable.
+ """
+ def __repr__(self): # [xx] really use %s here?
+ if len(self) == 0: return '{/}' # distinguish from dict.
+ # n.b., we sort the string reprs of our elements, to ensure
+ # that our own repr is deterministic.
+ return '{%s}' % ', '.join(sorted('%s' % (b,) for b in self))
+ __str__ = __repr__
+
+ at python_2_unicode_compatible
+class FeatureValueUnion(SubstituteBindingsSequence, frozenset):
+ """
+ A base feature value that represents the union of two or more
+ ``FeatureValueSet`` or ``Variable``.
+ """
+ def __new__(cls, values):
+ # If values contains FeatureValueUnions, then collapse them.
+ values = _flatten(values, FeatureValueUnion)
+
+ # If the resulting list contains no variables, then
+ # use a simple FeatureValueSet instead.
+ if sum(isinstance(v, Variable) for v in values) == 0:
+ values = _flatten(values, FeatureValueSet)
+ return FeatureValueSet(values)
+
+ # If we contain a single variable, return that variable.
+ if len(values) == 1:
+ return list(values)[0]
+
+ # Otherwise, build the FeatureValueUnion.
+ return frozenset.__new__(cls, values)
+
+ def __repr__(self):
+ # n.b., we sort the string reprs of our elements, to ensure
+ # that our own repr is deterministic. also, note that len(self)
+ # is guaranteed to be 2 or more.
+ return '{%s}' % '+'.join(sorted('%s' % (b,) for b in self))
+
+ at python_2_unicode_compatible
+class FeatureValueConcat(SubstituteBindingsSequence, tuple):
+ """
+ A base feature value that represents the concatenation of two or
+ more ``FeatureValueTuple`` or ``Variable``.
+ """
+ def __new__(cls, values):
+ # If values contains FeatureValueConcats, then collapse them.
+ values = _flatten(values, FeatureValueConcat)
+
+ # If the resulting list contains no variables, then
+ # use a simple FeatureValueTuple instead.
+ if sum(isinstance(v, Variable) for v in values) == 0:
+ values = _flatten(values, FeatureValueTuple)
+ return FeatureValueTuple(values)
+
+ # If we contain a single variable, return that variable.
+ if len(values) == 1:
+ return list(values)[0]
+
+ # Otherwise, build the FeatureValueConcat.
+ return tuple.__new__(cls, values)
+
+ def __repr__(self):
+ # n.b.: len(self) is guaranteed to be 2 or more.
+ return '(%s)' % '+'.join('%s' % (b,) for b in self)
+
+
+def _flatten(lst, cls):
+ """
+ Helper function -- return a copy of list, with all elements of
+ type ``cls`` spliced in rather than appended in.
+ """
+ result = []
+ for elt in lst:
+ if isinstance(elt, cls): result.extend(elt)
+ else: result.append(elt)
+ return result
+
+######################################################################
+# Specialized Features
+######################################################################
+
+ at total_ordering
+ at python_2_unicode_compatible
+class Feature(object):
+ """
+ A feature identifier that's specialized to put additional
+ constraints, default values, etc.
+ """
+ def __init__(self, name, default=None, display=None):
+ assert display in (None, 'prefix', 'slash')
+
+ self._name = name # [xx] rename to .identifier?
+ self._default = default # [xx] not implemented yet.
+ self._display = display
+
+ if self._display == 'prefix':
+ self._sortkey = (-1, self._name)
+ elif self._display == 'slash':
+ self._sortkey = (1, self._name)
+ else:
+ self._sortkey = (0, self._name)
+
+ @property
+ def name(self):
+ """The name of this feature."""
+ return self._name
+
+ @property
+ def default(self):
+ """Default value for this feature."""
+ return self._default
+
+ @property
+ def display(self):
+ """Custom display location: can be prefix, or slash."""
+ return self._display
+
+ def __repr__(self):
+ return '*%s*' % self.name
+
+ def __lt__(self, other):
+ if isinstance(other, string_types):
+ return True
+ if not isinstance(other, Feature):
+ raise_unorderable_types("<", self, other)
+ return self._sortkey < other._sortkey
+
+ def __eq__(self, other):
+ return type(self) == type(other) and self._name == other._name
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ return hash(self._name)
+
+ #////////////////////////////////////////////////////////////
+ # These can be overridden by subclasses:
+ #////////////////////////////////////////////////////////////
+
+ def read_value(self, s, position, reentrances, parser):
+ return parser.read_value(s, position, reentrances)
+
+ def unify_base_values(self, fval1, fval2, bindings):
+ """
+ If possible, return a single value.. If not, return
+ the value ``UnificationFailure``.
+ """
+ if fval1 == fval2: return fval1
+ else: return UnificationFailure
+
+
+class SlashFeature(Feature):
+ def read_value(self, s, position, reentrances, parser):
+ return parser.read_partial(s, position, reentrances)
+
+class RangeFeature(Feature):
+ RANGE_RE = re.compile('(-?\d+):(-?\d+)')
+ def read_value(self, s, position, reentrances, parser):
+ m = self.RANGE_RE.match(s, position)
+ if not m: raise ValueError('range', position)
+ return (int(m.group(1)), int(m.group(2))), m.end()
+
+ def unify_base_values(self, fval1, fval2, bindings):
+ if fval1 is None: return fval2
+ if fval2 is None: return fval1
+ rng = max(fval1[0], fval2[0]), min(fval1[1], fval2[1])
+ if rng[1] < rng[0]: return UnificationFailure
+ return rng
+
+SLASH = SlashFeature('slash', default=False, display='slash')
+TYPE = Feature('type', display='prefix')
+
+######################################################################
+# Specialized Feature Values
+######################################################################
+
+ at total_ordering
+class CustomFeatureValue(object):
+ """
+ An abstract base class for base values that define a custom
+ unification method. The custom unification method of
+ ``CustomFeatureValue`` will be used during unification if:
+
+ - The ``CustomFeatureValue`` is unified with another base value.
+ - The ``CustomFeatureValue`` is not the value of a customized
+ ``Feature`` (which defines its own unification method).
+
+ If two ``CustomFeatureValue`` objects are unified with one another
+ during feature structure unification, then the unified base values
+ they return *must* be equal; otherwise, an ``AssertionError`` will
+ be raised.
+
+ Subclasses must define ``unify()``, ``__eq__()`` and ``__lt__()``.
+ Subclasses may also wish to define ``__hash__()``.
+ """
+ def unify(self, other):
+ """
+ If this base value unifies with ``other``, then return the
+ unified value. Otherwise, return ``UnificationFailure``.
+ """
+ raise NotImplementedError('abstract base class')
+
+ def __eq__(self, other):
+ raise NotImplementedError('abstract base class')
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ raise NotImplementedError('abstract base class')
+
+ def __hash__(self):
+ raise TypeError('%s objects or unhashable' % self.__class__.__name__)
+
+######################################################################
+# Feature Structure Reader
+######################################################################
+
+class FeatStructReader(object):
+ def __init__(self, features=(SLASH, TYPE), fdict_class=FeatStruct,
+ flist_class=FeatList, logic_parser=None):
+ self._features = dict((f.name,f) for f in features)
+ self._fdict_class = fdict_class
+ self._flist_class = flist_class
+ self._prefix_feature = None
+ self._slash_feature = None
+ for feature in features:
+ if feature.display == 'slash':
+ if self._slash_feature:
+ raise ValueError('Multiple features w/ display=slash')
+ self._slash_feature = feature
+ if feature.display == 'prefix':
+ if self._prefix_feature:
+ raise ValueError('Multiple features w/ display=prefix')
+ self._prefix_feature = feature
+ self._features_with_defaults = [feature for feature in features
+ if feature.default is not None]
+ if logic_parser is None:
+ logic_parser = _LogicParser()
+ self._logic_parser = logic_parser
+
+ def fromstring(self, s, fstruct=None):
+ """
+ Convert a string representation of a feature structure (as
+ displayed by repr) into a ``FeatStruct``. This process
+ imposes the following restrictions on the string
+ representation:
+
+ - Feature names cannot contain any of the following:
+ whitespace, parentheses, quote marks, equals signs,
+ dashes, commas, and square brackets. Feature names may
+ not begin with plus signs or minus signs.
+ - Only the following basic feature value are supported:
+ strings, integers, variables, None, and unquoted
+ alphanumeric strings.
+ - For reentrant values, the first mention must specify
+ a reentrance identifier and a value; and any subsequent
+ mentions must use arrows (``'->'``) to reference the
+ reentrance identifier.
+ """
+ s = s.strip()
+ value, position = self.read_partial(s, 0, {}, fstruct)
+ if position != len(s):
+ self._error(s, 'end of string', position)
+ return value
+
+ _START_FSTRUCT_RE = re.compile(r'\s*(?:\((\d+)\)\s*)?(\??[\w-]+)?(\[)')
+ _END_FSTRUCT_RE = re.compile(r'\s*]\s*')
+ _SLASH_RE = re.compile(r'/')
+ _FEATURE_NAME_RE = re.compile(r'\s*([+-]?)([^\s\(\)<>"\'\-=\[\],]+)\s*')
+ _REENTRANCE_RE = re.compile(r'\s*->\s*')
+ _TARGET_RE = re.compile(r'\s*\((\d+)\)\s*')
+ _ASSIGN_RE = re.compile(r'\s*=\s*')
+ _COMMA_RE = re.compile(r'\s*,\s*')
+ _BARE_PREFIX_RE = re.compile(r'\s*(?:\((\d+)\)\s*)?(\??[\w-]+\s*)()')
+ # This one is used to distinguish fdicts from flists:
+ _START_FDICT_RE = re.compile(r'(%s)|(%s\s*(%s\s*(=|->)|[+-]%s|\]))' % (
+ _BARE_PREFIX_RE.pattern, _START_FSTRUCT_RE.pattern,
+ _FEATURE_NAME_RE.pattern, _FEATURE_NAME_RE.pattern))
+
+ def read_partial(self, s, position=0, reentrances=None, fstruct=None):
+ """
+ Helper function that reads in a feature structure.
+
+ :param s: The string to read.
+ :param position: The position in the string to start parsing.
+ :param reentrances: A dictionary from reentrance ids to values.
+ Defaults to an empty dictionary.
+ :return: A tuple (val, pos) of the feature structure created by
+ parsing and the position where the parsed feature structure ends.
+ :rtype: bool
+ """
+ if reentrances is None: reentrances = {}
+ try:
+ return self._read_partial(s, position, reentrances, fstruct)
+ except ValueError as e:
+ if len(e.args) != 2: raise
+ self._error(s, *e.args)
+
+ def _read_partial(self, s, position, reentrances, fstruct=None):
+ # Create the new feature structure
+ if fstruct is None:
+ if self._START_FDICT_RE.match(s, position):
+ fstruct = self._fdict_class()
+ else:
+ fstruct = self._flist_class()
+
+ # Read up to the open bracket.
+ match = self._START_FSTRUCT_RE.match(s, position)
+ if not match:
+ match = self._BARE_PREFIX_RE.match(s, position)
+ if not match:
+ raise ValueError('open bracket or identifier', position)
+ position = match.end()
+
+ # If there as an identifier, record it.
+ if match.group(1):
+ identifier = match.group(1)
+ if identifier in reentrances:
+ raise ValueError('new identifier', match.start(1))
+ reentrances[identifier] = fstruct
+
+ if isinstance(fstruct, FeatDict):
+ fstruct.clear()
+ return self._read_partial_featdict(s, position, match,
+ reentrances, fstruct)
+ else:
+ del fstruct[:]
+ return self._read_partial_featlist(s, position, match,
+ reentrances, fstruct)
+
+ def _read_partial_featlist(self, s, position, match,
+ reentrances, fstruct):
+ # Prefix features are not allowed:
+ if match.group(2): raise ValueError('open bracket')
+ # Bare prefixes are not allowed:
+ if not match.group(3): raise ValueError('open bracket')
+
+ # Build a list of the features defined by the structure.
+ while position < len(s):
+ # Check for the close bracket.
+ match = self._END_FSTRUCT_RE.match(s, position)
+ if match is not None:
+ return fstruct, match.end()
+
+ # Reentances have the form "-> (target)"
+ match = self._REENTRANCE_RE.match(s, position)
+ if match:
+ position = match.end()
+ match = self._TARGET_RE.match(s, position)
+ if not match: raise ValueError('identifier', position)
+ target = match.group(1)
+ if target not in reentrances:
+ raise ValueError('bound identifier', position)
+ position = match.end()
+ fstruct.append(reentrances[target])
+
+ # Anything else is a value.
+ else:
+ value, position = (
+ self._read_value(0, s, position, reentrances))
+ fstruct.append(value)
+
+ # If there's a close bracket, handle it at the top of the loop.
+ if self._END_FSTRUCT_RE.match(s, position):
+ continue
+
+ # Otherwise, there should be a comma
+ match = self._COMMA_RE.match(s, position)
+ if match is None: raise ValueError('comma', position)
+ position = match.end()
+
+ # We never saw a close bracket.
+ raise ValueError('close bracket', position)
+
+ def _read_partial_featdict(self, s, position, match,
+ reentrances, fstruct):
+ # If there was a prefix feature, record it.
+ if match.group(2):
+ if self._prefix_feature is None:
+ raise ValueError('open bracket or identifier', match.start(2))
+ prefixval = match.group(2).strip()
+ if prefixval.startswith('?'):
+ prefixval = Variable(prefixval)
+ fstruct[self._prefix_feature] = prefixval
+
+ # If group 3 is empty, then we just have a bare prefix, so
+ # we're done.
+ if not match.group(3):
+ return self._finalize(s, match.end(), reentrances, fstruct)
+
+ # Build a list of the features defined by the structure.
+ # Each feature has one of the three following forms:
+ # name = value
+ # name -> (target)
+ # +name
+ # -name
+ while position < len(s):
+ # Use these variables to hold info about each feature:
+ name = value = None
+
+ # Check for the close bracket.
+ match = self._END_FSTRUCT_RE.match(s, position)
+ if match is not None:
+ return self._finalize(s, match.end(), reentrances, fstruct)
+
+ # Get the feature name's name
+ match = self._FEATURE_NAME_RE.match(s, position)
+ if match is None: raise ValueError('feature name', position)
+ name = match.group(2)
+ position = match.end()
+
+ # Check if it's a special feature.
+ if name[0] == '*' and name[-1] == '*':
+ name = self._features.get(name[1:-1])
+ if name is None:
+ raise ValueError('known special feature', match.start(2))
+
+ # Check if this feature has a value already.
+ if name in fstruct:
+ raise ValueError('new name', match.start(2))
+
+ # Boolean value ("+name" or "-name")
+ if match.group(1) == '+': value = True
+ if match.group(1) == '-': value = False
+
+ # Reentrance link ("-> (target)")
+ if value is None:
+ match = self._REENTRANCE_RE.match(s, position)
+ if match is not None:
+ position = match.end()
+ match = self._TARGET_RE.match(s, position)
+ if not match:
+ raise ValueError('identifier', position)
+ target = match.group(1)
+ if target not in reentrances:
+ raise ValueError('bound identifier', position)
+ position = match.end()
+ value = reentrances[target]
+
+ # Assignment ("= value").
+ if value is None:
+ match = self._ASSIGN_RE.match(s, position)
+ if match:
+ position = match.end()
+ value, position = (
+ self._read_value(name, s, position, reentrances))
+ # None of the above: error.
+ else:
+ raise ValueError('equals sign', position)
+
+ # Store the value.
+ fstruct[name] = value
+
+ # If there's a close bracket, handle it at the top of the loop.
+ if self._END_FSTRUCT_RE.match(s, position):
+ continue
+
+ # Otherwise, there should be a comma
+ match = self._COMMA_RE.match(s, position)
+ if match is None: raise ValueError('comma', position)
+ position = match.end()
+
+ # We never saw a close bracket.
+ raise ValueError('close bracket', position)
+
+ def _finalize(self, s, pos, reentrances, fstruct):
+ """
+ Called when we see the close brace -- checks for a slash feature,
+ and adds in default values.
+ """
+ # Add the slash feature (if any)
+ match = self._SLASH_RE.match(s, pos)
+ if match:
+ name = self._slash_feature
+ v, pos = self._read_value(name, s, match.end(), reentrances)
+ fstruct[name] = v
+ ## Add any default features. -- handle in unficiation instead?
+ #for feature in self._features_with_defaults:
+ # fstruct.setdefault(feature, feature.default)
+ # Return the value.
+ return fstruct, pos
+
+ def _read_value(self, name, s, position, reentrances):
+ if isinstance(name, Feature):
+ return name.read_value(s, position, reentrances, self)
+ else:
+ return self.read_value(s, position, reentrances)
+
+ def read_value(self, s, position, reentrances):
+ for (handler, regexp) in self.VALUE_HANDLERS:
+ match = regexp.match(s, position)
+ if match:
+ handler_func = getattr(self, handler)
+ return handler_func(s, position, reentrances, match)
+ raise ValueError('value', position)
+
+ def _error(self, s, expected, position):
+ lines = s.split('\n')
+ while position > len(lines[0]):
+ position -= len(lines.pop(0))+1 # +1 for the newline.
+ estr = ('Error parsing feature structure\n ' +
+ lines[0] + '\n ' + ' '*position + '^ ' +
+ 'Expected %s' % expected)
+ raise ValueError(estr)
+
+ #////////////////////////////////////////////////////////////
+ #{ Value Readers
+ #////////////////////////////////////////////////////////////
+
+ #: A table indicating how feature values should be processed. Each
+ #: entry in the table is a pair (handler, regexp). The first entry
+ #: with a matching regexp will have its handler called. Handlers
+ #: should have the following signature::
+ #:
+ #: def handler(s, position, reentrances, match): ...
+ #:
+ #: and should return a tuple (value, position), where position is
+ #: the string position where the value ended. (n.b.: order is
+ #: important here!)
+ VALUE_HANDLERS = [
+ ('read_fstruct_value', _START_FSTRUCT_RE),
+ ('read_var_value', re.compile(r'\?[a-zA-Z_][a-zA-Z0-9_]*')),
+ ('read_str_value', re.compile("[uU]?[rR]?(['\"])")),
+ ('read_int_value', re.compile(r'-?\d+')),
+ ('read_sym_value', re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*')),
+ ('read_app_value', re.compile(r'<(app)\((\?[a-z][a-z]*)\s*,'
+ r'\s*(\?[a-z][a-z]*)\)>')),
+# ('read_logic_value', re.compile(r'<([^>]*)>')),
+ #lazily match any character after '<' until we hit a '>' not preceded by '-'
+ ('read_logic_value', re.compile(r'<(.*?)(?<!-)>')),
+ ('read_set_value', re.compile(r'{')),
+ ('read_tuple_value', re.compile(r'\(')),
+ ]
+
+ def read_fstruct_value(self, s, position, reentrances, match):
+ return self.read_partial(s, position, reentrances)
+
+ def read_str_value(self, s, position, reentrances, match):
+ return read_str(s, position)
+
+ def read_int_value(self, s, position, reentrances, match):
+ return int(match.group()), match.end()
+
+ # Note: the '?' is included in the variable name.
+ def read_var_value(self, s, position, reentrances, match):
+ return Variable(match.group()), match.end()
+
+ _SYM_CONSTS = {'None':None, 'True':True, 'False':False}
+ def read_sym_value(self, s, position, reentrances, match):
+ val, end = match.group(), match.end()
+ return self._SYM_CONSTS.get(val, val), end
+
+ def read_app_value(self, s, position, reentrances, match):
+ """Mainly included for backwards compat."""
+ return self._logic_parser.parse('%s(%s)' % match.group(2,3)), match.end()
+
+ def read_logic_value(self, s, position, reentrances, match):
+ try:
+ try:
+ expr = self._logic_parser.parse(match.group(1))
+ except LogicalExpressionException:
+ raise ValueError()
+ return expr, match.end()
+ except ValueError:
+ raise ValueError('logic expression', match.start(1))
+
+ def read_tuple_value(self, s, position, reentrances, match):
+ return self._read_seq_value(s, position, reentrances, match, ')',
+ FeatureValueTuple, FeatureValueConcat)
+
+ def read_set_value(self, s, position, reentrances, match):
+ return self._read_seq_value(s, position, reentrances, match, '}',
+ FeatureValueSet, FeatureValueUnion)
+
+ def _read_seq_value(self, s, position, reentrances, match,
+ close_paren, seq_class, plus_class):
+ """
+ Helper function used by read_tuple_value and read_set_value.
+ """
+ cp = re.escape(close_paren)
+ position = match.end()
+ # Special syntax fo empty tuples:
+ m = re.compile(r'\s*/?\s*%s' % cp).match(s, position)
+ if m: return seq_class(), m.end()
+ # Read values:
+ values = []
+ seen_plus = False
+ while True:
+ # Close paren: return value.
+ m = re.compile(r'\s*%s' % cp).match(s, position)
+ if m:
+ if seen_plus: return plus_class(values), m.end()
+ else: return seq_class(values), m.end()
+
+ # Read the next value.
+ val, position = self.read_value(s, position, reentrances)
+ values.append(val)
+
+ # Comma or looking at close paren
+ m = re.compile(r'\s*(,|\+|(?=%s))\s*' % cp).match(s, position)
+ if not m: raise ValueError("',' or '+' or '%s'" % cp, position)
+ if m.group(1) == '+': seen_plus = True
+ position = m.end()
+
+######################################################################
+#{ Demo
+######################################################################
+
+def display_unification(fs1, fs2, indent=' '):
+ # Print the two input feature structures, side by side.
+ fs1_lines = ("%s" % fs1).split('\n')
+ fs2_lines = ("%s" % fs2).split('\n')
+ if len(fs1_lines) > len(fs2_lines):
+ blankline = '['+' '*(len(fs2_lines[0])-2)+']'
+ fs2_lines += [blankline]*len(fs1_lines)
+ else:
+ blankline = '['+' '*(len(fs1_lines[0])-2)+']'
+ fs1_lines += [blankline]*len(fs2_lines)
+ for (fs1_line, fs2_line) in zip(fs1_lines, fs2_lines):
+ print(indent + fs1_line + ' ' + fs2_line)
+ print(indent+'-'*len(fs1_lines[0])+' '+'-'*len(fs2_lines[0]))
+
+ linelen = len(fs1_lines[0])*2+3
+ print(indent+'| |'.center(linelen))
+ print(indent+'+-----UNIFY-----+'.center(linelen))
+ print(indent+'|'.center(linelen))
+ print(indent+'V'.center(linelen))
+
+ bindings = {}
+
+ result = fs1.unify(fs2, bindings)
+ if result is None:
+ print(indent+'(FAILED)'.center(linelen))
+ else:
+ print('\n'.join(indent+l.center(linelen)
+ for l in ("%s" % result).split('\n')))
+ if bindings and len(bindings.bound_variables()) > 0:
+ print(repr(bindings).center(linelen))
+ return result
+
+def interactive_demo(trace=False):
+ import random, sys
+
+ HELP = '''
+ 1-%d: Select the corresponding feature structure
+ q: Quit
+ t: Turn tracing on or off
+ l: List all feature structures
+ ?: Help
+ '''
+
+ print('''
+ This demo will repeatedly present you with a list of feature
+ structures, and ask you to choose two for unification. Whenever a
+ new feature structure is generated, it is added to the list of
+ choices that you can pick from. However, since this can be a
+ large number of feature structures, the demo will only print out a
+ random subset for you to choose between at a given time. If you
+ want to see the complete lists, type "l". For a list of valid
+ commands, type "?".
+ ''')
+ print('Press "Enter" to continue...')
+ sys.stdin.readline()
+
+ fstruct_strings = [
+ '[agr=[number=sing, gender=masc]]',
+ '[agr=[gender=masc, person=3]]',
+ '[agr=[gender=fem, person=3]]',
+ '[subj=[agr=(1)[]], agr->(1)]',
+ '[obj=?x]', '[subj=?x]',
+ '[/=None]', '[/=NP]',
+ '[cat=NP]', '[cat=VP]', '[cat=PP]',
+ '[subj=[agr=[gender=?y]], obj=[agr=[gender=?y]]]',
+ '[gender=masc, agr=?C]',
+ '[gender=?S, agr=[gender=?S,person=3]]'
+ ]
+
+ all_fstructs = [(i, FeatStruct(fstruct_strings[i]))
+ for i in range(len(fstruct_strings))]
+
+ def list_fstructs(fstructs):
+ for i, fstruct in fstructs:
+ print()
+ lines = ("%s" % fstruct).split('\n')
+ print('%3d: %s' % (i+1, lines[0]))
+ for line in lines[1:]: print(' '+line)
+ print()
+
+
+ while True:
+ # Pick 5 feature structures at random from the master list.
+ MAX_CHOICES = 5
+ if len(all_fstructs) > MAX_CHOICES:
+ fstructs = sorted(random.sample(all_fstructs, MAX_CHOICES))
+ else:
+ fstructs = all_fstructs
+
+ print('_'*75)
+
+ print('Choose two feature structures to unify:')
+ list_fstructs(fstructs)
+
+ selected = [None,None]
+ for (nth,i) in (('First',0), ('Second',1)):
+ while selected[i] is None:
+ print(('%s feature structure (1-%d,q,t,l,?): '
+ % (nth, len(all_fstructs))), end=' ')
+ try:
+ input = sys.stdin.readline().strip()
+ if input in ('q', 'Q', 'x', 'X'): return
+ if input in ('t', 'T'):
+ trace = not trace
+ print(' Trace = %s' % trace)
+ continue
+ if input in ('h', 'H', '?'):
+ print(HELP % len(fstructs)); continue
+ if input in ('l', 'L'):
+ list_fstructs(all_fstructs); continue
+ num = int(input)-1
+ selected[i] = all_fstructs[num][1]
+ print()
+ except:
+ print('Bad sentence number')
+ continue
+
+ if trace:
+ result = selected[0].unify(selected[1], trace=1)
+ else:
+ result = display_unification(selected[0], selected[1])
+ if result is not None:
+ for i, fstruct in all_fstructs:
+ if repr(result) == repr(fstruct): break
+ else:
+ all_fstructs.append((len(all_fstructs), result))
+
+ print('\nType "Enter" to continue unifying; or "q" to quit.')
+ input = sys.stdin.readline().strip()
+ if input in ('q', 'Q', 'x', 'X'): return
+
+def demo(trace=False):
+ """
+ Just for testing
+ """
+ #import random
+
+ # processor breaks with values like '3rd'
+ fstruct_strings = [
+ '[agr=[number=sing, gender=masc]]',
+ '[agr=[gender=masc, person=3]]',
+ '[agr=[gender=fem, person=3]]',
+ '[subj=[agr=(1)[]], agr->(1)]',
+ '[obj=?x]', '[subj=?x]',
+ '[/=None]', '[/=NP]',
+ '[cat=NP]', '[cat=VP]', '[cat=PP]',
+ '[subj=[agr=[gender=?y]], obj=[agr=[gender=?y]]]',
+ '[gender=masc, agr=?C]',
+ '[gender=?S, agr=[gender=?S,person=3]]'
+ ]
+ all_fstructs = [FeatStruct(fss) for fss in fstruct_strings]
+ #MAX_CHOICES = 5
+ #if len(all_fstructs) > MAX_CHOICES:
+ #fstructs = random.sample(all_fstructs, MAX_CHOICES)
+ #fstructs.sort()
+ #else:
+ #fstructs = all_fstructs
+
+ for fs1 in all_fstructs:
+ for fs2 in all_fstructs:
+ print("\n*******************\nfs1 is:\n%s\n\nfs2 is:\n%s\n\nresult is:\n%s" % (fs1, fs2, unify(fs1, fs2)))
+
+
+if __name__ == '__main__':
+ demo()
+
+__all__ = ['FeatStruct', 'FeatDict', 'FeatList', 'unify', 'subsumes', 'conflicts',
+ 'Feature', 'SlashFeature', 'RangeFeature', 'SLASH', 'TYPE',
+ 'FeatStructReader']
diff --git a/nltk/grammar.py b/nltk/grammar.py
new file mode 100644
index 0000000..05d26f9
--- /dev/null
+++ b/nltk/grammar.py
@@ -0,0 +1,1515 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Context Free Grammars
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# Jason Narad <jason.narad at gmail.com>
+# Peter Ljunglöf <peter.ljunglof at heatherleaf.se>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+"""
+Basic data classes for representing context free grammars. A
+"grammar" specifies which trees can represent the structure of a
+given text. Each of these trees is called a "parse tree" for the
+text (or simply a "parse"). In a "context free" grammar, the set of
+parse trees for any piece of a text can depend only on that piece, and
+not on the rest of the text (i.e., the piece's context). Context free
+grammars are often used to find possible syntactic structures for
+sentences. In this context, the leaves of a parse tree are word
+tokens; and the node values are phrasal categories, such as ``NP``
+and ``VP``.
+
+The ``CFG`` class is used to encode context free grammars. Each
+``CFG`` consists of a start symbol and a set of productions.
+The "start symbol" specifies the root node value for parse trees. For example,
+the start symbol for syntactic parsing is usually ``S``. Start
+symbols are encoded using the ``Nonterminal`` class, which is discussed
+below.
+
+A Grammar's "productions" specify what parent-child relationships a parse
+tree can contain. Each production specifies that a particular
+node can be the parent of a particular set of children. For example,
+the production ``<S> -> <NP> <VP>`` specifies that an ``S`` node can
+be the parent of an ``NP`` node and a ``VP`` node.
+
+Grammar productions are implemented by the ``Production`` class.
+Each ``Production`` consists of a left hand side and a right hand
+side. The "left hand side" is a ``Nonterminal`` that specifies the
+node type for a potential parent; and the "right hand side" is a list
+that specifies allowable children for that parent. This lists
+consists of ``Nonterminals`` and text types: each ``Nonterminal``
+indicates that the corresponding child may be a ``TreeToken`` with the
+specified node type; and each text type indicates that the
+corresponding child may be a ``Token`` with the with that type.
+
+The ``Nonterminal`` class is used to distinguish node values from leaf
+values. This prevents the grammar from accidentally using a leaf
+value (such as the English word "A") as the node of a subtree. Within
+a ``CFG``, all node values are wrapped in the ``Nonterminal``
+class. Note, however, that the trees that are specified by the grammar do
+*not* include these ``Nonterminal`` wrappers.
+
+Grammars can also be given a more procedural interpretation. According to
+this interpretation, a Grammar specifies any tree structure *tree* that
+can be produced by the following procedure:
+
+| Set tree to the start symbol
+| Repeat until tree contains no more nonterminal leaves:
+| Choose a production prod with whose left hand side
+| lhs is a nonterminal leaf of tree.
+| Replace the nonterminal leaf with a subtree, whose node
+| value is the value wrapped by the nonterminal lhs, and
+| whose children are the right hand side of prod.
+
+The operation of replacing the left hand side (*lhs*) of a production
+with the right hand side (*rhs*) in a tree (*tree*) is known as
+"expanding" *lhs* to *rhs* in *tree*.
+"""
+from __future__ import print_function, unicode_literals
+
+import re
+
+from nltk.util import transitive_closure, invert_graph
+from nltk.compat import (string_types, total_ordering, text_type,
+ python_2_unicode_compatible, unicode_repr)
+from nltk.internals import raise_unorderable_types
+
+from nltk.probability import ImmutableProbabilisticMixIn
+from nltk.featstruct import FeatStruct, FeatDict, FeatStructReader, SLASH, TYPE
+
+#################################################################
+# Nonterminal
+#################################################################
+
+ at total_ordering
+ at python_2_unicode_compatible
+class Nonterminal(object):
+ """
+ A non-terminal symbol for a context free grammar. ``Nonterminal``
+ is a wrapper class for node values; it is used by ``Production``
+ objects to distinguish node values from leaf values.
+ The node value that is wrapped by a ``Nonterminal`` is known as its
+ "symbol". Symbols are typically strings representing phrasal
+ categories (such as ``"NP"`` or ``"VP"``). However, more complex
+ symbol types are sometimes used (e.g., for lexicalized grammars).
+ Since symbols are node values, they must be immutable and
+ hashable. Two ``Nonterminals`` are considered equal if their
+ symbols are equal.
+
+ :see: ``CFG``, ``Production``
+ :type _symbol: any
+ :ivar _symbol: The node value corresponding to this
+ ``Nonterminal``. This value must be immutable and hashable.
+ """
+ def __init__(self, symbol):
+ """
+ Construct a new non-terminal from the given symbol.
+
+ :type symbol: any
+ :param symbol: The node value corresponding to this
+ ``Nonterminal``. This value must be immutable and
+ hashable.
+ """
+ self._symbol = symbol
+ self._hash = hash(symbol)
+
+ def symbol(self):
+ """
+ Return the node value corresponding to this ``Nonterminal``.
+
+ :rtype: (any)
+ """
+ return self._symbol
+
+ def __eq__(self, other):
+ """
+ Return True if this non-terminal is equal to ``other``. In
+ particular, return True if ``other`` is a ``Nonterminal``
+ and this non-terminal's symbol is equal to ``other`` 's symbol.
+
+ :rtype: bool
+ """
+ return type(self) == type(other) and self._symbol == other._symbol
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, Nonterminal):
+ raise_unorderable_types("<", self, other)
+ return self._symbol < other._symbol
+
+ def __hash__(self):
+ return self._hash
+
+ def __repr__(self):
+ """
+ Return a string representation for this ``Nonterminal``.
+
+ :rtype: str
+ """
+ if isinstance(self._symbol, string_types):
+ return '%s' % self._symbol
+ else:
+ return '%s' % unicode_repr(self._symbol)
+
+ def __str__(self):
+ """
+ Return a string representation for this ``Nonterminal``.
+
+ :rtype: str
+ """
+ if isinstance(self._symbol, string_types):
+ return '%s' % self._symbol
+ else:
+ return '%s' % unicode_repr(self._symbol)
+
+ def __div__(self, rhs):
+ """
+ Return a new nonterminal whose symbol is ``A/B``, where ``A`` is
+ the symbol for this nonterminal, and ``B`` is the symbol for rhs.
+
+ :param rhs: The nonterminal used to form the right hand side
+ of the new nonterminal.
+ :type rhs: Nonterminal
+ :rtype: Nonterminal
+ """
+ return Nonterminal('%s/%s' % (self._symbol, rhs._symbol))
+
+def nonterminals(symbols):
+ """
+ Given a string containing a list of symbol names, return a list of
+ ``Nonterminals`` constructed from those symbols.
+
+ :param symbols: The symbol name string. This string can be
+ delimited by either spaces or commas.
+ :type symbols: str
+ :return: A list of ``Nonterminals`` constructed from the symbol
+ names given in ``symbols``. The ``Nonterminals`` are sorted
+ in the same order as the symbols names.
+ :rtype: list(Nonterminal)
+ """
+ if ',' in symbols: symbol_list = symbols.split(',')
+ else: symbol_list = symbols.split()
+ return [Nonterminal(s.strip()) for s in symbol_list]
+
+class FeatStructNonterminal(FeatDict, Nonterminal):
+ """A feature structure that's also a nonterminal. It acts as its
+ own symbol, and automatically freezes itself when hashed."""
+ def __hash__(self):
+ self.freeze()
+ return FeatStruct.__hash__(self)
+ def symbol(self):
+ return self
+
+def is_nonterminal(item):
+ """
+ :return: True if the item is a ``Nonterminal``.
+ :rtype: bool
+ """
+ return isinstance(item, Nonterminal)
+
+
+#################################################################
+# Terminals
+#################################################################
+
+def is_terminal(item):
+ """
+ Return True if the item is a terminal, which currently is
+ if it is hashable and not a ``Nonterminal``.
+
+ :rtype: bool
+ """
+ return hasattr(item, '__hash__') and not isinstance(item, Nonterminal)
+
+
+#################################################################
+# Productions
+#################################################################
+
+ at total_ordering
+ at python_2_unicode_compatible
+class Production(object):
+ """
+ A grammar production. Each production maps a single symbol
+ on the "left-hand side" to a sequence of symbols on the
+ "right-hand side". (In the case of context-free productions,
+ the left-hand side must be a ``Nonterminal``, and the right-hand
+ side is a sequence of terminals and ``Nonterminals``.)
+ "terminals" can be any immutable hashable object that is
+ not a ``Nonterminal``. Typically, terminals are strings
+ representing words, such as ``"dog"`` or ``"under"``.
+
+ :see: ``CFG``
+ :see: ``DependencyGrammar``
+ :see: ``Nonterminal``
+ :type _lhs: Nonterminal
+ :ivar _lhs: The left-hand side of the production.
+ :type _rhs: tuple(Nonterminal, terminal)
+ :ivar _rhs: The right-hand side of the production.
+ """
+
+ def __init__(self, lhs, rhs):
+ """
+ Construct a new ``Production``.
+
+ :param lhs: The left-hand side of the new ``Production``.
+ :type lhs: Nonterminal
+ :param rhs: The right-hand side of the new ``Production``.
+ :type rhs: sequence(Nonterminal and terminal)
+ """
+ if isinstance(rhs, string_types):
+ raise TypeError('production right hand side should be a list, '
+ 'not a string')
+ self._lhs = lhs
+ self._rhs = tuple(rhs)
+ self._hash = hash((self._lhs, self._rhs))
+
+ def lhs(self):
+ """
+ Return the left-hand side of this ``Production``.
+
+ :rtype: Nonterminal
+ """
+ return self._lhs
+
+ def rhs(self):
+ """
+ Return the right-hand side of this ``Production``.
+
+ :rtype: sequence(Nonterminal and terminal)
+ """
+ return self._rhs
+
+ def __len__(self):
+ """
+ Return the length of the right-hand side.
+
+ :rtype: int
+ """
+ return len(self._rhs)
+
+ def is_nonlexical(self):
+ """
+ Return True if the right-hand side only contains ``Nonterminals``
+
+ :rtype: bool
+ """
+ return all(is_nonterminal(n) for n in self._rhs)
+
+ def is_lexical(self):
+ """
+ Return True if the right-hand contain at least one terminal token.
+
+ :rtype: bool
+ """
+ return not self.is_nonlexical()
+
+ def __str__(self):
+ """
+ Return a verbose string representation of the ``Production``.
+
+ :rtype: str
+ """
+ result = '%s -> ' % unicode_repr(self._lhs)
+ result += " ".join(unicode_repr(el) for el in self._rhs)
+ return result
+
+ def __repr__(self):
+ """
+ Return a concise string representation of the ``Production``.
+
+ :rtype: str
+ """
+ return '%s' % self
+
+ def __eq__(self, other):
+ """
+ Return True if this ``Production`` is equal to ``other``.
+
+ :rtype: bool
+ """
+ return (type(self) == type(other) and
+ self._lhs == other._lhs and
+ self._rhs == other._rhs)
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, Production):
+ raise_unorderable_types("<", self, other)
+ return (self._lhs, self._rhs) < (other._lhs, other._rhs)
+
+ def __hash__(self):
+ """
+ Return a hash value for the ``Production``.
+
+ :rtype: int
+ """
+ return self._hash
+
+
+ at python_2_unicode_compatible
+class DependencyProduction(Production):
+ """
+ A dependency grammar production. Each production maps a single
+ head word to an unordered list of one or more modifier words.
+ """
+ def __str__(self):
+ """
+ Return a verbose string representation of the ``DependencyProduction``.
+
+ :rtype: str
+ """
+ result = '\'%s\' ->' % (self._lhs,)
+ for elt in self._rhs:
+ result += ' \'%s\'' % (elt,)
+ return result
+
+
+ at python_2_unicode_compatible
+class ProbabilisticProduction(Production, ImmutableProbabilisticMixIn):
+ """
+ A probabilistic context free grammar production.
+ A PCFG ``ProbabilisticProduction`` is essentially just a ``Production`` that
+ has an associated probability, which represents how likely it is that
+ this production will be used. In particular, the probability of a
+ ``ProbabilisticProduction`` records the likelihood that its right-hand side is
+ the correct instantiation for any given occurrence of its left-hand side.
+
+ :see: ``Production``
+ """
+ def __init__(self, lhs, rhs, **prob):
+ """
+ Construct a new ``ProbabilisticProduction``.
+
+ :param lhs: The left-hand side of the new ``ProbabilisticProduction``.
+ :type lhs: Nonterminal
+ :param rhs: The right-hand side of the new ``ProbabilisticProduction``.
+ :type rhs: sequence(Nonterminal and terminal)
+ :param prob: Probability parameters of the new ``ProbabilisticProduction``.
+ """
+ ImmutableProbabilisticMixIn.__init__(self, **prob)
+ Production.__init__(self, lhs, rhs)
+
+ def __str__(self):
+ return Production.__unicode__(self) + ' [%.6g]' % self.prob()
+
+ def __eq__(self, other):
+ return (type(self) == type(other) and
+ self._lhs == other._lhs and
+ self._rhs == other._rhs and
+ self.prob() == other.prob())
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __hash__(self):
+ return hash((self._lhs, self._rhs, self.prob()))
+
+#################################################################
+# Grammars
+#################################################################
+
+ at python_2_unicode_compatible
+class CFG(object):
+ """
+ A context-free grammar. A grammar consists of a start state and
+ a set of productions. The set of terminals and nonterminals is
+ implicitly specified by the productions.
+
+ If you need efficient key-based access to productions, you
+ can use a subclass to implement it.
+ """
+ def __init__(self, start, productions, calculate_leftcorners=True):
+ """
+ Create a new context-free grammar, from the given start state
+ and set of ``Production``s.
+
+ :param start: The start symbol
+ :type start: Nonterminal
+ :param productions: The list of productions that defines the grammar
+ :type productions: list(Production)
+ :param calculate_leftcorners: False if we don't want to calculate the
+ leftcorner relation. In that case, some optimized chart parsers won't work.
+ :type calculate_leftcorners: bool
+ """
+ if not is_nonterminal(start):
+ raise TypeError("start should be a Nonterminal object,"
+ " not a %s" % type(start).__name__)
+
+ self._start = start
+ self._productions = productions
+ self._categories = set(prod.lhs() for prod in productions)
+ self._calculate_indexes()
+ self._calculate_grammar_forms()
+ if calculate_leftcorners:
+ self._calculate_leftcorners()
+
+ def _calculate_indexes(self):
+ self._lhs_index = {}
+ self._rhs_index = {}
+ self._empty_index = {}
+ self._lexical_index = {}
+ for prod in self._productions:
+ # Left hand side.
+ lhs = prod._lhs
+ if lhs not in self._lhs_index:
+ self._lhs_index[lhs] = []
+ self._lhs_index[lhs].append(prod)
+ if prod._rhs:
+ # First item in right hand side.
+ rhs0 = prod._rhs[0]
+ if rhs0 not in self._rhs_index:
+ self._rhs_index[rhs0] = []
+ self._rhs_index[rhs0].append(prod)
+ else:
+ # The right hand side is empty.
+ self._empty_index[prod.lhs()] = prod
+ # Lexical tokens in the right hand side.
+ for token in prod._rhs:
+ if is_terminal(token):
+ self._lexical_index.setdefault(token, set()).add(prod)
+
+ def _calculate_leftcorners(self):
+ # Calculate leftcorner relations, for use in optimized parsing.
+ self._immediate_leftcorner_categories = dict((cat, set([cat])) for cat in self._categories)
+ self._immediate_leftcorner_words = dict((cat, set()) for cat in self._categories)
+ for prod in self.productions():
+ if len(prod) > 0:
+ cat, left = prod.lhs(), prod.rhs()[0]
+ if is_nonterminal(left):
+ self._immediate_leftcorner_categories[cat].add(left)
+ else:
+ self._immediate_leftcorner_words[cat].add(left)
+
+ lc = transitive_closure(self._immediate_leftcorner_categories, reflexive=True)
+ self._leftcorners = lc
+ self._leftcorner_parents = invert_graph(lc)
+
+ nr_leftcorner_categories = sum(map(len, self._immediate_leftcorner_categories.values()))
+ nr_leftcorner_words = sum(map(len, self._immediate_leftcorner_words.values()))
+ if nr_leftcorner_words > nr_leftcorner_categories > 10000:
+ # If the grammar is big, the leftcorner-word dictionary will be too large.
+ # In that case it is better to calculate the relation on demand.
+ self._leftcorner_words = None
+ return
+
+ self._leftcorner_words = {}
+ for cat in self._leftcorners:
+ lefts = self._leftcorners[cat]
+ lc = self._leftcorner_words[cat] = set()
+ for left in lefts:
+ lc.update(self._immediate_leftcorner_words.get(left, set()))
+
+ @classmethod
+ def fromstring(cls, input, encoding=None):
+ """
+ Return the ``CFG`` corresponding to the input string(s).
+
+ :param input: a grammar, either in the form of a string or as a list of strings.
+ """
+ start, productions = read_grammar(input, standard_nonterm_parser,
+ encoding=encoding)
+ return CFG(start, productions)
+
+ def start(self):
+ """
+ Return the start symbol of the grammar
+
+ :rtype: Nonterminal
+ """
+ return self._start
+
+ # tricky to balance readability and efficiency here!
+ # can't use set operations as they don't preserve ordering
+ def productions(self, lhs=None, rhs=None, empty=False):
+ """
+ Return the grammar productions, filtered by the left-hand side
+ or the first item in the right-hand side.
+
+ :param lhs: Only return productions with the given left-hand side.
+ :param rhs: Only return productions with the given first item
+ in the right-hand side.
+ :param empty: Only return productions with an empty right-hand side.
+ :return: A list of productions matching the given constraints.
+ :rtype: list(Production)
+ """
+ if rhs and empty:
+ raise ValueError("You cannot select empty and non-empty "
+ "productions at the same time.")
+
+ # no constraints so return everything
+ if not lhs and not rhs:
+ if not empty:
+ return self._productions
+ else:
+ return self._empty_index.values()
+
+ # only lhs specified so look up its index
+ elif lhs and not rhs:
+ if not empty:
+ return self._lhs_index.get(lhs, [])
+ elif lhs in self._empty_index:
+ return [self._empty_index[lhs]]
+ else:
+ return []
+
+ # only rhs specified so look up its index
+ elif rhs and not lhs:
+ return self._rhs_index.get(rhs, [])
+
+ # intersect
+ else:
+ return [prod for prod in self._lhs_index.get(lhs, [])
+ if prod in self._rhs_index.get(rhs, [])]
+
+ def leftcorners(self, cat):
+ """
+ Return the set of all nonterminals that the given nonterminal
+ can start with, including itself.
+
+ This is the reflexive, transitive closure of the immediate
+ leftcorner relation: (A > B) iff (A -> B beta)
+
+ :param cat: the parent of the leftcorners
+ :type cat: Nonterminal
+ :return: the set of all leftcorners
+ :rtype: set(Nonterminal)
+ """
+ return self._leftcorners.get(cat, set([cat]))
+
+ def is_leftcorner(self, cat, left):
+ """
+ True if left is a leftcorner of cat, where left can be a
+ terminal or a nonterminal.
+
+ :param cat: the parent of the leftcorner
+ :type cat: Nonterminal
+ :param left: the suggested leftcorner
+ :type left: Terminal or Nonterminal
+ :rtype: bool
+ """
+ if is_nonterminal(left):
+ return left in self.leftcorners(cat)
+ elif self._leftcorner_words:
+ return left in self._leftcorner_words.get(cat, set())
+ else:
+ return any(left in self._immediate_leftcorner_words.get(parent, set())
+ for parent in self.leftcorners(cat))
+
+ def leftcorner_parents(self, cat):
+ """
+ Return the set of all nonterminals for which the given category
+ is a left corner. This is the inverse of the leftcorner relation.
+
+ :param cat: the suggested leftcorner
+ :type cat: Nonterminal
+ :return: the set of all parents to the leftcorner
+ :rtype: set(Nonterminal)
+ """
+ return self._leftcorner_parents.get(cat, set([cat]))
+
+ def check_coverage(self, tokens):
+ """
+ Check whether the grammar rules cover the given list of tokens.
+ If not, then raise an exception.
+
+ :type tokens: list(str)
+ """
+ missing = [tok for tok in tokens
+ if not self._lexical_index.get(tok)]
+ if missing:
+ missing = ', '.join('%r' % (w,) for w in missing)
+ raise ValueError("Grammar does not cover some of the "
+ "input words: %r." % missing)
+
+ def _calculate_grammar_forms(self):
+ """
+ Pre-calculate of which form(s) the grammar is.
+ """
+ prods = self._productions
+ self._is_lexical = all(p.is_lexical() for p in prods)
+ self._is_nonlexical = all(p.is_nonlexical() for p in prods
+ if len(p) != 1)
+ self._min_len = min(len(p) for p in prods)
+ self._max_len = max(len(p) for p in prods)
+ self._all_unary_are_lexical = all(p.is_lexical() for p in prods
+ if len(p) == 1)
+
+ def is_lexical(self):
+ """
+ Return True if all productions are lexicalised.
+ """
+ return self._is_lexical
+
+ def is_nonlexical(self):
+ """
+ Return True if all lexical rules are "preterminals", that is,
+ unary rules which can be separated in a preprocessing step.
+
+ This means that all productions are of the forms
+ A -> B1 ... Bn (n>=0), or A -> "s".
+
+ Note: is_lexical() and is_nonlexical() are not opposites.
+ There are grammars which are neither, and grammars which are both.
+ """
+ return self._is_nonlexical
+
+ def min_len(self):
+ """
+ Return the right-hand side length of the shortest grammar production.
+ """
+ return self._min_len
+
+ def max_len(self):
+ """
+ Return the right-hand side length of the longest grammar production.
+ """
+ return self._max_len
+
+ def is_nonempty(self):
+ """
+ Return True if there are no empty productions.
+ """
+ return self._min_len > 0
+
+ def is_binarised(self):
+ """
+ Return True if all productions are at most binary.
+ Note that there can still be empty and unary productions.
+ """
+ return self._max_len <= 2
+
+ def is_flexible_chomsky_normal_form(self):
+ """
+ Return True if all productions are of the forms
+ A -> B C, A -> B, or A -> "s".
+ """
+ return self.is_nonempty() and self.is_nonlexical() and self.is_binarised()
+
+ def is_chomsky_normal_form(self):
+ """
+ Return True if the grammar is of Chomsky Normal Form, i.e. all productions
+ are of the form A -> B C, or A -> "s".
+ """
+ return (self.is_flexible_chomsky_normal_form() and
+ self._all_unary_are_lexical)
+
+ def __repr__(self):
+ return '<Grammar with %d productions>' % len(self._productions)
+
+ def __str__(self):
+ result = 'Grammar with %d productions' % len(self._productions)
+ result += ' (start state = %r)' % self._start
+ for production in self._productions:
+ result += '\n %s' % production
+ return result
+
+
+class FeatureGrammar(CFG):
+ """
+ A feature-based grammar. This is equivalent to a
+ ``CFG`` whose nonterminals are all
+ ``FeatStructNonterminal``.
+
+ A grammar consists of a start state and a set of
+ productions. The set of terminals and nonterminals
+ is implicitly specified by the productions.
+ """
+ def __init__(self, start, productions):
+ """
+ Create a new feature-based grammar, from the given start
+ state and set of ``Productions``.
+
+ :param start: The start symbol
+ :type start: FeatStructNonterminal
+ :param productions: The list of productions that defines the grammar
+ :type productions: list(Production)
+ """
+ CFG.__init__(self, start, productions)
+
+ # The difference with CFG is that the productions are
+ # indexed on the TYPE feature of the nonterminals.
+ # This is calculated by the method _get_type_if_possible().
+
+ def _calculate_indexes(self):
+ self._lhs_index = {}
+ self._rhs_index = {}
+ self._empty_index = {}
+ self._empty_productions = []
+ self._lexical_index = {}
+ for prod in self._productions:
+ # Left hand side.
+ lhs = self._get_type_if_possible(prod._lhs)
+ if lhs not in self._lhs_index:
+ self._lhs_index[lhs] = []
+ self._lhs_index[lhs].append(prod)
+ if prod._rhs:
+ # First item in right hand side.
+ rhs0 = self._get_type_if_possible(prod._rhs[0])
+ if rhs0 not in self._rhs_index:
+ self._rhs_index[rhs0] = []
+ self._rhs_index[rhs0].append(prod)
+ else:
+ # The right hand side is empty.
+ if lhs not in self._empty_index:
+ self._empty_index[lhs] = []
+ self._empty_index[lhs].append(prod)
+ self._empty_productions.append(prod)
+ # Lexical tokens in the right hand side.
+ for token in prod._rhs:
+ if is_terminal(token):
+ self._lexical_index.setdefault(token, set()).add(prod)
+
+ @classmethod
+ def fromstring(cls, input, features=None, logic_parser=None, fstruct_reader=None,
+ encoding=None):
+ """
+ Return a feature structure based ``FeatureGrammar``.
+
+ :param input: a grammar, either in the form of a string or else
+ as a list of strings.
+ :param features: a tuple of features (default: SLASH, TYPE)
+ :param logic_parser: a parser for lambda-expressions,
+ by default, ``LogicParser()``
+ :param fstruct_reader: a feature structure parser
+ (only if features and logic_parser is None)
+ """
+ if features is None:
+ features = (SLASH, TYPE)
+
+ if fstruct_reader is None:
+ fstruct_reader = FeatStructReader(features, FeatStructNonterminal,
+ logic_parser=logic_parser)
+ elif logic_parser is not None:
+ raise Exception('\'logic_parser\' and \'fstruct_reader\' must '
+ 'not both be set')
+
+ start, productions = read_grammar(input, fstruct_reader.read_partial,
+ encoding=encoding)
+ return FeatureGrammar(start, productions)
+
+
+ def productions(self, lhs=None, rhs=None, empty=False):
+ """
+ Return the grammar productions, filtered by the left-hand side
+ or the first item in the right-hand side.
+
+ :param lhs: Only return productions with the given left-hand side.
+ :param rhs: Only return productions with the given first item
+ in the right-hand side.
+ :param empty: Only return productions with an empty right-hand side.
+ :rtype: list(Production)
+ """
+ if rhs and empty:
+ raise ValueError("You cannot select empty and non-empty "
+ "productions at the same time.")
+
+ # no constraints so return everything
+ if not lhs and not rhs:
+ if empty:
+ return self._empty_productions
+ else:
+ return self._productions
+
+ # only lhs specified so look up its index
+ elif lhs and not rhs:
+ if empty:
+ return self._empty_index.get(self._get_type_if_possible(lhs), [])
+ else:
+ return self._lhs_index.get(self._get_type_if_possible(lhs), [])
+
+ # only rhs specified so look up its index
+ elif rhs and not lhs:
+ return self._rhs_index.get(self._get_type_if_possible(rhs), [])
+
+ # intersect
+ else:
+ return [prod for prod in self._lhs_index.get(self._get_type_if_possible(lhs), [])
+ if prod in self._rhs_index.get(self._get_type_if_possible(rhs), [])]
+
+ def leftcorners(self, cat):
+ """
+ Return the set of all words that the given category can start with.
+ Also called the "first set" in compiler construction.
+ """
+ raise NotImplementedError("Not implemented yet")
+
+ def leftcorner_parents(self, cat):
+ """
+ Return the set of all categories for which the given category
+ is a left corner.
+ """
+ raise NotImplementedError("Not implemented yet")
+
+ def _get_type_if_possible(self, item):
+ """
+ Helper function which returns the ``TYPE`` feature of the ``item``,
+ if it exists, otherwise it returns the ``item`` itself
+ """
+ if isinstance(item, dict) and TYPE in item:
+ return FeatureValueType(item[TYPE])
+ else:
+ return item
+
+ at total_ordering
+ at python_2_unicode_compatible
+class FeatureValueType(object):
+ """
+ A helper class for ``FeatureGrammars``, designed to be different
+ from ordinary strings. This is to stop the ``FeatStruct``
+ ``FOO[]`` from being compare equal to the terminal "FOO".
+ """
+ def __init__(self, value):
+ self._value = value
+ self._hash = hash(value)
+
+ def __repr__(self):
+ return '<%s>' % self._value
+
+ def __eq__(self, other):
+ return type(self) == type(other) and self._value == other._value
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, FeatureValueType):
+ raise_unorderable_types("<", self, other)
+ return self._value < other._value
+
+ def __hash__(self):
+ return self._hash
+
+
+ at python_2_unicode_compatible
+class DependencyGrammar(object):
+ """
+ A dependency grammar. A DependencyGrammar consists of a set of
+ productions. Each production specifies a head/modifier relationship
+ between a pair of words.
+ """
+ def __init__(self, productions):
+ """
+ Create a new dependency grammar, from the set of ``Productions``.
+
+ :param productions: The list of productions that defines the grammar
+ :type productions: list(Production)
+ """
+ self._productions = productions
+
+ @classmethod
+ def fromstring(cls, input):
+ productions = []
+ for linenum, line in enumerate(input.split('\n')):
+ line = line.strip()
+ if line.startswith('#') or line=='': continue
+ try: productions += _read_dependency_production(line)
+ except ValueError:
+ raise ValueError('Unable to parse line %s: %s' % (linenum, line))
+ if len(productions) == 0:
+ raise ValueError('No productions found!')
+ return DependencyGrammar(productions)
+
+ def contains(self, head, mod):
+ """
+ :param head: A head word.
+ :type head: str
+ :param mod: A mod word, to test as a modifier of 'head'.
+ :type mod: str
+
+ :return: true if this ``DependencyGrammar`` contains a
+ ``DependencyProduction`` mapping 'head' to 'mod'.
+ :rtype: bool
+ """
+ for production in self._productions:
+ for possibleMod in production._rhs:
+ if(production._lhs == head and possibleMod == mod):
+ return True
+ return False
+
+ def __contains__(self, head, mod):
+ """
+ Return True if this ``DependencyGrammar`` contains a
+ ``DependencyProduction`` mapping 'head' to 'mod'.
+
+ :param head: A head word.
+ :type head: str
+ :param mod: A mod word, to test as a modifier of 'head'.
+ :type mod: str
+ :rtype: bool
+ """
+ for production in self._productions:
+ for possibleMod in production._rhs:
+ if(production._lhs == head and possibleMod == mod):
+ return True
+ return False
+
+ # # should be rewritten, the set comp won't work in all comparisons
+ # def contains_exactly(self, head, modlist):
+ # for production in self._productions:
+ # if(len(production._rhs) == len(modlist)):
+ # if(production._lhs == head):
+ # set1 = Set(production._rhs)
+ # set2 = Set(modlist)
+ # if(set1 == set2):
+ # return True
+ # return False
+
+
+ def __str__(self):
+ """
+ Return a verbose string representation of the ``DependencyGrammar``
+
+ :rtype: str
+ """
+ str = 'Dependency grammar with %d productions' % len(self._productions)
+ for production in self._productions:
+ str += '\n %s' % production
+ return str
+
+ def __repr__(self):
+ """
+ Return a concise string representation of the ``DependencyGrammar``
+ """
+ return 'Dependency grammar with %d productions' % len(self._productions)
+
+
+ at python_2_unicode_compatible
+class ProbabilisticDependencyGrammar(object):
+ """
+
+ """
+
+ def __init__(self, productions, events, tags):
+ self._productions = productions
+ self._events = events
+ self._tags = tags
+
+ def contains(self, head, mod):
+ """
+ Return True if this ``DependencyGrammar`` contains a
+ ``DependencyProduction`` mapping 'head' to 'mod'.
+
+ :param head: A head word.
+ :type head: str
+ :param mod: A mod word, to test as a modifier of 'head'.
+ :type mod: str
+ :rtype: bool
+ """
+ for production in self._productions:
+ for possibleMod in production._rhs:
+ if(production._lhs == head and possibleMod == mod):
+ return True
+ return False
+
+ def __str__(self):
+ """
+ Return a verbose string representation of the ``ProbabilisticDependencyGrammar``
+
+ :rtype: str
+ """
+ str = 'Statistical dependency grammar with %d productions' % len(self._productions)
+ for production in self._productions:
+ str += '\n %s' % production
+ str += '\nEvents:'
+ for event in self._events:
+ str += '\n %d:%s' % (self._events[event], event)
+ str += '\nTags:'
+ for tag_word in self._tags:
+ str += '\n %s:\t(%s)' % (tag_word, self._tags[tag_word])
+ return str
+
+ def __repr__(self):
+ """
+ Return a concise string representation of the ``ProbabilisticDependencyGrammar``
+ """
+ return 'Statistical Dependency grammar with %d productions' % len(self._productions)
+
+
+class PCFG(CFG):
+ """
+ A probabilistic context-free grammar. A PCFG consists of a
+ start state and a set of productions with probabilities. The set of
+ terminals and nonterminals is implicitly specified by the productions.
+
+ PCFG productions use the ``ProbabilisticProduction`` class.
+ ``PCFGs`` impose the constraint that the set of productions with
+ any given left-hand-side must have probabilities that sum to 1
+ (allowing for a small margin of error).
+
+ If you need efficient key-based access to productions, you can use
+ a subclass to implement it.
+
+ :type EPSILON: float
+ :cvar EPSILON: The acceptable margin of error for checking that
+ productions with a given left-hand side have probabilities
+ that sum to 1.
+ """
+ EPSILON = 0.01
+
+ def __init__(self, start, productions, calculate_leftcorners=True):
+ """
+ Create a new context-free grammar, from the given start state
+ and set of ``ProbabilisticProductions``.
+
+ :param start: The start symbol
+ :type start: Nonterminal
+ :param productions: The list of productions that defines the grammar
+ :type productions: list(Production)
+ :raise ValueError: if the set of productions with any left-hand-side
+ do not have probabilities that sum to a value within
+ EPSILON of 1.
+ :param calculate_leftcorners: False if we don't want to calculate the
+ leftcorner relation. In that case, some optimized chart parsers won't work.
+ :type calculate_leftcorners: bool
+ """
+ CFG.__init__(self, start, productions, calculate_leftcorners)
+
+ # Make sure that the probabilities sum to one.
+ probs = {}
+ for production in productions:
+ probs[production.lhs()] = (probs.get(production.lhs(), 0) +
+ production.prob())
+ for (lhs, p) in probs.items():
+ if not ((1-PCFG.EPSILON) < p <
+ (1+PCFG.EPSILON)):
+ raise ValueError("Productions for %r do not sum to 1" % lhs)
+
+
+ @classmethod
+ def fromstring(cls, input, encoding=None):
+ """
+ Return a probabilistic ``PCFG`` corresponding to the
+ input string(s).
+
+ :param input: a grammar, either in the form of a string or else
+ as a list of strings.
+ """
+ start, productions = read_grammar(input, standard_nonterm_parser,
+ probabilistic=True, encoding=encoding)
+ return PCFG(start, productions)
+
+
+#################################################################
+# Inducing Grammars
+#################################################################
+
+# Contributed by Nathan Bodenstab <bodenstab at cslu.ogi.edu>
+
+def induce_pcfg(start, productions):
+ """
+ Induce a PCFG grammar from a list of productions.
+
+ The probability of a production A -> B C in a PCFG is:
+
+ | count(A -> B C)
+ | P(B, C | A) = --------------- where \* is any right hand side
+ | count(A -> \*)
+
+ :param start: The start symbol
+ :type start: Nonterminal
+ :param productions: The list of productions that defines the grammar
+ :type productions: list(Production)
+ """
+ # Production count: the number of times a given production occurs
+ pcount = {}
+
+ # LHS-count: counts the number of times a given lhs occurs
+ lcount = {}
+
+ for prod in productions:
+ lcount[prod.lhs()] = lcount.get(prod.lhs(), 0) + 1
+ pcount[prod] = pcount.get(prod, 0) + 1
+
+ prods = [ProbabilisticProduction(p.lhs(), p.rhs(),
+ prob=float(pcount[p]) / lcount[p.lhs()])
+ for p in pcount]
+ return PCFG(start, prods)
+
+
+#################################################################
+# Helper functions for reading productions
+#################################################################
+
+def _read_cfg_production(input):
+ """
+ Return a list of context-free ``Productions``.
+ """
+ return _read_production(input, standard_nonterm_parser)
+
+def _read_pcfg_production(input):
+ """
+ Return a list of PCFG ``ProbabilisticProductions``.
+ """
+ return _read_production(input, standard_nonterm_parser, probabilistic=True)
+
+def _read_fcfg_production(input, fstruct_reader):
+ """
+ Return a list of feature-based ``Productions``.
+ """
+ return _read_production(input, fstruct_reader)
+
+
+# Parsing generic grammars
+
+_ARROW_RE = re.compile(r'\s* -> \s*', re.VERBOSE)
+_PROBABILITY_RE = re.compile(r'( \[ [\d\.]+ \] ) \s*', re.VERBOSE)
+_TERMINAL_RE = re.compile(r'( "[^"]+" | \'[^\']+\' ) \s*', re.VERBOSE)
+_DISJUNCTION_RE = re.compile(r'\| \s*', re.VERBOSE)
+
+def _read_production(line, nonterm_parser, probabilistic=False):
+ """
+ Parse a grammar rule, given as a string, and return
+ a list of productions.
+ """
+ pos = 0
+
+ # Parse the left-hand side.
+ lhs, pos = nonterm_parser(line, pos)
+
+ # Skip over the arrow.
+ m = _ARROW_RE.match(line, pos)
+ if not m: raise ValueError('Expected an arrow')
+ pos = m.end()
+
+ # Parse the right hand side.
+ probabilities = [0.0]
+ rhsides = [[]]
+ while pos < len(line):
+ # Probability.
+ m = _PROBABILITY_RE.match(line, pos)
+ if probabilistic and m:
+ pos = m.end()
+ probabilities[-1] = float(m.group(1)[1:-1])
+ if probabilities[-1] > 1.0:
+ raise ValueError('Production probability %f, '
+ 'should not be greater than 1.0' %
+ (probabilities[-1],))
+
+ # String -- add terminal.
+ elif line[pos] in "\'\"":
+ m = _TERMINAL_RE.match(line, pos)
+ if not m: raise ValueError('Unterminated string')
+ rhsides[-1].append(m.group(1)[1:-1])
+ pos = m.end()
+
+ # Vertical bar -- start new rhside.
+ elif line[pos] == '|':
+ m = _DISJUNCTION_RE.match(line, pos)
+ probabilities.append(0.0)
+ rhsides.append([])
+ pos = m.end()
+
+ # Anything else -- nonterminal.
+ else:
+ nonterm, pos = nonterm_parser(line, pos)
+ rhsides[-1].append(nonterm)
+
+ if probabilistic:
+ return [ProbabilisticProduction(lhs, rhs, prob=probability)
+ for (rhs, probability) in zip(rhsides, probabilities)]
+ else:
+ return [Production(lhs, rhs) for rhs in rhsides]
+
+
+#################################################################
+# Reading Phrase Structure Grammars
+#################################################################
+
+def read_grammar(input, nonterm_parser, probabilistic=False, encoding=None):
+ """
+ Return a pair consisting of a starting category and a list of
+ ``Productions``.
+
+ :param input: a grammar, either in the form of a string or else
+ as a list of strings.
+ :param nonterm_parser: a function for parsing nonterminals.
+ It should take a ``(string, position)`` as argument and
+ return a ``(nonterminal, position)`` as result.
+ :param probabilistic: are the grammar rules probabilistic?
+ :type probabilistic: bool
+ :param encoding: the encoding of the grammar, if it is a binary string
+ :type encoding: str
+ """
+ if encoding is not None:
+ input = input.decode(encoding)
+ if isinstance(input, string_types):
+ lines = input.split('\n')
+ else:
+ lines = input
+
+ start = None
+ productions = []
+ continue_line = ''
+ for linenum, line in enumerate(lines):
+ line = continue_line + line.strip()
+ if line.startswith('#') or line=='': continue
+ if line.endswith('\\'):
+ continue_line = line[:-1].rstrip()+' '
+ continue
+ continue_line = ''
+ try:
+ if line[0] == '%':
+ directive, args = line[1:].split(None, 1)
+ if directive == 'start':
+ start, pos = nonterm_parser(args, 0)
+ if pos != len(args):
+ raise ValueError('Bad argument to start directive')
+ else:
+ raise ValueError('Bad directive')
+ else:
+ # expand out the disjunctions on the RHS
+ productions += _read_production(line, nonterm_parser, probabilistic)
+ except ValueError as e:
+ raise ValueError('Unable to parse line %s: %s\n%s' %
+ (linenum+1, line, e))
+
+ if not productions:
+ raise ValueError('No productions found!')
+ if not start:
+ start = productions[0].lhs()
+ return (start, productions)
+
+_STANDARD_NONTERM_RE = re.compile('( [\w/][\w/^<>-]* ) \s*', re.VERBOSE)
+
+def standard_nonterm_parser(string, pos):
+ m = _STANDARD_NONTERM_RE.match(string, pos)
+ if not m: raise ValueError('Expected a nonterminal, found: '
+ + string[pos:])
+ return (Nonterminal(m.group(1)), m.end())
+
+
+#################################################################
+# Reading Dependency Grammars
+#################################################################
+
+_READ_DG_RE = re.compile(r'''^\s* # leading whitespace
+ ('[^']+')\s* # single-quoted lhs
+ (?:[-=]+>)\s* # arrow
+ (?:( # rhs:
+ "[^"]+" # doubled-quoted terminal
+ | '[^']+' # single-quoted terminal
+ | \| # disjunction
+ )
+ \s*) # trailing space
+ *$''', # zero or more copies
+ re.VERBOSE)
+_SPLIT_DG_RE = re.compile(r'''('[^']'|[-=]+>|"[^"]+"|'[^']+'|\|)''')
+
+def _read_dependency_production(s):
+ if not _READ_DG_RE.match(s):
+ raise ValueError('Bad production string')
+ pieces = _SPLIT_DG_RE.split(s)
+ pieces = [p for i,p in enumerate(pieces) if i%2==1]
+ lhside = pieces[0].strip('\'\"')
+ rhsides = [[]]
+ for piece in pieces[2:]:
+ if piece == '|':
+ rhsides.append([])
+ else:
+ rhsides[-1].append(piece.strip('\'\"'))
+ return [DependencyProduction(lhside, rhside) for rhside in rhsides]
+
+
+#################################################################
+# Demonstration
+#################################################################
+
+def cfg_demo():
+ """
+ A demonstration showing how ``CFGs`` can be created and used.
+ """
+
+ from nltk import nonterminals, Production, CFG
+
+ # Create some nonterminals
+ S, NP, VP, PP = nonterminals('S, NP, VP, PP')
+ N, V, P, Det = nonterminals('N, V, P, Det')
+ VP_slash_NP = VP/NP
+
+ print('Some nonterminals:', [S, NP, VP, PP, N, V, P, Det, VP/NP])
+ print(' S.symbol() =>', repr(S.symbol()))
+ print()
+
+ print(Production(S, [NP]))
+
+ # Create some Grammar Productions
+ grammar = CFG.fromstring("""
+ S -> NP VP
+ PP -> P NP
+ NP -> Det N | NP PP
+ VP -> V NP | VP PP
+ Det -> 'a' | 'the'
+ N -> 'dog' | 'cat'
+ V -> 'chased' | 'sat'
+ P -> 'on' | 'in'
+ """)
+
+ print('A Grammar:', repr(grammar))
+ print(' grammar.start() =>', repr(grammar.start()))
+ print(' grammar.productions() =>', end=' ')
+ # Use string.replace(...) is to line-wrap the output.
+ print(repr(grammar.productions()).replace(',', ',\n'+' '*25))
+ print()
+
+toy_pcfg1 = PCFG.fromstring("""
+ S -> NP VP [1.0]
+ NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
+ Det -> 'the' [0.8] | 'my' [0.2]
+ N -> 'man' [0.5] | 'telescope' [0.5]
+ VP -> VP PP [0.1] | V NP [0.7] | V [0.2]
+ V -> 'ate' [0.35] | 'saw' [0.65]
+ PP -> P NP [1.0]
+ P -> 'with' [0.61] | 'under' [0.39]
+ """)
+
+toy_pcfg2 = PCFG.fromstring("""
+ S -> NP VP [1.0]
+ VP -> V NP [.59]
+ VP -> V [.40]
+ VP -> VP PP [.01]
+ NP -> Det N [.41]
+ NP -> Name [.28]
+ NP -> NP PP [.31]
+ PP -> P NP [1.0]
+ V -> 'saw' [.21]
+ V -> 'ate' [.51]
+ V -> 'ran' [.28]
+ N -> 'boy' [.11]
+ N -> 'cookie' [.12]
+ N -> 'table' [.13]
+ N -> 'telescope' [.14]
+ N -> 'hill' [.5]
+ Name -> 'Jack' [.52]
+ Name -> 'Bob' [.48]
+ P -> 'with' [.61]
+ P -> 'under' [.39]
+ Det -> 'the' [.41]
+ Det -> 'a' [.31]
+ Det -> 'my' [.28]
+ """)
+
+def pcfg_demo():
+ """
+ A demonstration showing how a ``PCFG`` can be created and used.
+ """
+
+ from nltk.corpus import treebank
+ from nltk import treetransforms
+ from nltk import induce_pcfg
+ from nltk.parse import pchart
+
+ pcfg_prods = toy_pcfg1.productions()
+
+ pcfg_prod = pcfg_prods[2]
+ print('A PCFG production:', repr(pcfg_prod))
+ print(' pcfg_prod.lhs() =>', repr(pcfg_prod.lhs()))
+ print(' pcfg_prod.rhs() =>', repr(pcfg_prod.rhs()))
+ print(' pcfg_prod.prob() =>', repr(pcfg_prod.prob()))
+ print()
+
+ grammar = toy_pcfg2
+ print('A PCFG grammar:', repr(grammar))
+ print(' grammar.start() =>', repr(grammar.start()))
+ print(' grammar.productions() =>', end=' ')
+ # Use .replace(...) is to line-wrap the output.
+ print(repr(grammar.productions()).replace(',', ',\n'+' '*26))
+ print()
+
+ # extract productions from three trees and induce the PCFG
+ print("Induce PCFG grammar from treebank data:")
+
+ productions = []
+ item = treebank._fileids[0]
+ for tree in treebank.parsed_sents(item)[:3]:
+ # perform optional tree transformations, e.g.:
+ tree.collapse_unary(collapsePOS = False)
+ tree.chomsky_normal_form(horzMarkov = 2)
+
+ productions += tree.productions()
+
+ S = Nonterminal('S')
+ grammar = induce_pcfg(S, productions)
+ print(grammar)
+ print()
+
+ print("Parse sentence using induced grammar:")
+
+ parser = pchart.InsideChartParser(grammar)
+ parser.trace(3)
+
+ # doesn't work as tokens are different:
+ #sent = treebank.tokenized('wsj_0001.mrg')[0]
+
+ sent = treebank.parsed_sents(item)[0].leaves()
+ print(sent)
+ for parse in parser.parse(sent):
+ print(parse)
+
+def fcfg_demo():
+ import nltk.data
+ g = nltk.data.load('grammars/book_grammars/feat0.fcfg')
+ print(g)
+ print()
+
+def dg_demo():
+ """
+ A demonstration showing the creation and inspection of a
+ ``DependencyGrammar``.
+ """
+ grammar = DependencyGrammar.fromstring("""
+ 'scratch' -> 'cats' | 'walls'
+ 'walls' -> 'the'
+ 'cats' -> 'the'
+ """)
+ print(grammar)
+
+def sdg_demo():
+ """
+ A demonstration of how to read a string representation of
+ a CoNLL format dependency tree.
+ """
+ from nltk.parse import DependencyGraph
+
+ dg = DependencyGraph("""
+ 1 Ze ze Pron Pron per|3|evofmv|nom 2 su _ _
+ 2 had heb V V trans|ovt|1of2of3|ev 0 ROOT _ _
+ 3 met met Prep Prep voor 8 mod _ _
+ 4 haar haar Pron Pron bez|3|ev|neut|attr 5 det _ _
+ 5 moeder moeder N N soort|ev|neut 3 obj1 _ _
+ 6 kunnen kan V V hulp|ott|1of2of3|mv 2 vc _ _
+ 7 gaan ga V V hulp|inf 6 vc _ _
+ 8 winkelen winkel V V intrans|inf 11 cnj _ _
+ 9 , , Punc Punc komma 8 punct _ _
+ 10 zwemmen zwem V V intrans|inf 11 cnj _ _
+ 11 of of Conj Conj neven 7 vc _ _
+ 12 terrassen terras N N soort|mv|neut 11 cnj _ _
+ 13 . . Punc Punc punt 12 punct _ _
+ """)
+ tree = dg.tree()
+ print(tree.pprint())
+
+def demo():
+ cfg_demo()
+ pcfg_demo()
+ fcfg_demo()
+ dg_demo()
+ sdg_demo()
+
+if __name__ == '__main__':
+ demo()
+
+__all__ = ['Nonterminal', 'nonterminals',
+ 'CFG', 'Production',
+ 'PCFG', 'ProbabilisticProduction',
+ 'DependencyGrammar', 'DependencyProduction',
+ 'ProbabilisticDependencyGrammar',
+ 'induce_pcfg', 'read_grammar']
+
diff --git a/nltk/help.py b/nltk/help.py
new file mode 100644
index 0000000..736b18f
--- /dev/null
+++ b/nltk/help.py
@@ -0,0 +1,56 @@
+# Natural Language Toolkit (NLTK) Help
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Authors: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Provide structured access to documentation.
+"""
+from __future__ import print_function
+
+import re
+from textwrap import wrap
+
+from nltk.data import load
+
+def brown_tagset(tagpattern=None):
+ _format_tagset("brown_tagset", tagpattern)
+
+def claws5_tagset(tagpattern=None):
+ _format_tagset("claws5_tagset", tagpattern)
+
+def upenn_tagset(tagpattern=None):
+ _format_tagset("upenn_tagset", tagpattern)
+
+#####################################################################
+# UTILITIES
+#####################################################################
+
+def _print_entries(tags, tagdict):
+ for tag in tags:
+ entry = tagdict[tag]
+ defn = [tag + ": " + entry[0]]
+ examples = wrap(entry[1], width=75, initial_indent=' ', subsequent_indent=' ')
+ print("\n".join(defn + examples))
+
+def _format_tagset(tagset, tagpattern=None):
+ tagdict = load("help/tagsets/" + tagset + ".pickle")
+ if not tagpattern:
+ _print_entries(sorted(tagdict), tagdict)
+ elif tagpattern in tagdict:
+ _print_entries([tagpattern], tagdict)
+ else:
+ tagpattern = re.compile(tagpattern)
+ tags = [tag for tag in sorted(tagdict) if tagpattern.match(tag)]
+ if tags:
+ _print_entries(tags, tagdict)
+ else:
+ print("No matching tags found.")
+
+if __name__ == '__main__':
+ brown_tagset(r'NN.*')
+ upenn_tagset(r'.*\$')
+ claws5_tagset('UNDEFINED')
+ brown_tagset(r'NN')
diff --git a/nltk/inference/__init__.py b/nltk/inference/__init__.py
new file mode 100644
index 0000000..7ba76e0
--- /dev/null
+++ b/nltk/inference/__init__.py
@@ -0,0 +1,20 @@
+# Natural Language Toolkit: Inference
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Dan Garrette <dhgarrette at gmail.com>
+# Ewan Klein <ewan at inf.ed.ac.uk>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classes and interfaces for theorem proving and model building.
+"""
+
+from nltk.inference.api import ParallelProverBuilder, ParallelProverBuilderCommand
+from nltk.inference.mace import Mace, MaceCommand
+from nltk.inference.prover9 import Prover9, Prover9Command
+from nltk.inference.resolution import ResolutionProver, ResolutionProverCommand
+from nltk.inference.tableau import TableauProver, TableauProverCommand
+from nltk.inference.discourse import (ReadingCommand, CfgReadingCommand,
+ DrtGlueReadingCommand, DiscourseTester)
diff --git a/nltk/inference/api.py b/nltk/inference/api.py
new file mode 100644
index 0000000..d8185ef
--- /dev/null
+++ b/nltk/inference/api.py
@@ -0,0 +1,589 @@
+# Natural Language Toolkit: Classifier Interface
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# Dan Garrette <dhgarrette at gmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interfaces and base classes for theorem provers and model builders.
+
+``Prover`` is a standard interface for a theorem prover which tries to prove a goal from a
+list of assumptions.
+
+``ModelBuilder`` is a standard interface for a model builder. Given just a set of assumptions.
+the model builder tries to build a model for the assumptions. Given a set of assumptions and a
+goal *G*, the model builder tries to find a counter-model, in the sense of a model that will satisfy
+the assumptions plus the negation of *G*.
+"""
+from __future__ import print_function
+
+import threading
+import time
+
+class Prover(object):
+ """
+ Interface for trying to prove a goal from assumptions. Both the goal and
+ the assumptions are constrained to be formulas of ``logic.Expression``.
+ """
+ def prove(self, goal=None, assumptions=None, verbose=False):
+ """
+ :return: Whether the proof was successful or not.
+ :rtype: bool
+ """
+ return self._prove(goal, assumptions, verbose)[0]
+
+ def _prove(self, goal=None, assumptions=None, verbose=False):
+ """
+ :return: Whether the proof was successful or not, along with the proof
+ :rtype: tuple: (bool, str)
+ """
+ raise NotImplementedError()
+
+class ModelBuilder(object):
+ """
+ Interface for trying to build a model of set of formulas.
+ Open formulas are assumed to be universally quantified.
+ Both the goal and the assumptions are constrained to be formulas
+ of ``logic.Expression``.
+ """
+ def build_model(self, goal=None, assumptions=None, verbose=False):
+ """
+ Perform the actual model building.
+ :return: Whether a model was generated
+ :rtype: bool
+ """
+ return self._build_model(goal, assumptions, verbose)[0]
+
+ def _build_model(self, goal=None, assumptions=None, verbose=False):
+ """
+ Perform the actual model building.
+ :return: Whether a model was generated, and the model itself
+ :rtype: tuple(bool, sem.Valuation)
+ """
+ raise NotImplementedError()
+
+
+class TheoremToolCommand(object):
+ """
+ This class holds a goal and a list of assumptions to be used in proving
+ or model building.
+ """
+ def add_assumptions(self, new_assumptions):
+ """
+ Add new assumptions to the assumption list.
+
+ :param new_assumptions: new assumptions
+ :type new_assumptions: list(sem.Expression)
+ """
+ raise NotImplementedError()
+
+ def retract_assumptions(self, retracted, debug=False):
+ """
+ Retract assumptions from the assumption list.
+
+ :param debug: If True, give warning when ``retracted`` is not present on
+ assumptions list.
+ :type debug: bool
+ :param retracted: assumptions to be retracted
+ :type retracted: list(sem.Expression)
+ """
+ raise NotImplementedError()
+
+ def assumptions(self):
+ """
+ List the current assumptions.
+
+ :return: list of ``Expression``
+ """
+ raise NotImplementedError()
+
+ def goal(self):
+ """
+ Return the goal
+
+ :return: ``Expression``
+ """
+ raise NotImplementedError()
+
+ def print_assumptions(self):
+ """
+ Print the list of the current assumptions.
+ """
+ raise NotImplementedError()
+
+
+class ProverCommand(TheoremToolCommand):
+ """
+ This class holds a ``Prover``, a goal, and a list of assumptions. When
+ prove() is called, the ``Prover`` is executed with the goal and assumptions.
+ """
+ def prove(self, verbose=False):
+ """
+ Perform the actual proof.
+ """
+ raise NotImplementedError()
+
+ def proof(self, simplify=True):
+ """
+ Return the proof string
+ :param simplify: bool simplify the proof?
+ :return: str
+ """
+ raise NotImplementedError()
+
+ def get_prover(self):
+ """
+ Return the prover object
+ :return: ``Prover``
+ """
+ raise NotImplementedError()
+
+
+class ModelBuilderCommand(TheoremToolCommand):
+ """
+ This class holds a ``ModelBuilder``, a goal, and a list of assumptions.
+ When build_model() is called, the ``ModelBuilder`` is executed with the goal
+ and assumptions.
+ """
+ def build_model(self, verbose=False):
+ """
+ Perform the actual model building.
+ :return: A model if one is generated; None otherwise.
+ :rtype: sem.Valuation
+ """
+ raise NotImplementedError()
+
+ def model(self, format=None):
+ """
+ Return a string representation of the model
+
+ :param simplify: bool simplify the proof?
+ :return: str
+ """
+ raise NotImplementedError()
+
+ def get_model_builder(self):
+ """
+ Return the model builder object
+ :return: ``ModelBuilder``
+ """
+ raise NotImplementedError()
+
+
+class BaseTheoremToolCommand(TheoremToolCommand):
+ """
+ This class holds a goal and a list of assumptions to be used in proving
+ or model building.
+ """
+ def __init__(self, goal=None, assumptions=None):
+ """
+ :param goal: Input expression to prove
+ :type goal: sem.Expression
+ :param assumptions: Input expressions to use as assumptions in
+ the proof.
+ :type assumptions: list(sem.Expression)
+ """
+ self._goal = goal
+
+ if not assumptions:
+ self._assumptions = []
+ else:
+ self._assumptions = list(assumptions)
+
+ self._result = None
+ """A holder for the result, to prevent unnecessary re-proving"""
+
+ def add_assumptions(self, new_assumptions):
+ """
+ Add new assumptions to the assumption list.
+
+ :param new_assumptions: new assumptions
+ :type new_assumptions: list(sem.Expression)
+ """
+ self._assumptions.extend(new_assumptions)
+ self._result = None
+
+ def retract_assumptions(self, retracted, debug=False):
+ """
+ Retract assumptions from the assumption list.
+
+ :param debug: If True, give warning when ``retracted`` is not present on
+ assumptions list.
+ :type debug: bool
+ :param retracted: assumptions to be retracted
+ :type retracted: list(sem.Expression)
+ """
+ retracted = set(retracted)
+ result_list = list(filter(lambda a: a not in retracted, self._assumptions))
+ if debug and result_list == self._assumptions:
+ print(Warning("Assumptions list has not been changed:"))
+ self.print_assumptions()
+
+ self._assumptions = result_list
+
+ self._result = None
+
+ def assumptions(self):
+ """
+ List the current assumptions.
+
+ :return: list of ``Expression``
+ """
+ return self._assumptions
+
+ def goal(self):
+ """
+ Return the goal
+
+ :return: ``Expression``
+ """
+ return self._goal
+
+ def print_assumptions(self):
+ """
+ Print the list of the current assumptions.
+ """
+ for a in self.assumptions():
+ print(a)
+
+
+class BaseProverCommand(BaseTheoremToolCommand, ProverCommand):
+ """
+ This class holds a ``Prover``, a goal, and a list of assumptions. When
+ prove() is called, the ``Prover`` is executed with the goal and assumptions.
+ """
+ def __init__(self, prover, goal=None, assumptions=None):
+ """
+ :param prover: The theorem tool to execute with the assumptions
+ :type prover: Prover
+ :see: ``BaseTheoremToolCommand``
+ """
+ self._prover = prover
+ """The theorem tool to execute with the assumptions"""
+
+ BaseTheoremToolCommand.__init__(self, goal, assumptions)
+
+ self._proof = None
+
+ def prove(self, verbose=False):
+ """
+ Perform the actual proof. Store the result to prevent unnecessary
+ re-proving.
+ """
+ if self._result is None:
+ self._result, self._proof = self._prover._prove(self.goal(),
+ self.assumptions(),
+ verbose)
+ return self._result
+
+ def proof(self, simplify=True):
+ """
+ Return the proof string
+ :param simplify: bool simplify the proof?
+ :return: str
+ """
+ if self._result is None:
+ raise LookupError("You have to call prove() first to get a proof!")
+ else:
+ return self.decorate_proof(self._proof, simplify)
+
+ def decorate_proof(self, proof_string, simplify=True):
+ """
+ Modify and return the proof string
+ :param proof_string: str the proof to decorate
+ :param simplify: bool simplify the proof?
+ :return: str
+ """
+ return proof_string
+
+ def get_prover(self):
+ return self._prover
+
+
+class BaseModelBuilderCommand(BaseTheoremToolCommand, ModelBuilderCommand):
+ """
+ This class holds a ``ModelBuilder``, a goal, and a list of assumptions. When
+ build_model() is called, the ``ModelBuilder`` is executed with the goal and
+ assumptions.
+ """
+ def __init__(self, modelbuilder, goal=None, assumptions=None):
+ """
+ :param modelbuilder: The theorem tool to execute with the assumptions
+ :type modelbuilder: ModelBuilder
+ :see: ``BaseTheoremToolCommand``
+ """
+ self._modelbuilder = modelbuilder
+ """The theorem tool to execute with the assumptions"""
+
+ BaseTheoremToolCommand.__init__(self, goal, assumptions)
+
+ self._model = None
+
+ def build_model(self, verbose=False):
+ """
+ Attempt to build a model. Store the result to prevent unnecessary
+ re-building.
+ """
+ if self._result is None:
+ self._result, self._model = \
+ self._modelbuilder._build_model(self.goal(),
+ self.assumptions(),
+ verbose)
+ return self._result
+
+ def model(self, format=None):
+ """
+ Return a string representation of the model
+
+ :param simplify: bool simplify the proof?
+ :return: str
+ """
+ if self._result is None:
+ raise LookupError('You have to call build_model() first to '
+ 'get a model!')
+ else:
+ return self._decorate_model(self._model, format)
+
+ def _decorate_model(self, valuation_str, format=None):
+ """
+ :param valuation_str: str with the model builder's output
+ :param format: str indicating the format for displaying
+ :return: str
+ """
+ return valuation_str
+
+ def get_model_builder(self):
+ return self._modelbuilder
+
+
+class TheoremToolCommandDecorator(TheoremToolCommand):
+ """
+ A base decorator for the ``ProverCommandDecorator`` and
+ ``ModelBuilderCommandDecorator`` classes from which decorators can extend.
+ """
+ def __init__(self, command):
+ """
+ :param command: ``TheoremToolCommand`` to decorate
+ """
+ self._command = command
+
+ #The decorator has its own versions of 'result' different from the
+ #underlying command
+ self._result = None
+
+ def assumptions(self):
+ return self._command.assumptions()
+
+ def goal(self):
+ return self._command.goal()
+
+ def add_assumptions(self, new_assumptions):
+ self._command.add_assumptions(new_assumptions)
+ self._result = None
+
+ def retract_assumptions(self, retracted, debug=False):
+ self._command.retract_assumptions(retracted, debug)
+ self._result = None
+
+ def print_assumptions(self):
+ self._command.print_assumptions()
+
+
+class ProverCommandDecorator(TheoremToolCommandDecorator, ProverCommand):
+ """
+ A base decorator for the ``ProverCommand`` class from which other
+ prover command decorators can extend.
+ """
+ def __init__(self, proverCommand):
+ """
+ :param proverCommand: ``ProverCommand`` to decorate
+ """
+ TheoremToolCommandDecorator.__init__(self, proverCommand)
+
+ #The decorator has its own versions of 'result' and 'proof'
+ #because they may be different from the underlying command
+ self._proof = None
+
+ def prove(self, verbose=False):
+ if self._result is None:
+ prover = self.get_prover()
+ self._result, self._proof = prover._prove(self.goal(),
+ self.assumptions(),
+ verbose)
+ return self._result
+
+ def proof(self, simplify=True):
+ """
+ Return the proof string
+ :param simplify: bool simplify the proof?
+ :return: str
+ """
+ if self._result is None:
+ raise LookupError("You have to call prove() first to get a proof!")
+ else:
+ return self.decorate_proof(self._proof, simplify)
+
+ def decorate_proof(self, proof_string, simplify=True):
+ """
+ Modify and return the proof string
+ :param proof_string: str the proof to decorate
+ :param simplify: bool simplify the proof?
+ :return: str
+ """
+ return self._command.decorate_proof(proof_string, simplify)
+
+ def get_prover(self):
+ return self._command.get_prover()
+
+
+class ModelBuilderCommandDecorator(TheoremToolCommandDecorator, ModelBuilderCommand):
+ """
+ A base decorator for the ``ModelBuilderCommand`` class from which other
+ prover command decorators can extend.
+ """
+ def __init__(self, modelBuilderCommand):
+ """
+ :param modelBuilderCommand: ``ModelBuilderCommand`` to decorate
+ """
+ TheoremToolCommandDecorator.__init__(self, modelBuilderCommand)
+
+ #The decorator has its own versions of 'result' and 'valuation'
+ #because they may be different from the underlying command
+ self._model = None
+
+ def build_model(self, verbose=False):
+ """
+ Attempt to build a model. Store the result to prevent unnecessary
+ re-building.
+ """
+ if self._result is None:
+ modelbuilder = self.get_model_builder()
+ self._result, self._model = \
+ modelbuilder._build_model(self.goal(),
+ self.assumptions(),
+ verbose)
+ return self._result
+
+ def model(self, format=None):
+ """
+ Return a string representation of the model
+
+ :param simplify: bool simplify the proof?
+ :return: str
+ """
+ if self._result is None:
+ raise LookupError('You have to call build_model() first to '
+ 'get a model!')
+ else:
+ return self._decorate_model(self._model, format)
+
+ def _decorate_model(self, valuation_str, format=None):
+ """
+ Modify and return the proof string
+ :param valuation_str: str with the model builder's output
+ :param format: str indicating the format for displaying
+ :return: str
+ """
+ return self._command._decorate_model(valuation_str, format)
+
+ def get_model_builder(self):
+ return self._command.get_prover()
+
+
+class ParallelProverBuilder(Prover, ModelBuilder):
+ """
+ This class stores both a prover and a model builder and when either
+ prove() or build_model() is called, then both theorem tools are run in
+ parallel. Whichever finishes first, the prover or the model builder, is the
+ result that will be used.
+ """
+ def __init__(self, prover, modelbuilder):
+ self._prover = prover
+ self._modelbuilder = modelbuilder
+
+ def _prove(self, goal=None, assumptions=None, verbose=False):
+ return self._run(goal, assumptions, verbose), ''
+
+ def _build_model(self, goal=None, assumptions=None, verbose=False):
+ return not self._run(goal, assumptions, verbose), ''
+
+ def _run(self, goal, assumptions, verbose):
+ # Set up two thread, Prover and ModelBuilder to run in parallel
+ tp_thread = TheoremToolThread(lambda: self._prover.prove(goal, assumptions, verbose), verbose, 'TP')
+ mb_thread = TheoremToolThread(lambda: self._modelbuilder.build_model(goal, assumptions, verbose), verbose, 'MB')
+
+ tp_thread.start()
+ mb_thread.start()
+
+ while tp_thread.isAlive() and mb_thread.isAlive():
+ # wait until either the prover or the model builder is done
+ pass
+
+ if tp_thread.result is not None:
+ return tp_thread.result
+ elif mb_thread.result is not None:
+ return not mb_thread.result
+ else:
+ return None
+
+class ParallelProverBuilderCommand(BaseProverCommand, BaseModelBuilderCommand):
+ """
+ This command stores both a prover and a model builder and when either
+ prove() or build_model() is called, then both theorem tools are run in
+ parallel. Whichever finishes first, the prover or the model builder, is the
+ result that will be used.
+
+ Because the theorem prover result is the opposite of the model builder
+ result, we will treat self._result as meaning "proof found/no model found".
+ """
+ def __init__(self, prover, modelbuilder, goal=None, assumptions=None):
+ BaseProverCommand.__init__(self, prover, goal, assumptions)
+ BaseModelBuilderCommand.__init__(self, modelbuilder, goal, assumptions)
+
+ def prove(self, verbose=False):
+ return self._run(verbose)
+
+ def build_model(self, verbose=False):
+ return not self._run(verbose)
+
+ def _run(self, verbose):
+ # Set up two thread, Prover and ModelBuilder to run in parallel
+ tp_thread = TheoremToolThread(lambda: BaseProverCommand.prove(self, verbose), verbose, 'TP')
+ mb_thread = TheoremToolThread(lambda: BaseModelBuilderCommand.build_model(self, verbose), verbose, 'MB')
+
+ tp_thread.start()
+ mb_thread.start()
+
+ while tp_thread.isAlive() and mb_thread.isAlive():
+ # wait until either the prover or the model builder is done
+ pass
+
+ if tp_thread.result is not None:
+ self._result = tp_thread.result
+ elif mb_thread.result is not None:
+ self._result = not mb_thread.result
+ return self._result
+
+
+class TheoremToolThread(threading.Thread):
+ def __init__(self, command, verbose, name=None):
+ threading.Thread.__init__(self)
+ self._command = command
+ self._result = None
+ self._verbose = verbose
+ self._name = name
+
+ def run(self):
+ try:
+ self._result = self._command()
+ if self._verbose:
+ print('Thread %s finished with result %s at %s' % \
+ (self._name, self._result, time.localtime(time.time())))
+ except Exception as e:
+ print(e)
+ print('Thread %s completed abnormally' % (self._name))
+
+ @property
+ def result(self): return self._result
diff --git a/nltk/inference/discourse.py b/nltk/inference/discourse.py
new file mode 100644
index 0000000..1af2754
--- /dev/null
+++ b/nltk/inference/discourse.py
@@ -0,0 +1,608 @@
+# Natural Language Toolkit: Discourse Processing
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# Dan Garrette <dhgarrette at gmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Module for incrementally developing simple discourses, and checking for semantic ambiguity,
+consistency and informativeness.
+
+Many of the ideas are based on the CURT family of programs of Blackburn and Bos
+(see http://homepages.inf.ed.ac.uk/jbos/comsem/book1.html).
+
+Consistency checking is carried out by using the ``mace`` module to call the Mace4 model builder.
+Informativeness checking is carried out with a call to ``Prover.prove()`` from
+the ``inference`` module.
+
+``DiscourseTester`` is a constructor for discourses.
+The basic data structure is a list of sentences, stored as ``self._sentences``. Each sentence in the list
+is assigned a "sentence ID" (``sid``) of the form ``s``\ *i*. For example::
+
+ s0: A boxer walks
+ s1: Every boxer chases a girl
+
+Each sentence can be ambiguous between a number of readings, each of which receives a
+"reading ID" (``rid``) of the form ``s``\ *i* -``r``\ *j*. For example::
+
+ s0 readings:
+
+ s0-r1: some x.(boxer(x) & walk(x))
+ s0-r0: some x.(boxerdog(x) & walk(x))
+
+A "thread" is a list of readings, represented as a list of ``rid``\ s.
+Each thread receives a "thread ID" (``tid``) of the form ``d``\ *i*.
+For example::
+
+ d0: ['s0-r0', 's1-r0']
+
+The set of all threads for a discourse is the Cartesian product of all the readings of the sequences of sentences.
+(This is not intended to scale beyond very short discourses!) The method ``readings(filter=True)`` will only show
+those threads which are consistent (taking into account any background assumptions).
+"""
+from __future__ import print_function
+import os
+
+from operator import and_, add
+from functools import reduce
+
+from nltk.data import show_cfg
+from nltk.tag import RegexpTagger
+from nltk.parse import load_parser
+from nltk.parse.malt import MaltParser
+from nltk.sem.drt import resolve_anaphora, AnaphoraResolutionException
+from nltk.sem.glue import DrtGlue
+from nltk.sem.logic import Expression
+
+from nltk.inference.mace import MaceCommand
+from nltk.inference.prover9 import Prover9Command
+
+
+class ReadingCommand(object):
+ def parse_to_readings(self, sentence):
+ """
+ :param sentence: the sentence to read
+ :type sentence: str
+ """
+ raise NotImplementedError()
+
+ def process_thread(self, sentence_readings):
+ """
+ This method should be used to handle dependencies between readings such
+ as resolving anaphora.
+
+ :param sentence_readings: readings to process
+ :type sentence_readings: list(Expression)
+ :return: the list of readings after processing
+ :rtype: list(Expression)
+ """
+ return sentence_readings
+
+ def combine_readings(self, readings):
+ """
+ :param readings: readings to combine
+ :type readings: list(Expression)
+ :return: one combined reading
+ :rtype: Expression
+ """
+ raise NotImplementedError()
+
+ def to_fol(self, expression):
+ """
+ Convert this expression into a First-Order Logic expression.
+
+ :param expression: an expression
+ :type expression: Expression
+ :return: a FOL version of the input expression
+ :rtype: Expression
+ """
+ raise NotImplementedError()
+
+
+class CfgReadingCommand(ReadingCommand):
+ def __init__(self, gramfile=None):
+ """
+ :param gramfile: name of file where grammar can be loaded
+ :type gramfile: str
+ """
+ self._gramfile = (gramfile if gramfile else 'grammars/book_grammars/discourse.fcfg')
+ self._parser = load_parser(self._gramfile)
+
+ def parse_to_readings(self, sentence):
+ """:see: ReadingCommand.parse_to_readings()"""
+ from nltk.sem import root_semrep
+ tokens = sentence.split()
+ trees = self._parser.parse(tokens)
+ return [root_semrep(tree) for tree in trees]
+
+ def combine_readings(self, readings):
+ """:see: ReadingCommand.combine_readings()"""
+ return reduce(and_, readings)
+
+ def to_fol(self, expression):
+ """:see: ReadingCommand.to_fol()"""
+ return expression
+
+
+class DrtGlueReadingCommand(ReadingCommand):
+ def __init__(self, semtype_file=None, remove_duplicates=False,
+ depparser=None):
+ """
+ :param semtype_file: name of file where grammar can be loaded
+ :param remove_duplicates: should duplicates be removed?
+ :param depparser: the dependency parser
+ """
+ if semtype_file is None:
+ semtype_file = os.path.join('grammars', 'sample_grammars','drt_glue.semtype')
+ self._glue = DrtGlue(semtype_file=semtype_file,
+ remove_duplicates=remove_duplicates,
+ depparser=depparser)
+
+ def parse_to_readings(self, sentence):
+ """:see: ReadingCommand.parse_to_readings()"""
+ return self._glue.parse_to_meaning(sentence)
+
+ def process_thread(self, sentence_readings):
+ """:see: ReadingCommand.process_thread()"""
+ try:
+ return [self.combine_readings(sentence_readings)]
+ except AnaphoraResolutionException:
+ return []
+
+ def combine_readings(self, readings):
+ """:see: ReadingCommand.combine_readings()"""
+ thread_reading = reduce(add, readings)
+ return resolve_anaphora(thread_reading.simplify())
+
+ def to_fol(self, expression):
+ """:see: ReadingCommand.to_fol()"""
+ return expression.fol()
+
+
+class DiscourseTester(object):
+ """
+ Check properties of an ongoing discourse.
+ """
+ def __init__(self, input, reading_command=None, background=None):
+ """
+ Initialize a ``DiscourseTester``.
+
+ :param input: the discourse sentences
+ :type input: list of str
+ :param background: Formulas which express background assumptions
+ :type background: list(Expression)
+ """
+ self._input = input
+ self._sentences = dict([('s%s' % i, sent) for i, sent in enumerate(input)])
+ self._models = None
+ self._readings = {}
+ self._reading_command = (reading_command if reading_command else CfgReadingCommand())
+ self._threads = {}
+ self._filtered_threads = {}
+ if background is not None:
+ from nltk.sem.logic import Expression
+ for e in background:
+ assert isinstance(e, Expression)
+ self._background = background
+ else:
+ self._background = []
+
+ ###############################
+ # Sentences
+ ###############################
+
+ def sentences(self):
+ """
+ Display the list of sentences in the current discourse.
+ """
+ for id in sorted(self._sentences):
+ print("%s: %s" % (id, self._sentences[id]))
+
+ def add_sentence(self, sentence, informchk=False, consistchk=False,):
+ """
+ Add a sentence to the current discourse.
+
+ Updates ``self._input`` and ``self._sentences``.
+ :param sentence: An input sentence
+ :type sentence: str
+ :param informchk: if ``True``, check that the result of adding the sentence is thread-informative. Updates ``self._readings``.
+ :param consistchk: if ``True``, check that the result of adding the sentence is thread-consistent. Updates ``self._readings``.
+
+ """
+ # check whether the new sentence is informative (i.e. not entailed by the previous discourse)
+ if informchk:
+ self.readings(verbose=False)
+ for tid in sorted(self._threads):
+ assumptions = [reading for (rid, reading) in self.expand_threads(tid)]
+ assumptions += self._background
+ for sent_reading in self._get_readings(sentence):
+ tp = Prover9Command(goal=sent_reading, assumptions=assumptions)
+ if tp.prove():
+ print("Sentence '%s' under reading '%s':" % (sentence, str(sent_reading)))
+ print("Not informative relative to thread '%s'" % tid)
+
+ self._input.append(sentence)
+ self._sentences = dict([('s%s' % i, sent) for i, sent in enumerate(self._input)])
+ # check whether adding the new sentence to the discourse preserves consistency (i.e. a model can be found for the combined set of
+ # of assumptions
+ if consistchk:
+ self.readings(verbose=False)
+ self.models(show=False)
+
+ def retract_sentence(self, sentence, verbose=True):
+ """
+ Remove a sentence from the current discourse.
+
+ Updates ``self._input``, ``self._sentences`` and ``self._readings``.
+ :param sentence: An input sentence
+ :type sentence: str
+ :param verbose: If ``True``, report on the updated list of sentences.
+ """
+ try:
+ self._input.remove(sentence)
+ except ValueError:
+ print("Retraction failed. The sentence '%s' is not part of the current discourse:" % sentence)
+ self.sentences()
+ return None
+ self._sentences = dict([('s%s' % i, sent) for i, sent in enumerate(self._input)])
+ self.readings(verbose=False)
+ if verbose:
+ print("Current sentences are ")
+ self.sentences()
+
+ def grammar(self):
+ """
+ Print out the grammar in use for parsing input sentences
+ """
+ show_cfg(self._reading_command._gramfile)
+
+ ###############################
+ # Readings and Threads
+ ###############################
+
+ def _get_readings(self, sentence):
+ """
+ Build a list of semantic readings for a sentence.
+
+ :rtype: list(Expression)
+ """
+ return self._reading_command.parse_to_readings(sentence)
+
+ def _construct_readings(self):
+ """
+ Use ``self._sentences`` to construct a value for ``self._readings``.
+ """
+ # re-initialize self._readings in case we have retracted a sentence
+ self._readings = {}
+ for sid in sorted(self._sentences):
+ sentence = self._sentences[sid]
+ readings = self._get_readings(sentence)
+ self._readings[sid] = dict([("%s-r%s" % (sid, rid), reading.simplify())
+ for rid, reading in enumerate(sorted(readings, key=str))])
+
+ def _construct_threads(self):
+ """
+ Use ``self._readings`` to construct a value for ``self._threads``
+ and use the model builder to construct a value for ``self._filtered_threads``
+ """
+ thread_list = [[]]
+ for sid in sorted(self._readings):
+ thread_list = self.multiply(thread_list, sorted(self._readings[sid]))
+ self._threads = dict([("d%s" % tid, thread) for tid, thread in enumerate(thread_list)])
+ # re-initialize the filtered threads
+ self._filtered_threads = {}
+ # keep the same ids, but only include threads which get models
+ consistency_checked = self._check_consistency(self._threads)
+ for (tid, thread) in self._threads.items():
+ if (tid, True) in consistency_checked:
+ self._filtered_threads[tid] = thread
+
+
+ def _show_readings(self, sentence=None):
+ """
+ Print out the readings for the discourse (or a single sentence).
+ """
+ if sentence is not None:
+ print("The sentence '%s' has these readings:" % sentence)
+ for r in [str(reading) for reading in (self._get_readings(sentence))]:
+ print(" %s" % r)
+ else:
+ for sid in sorted(self._readings):
+ print()
+ print('%s readings:' % sid)
+ print() #'-' * 30
+ for rid in sorted(self._readings[sid]):
+ lf = self._readings[sid][rid]
+ print("%s: %s" % (rid, lf.normalize()))
+
+ def _show_threads(self, filter=False, show_thread_readings=False):
+ """
+ Print out the value of ``self._threads`` or ``self._filtered_hreads``
+ """
+ threads = (self._filtered_threads if filter else self._threads)
+ for tid in sorted(threads):
+ if show_thread_readings:
+ readings = [self._readings[rid.split('-')[0]][rid]
+ for rid in self._threads[tid]]
+ try:
+ thread_reading = ": %s" % \
+ self._reading_command.combine_readings(readings).normalize()
+ except Exception as e:
+ thread_reading = ': INVALID: %s' % e.__class__.__name__
+ else:
+ thread_reading = ''
+
+ print("%s:" % tid, self._threads[tid], thread_reading)
+
+
+ def readings(self, sentence=None, threaded=False, verbose=True,
+ filter=False, show_thread_readings=False):
+ """
+ Construct and show the readings of the discourse (or of a single sentence).
+
+ :param sentence: test just this sentence
+ :type sentence: str
+ :param threaded: if ``True``, print out each thread ID and the corresponding thread.
+ :param filter: if ``True``, only print out consistent thread IDs and threads.
+ """
+ self._construct_readings()
+ self._construct_threads()
+
+ # if we are filtering or showing thread readings, show threads
+ if filter or show_thread_readings:
+ threaded = True
+
+ if verbose:
+ if not threaded:
+ self._show_readings(sentence=sentence)
+ else:
+ self._show_threads(filter=filter,
+ show_thread_readings=show_thread_readings)
+
+ def expand_threads(self, thread_id, threads=None):
+ """
+ Given a thread ID, find the list of ``logic.Expression`` objects corresponding to the reading IDs in that thread.
+
+ :param thread_id: thread ID
+ :type thread_id: str
+ :param threads: a mapping from thread IDs to lists of reading IDs
+ :type threads: dict
+ :return: A list of pairs ``(rid, reading)`` where reading is the ``logic.Expression`` associated with a reading ID
+ :rtype: list of tuple
+ """
+ if threads is None:
+ threads = self._threads
+ return [(rid, self._readings[sid][rid]) for rid in threads[thread_id] for sid in rid.split('-')[:1]]
+
+
+ ###############################
+ # Models and Background
+ ###############################
+
+ def _check_consistency(self, threads, show=False, verbose=False):
+ results = []
+ for tid in sorted(threads):
+ assumptions = [reading for (rid, reading) in self.expand_threads(tid, threads=threads)]
+ assumptions = list(map(self._reading_command.to_fol, self._reading_command.process_thread(assumptions)))
+ if assumptions:
+ assumptions += self._background
+ # if Mace4 finds a model, it always seems to find it quickly
+ mb = MaceCommand(None, assumptions, max_models=20)
+ modelfound = mb.build_model()
+ else:
+ modelfound = False
+ results.append((tid, modelfound))
+ if show:
+ spacer(80)
+ print("Model for Discourse Thread %s" % tid)
+ spacer(80)
+ if verbose:
+ for a in assumptions:
+ print(a)
+ spacer(80)
+ if modelfound:
+ print(mb.model(format='cooked'))
+ else:
+ print("No model found!\n")
+ return results
+
+ def models(self, thread_id=None, show=True, verbose=False):
+ """
+ Call Mace4 to build a model for each current discourse thread.
+
+ :param thread_id: thread ID
+ :type thread_id: str
+ :param show: If ``True``, display the model that has been found.
+ """
+ self._construct_readings()
+ self._construct_threads()
+ threads = ({thread_id: self._threads[thread_id]} if thread_id else self._threads)
+
+ for (tid, modelfound) in self._check_consistency(threads, show=show, verbose=verbose):
+ idlist = [rid for rid in threads[tid]]
+
+ if not modelfound:
+ print("Inconsistent discourse: %s %s:" % (tid, idlist))
+ for rid, reading in self.expand_threads(tid):
+ print(" %s: %s" % (rid, reading.normalize()))
+ print()
+ else:
+ print("Consistent discourse: %s %s:" % (tid, idlist))
+ for rid, reading in self.expand_threads(tid):
+ print(" %s: %s" % (rid, reading.normalize()))
+ print()
+
+ def add_background(self, background, verbose=False):
+ """
+ Add a list of background assumptions for reasoning about the discourse.
+
+ When called, this method also updates the discourse model's set of readings and threads.
+ :param background: Formulas which contain background information
+ :type background: list(Expression)
+ """
+ from nltk.sem.logic import Expression
+ for (count, e) in enumerate(background):
+ assert isinstance(e, Expression)
+ if verbose:
+ print("Adding assumption %s to background" % count)
+ self._background.append(e)
+
+ #update the state
+ self._construct_readings()
+ self._construct_threads()
+
+ def background(self):
+ """
+ Show the current background assumptions.
+ """
+ for e in self._background:
+ print(str(e))
+
+ ###############################
+ # Misc
+ ###############################
+
+ @staticmethod
+ def multiply(discourse, readings):
+ """
+ Multiply every thread in ``discourse`` by every reading in ``readings``.
+
+ Given discourse = [['A'], ['B']], readings = ['a', 'b', 'c'] , returns
+ [['A', 'a'], ['A', 'b'], ['A', 'c'], ['B', 'a'], ['B', 'b'], ['B', 'c']]
+
+ :param discourse: the current list of readings
+ :type discourse: list of lists
+ :param readings: an additional list of readings
+ :type readings: list(Expression)
+ :rtype: A list of lists
+ """
+ result = []
+ for sublist in discourse:
+ for r in readings:
+ new = []
+ new += sublist
+ new.append(r)
+ result.append(new)
+ return result
+
+#multiply = DiscourseTester.multiply
+#L1 = [['A'], ['B']]
+#L2 = ['a', 'b', 'c']
+#print multiply(L1,L2)
+
+def parse_fol(s):
+ """
+ Temporarily duplicated from ``nltk.sem.util``.
+ Convert a file of first order formulas into a list of ``Expression`` objects.
+
+ :param s: the contents of the file
+ :type s: str
+ :return: a list of parsed formulas.
+ :rtype: list(Expression)
+ """
+ statements = []
+ for linenum, line in enumerate(s.splitlines()):
+ line = line.strip()
+ if line.startswith('#') or line=='': continue
+ try:
+ statements.append(Expression.fromstring(line))
+ except Exception:
+ raise ValueError('Unable to parse line %s: %s' % (linenum, line))
+ return statements
+
+###############################
+# Demo
+###############################
+
+def discourse_demo(reading_command=None):
+ """
+ Illustrate the various methods of ``DiscourseTester``
+ """
+ dt = DiscourseTester(['A boxer walks', 'Every boxer chases a girl'],
+ reading_command)
+ dt.models()
+ print()
+ #dt.grammar()
+ print()
+ dt.sentences()
+ print()
+ dt.readings()
+ print()
+ dt.readings(threaded=True)
+ print()
+ dt.models('d1')
+ dt.add_sentence('John is a boxer')
+ print()
+ dt.sentences()
+ print()
+ dt.readings(threaded=True)
+ print()
+ dt = DiscourseTester(['A student dances', 'Every student is a person'],
+ reading_command)
+ print()
+ dt.add_sentence('No person dances', consistchk=True)
+ print()
+ dt.readings()
+ print()
+ dt.retract_sentence('No person dances', verbose=True)
+ print()
+ dt.models()
+ print()
+ dt.readings('A person dances')
+ print()
+ dt.add_sentence('A person dances', informchk=True)
+ dt = DiscourseTester(['Vincent is a boxer', 'Fido is a boxer',
+ 'Vincent is married', 'Fido barks'],
+ reading_command)
+ dt.readings(filter=True)
+ import nltk.data
+ background_file = os.path.join('grammars', 'book_grammars', 'background.fol')
+ background = nltk.data.load(background_file)
+
+ print()
+ dt.add_background(background, verbose=False)
+ dt.background()
+ print()
+ dt.readings(filter=True)
+ print()
+ dt.models()
+
+
+def drt_discourse_demo(reading_command=None):
+ """
+ Illustrate the various methods of ``DiscourseTester``
+ """
+ dt = DiscourseTester(['every dog chases a boy', 'he runs'],
+ reading_command)
+ dt.models()
+ print()
+ dt.sentences()
+ print()
+ dt.readings()
+ print()
+ dt.readings(show_thread_readings=True)
+ print()
+ dt.readings(filter=True, show_thread_readings=True)
+
+
+def spacer(num=30):
+ print('-' * num)
+
+def demo():
+ discourse_demo()
+
+ tagger = RegexpTagger(
+ [('^(chases|runs)$', 'VB'),
+ ('^(a)$', 'ex_quant'),
+ ('^(every)$', 'univ_quant'),
+ ('^(dog|boy)$', 'NN'),
+ ('^(he)$', 'PRP')
+ ])
+ depparser = MaltParser(tagger=tagger)
+ drt_discourse_demo(DrtGlueReadingCommand(remove_duplicates=False,
+ depparser=depparser))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/inference/mace.py b/nltk/inference/mace.py
new file mode 100644
index 0000000..83e841b
--- /dev/null
+++ b/nltk/inference/mace.py
@@ -0,0 +1,311 @@
+# Natural Language Toolkit: Interface to the Mace4 Model Builder
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+# Ewan Klein <ewan at inf.ed.ac.uk>
+
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A model builder that makes use of the external 'Mace4' package.
+"""
+from __future__ import print_function
+
+import os
+import tempfile
+
+from nltk.sem.logic import is_indvar
+from nltk.sem import Valuation, Expression
+
+from nltk.inference.api import ModelBuilder, BaseModelBuilderCommand
+from nltk.inference.prover9 import Prover9CommandParent, Prover9Parent
+
+
+class MaceCommand(Prover9CommandParent, BaseModelBuilderCommand):
+ """
+ A ``MaceCommand`` specific to the ``Mace`` model builder. It contains
+ a print_assumptions() method that is used to print the list
+ of assumptions in multiple formats.
+ """
+ _interpformat_bin = None
+
+ def __init__(self, goal=None, assumptions=None, max_models=500, model_builder=None):
+ """
+ :param goal: Input expression to prove
+ :type goal: sem.Expression
+ :param assumptions: Input expressions to use as assumptions in
+ the proof.
+ :type assumptions: list(sem.Expression)
+ :param max_models: The maximum number of models that Mace will try before
+ simply returning false. (Use 0 for no maximum.)
+ :type max_models: int
+ """
+ if model_builder is not None:
+ assert isinstance(model_builder, Mace)
+ else:
+ model_builder = Mace(max_models)
+
+ BaseModelBuilderCommand.__init__(self, model_builder, goal, assumptions)
+
+ @property
+ def valuation(mbc): return mbc.model('valuation')
+
+ def _convert2val(self, valuation_str):
+ """
+ Transform the output file into an NLTK-style Valuation.
+
+ :return: A model if one is generated; None otherwise.
+ :rtype: sem.Valuation
+ """
+ valuation_standard_format = self._transform_output(valuation_str, 'standard')
+
+ val = []
+ for line in valuation_standard_format.splitlines(False):
+ l = line.strip()
+
+ if l.startswith('interpretation'):
+ # find the number of entities in the model
+ num_entities = int(l[l.index('(')+1:l.index(',')].strip())
+
+ elif l.startswith('function') and l.find('_') == -1:
+ # replace the integer identifier with a corresponding alphabetic character
+ name = l[l.index('(')+1:l.index(',')].strip()
+ if is_indvar(name):
+ name = name.upper()
+ value = int(l[l.index('[')+1:l.index(']')].strip())
+ val.append((name, MaceCommand._make_model_var(value)))
+
+ elif l.startswith('relation'):
+ l = l[l.index('(')+1:]
+ if '(' in l:
+ #relation is not nullary
+ name = l[:l.index('(')].strip()
+ values = [int(v.strip()) for v in l[l.index('[')+1:l.index(']')].split(',')]
+ val.append((name, MaceCommand._make_relation_set(num_entities, values)))
+ else:
+ #relation is nullary
+ name = l[:l.index(',')].strip()
+ value = int(l[l.index('[')+1:l.index(']')].strip())
+ val.append((name, value == 1))
+
+ return Valuation(val)
+
+ @staticmethod
+ def _make_relation_set(num_entities, values):
+ """
+ Convert a Mace4-style relation table into a dictionary.
+
+ :param num_entities: the number of entities in the model; determines the row length in the table.
+ :type num_entities: int
+ :param values: a list of 1's and 0's that represent whether a relation holds in a Mace4 model.
+ :type values: list of int
+ """
+ r = set()
+ for position in [pos for (pos,v) in enumerate(values) if v == 1]:
+ r.add(tuple(MaceCommand._make_relation_tuple(position, values, num_entities)))
+ return r
+
+ @staticmethod
+ def _make_relation_tuple(position, values, num_entities):
+ if len(values) == 1:
+ return []
+ else:
+ sublist_size = len(values) // num_entities
+ sublist_start = position // sublist_size
+ sublist_position = int(position % sublist_size)
+
+ sublist = values[sublist_start*sublist_size:(sublist_start+1)*sublist_size]
+ return [MaceCommand._make_model_var(sublist_start)] + \
+ MaceCommand._make_relation_tuple(sublist_position,
+ sublist,
+ num_entities)
+
+ @staticmethod
+ def _make_model_var(value):
+ """
+ Pick an alphabetic character as identifier for an entity in the model.
+
+ :param value: where to index into the list of characters
+ :type value: int
+ """
+ letter = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n',
+ 'o','p','q','r','s','t','u','v','w','x','y','z'][value]
+ num = value // 26
+ return (letter + str(num) if num > 0 else letter)
+
+ def _decorate_model(self, valuation_str, format):
+ """
+ Print out a Mace4 model using any Mace4 ``interpformat`` format.
+ See http://www.cs.unm.edu/~mccune/mace4/manual/ for details.
+
+ :param valuation_str: str with the model builder's output
+ :param format: str indicating the format for displaying
+ models. Defaults to 'standard' format.
+ :return: str
+ """
+ if not format:
+ return valuation_str
+ elif format == 'valuation':
+ return self._convert2val(valuation_str)
+ else:
+ return self._transform_output(valuation_str, format)
+
+ def _transform_output(self, valuation_str, format):
+ """
+ Transform the output file into any Mace4 ``interpformat`` format.
+
+ :param format: Output format for displaying models.
+ :type format: str
+ """
+ if format in ['standard', 'standard2', 'portable', 'tabular',
+ 'raw', 'cooked', 'xml', 'tex']:
+ return self._call_interpformat(valuation_str, [format])[0]
+ else:
+ raise LookupError("The specified format does not exist")
+
+ def _call_interpformat(self, input_str, args=[], verbose=False):
+ """
+ Call the ``interpformat`` binary with the given input.
+
+ :param input_str: A string whose contents are used as stdin.
+ :param args: A list of command-line arguments.
+ :return: A tuple (stdout, returncode)
+ :see: ``config_prover9``
+ """
+ if self._interpformat_bin is None:
+ self._interpformat_bin = self._modelbuilder._find_binary(
+ 'interpformat', verbose)
+
+ return self._modelbuilder._call(input_str, self._interpformat_bin,
+ args, verbose)
+
+
+class Mace(Prover9Parent, ModelBuilder):
+ _mace4_bin = None
+
+ def __init__(self, end_size=500):
+ self._end_size = end_size
+ """The maximum model size that Mace will try before
+ simply returning false. (Use -1 for no maximum.)"""
+
+ def _build_model(self, goal=None, assumptions=None, verbose=False):
+ """
+ Use Mace4 to build a first order model.
+
+ :return: ``True`` if a model was found (i.e. Mace returns value of 0),
+ else ``False``
+ """
+ if not assumptions:
+ assumptions = []
+
+ stdout, returncode = self._call_mace4(self.prover9_input(goal, assumptions),
+ verbose=verbose)
+ return (returncode == 0, stdout)
+
+ def _call_mace4(self, input_str, args=[], verbose=False):
+ """
+ Call the ``mace4`` binary with the given input.
+
+ :param input_str: A string whose contents are used as stdin.
+ :param args: A list of command-line arguments.
+ :return: A tuple (stdout, returncode)
+ :see: ``config_prover9``
+ """
+ if self._mace4_bin is None:
+ self._mace4_bin = self._find_binary('mace4', verbose)
+
+ updated_input_str = ''
+ if self._end_size > 0:
+ updated_input_str += 'assign(end_size, %d).\n\n' % self._end_size
+ updated_input_str += input_str
+
+ return self._call(updated_input_str, self._mace4_bin, args, verbose)
+
+
+def spacer(num=30):
+ print('-' * num)
+
+def decode_result(found):
+ """
+ Decode the result of model_found()
+
+ :param found: The output of model_found()
+ :type found: bool
+ """
+ return {True: 'Countermodel found', False: 'No countermodel found', None: 'None'}[found]
+
+def test_model_found(arguments):
+ """
+ Try some proofs and exhibit the results.
+ """
+ for (goal, assumptions) in arguments:
+ g = Expression.fromstring(goal)
+ alist = [lp.parse(a) for a in assumptions]
+ m = MaceCommand(g, assumptions=alist, max_models=50)
+ found = m.build_model()
+ for a in alist:
+ print(' %s' % a)
+ print('|- %s: %s\n' % (g, decode_result(found)))
+
+
+def test_build_model(arguments):
+ """
+ Try to build a ``nltk.sem.Valuation``.
+ """
+ g = Expression.fromstring('all x.man(x)')
+ alist = [Expression.fromstring(a) for a in ['man(John)',
+ 'man(Socrates)',
+ 'man(Bill)',
+ 'some x.(-(x = John) & man(x) & sees(John,x))',
+ 'some x.(-(x = Bill) & man(x))',
+ 'all x.some y.(man(x) -> gives(Socrates,x,y))']]
+
+ m = MaceCommand(g, assumptions=alist)
+ m.build_model()
+ spacer()
+ print("Assumptions and Goal")
+ spacer()
+ for a in alist:
+ print(' %s' % a)
+ print('|- %s: %s\n' % (g, decode_result(m.build_model())))
+ spacer()
+ #print m.model('standard')
+ #print m.model('cooked')
+ print("Valuation")
+ spacer()
+ print(m.valuation, '\n')
+
+def test_transform_output(argument_pair):
+ """
+ Transform the model into various Mace4 ``interpformat`` formats.
+ """
+ g = Expression.fromstring(argument_pair[0])
+ alist = [lp.parse(a) for a in argument_pair[1]]
+ m = MaceCommand(g, assumptions=alist)
+ m.build_model()
+ for a in alist:
+ print(' %s' % a)
+ print('|- %s: %s\n' % (g, m.build_model()))
+ for format in ['standard', 'portable', 'xml', 'cooked']:
+ spacer()
+ print("Using '%s' format" % format)
+ spacer()
+ print(m.model(format=format))
+
+def test_make_relation_set():
+ print(MaceCommand._make_relation_set(num_entities=3, values=[1,0,1]) == set([('c',), ('a',)]))
+ print(MaceCommand._make_relation_set(num_entities=3, values=[0,0,0,0,0,0,1,0,0]) == set([('c', 'a')]))
+ print(MaceCommand._make_relation_set(num_entities=2, values=[0,0,1,0,0,0,1,0]) == set([('a', 'b', 'a'), ('b', 'b', 'a')]))
+
+arguments = [
+ ('mortal(Socrates)', ['all x.(man(x) -> mortal(x))', 'man(Socrates)']),
+ ('(not mortal(Socrates))', ['all x.(man(x) -> mortal(x))', 'man(Socrates)'])
+]
+
+def demo():
+ test_model_found(arguments)
+ test_build_model(arguments)
+ test_transform_output(arguments[1])
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/inference/nonmonotonic.py b/nltk/inference/nonmonotonic.py
new file mode 100644
index 0000000..0d168a5
--- /dev/null
+++ b/nltk/inference/nonmonotonic.py
@@ -0,0 +1,509 @@
+# Natural Language Toolkit: Nonmonotonic Reasoning
+#
+# Author: Daniel H. Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+
+"""
+A module to perform nonmonotonic reasoning. The ideas and demonstrations in
+this module are based on "Logical Foundations of Artificial Intelligence" by
+Michael R. Genesereth and Nils J. Nilsson.
+"""
+from __future__ import print_function, unicode_literals
+
+from nltk.inference.prover9 import Prover9, Prover9Command
+from collections import defaultdict
+from functools import reduce
+
+from nltk.sem.logic import (VariableExpression, EqualityExpression,
+ ApplicationExpression, Expression,
+ AbstractVariableExpression, AllExpression,
+ BooleanExpression, NegatedExpression,
+ ExistsExpression, Variable, ImpExpression,
+ AndExpression, unique_variable, operator)
+
+from nltk.inference.api import Prover, ProverCommandDecorator
+from nltk.compat import python_2_unicode_compatible
+
+class ProverParseError(Exception): pass
+
+def get_domain(goal, assumptions):
+ if goal is None:
+ all_expressions = assumptions
+ else:
+ all_expressions = assumptions + [-goal]
+ return reduce(operator.or_, (a.constants() for a in all_expressions), set())
+
+class ClosedDomainProver(ProverCommandDecorator):
+ """
+ This is a prover decorator that adds domain closure assumptions before
+ proving.
+ """
+ def assumptions(self):
+ assumptions = [a for a in self._command.assumptions()]
+ goal = self._command.goal()
+ domain = get_domain(goal, assumptions)
+ return [self.replace_quants(ex, domain) for ex in assumptions]
+
+ def goal(self):
+ goal = self._command.goal()
+ domain = get_domain(goal, self._command.assumptions())
+ return self.replace_quants(goal, domain)
+
+ def replace_quants(self, ex, domain):
+ """
+ Apply the closed domain assumption to the expression
+ - Domain = union([e.free()|e.constants() for e in all_expressions])
+ - translate "exists x.P" to "(z=d1 | z=d2 | ... ) & P.replace(x,z)" OR
+ "P.replace(x, d1) | P.replace(x, d2) | ..."
+ - translate "all x.P" to "P.replace(x, d1) & P.replace(x, d2) & ..."
+ :param ex: ``Expression``
+ :param domain: set of {Variable}s
+ :return: ``Expression``
+ """
+ if isinstance(ex, AllExpression):
+ conjuncts = [ex.term.replace(ex.variable, VariableExpression(d))
+ for d in domain]
+ conjuncts = [self.replace_quants(c, domain) for c in conjuncts]
+ return reduce(lambda x,y: x&y, conjuncts)
+ elif isinstance(ex, BooleanExpression):
+ return ex.__class__(self.replace_quants(ex.first, domain),
+ self.replace_quants(ex.second, domain) )
+ elif isinstance(ex, NegatedExpression):
+ return -self.replace_quants(ex.term, domain)
+ elif isinstance(ex, ExistsExpression):
+ disjuncts = [ex.term.replace(ex.variable, VariableExpression(d))
+ for d in domain]
+ disjuncts = [self.replace_quants(d, domain) for d in disjuncts]
+ return reduce(lambda x,y: x|y, disjuncts)
+ else:
+ return ex
+
+class UniqueNamesProver(ProverCommandDecorator):
+ """
+ This is a prover decorator that adds unique names assumptions before
+ proving.
+ """
+ def assumptions(self):
+ """
+ - Domain = union([e.free()|e.constants() for e in all_expressions])
+ - if "d1 = d2" cannot be proven from the premises, then add "d1 != d2"
+ """
+ assumptions = self._command.assumptions()
+
+ domain = list(get_domain(self._command.goal(), assumptions))
+
+ #build a dictionary of obvious equalities
+ eq_sets = SetHolder()
+ for a in assumptions:
+ if isinstance(a, EqualityExpression):
+ av = a.first.variable
+ bv = a.second.variable
+ #put 'a' and 'b' in the same set
+ eq_sets[av].add(bv)
+
+ new_assumptions = []
+ for i,a in enumerate(domain):
+ for b in domain[i+1:]:
+ #if a and b are not already in the same equality set
+ if b not in eq_sets[a]:
+ newEqEx = EqualityExpression(VariableExpression(a),
+ VariableExpression(b))
+ if Prover9().prove(newEqEx, assumptions):
+ #we can prove that the names are the same entity.
+ #remember that they are equal so we don't re-check.
+ eq_sets[a].add(b)
+ else:
+ #we can't prove it, so assume unique names
+ new_assumptions.append(-newEqEx)
+
+ return assumptions + new_assumptions
+
+class SetHolder(list):
+ """
+ A list of sets of Variables.
+ """
+ def __getitem__(self, item):
+ """
+ :param item: ``Variable``
+ :return: the set containing 'item'
+ """
+ assert isinstance(item, Variable)
+ for s in self:
+ if item in s:
+ return s
+ #item is not found in any existing set. so create a new set
+ new = set([item])
+ self.append(new)
+ return new
+
+class ClosedWorldProver(ProverCommandDecorator):
+ """
+ This is a prover decorator that completes predicates before proving.
+
+ If the assumptions contain "P(A)", then "all x.(P(x) -> (x=A))" is the completion of "P".
+ If the assumptions contain "all x.(ostrich(x) -> bird(x))", then "all x.(bird(x) -> ostrich(x))" is the completion of "bird".
+ If the assumptions don't contain anything that are "P", then "all x.-P(x)" is the completion of "P".
+
+ walk(Socrates)
+ Socrates != Bill
+ + all x.(walk(x) -> (x=Socrates))
+ ----------------
+ -walk(Bill)
+
+ see(Socrates, John)
+ see(John, Mary)
+ Socrates != John
+ John != Mary
+ + all x.all y.(see(x,y) -> ((x=Socrates & y=John) | (x=John & y=Mary)))
+ ----------------
+ -see(Socrates, Mary)
+
+ all x.(ostrich(x) -> bird(x))
+ bird(Tweety)
+ -ostrich(Sam)
+ Sam != Tweety
+ + all x.(bird(x) -> (ostrich(x) | x=Tweety))
+ + all x.-ostrich(x)
+ -------------------
+ -bird(Sam)
+ """
+ def assumptions(self):
+ assumptions = self._command.assumptions()
+
+ predicates = self._make_predicate_dict(assumptions)
+
+ new_assumptions = []
+ for p in predicates:
+ predHolder = predicates[p]
+ new_sig = self._make_unique_signature(predHolder)
+ new_sig_exs = [VariableExpression(v) for v in new_sig]
+
+ disjuncts = []
+
+ #Turn the signatures into disjuncts
+ for sig in predHolder.signatures:
+ equality_exs = []
+ for v1,v2 in zip(new_sig_exs, sig):
+ equality_exs.append(EqualityExpression(v1,v2))
+ disjuncts.append(reduce(lambda x,y: x&y, equality_exs))
+
+ #Turn the properties into disjuncts
+ for prop in predHolder.properties:
+ #replace variables from the signature with new sig variables
+ bindings = {}
+ for v1,v2 in zip(new_sig_exs, prop[0]):
+ bindings[v2] = v1
+ disjuncts.append(prop[1].substitute_bindings(bindings))
+
+ #make the assumption
+ if disjuncts:
+ #disjuncts exist, so make an implication
+ antecedent = self._make_antecedent(p, new_sig)
+ consequent = reduce(lambda x,y: x|y, disjuncts)
+ accum = ImpExpression(antecedent, consequent)
+ else:
+ #nothing has property 'p'
+ accum = NegatedExpression(self._make_antecedent(p, new_sig))
+
+ #quantify the implication
+ for new_sig_var in new_sig[::-1]:
+ accum = AllExpression(new_sig_var, accum)
+ new_assumptions.append(accum)
+
+ return assumptions + new_assumptions
+
+ def _make_unique_signature(self, predHolder):
+ """
+ This method figures out how many arguments the predicate takes and
+ returns a tuple containing that number of unique variables.
+ """
+ return tuple(unique_variable() for i in range(predHolder.signature_len))
+
+ def _make_antecedent(self, predicate, signature):
+ """
+ Return an application expression with 'predicate' as the predicate
+ and 'signature' as the list of arguments.
+ """
+ antecedent = predicate
+ for v in signature:
+ antecedent = antecedent(VariableExpression(v))
+ return antecedent
+
+ def _make_predicate_dict(self, assumptions):
+ """
+ Create a dictionary of predicates from the assumptions.
+
+ :param assumptions: a list of ``Expression``s
+ :return: dict mapping ``AbstractVariableExpression`` to ``PredHolder``
+ """
+ predicates = defaultdict(PredHolder)
+ for a in assumptions:
+ self._map_predicates(a, predicates)
+ return predicates
+
+ def _map_predicates(self, expression, predDict):
+ if isinstance(expression, ApplicationExpression):
+ func, args = expression.uncurry()
+ if isinstance(func, AbstractVariableExpression):
+ predDict[func].append_sig(tuple(args))
+ elif isinstance(expression, AndExpression):
+ self._map_predicates(expression.first, predDict)
+ self._map_predicates(expression.second, predDict)
+ elif isinstance(expression, AllExpression):
+ #collect all the universally quantified variables
+ sig = [expression.variable]
+ term = expression.term
+ while isinstance(term, AllExpression):
+ sig.append(term.variable)
+ term = term.term
+ if isinstance(term, ImpExpression):
+ if isinstance(term.first, ApplicationExpression) and \
+ isinstance(term.second, ApplicationExpression):
+ func1, args1 = term.first.uncurry()
+ func2, args2 = term.second.uncurry()
+ if isinstance(func1, AbstractVariableExpression) and \
+ isinstance(func2, AbstractVariableExpression) and \
+ sig == [v.variable for v in args1] and \
+ sig == [v.variable for v in args2]:
+ predDict[func2].append_prop((tuple(sig), term.first))
+ predDict[func1].validate_sig_len(sig)
+
+ at python_2_unicode_compatible
+class PredHolder(object):
+ """
+ This class will be used by a dictionary that will store information
+ about predicates to be used by the ``ClosedWorldProver``.
+
+ The 'signatures' property is a list of tuples defining signatures for
+ which the predicate is true. For instance, 'see(john, mary)' would be
+ result in the signature '(john,mary)' for 'see'.
+
+ The second element of the pair is a list of pairs such that the first
+ element of the pair is a tuple of variables and the second element is an
+ expression of those variables that makes the predicate true. For instance,
+ 'all x.all y.(see(x,y) -> know(x,y))' would result in "((x,y),('see(x,y)'))"
+ for 'know'.
+ """
+ def __init__(self):
+ self.signatures = []
+ self.properties = []
+ self.signature_len = None
+
+ def append_sig(self, new_sig):
+ self.validate_sig_len(new_sig)
+ self.signatures.append(new_sig)
+
+ def append_prop(self, new_prop):
+ self.validate_sig_len(new_prop[0])
+ self.properties.append(new_prop)
+
+ def validate_sig_len(self, new_sig):
+ if self.signature_len is None:
+ self.signature_len = len(new_sig)
+ elif self.signature_len != len(new_sig):
+ raise Exception("Signature lengths do not match")
+
+ def __str__(self):
+ return '(%s,%s,%s)' % (self.signatures, self.properties,
+ self.signature_len)
+
+ def __repr__(self):
+ return "%s" % self
+
+def closed_domain_demo():
+ lexpr = Expression.fromstring
+
+ p1 = lexpr(r'exists x.walk(x)')
+ p2 = lexpr(r'man(Socrates)')
+ c = lexpr(r'walk(Socrates)')
+ prover = Prover9Command(c, [p1,p2])
+ print(prover.prove())
+ cdp = ClosedDomainProver(prover)
+ print('assumptions:')
+ for a in cdp.assumptions(): print(' ', a)
+ print('goal:', cdp.goal())
+ print(cdp.prove())
+
+ p1 = lexpr(r'exists x.walk(x)')
+ p2 = lexpr(r'man(Socrates)')
+ p3 = lexpr(r'-walk(Bill)')
+ c = lexpr(r'walk(Socrates)')
+ prover = Prover9Command(c, [p1,p2,p3])
+ print(prover.prove())
+ cdp = ClosedDomainProver(prover)
+ print('assumptions:')
+ for a in cdp.assumptions(): print(' ', a)
+ print('goal:', cdp.goal())
+ print(cdp.prove())
+
+ p1 = lexpr(r'exists x.walk(x)')
+ p2 = lexpr(r'man(Socrates)')
+ p3 = lexpr(r'-walk(Bill)')
+ c = lexpr(r'walk(Socrates)')
+ prover = Prover9Command(c, [p1,p2,p3])
+ print(prover.prove())
+ cdp = ClosedDomainProver(prover)
+ print('assumptions:')
+ for a in cdp.assumptions(): print(' ', a)
+ print('goal:', cdp.goal())
+ print(cdp.prove())
+
+ p1 = lexpr(r'walk(Socrates)')
+ p2 = lexpr(r'walk(Bill)')
+ c = lexpr(r'all x.walk(x)')
+ prover = Prover9Command(c, [p1,p2])
+ print(prover.prove())
+ cdp = ClosedDomainProver(prover)
+ print('assumptions:')
+ for a in cdp.assumptions(): print(' ', a)
+ print('goal:', cdp.goal())
+ print(cdp.prove())
+
+ p1 = lexpr(r'girl(mary)')
+ p2 = lexpr(r'dog(rover)')
+ p3 = lexpr(r'all x.(girl(x) -> -dog(x))')
+ p4 = lexpr(r'all x.(dog(x) -> -girl(x))')
+ p5 = lexpr(r'chase(mary, rover)')
+ c = lexpr(r'exists y.(dog(y) & all x.(girl(x) -> chase(x,y)))')
+ prover = Prover9Command(c, [p1,p2,p3,p4,p5])
+ print(prover.prove())
+ cdp = ClosedDomainProver(prover)
+ print('assumptions:')
+ for a in cdp.assumptions(): print(' ', a)
+ print('goal:', cdp.goal())
+ print(cdp.prove())
+
+def unique_names_demo():
+ lexpr = Expression.fromstring
+
+ p1 = lexpr(r'man(Socrates)')
+ p2 = lexpr(r'man(Bill)')
+ c = lexpr(r'exists x.exists y.(x != y)')
+ prover = Prover9Command(c, [p1,p2])
+ print(prover.prove())
+ unp = UniqueNamesProver(prover)
+ print('assumptions:')
+ for a in unp.assumptions(): print(' ', a)
+ print('goal:', unp.goal())
+ print(unp.prove())
+
+ p1 = lexpr(r'all x.(walk(x) -> (x = Socrates))')
+ p2 = lexpr(r'Bill = William')
+ p3 = lexpr(r'Bill = Billy')
+ c = lexpr(r'-walk(William)')
+ prover = Prover9Command(c, [p1,p2,p3])
+ print(prover.prove())
+ unp = UniqueNamesProver(prover)
+ print('assumptions:')
+ for a in unp.assumptions(): print(' ', a)
+ print('goal:', unp.goal())
+ print(unp.prove())
+
+def closed_world_demo():
+ lexpr = Expression.fromstring
+
+ p1 = lexpr(r'walk(Socrates)')
+ p2 = lexpr(r'(Socrates != Bill)')
+ c = lexpr(r'-walk(Bill)')
+ prover = Prover9Command(c, [p1,p2])
+ print(prover.prove())
+ cwp = ClosedWorldProver(prover)
+ print('assumptions:')
+ for a in cwp.assumptions(): print(' ', a)
+ print('goal:', cwp.goal())
+ print(cwp.prove())
+
+ p1 = lexpr(r'see(Socrates, John)')
+ p2 = lexpr(r'see(John, Mary)')
+ p3 = lexpr(r'(Socrates != John)')
+ p4 = lexpr(r'(John != Mary)')
+ c = lexpr(r'-see(Socrates, Mary)')
+ prover = Prover9Command(c, [p1,p2,p3,p4])
+ print(prover.prove())
+ cwp = ClosedWorldProver(prover)
+ print('assumptions:')
+ for a in cwp.assumptions(): print(' ', a)
+ print('goal:', cwp.goal())
+ print(cwp.prove())
+
+ p1 = lexpr(r'all x.(ostrich(x) -> bird(x))')
+ p2 = lexpr(r'bird(Tweety)')
+ p3 = lexpr(r'-ostrich(Sam)')
+ p4 = lexpr(r'Sam != Tweety')
+ c = lexpr(r'-bird(Sam)')
+ prover = Prover9Command(c, [p1,p2,p3,p4])
+ print(prover.prove())
+ cwp = ClosedWorldProver(prover)
+ print('assumptions:')
+ for a in cwp.assumptions(): print(' ', a)
+ print('goal:', cwp.goal())
+ print(cwp.prove())
+
+def combination_prover_demo():
+ lexpr = Expression.fromstring
+
+ p1 = lexpr(r'see(Socrates, John)')
+ p2 = lexpr(r'see(John, Mary)')
+ c = lexpr(r'-see(Socrates, Mary)')
+ prover = Prover9Command(c, [p1,p2])
+ print(prover.prove())
+ command = ClosedDomainProver(
+ UniqueNamesProver(
+ ClosedWorldProver(prover)))
+ for a in command.assumptions(): print(a)
+ print(command.prove())
+
+def default_reasoning_demo():
+ lexpr = Expression.fromstring
+
+ premises = []
+
+ #define taxonomy
+ premises.append(lexpr(r'all x.(elephant(x) -> animal(x))'))
+ premises.append(lexpr(r'all x.(bird(x) -> animal(x))'))
+ premises.append(lexpr(r'all x.(dove(x) -> bird(x))'))
+ premises.append(lexpr(r'all x.(ostrich(x) -> bird(x))'))
+ premises.append(lexpr(r'all x.(flying_ostrich(x) -> ostrich(x))'))
+
+ #default properties
+ premises.append(lexpr(r'all x.((animal(x) & -Ab1(x)) -> -fly(x))')) #normal animals don't fly
+ premises.append(lexpr(r'all x.((bird(x) & -Ab2(x)) -> fly(x))')) #normal birds fly
+ premises.append(lexpr(r'all x.((ostrich(x) & -Ab3(x)) -> -fly(x))')) #normal ostriches don't fly
+
+ #specify abnormal entities
+ premises.append(lexpr(r'all x.(bird(x) -> Ab1(x))')) #flight
+ premises.append(lexpr(r'all x.(ostrich(x) -> Ab2(x))')) #non-flying bird
+ premises.append(lexpr(r'all x.(flying_ostrich(x) -> Ab3(x))')) #flying ostrich
+
+ #define entities
+ premises.append(lexpr(r'elephant(E)'))
+ premises.append(lexpr(r'dove(D)'))
+ premises.append(lexpr(r'ostrich(O)'))
+
+ #print the assumptions
+ prover = Prover9Command(None, premises)
+ command = UniqueNamesProver(ClosedWorldProver(prover))
+ for a in command.assumptions(): print(a)
+
+ print_proof('-fly(E)', premises)
+ print_proof('fly(D)', premises)
+ print_proof('-fly(O)', premises)
+
+def print_proof(goal, premises):
+ lexpr = Expression.fromstring
+ prover = Prover9Command(lexpr(goal), premises)
+ command = UniqueNamesProver(ClosedWorldProver(prover))
+ print(goal, prover.prove(), command.prove())
+
+def demo():
+ closed_domain_demo()
+ unique_names_demo()
+ closed_world_demo()
+ combination_prover_demo()
+ default_reasoning_demo()
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/inference/prover9.py b/nltk/inference/prover9.py
new file mode 100644
index 0000000..c4c76a3
--- /dev/null
+++ b/nltk/inference/prover9.py
@@ -0,0 +1,431 @@
+# Natural Language Toolkit: Interface to the Prover9 Theorem Prover
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Dan Garrette <dhgarrette at gmail.com>
+# Ewan Klein <ewan at inf.ed.ac.uk>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+A theorem prover that makes use of the external 'Prover9' package.
+"""
+from __future__ import print_function
+
+import os
+import subprocess
+
+import nltk
+from nltk.sem.logic import Expression, ExistsExpression, AllExpression, \
+ NegatedExpression, AndExpression, IffExpression, OrExpression, \
+ EqualityExpression, ImpExpression
+from nltk.inference.api import BaseProverCommand, Prover
+
+#
+# Following is not yet used. Return code for 2 actually realized as 512.
+#
+p9_return_codes = {
+ 0: True,
+ 1: "(FATAL)", #A fatal error occurred (user's syntax error).
+ 2: False, # (SOS_EMPTY) Prover9 ran out of things to do
+ # (sos list exhausted).
+ 3: "(MAX_MEGS)", # The max_megs (memory limit) parameter was exceeded.
+ 4: "(MAX_SECONDS)", # The max_seconds parameter was exceeded.
+ 5: "(MAX_GIVEN)", # The max_given parameter was exceeded.
+ 6: "(MAX_KEPT)", # The max_kept parameter was exceeded.
+ 7: "(ACTION)", # A Prover9 action terminated the search.
+ 101: "(SIGSEGV)", # Prover9 crashed, most probably due to a bug.
+ }
+
+
+class Prover9CommandParent(object):
+ """
+ A common base class used by both ``Prover9Command`` and ``MaceCommand``,
+ which is responsible for maintaining a goal and a set of assumptions,
+ and generating prover9-style input files from them.
+ """
+ def print_assumptions(self, output_format='nltk'):
+ """
+ Print the list of the current assumptions.
+ """
+ if output_format.lower() == 'nltk':
+ for a in self.assumptions():
+ print(a)
+ elif output_format.lower() == 'prover9':
+ for a in convert_to_prover9(self.assumptions()):
+ print(a)
+ else:
+ raise NameError("Unrecognized value for 'output_format': %s" %
+ output_format)
+
+class Prover9Command(Prover9CommandParent, BaseProverCommand):
+ """
+ A ``ProverCommand`` specific to the ``Prover9`` prover. It contains
+ the a print_assumptions() method that is used to print the list
+ of assumptions in multiple formats.
+ """
+ def __init__(self, goal=None, assumptions=None, timeout=60, prover=None):
+ """
+ :param goal: Input expression to prove
+ :type goal: sem.Expression
+ :param assumptions: Input expressions to use as assumptions in
+ the proof.
+ :type assumptions: list(sem.Expression)
+ :param timeout: number of seconds before timeout; set to 0 for
+ no timeout.
+ :type timeout: int
+ :param prover: a prover. If not set, one will be created.
+ :type prover: Prover9
+ """
+ if not assumptions:
+ assumptions = []
+
+ if prover is not None:
+ assert isinstance(prover, Prover9)
+ else:
+ prover = Prover9(timeout)
+
+ BaseProverCommand.__init__(self, prover, goal, assumptions)
+
+ def decorate_proof(self, proof_string, simplify=True):
+ """
+ :see BaseProverCommand.decorate_proof()
+ """
+ if simplify:
+ return self._prover._call_prooftrans(proof_string, ['striplabels'])[0].rstrip()
+ else:
+ return proof_string.rstrip()
+
+
+class Prover9Parent(object):
+ """
+ A common class extended by both ``Prover9`` and ``Mace <mace.Mace>``.
+ It contains the functionality required to convert NLTK-style
+ expressions into Prover9-style expressions.
+ """
+
+ _binary_location = None
+
+ def config_prover9(self, binary_location, verbose=False):
+ if binary_location is None:
+ self._binary_location = None
+ self._prover9_bin = None
+ else:
+ name = 'prover9'
+ self._prover9_bin = nltk.internals.find_binary(
+ name,
+ path_to_bin=binary_location,
+ env_vars=['PROVER9HOME'],
+ url='http://www.cs.unm.edu/~mccune/prover9/',
+ binary_names=[name, name + '.exe'],
+ verbose=verbose)
+ self._binary_location = self._prover9_bin.rsplit(os.path.sep, 1)
+
+ def prover9_input(self, goal, assumptions):
+ """
+ :return: The input string that should be provided to the
+ prover9 binary. This string is formed based on the goal,
+ assumptions, and timeout value of this object.
+ """
+ s = ''
+
+ if assumptions:
+ s += 'formulas(assumptions).\n'
+ for p9_assumption in convert_to_prover9(assumptions):
+ s += ' %s.\n' % p9_assumption
+ s += 'end_of_list.\n\n'
+
+ if goal:
+ s += 'formulas(goals).\n'
+ s += ' %s.\n' % convert_to_prover9(goal)
+ s += 'end_of_list.\n\n'
+
+ return s
+
+ def binary_locations(self):
+ """
+ A list of directories that should be searched for the prover9
+ executables. This list is used by ``config_prover9`` when searching
+ for the prover9 executables.
+ """
+ return ['/usr/local/bin/prover9',
+ '/usr/local/bin/prover9/bin',
+ '/usr/local/bin',
+ '/usr/bin',
+ '/usr/local/prover9',
+ '/usr/local/share/prover9']
+
+ def _find_binary(self, name, verbose=False):
+ binary_locations = self.binary_locations()
+ if self._binary_location is not None:
+ binary_locations += [self._binary_location]
+ return nltk.internals.find_binary(name,
+ searchpath=binary_locations,
+ env_vars=['PROVER9HOME'],
+ url='http://www.cs.unm.edu/~mccune/prover9/',
+ binary_names=[name, name + '.exe'],
+ verbose=verbose)
+
+ def _call(self, input_str, binary, args=[], verbose=False):
+ """
+ Call the binary with the given input.
+
+ :param input_str: A string whose contents are used as stdin.
+ :param binary: The location of the binary to call
+ :param args: A list of command-line arguments.
+ :return: A tuple (stdout, returncode)
+ :see: ``config_prover9``
+ """
+ if verbose:
+ print('Calling:', binary)
+ print('Args:', args)
+ print('Input:\n', input_str, '\n')
+
+ # Call prover9 via a subprocess
+ cmd = [binary] + args
+ try:
+ input_str = input_str.encode("utf8")
+ except AttributeError:
+ pass
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ stdin=subprocess.PIPE)
+ (stdout, stderr) = p.communicate(input=input_str)
+
+ if verbose:
+ print('Return code:', p.returncode)
+ if stdout: print('stdout:\n', stdout, '\n')
+ if stderr: print('stderr:\n', stderr, '\n')
+
+ return (stdout.decode("utf-8"), p.returncode)
+
+
+def convert_to_prover9(input):
+ """
+ Convert a ``logic.Expression`` to Prover9 format.
+ """
+ if isinstance(input, list):
+ result = []
+ for s in input:
+ try:
+ result.append(_convert_to_prover9(s.simplify()))
+ except:
+ print('input %s cannot be converted to Prover9 input syntax' % input)
+ raise
+ return result
+ else:
+ try:
+ return _convert_to_prover9(input.simplify())
+ except:
+ print('input %s cannot be converted to Prover9 input syntax' % input)
+ raise
+
+def _convert_to_prover9(expression):
+ """
+ Convert ``logic.Expression`` to Prover9 formatted string.
+ """
+ if isinstance(expression, ExistsExpression):
+ return 'exists ' + str(expression.variable) + ' ' + _convert_to_prover9(expression.term)
+ elif isinstance(expression, AllExpression):
+ return 'all ' + str(expression.variable) + ' ' + _convert_to_prover9(expression.term)
+ elif isinstance(expression, NegatedExpression):
+ return '-(' + _convert_to_prover9(expression.term) + ')'
+ elif isinstance(expression, AndExpression):
+ return '(' + _convert_to_prover9(expression.first) + ' & ' + \
+ _convert_to_prover9(expression.second) + ')'
+ elif isinstance(expression, OrExpression):
+ return '(' + _convert_to_prover9(expression.first) + ' | ' + \
+ _convert_to_prover9(expression.second) + ')'
+ elif isinstance(expression, ImpExpression):
+ return '(' + _convert_to_prover9(expression.first) + ' -> ' + \
+ _convert_to_prover9(expression.second) + ')'
+ elif isinstance(expression, IffExpression):
+ return '(' + _convert_to_prover9(expression.first) + ' <-> ' + \
+ _convert_to_prover9(expression.second) + ')'
+ elif isinstance(expression, EqualityExpression):
+ return '(' + _convert_to_prover9(expression.first) + ' = ' + \
+ _convert_to_prover9(expression.second) + ')'
+ else:
+ return str(expression)
+
+
+class Prover9(Prover9Parent, Prover):
+ _prover9_bin = None
+ _prooftrans_bin = None
+
+ def __init__(self, timeout=60):
+ self._timeout = timeout
+ """The timeout value for prover9. If a proof can not be found
+ in this amount of time, then prover9 will return false.
+ (Use 0 for no timeout.)"""
+
+ def _prove(self, goal=None, assumptions=None, verbose=False):
+ """
+ Use Prover9 to prove a theorem.
+ :return: A pair whose first element is a boolean indicating if the
+ proof was successful (i.e. returns value of 0) and whose second element
+ is the output of the prover.
+ """
+ if not assumptions:
+ assumptions = []
+
+ stdout, returncode = self._call_prover9(self.prover9_input(goal, assumptions),
+ verbose=verbose)
+ return (returncode == 0, stdout)
+
+ def prover9_input(self, goal, assumptions):
+ """
+ :see: Prover9Parent.prover9_input
+ """
+ s = 'clear(auto_denials).\n' #only one proof required
+ return s + Prover9Parent.prover9_input(self, goal, assumptions)
+
+ def _call_prover9(self, input_str, args=[], verbose=False):
+ """
+ Call the ``prover9`` binary with the given input.
+
+ :param input_str: A string whose contents are used as stdin.
+ :param args: A list of command-line arguments.
+ :return: A tuple (stdout, returncode)
+ :see: ``config_prover9``
+ """
+ if self._prover9_bin is None:
+ self._prover9_bin = self._find_binary('prover9', verbose)
+
+ updated_input_str = ''
+ if self._timeout > 0:
+ updated_input_str += 'assign(max_seconds, %d).\n\n' % self._timeout
+ updated_input_str += input_str
+
+ stdout, returncode = self._call(updated_input_str, self._prover9_bin, args, verbose)
+
+ if returncode not in [0,2]:
+ errormsgprefix = '%%ERROR:'
+ if errormsgprefix in stdout:
+ msgstart = stdout.index(errormsgprefix)
+ errormsg = stdout[msgstart:].strip()
+ else:
+ errormsg = None
+ if returncode in [3,4,5,6]:
+ raise Prover9LimitExceededException(returncode, errormsg)
+ else:
+ raise Prover9FatalException(returncode, errormsg)
+
+ return stdout, returncode
+
+ def _call_prooftrans(self, input_str, args=[], verbose=False):
+ """
+ Call the ``prooftrans`` binary with the given input.
+
+ :param input_str: A string whose contents are used as stdin.
+ :param args: A list of command-line arguments.
+ :return: A tuple (stdout, returncode)
+ :see: ``config_prover9``
+ """
+ if self._prooftrans_bin is None:
+ self._prooftrans_bin = self._find_binary('prooftrans', verbose)
+
+ return self._call(input_str, self._prooftrans_bin, args, verbose)
+
+
+class Prover9Exception(Exception):
+ def __init__(self, returncode, message):
+ msg = p9_return_codes[returncode]
+ if message:
+ msg += '\n%s' % message
+ Exception.__init__(self, msg)
+
+class Prover9FatalException(Prover9Exception):
+ pass
+
+class Prover9LimitExceededException(Prover9Exception):
+ pass
+
+
+
+######################################################################
+#{ Tests and Demos
+######################################################################
+
+def test_config():
+
+ a = Expression.fromstring('(walk(j) & sing(j))')
+ g = Expression.fromstring('walk(j)')
+ p = Prover9Command(g, assumptions=[a])
+ p._executable_path = None
+ p.prover9_search=[]
+ p.prove()
+ #config_prover9('/usr/local/bin')
+ print(p.prove())
+ print(p.proof())
+
+def test_convert_to_prover9(expr):
+ """
+ Test that parsing works OK.
+ """
+ for t in expr:
+ e = Expression.fromstring(t)
+ print(convert_to_prover9(e))
+
+def test_prove(arguments):
+ """
+ Try some proofs and exhibit the results.
+ """
+ for (goal, assumptions) in arguments:
+ g = Expression.fromstring(goal)
+ alist = [Expression.fromstring(a) for a in assumptions]
+ p = Prover9Command(g, assumptions=alist).prove()
+ for a in alist:
+ print(' %s' % a)
+ print('|- %s: %s\n' % (g, p))
+
+arguments = [
+ ('(man(x) <-> (not (not man(x))))', []),
+ ('(not (man(x) & (not man(x))))', []),
+ ('(man(x) | (not man(x)))', []),
+ ('(man(x) & (not man(x)))', []),
+ ('(man(x) -> man(x))', []),
+ ('(not (man(x) & (not man(x))))', []),
+ ('(man(x) | (not man(x)))', []),
+ ('(man(x) -> man(x))', []),
+ ('(man(x) <-> man(x))', []),
+ ('(not (man(x) <-> (not man(x))))', []),
+ ('mortal(Socrates)', ['all x.(man(x) -> mortal(x))', 'man(Socrates)']),
+ ('((all x.(man(x) -> walks(x)) & man(Socrates)) -> some y.walks(y))', []),
+ ('(all x.man(x) -> all x.man(x))', []),
+ ('some x.all y.sees(x,y)', []),
+ ('some e3.(walk(e3) & subj(e3, mary))',
+ ['some e1.(see(e1) & subj(e1, john) & some e2.(pred(e1, e2) & walk(e2) & subj(e2, mary)))']),
+ ('some x e1.(see(e1) & subj(e1, x) & some e2.(pred(e1, e2) & walk(e2) & subj(e2, mary)))',
+ ['some e1.(see(e1) & subj(e1, john) & some e2.(pred(e1, e2) & walk(e2) & subj(e2, mary)))'])
+]
+
+expressions = [r'some x y.sees(x,y)',
+ r'some x.(man(x) & walks(x))',
+ r'\x.(man(x) & walks(x))',
+ r'\x y.sees(x,y)',
+ r'walks(john)',
+ r'\x.big(x, \y.mouse(y))',
+ r'(walks(x) & (runs(x) & (threes(x) & fours(x))))',
+ r'(walks(x) -> runs(x))',
+ r'some x.(PRO(x) & sees(John, x))',
+ r'some x.(man(x) & (not walks(x)))',
+ r'all x.(man(x) -> walks(x))']
+
+def spacer(num=45):
+ print('-' * num)
+
+def demo():
+ print("Testing configuration")
+ spacer()
+ test_config()
+ print()
+ print("Testing conversion to Prover9 format")
+ spacer()
+ test_convert_to_prover9(expressions)
+ print()
+ print("Testing proofs")
+ spacer()
+ test_prove(arguments)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/inference/resolution.py b/nltk/inference/resolution.py
new file mode 100755
index 0000000..4751cf8
--- /dev/null
+++ b/nltk/inference/resolution.py
@@ -0,0 +1,687 @@
+# Natural Language Toolkit: First-order Resolution-based Theorem Prover
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+
+"""
+Module for a resolution-based First Order theorem prover.
+"""
+from __future__ import print_function, unicode_literals
+
+import operator
+from collections import defaultdict
+from functools import reduce
+
+from nltk.sem import skolemize
+from nltk.sem.logic import (VariableExpression, EqualityExpression,
+ ApplicationExpression, Expression,
+ NegatedExpression, Variable,
+ AndExpression, unique_variable, OrExpression,
+ is_indvar, IndividualVariableExpression, Expression)
+
+from nltk.inference.api import Prover, BaseProverCommand
+from nltk.compat import python_2_unicode_compatible
+
+class ProverParseError(Exception): pass
+
+class ResolutionProver(Prover):
+ ANSWER_KEY = 'ANSWER'
+ _assume_false=True
+
+ def _prove(self, goal=None, assumptions=None, verbose=False):
+ """
+ :param goal: Input expression to prove
+ :type goal: sem.Expression
+ :param assumptions: Input expressions to use as assumptions in the proof
+ :type assumptions: list(sem.Expression)
+ """
+ if not assumptions:
+ assumptions = []
+
+ result = None
+ try:
+ clauses = []
+ if goal:
+ clauses.extend(clausify(-goal))
+ for a in assumptions:
+ clauses.extend(clausify(a))
+ result, clauses = self._attempt_proof(clauses)
+ if verbose:
+ print(ResolutionProverCommand._decorate_clauses(clauses))
+ except RuntimeError as e:
+ if self._assume_false and str(e).startswith('maximum recursion depth exceeded'):
+ result = False
+ clauses = []
+ else:
+ if verbose:
+ print(e)
+ else:
+ raise e
+ return (result, clauses)
+
+ def _attempt_proof(self, clauses):
+ #map indices to lists of indices, to store attempted unifications
+ tried = defaultdict(list)
+
+ i = 0
+ while i < len(clauses):
+ if not clauses[i].is_tautology():
+ #since we try clauses in order, we should start after the last
+ #index tried
+ if tried[i]:
+ j = tried[i][-1] + 1
+ else:
+ j = i + 1 #nothing tried yet for 'i', so start with the next
+
+ while j < len(clauses):
+ #don't: 1) unify a clause with itself,
+ # 2) use tautologies
+ if i != j and j and not clauses[j].is_tautology():
+ tried[i].append(j)
+ newclauses = clauses[i].unify(clauses[j])
+ if newclauses:
+ for newclause in newclauses:
+ newclause._parents = (i+1, j+1)
+ clauses.append(newclause)
+ if not len(newclause): #if there's an empty clause
+ return (True, clauses)
+ i=-1 #since we added a new clause, restart from the top
+ break
+ j += 1
+ i += 1
+ return (False, clauses)
+
+class ResolutionProverCommand(BaseProverCommand):
+ def __init__(self, goal=None, assumptions=None, prover=None):
+ """
+ :param goal: Input expression to prove
+ :type goal: sem.Expression
+ :param assumptions: Input expressions to use as assumptions in
+ the proof.
+ :type assumptions: list(sem.Expression)
+ """
+ if prover is not None:
+ assert isinstance(prover, ResolutionProver)
+ else:
+ prover = ResolutionProver()
+
+ BaseProverCommand.__init__(self, prover, goal, assumptions)
+ self._clauses = None
+
+ def prove(self, verbose=False):
+ """
+ Perform the actual proof. Store the result to prevent unnecessary
+ re-proving.
+ """
+ if self._result is None:
+ self._result, clauses = self._prover._prove(self.goal(),
+ self.assumptions(),
+ verbose)
+ self._clauses = clauses
+ self._proof = ResolutionProverCommand._decorate_clauses(clauses)
+ return self._result
+
+ def find_answers(self, verbose=False):
+ self.prove(verbose)
+
+ answers = set()
+ answer_ex = VariableExpression(Variable(ResolutionProver.ANSWER_KEY))
+ for clause in self._clauses:
+ for term in clause:
+ if isinstance(term, ApplicationExpression) and\
+ term.function == answer_ex and\
+ not isinstance(term.argument, IndividualVariableExpression):
+ answers.add(term.argument)
+ return answers
+
+ @staticmethod
+ def _decorate_clauses(clauses):
+ """
+ Decorate the proof output.
+ """
+ out = ''
+ max_clause_len = max([len(str(clause)) for clause in clauses])
+ max_seq_len = len(str(len(clauses)))
+ for i in range(len(clauses)):
+ parents = 'A'
+ taut = ''
+ if clauses[i].is_tautology():
+ taut = 'Tautology'
+ if clauses[i]._parents:
+ parents = str(clauses[i]._parents)
+ parents = ' '*(max_clause_len-len(str(clauses[i]))+1) + parents
+ seq = ' '*(max_seq_len-len(str(i+1))) + str(i+1)
+ out += '[%s] %s %s %s\n' % (seq, clauses[i], parents, taut)
+ return out
+
+ at python_2_unicode_compatible
+class Clause(list):
+ def __init__(self, data):
+ list.__init__(self, data)
+ self._is_tautology = None
+ self._parents = None
+
+ def unify(self, other, bindings=None, used=None, skipped=None, debug=False):
+ """
+ Attempt to unify this Clause with the other, returning a list of
+ resulting, unified, Clauses.
+
+ :param other: ``Clause`` with which to unify
+ :param bindings: ``BindingDict`` containing bindings that should be used
+ during the unification
+ :param used: tuple of two lists of atoms. The first lists the
+ atoms from 'self' that were successfully unified with atoms from
+ 'other'. The second lists the atoms from 'other' that were successfully
+ unified with atoms from 'self'.
+ :param skipped: tuple of two ``Clause`` objects. The first is a list of all
+ the atoms from the 'self' Clause that have not been unified with
+ anything on the path. The second is same thing for the 'other' Clause.
+ :param debug: bool indicating whether debug statements should print
+ :return: list containing all the resulting ``Clause`` objects that could be
+ obtained by unification
+ """
+ if bindings is None: bindings = BindingDict()
+ if used is None: used = ([],[])
+ if skipped is None: skipped = ([],[])
+ if isinstance(debug, bool): debug = DebugObject(debug)
+
+ newclauses = _iterate_first(self, other, bindings, used, skipped, _complete_unify_path, debug)
+
+ #remove subsumed clauses. make a list of all indices of subsumed
+ #clauses, and then remove them from the list
+ subsumed = []
+ for i, c1 in enumerate(newclauses):
+ if i not in subsumed:
+ for j, c2 in enumerate(newclauses):
+ if i!=j and j not in subsumed and c1.subsumes(c2):
+ subsumed.append(j)
+ result = []
+ for i in range(len(newclauses)):
+ if i not in subsumed:
+ result.append(newclauses[i])
+
+ return result
+
+ def isSubsetOf(self, other):
+ """
+ Return True iff every term in 'self' is a term in 'other'.
+
+ :param other: ``Clause``
+ :return: bool
+ """
+ for a in self:
+ if a not in other:
+ return False
+ return True
+
+ def subsumes(self, other):
+ """
+ Return True iff 'self' subsumes 'other', this is, if there is a
+ substitution such that every term in 'self' can be unified with a term
+ in 'other'.
+
+ :param other: ``Clause``
+ :return: bool
+ """
+ negatedother = []
+ for atom in other:
+ if isinstance(atom, NegatedExpression):
+ negatedother.append(atom.term)
+ else:
+ negatedother.append(-atom)
+
+ negatedotherClause = Clause(negatedother)
+
+ bindings = BindingDict()
+ used = ([],[])
+ skipped = ([],[])
+ debug = DebugObject(False)
+
+ return len(_iterate_first(self, negatedotherClause, bindings, used,
+ skipped, _subsumes_finalize,
+ debug)) > 0
+
+ def __getslice__(self, start, end):
+ return Clause(list.__getslice__(self, start, end))
+
+ def __sub__(self, other):
+ return Clause([a for a in self if a not in other])
+
+ def __add__(self, other):
+ return Clause(list.__add__(self, other))
+
+ def is_tautology(self):
+ """
+ Self is a tautology if it contains ground terms P and -P. The ground
+ term, P, must be an exact match, ie, not using unification.
+ """
+ if self._is_tautology is not None:
+ return self._is_tautology
+ for i,a in enumerate(self):
+ if not isinstance(a, EqualityExpression):
+ j = len(self)-1
+ while j > i:
+ b = self[j]
+ if isinstance(a, NegatedExpression):
+ if a.term == b:
+ self._is_tautology = True
+ return True
+ elif isinstance(b, NegatedExpression):
+ if a == b.term:
+ self._is_tautology = True
+ return True
+ j -= 1
+ self._is_tautology = False
+ return False
+
+ def free(self):
+ return reduce(operator.or_, ((atom.free() | atom.constants()) for atom in self))
+
+ def replace(self, variable, expression):
+ """
+ Replace every instance of variable with expression across every atom
+ in the clause
+
+ :param variable: ``Variable``
+ :param expression: ``Expression``
+ """
+ return Clause([atom.replace(variable, expression) for atom in self])
+
+ def substitute_bindings(self, bindings):
+ """
+ Replace every binding
+
+ :param bindings: A list of tuples mapping Variable Expressions to the
+ Expressions to which they are bound
+ :return: ``Clause``
+ """
+ return Clause([atom.substitute_bindings(bindings) for atom in self])
+
+ def __str__(self):
+ return '{' + ', '.join("%s" % item for item in self) + '}'
+
+ def __repr__(self):
+ return "%s" % self
+
+def _iterate_first(first, second, bindings, used, skipped, finalize_method, debug):
+ """
+ This method facilitates movement through the terms of 'self'
+ """
+ debug.line('unify(%s,%s) %s'%(first, second, bindings))
+
+ if not len(first) or not len(second): #if no more recursions can be performed
+ return finalize_method(first, second, bindings, used, skipped, debug)
+ else:
+ #explore this 'self' atom
+ result = _iterate_second(first, second, bindings, used, skipped, finalize_method, debug+1)
+
+ #skip this possible 'self' atom
+ newskipped = (skipped[0]+[first[0]], skipped[1])
+ result += _iterate_first(first[1:], second, bindings, used, newskipped, finalize_method, debug+1)
+
+ try:
+ newbindings, newused, unused = _unify_terms(first[0], second[0], bindings, used)
+ #Unification found, so progress with this line of unification
+ #put skipped and unused terms back into play for later unification.
+ newfirst = first[1:] + skipped[0] + unused[0]
+ newsecond = second[1:] + skipped[1] + unused[1]
+ result += _iterate_first(newfirst, newsecond, newbindings, newused, ([],[]), finalize_method, debug+1)
+ except BindingException:
+ #the atoms could not be unified,
+ pass
+
+ return result
+
+def _iterate_second(first, second, bindings, used, skipped, finalize_method, debug):
+ """
+ This method facilitates movement through the terms of 'other'
+ """
+ debug.line('unify(%s,%s) %s'%(first, second, bindings))
+
+ if not len(first) or not len(second): #if no more recursions can be performed
+ return finalize_method(first, second, bindings, used, skipped, debug)
+ else:
+ #skip this possible pairing and move to the next
+ newskipped = (skipped[0], skipped[1]+[second[0]])
+ result = _iterate_second(first, second[1:], bindings, used, newskipped, finalize_method, debug+1)
+
+ try:
+ newbindings, newused, unused = _unify_terms(first[0], second[0], bindings, used)
+ #Unification found, so progress with this line of unification
+ #put skipped and unused terms back into play for later unification.
+ newfirst = first[1:] + skipped[0] + unused[0]
+ newsecond = second[1:] + skipped[1] + unused[1]
+ result += _iterate_second(newfirst, newsecond, newbindings, newused, ([],[]), finalize_method, debug+1)
+ except BindingException:
+ #the atoms could not be unified,
+ pass
+
+ return result
+
+def _unify_terms(a, b, bindings=None, used=None):
+ """
+ This method attempts to unify two terms. Two expressions are unifiable
+ if there exists a substitution function S such that S(a) == S(-b).
+
+ :param a: ``Expression``
+ :param b: ``Expression``
+ :param bindings: ``BindingDict`` a starting set of bindings with which
+ the unification must be consistent
+ :return: ``BindingDict`` A dictionary of the bindings required to unify
+ :raise ``BindingException``: If the terms cannot be unified
+ """
+ assert isinstance(a, Expression)
+ assert isinstance(b, Expression)
+
+ if bindings is None: bindings = BindingDict()
+ if used is None: used = ([],[])
+
+ # Use resolution
+ if isinstance(a, NegatedExpression) and isinstance(b, ApplicationExpression):
+ newbindings = most_general_unification(a.term, b, bindings)
+ newused = (used[0]+[a], used[1]+[b])
+ unused = ([],[])
+ elif isinstance(a, ApplicationExpression) and isinstance(b, NegatedExpression):
+ newbindings = most_general_unification(a, b.term, bindings)
+ newused = (used[0]+[a], used[1]+[b])
+ unused = ([],[])
+
+ # Use demodulation
+ elif isinstance(a, EqualityExpression):
+ newbindings = BindingDict([(a.first.variable, a.second)])
+ newused = (used[0]+[a], used[1])
+ unused = ([],[b])
+ elif isinstance(b, EqualityExpression):
+ newbindings = BindingDict([(b.first.variable, b.second)])
+ newused = (used[0], used[1]+[b])
+ unused = ([a],[])
+
+ else:
+ raise BindingException((a, b))
+
+ return newbindings, newused, unused
+
+def _complete_unify_path(first, second, bindings, used, skipped, debug):
+ if used[0] or used[1]: #if bindings were made along the path
+ newclause = Clause(skipped[0] + skipped[1] + first + second)
+ debug.line(' -> New Clause: %s' % newclause)
+ return [newclause.substitute_bindings(bindings)]
+ else: #no bindings made means no unification occurred. so no result
+ debug.line(' -> End')
+ return []
+
+def _subsumes_finalize(first, second, bindings, used, skipped, debug):
+ if not len(skipped[0]) and not len(first):
+ #If there are no skipped terms and no terms left in 'first', then
+ #all of the terms in the original 'self' were unified with terms
+ #in 'other'. Therefore, there exists a binding (this one) such that
+ #every term in self can be unified with a term in other, which
+ #is the definition of subsumption.
+ return [True]
+ else:
+ return []
+
+def clausify(expression):
+ """
+ Skolemize, clausify, and standardize the variables apart.
+ """
+ clause_list = []
+ for clause in _clausify(skolemize(expression)):
+ for free in clause.free():
+ if is_indvar(free.name):
+ newvar = VariableExpression(unique_variable())
+ clause = clause.replace(free, newvar)
+ clause_list.append(clause)
+ return clause_list
+
+def _clausify(expression):
+ """
+ :param expression: a skolemized expression in CNF
+ """
+ if isinstance(expression, AndExpression):
+ return _clausify(expression.first) + _clausify(expression.second)
+ elif isinstance(expression, OrExpression):
+ first = _clausify(expression.first)
+ second = _clausify(expression.second)
+ assert len(first) == 1
+ assert len(second) == 1
+ return [first[0] + second[0]]
+ elif isinstance(expression, EqualityExpression):
+ return [Clause([expression])]
+ elif isinstance(expression, ApplicationExpression):
+ return [Clause([expression])]
+ elif isinstance(expression, NegatedExpression):
+ if isinstance(expression.term, ApplicationExpression):
+ return [Clause([expression])]
+ elif isinstance(expression.term, EqualityExpression):
+ return [Clause([expression])]
+ raise ProverParseError()
+
+
+ at python_2_unicode_compatible
+class BindingDict(object):
+ def __init__(self, binding_list=None):
+ """
+ :param binding_list: list of (``AbstractVariableExpression``, ``AtomicExpression``) to initialize the dictionary
+ """
+ self.d = {}
+
+ if binding_list:
+ for (v, b) in binding_list:
+ self[v] = b
+
+ def __setitem__(self, variable, binding):
+ """
+ A binding is consistent with the dict if its variable is not already bound, OR if its
+ variable is already bound to its argument.
+
+ :param variable: ``Variable`` The variable to bind
+ :param binding: ``Expression`` The atomic to which 'variable' should be bound
+ :raise BindingException: If the variable cannot be bound in this dictionary
+ """
+ assert isinstance(variable, Variable)
+ assert isinstance(binding, Expression)
+
+ try:
+ existing = self[variable]
+ except KeyError:
+ existing = None
+
+ if not existing or binding == existing:
+ self.d[variable] = binding
+ elif isinstance(binding, IndividualVariableExpression):
+ # Since variable is already bound, try to bind binding to variable
+ try:
+ existing = self[binding.variable]
+ except KeyError:
+ existing = None
+
+ binding2 = VariableExpression(variable)
+
+ if not existing or binding2 == existing:
+ self.d[binding.variable] = binding2
+ else:
+ raise BindingException('Variable %s already bound to another '
+ 'value' % (variable))
+ else:
+ raise BindingException('Variable %s already bound to another '
+ 'value' % (variable))
+
+ def __getitem__(self, variable):
+ """
+ Return the expression to which 'variable' is bound
+ """
+ assert isinstance(variable, Variable)
+
+ intermediate = self.d[variable]
+ while intermediate:
+ try:
+ intermediate = self.d[intermediate]
+ except KeyError:
+ return intermediate
+
+ def __contains__(self, item):
+ return item in self.d
+
+ def __add__(self, other):
+ """
+ :param other: ``BindingDict`` The dict with which to combine self
+ :return: ``BindingDict`` A new dict containing all the elements of both parameters
+ :raise BindingException: If the parameter dictionaries are not consistent with each other
+ """
+ try:
+ combined = BindingDict()
+ for v in self.d:
+ combined[v] = self.d[v]
+ for v in other.d:
+ combined[v] = other.d[v]
+ return combined
+ except BindingException:
+ raise BindingException("Attempting to add two contradicting "
+ "BindingDicts: '%s' and '%s'"
+ % (self, other))
+
+ def __len__(self):
+ return len(self.d)
+
+ def __str__(self):
+ data_str = ', '.join('%s: %s' % (v, self.d[v]) for v in sorted(self.d.keys()))
+ return '{' + data_str + '}'
+
+ def __repr__(self):
+ return "%s" % self
+
+
+def most_general_unification(a, b, bindings=None):
+ """
+ Find the most general unification of the two given expressions
+
+ :param a: ``Expression``
+ :param b: ``Expression``
+ :param bindings: ``BindingDict`` a starting set of bindings with which the
+ unification must be consistent
+ :return: a list of bindings
+ :raise BindingException: if the Expressions cannot be unified
+ """
+ if bindings is None:
+ bindings = BindingDict()
+
+ if a == b:
+ return bindings
+ elif isinstance(a, IndividualVariableExpression):
+ return _mgu_var(a, b, bindings)
+ elif isinstance(b, IndividualVariableExpression):
+ return _mgu_var(b, a, bindings)
+ elif isinstance(a, ApplicationExpression) and\
+ isinstance(b, ApplicationExpression):
+ return most_general_unification(a.function, b.function, bindings) +\
+ most_general_unification(a.argument, b.argument, bindings)
+ raise BindingException((a, b))
+
+def _mgu_var(var, expression, bindings):
+ if var.variable in expression.free()|expression.constants():
+ raise BindingException((var, expression))
+ else:
+ return BindingDict([(var.variable, expression)]) + bindings
+
+
+class BindingException(Exception):
+ def __init__(self, arg):
+ if isinstance(arg, tuple):
+ Exception.__init__(self, "'%s' cannot be bound to '%s'" % arg)
+ else:
+ Exception.__init__(self, arg)
+
+class UnificationException(Exception):
+ def __init__(self, a, b):
+ Exception.__init__(self, "'%s' cannot unify with '%s'" % (a,b))
+
+
+class DebugObject(object):
+ def __init__(self, enabled=True, indent=0):
+ self.enabled = enabled
+ self.indent = indent
+
+ def __add__(self, i):
+ return DebugObject(self.enabled, self.indent+i)
+
+ def line(self, line):
+ if self.enabled:
+ print(' '*self.indent + line)
+
+
+def testResolutionProver():
+ resolution_test(r'man(x)')
+ resolution_test(r'(man(x) -> man(x))')
+ resolution_test(r'(man(x) -> --man(x))')
+ resolution_test(r'-(man(x) and -man(x))')
+ resolution_test(r'(man(x) or -man(x))')
+ resolution_test(r'(man(x) -> man(x))')
+ resolution_test(r'-(man(x) and -man(x))')
+ resolution_test(r'(man(x) or -man(x))')
+ resolution_test(r'(man(x) -> man(x))')
+ resolution_test(r'(man(x) iff man(x))')
+ resolution_test(r'-(man(x) iff -man(x))')
+ resolution_test('all x.man(x)')
+ resolution_test('-all x.some y.F(x,y) & some x.all y.(-F(x,y))')
+ resolution_test('some x.all y.sees(x,y)')
+
+ p1 = Expression.fromstring(r'all x.(man(x) -> mortal(x))')
+ p2 = Expression.fromstring(r'man(Socrates)')
+ c = Expression.fromstring(r'mortal(Socrates)')
+ print('%s, %s |- %s: %s' % (p1, p2, c, ResolutionProver().prove(c, [p1,p2])))
+
+ p1 = Expression.fromstring(r'all x.(man(x) -> walks(x))')
+ p2 = Expression.fromstring(r'man(John)')
+ c = Expression.fromstring(r'some y.walks(y)')
+ print('%s, %s |- %s: %s' % (p1, p2, c, ResolutionProver().prove(c, [p1,p2])))
+
+ p = Expression.fromstring(r'some e1.some e2.(believe(e1,john,e2) & walk(e2,mary))')
+ c = Expression.fromstring(r'some e0.walk(e0,mary)')
+ print('%s |- %s: %s' % (p, c, ResolutionProver().prove(c, [p])))
+
+def resolution_test(e):
+ f = Expression.fromstring(e)
+ t = ResolutionProver().prove(f)
+ print('|- %s: %s' % (f, t))
+
+def test_clausify():
+ lexpr = Expression.fromstring
+
+ print(clausify(lexpr('P(x) | Q(x)')))
+ print(clausify(lexpr('(P(x) & Q(x)) | R(x)')))
+ print(clausify(lexpr('P(x) | (Q(x) & R(x))')))
+ print(clausify(lexpr('(P(x) & Q(x)) | (R(x) & S(x))')))
+
+ print(clausify(lexpr('P(x) | Q(x) | R(x)')))
+ print(clausify(lexpr('P(x) | (Q(x) & R(x)) | S(x)')))
+
+ print(clausify(lexpr('exists x.P(x) | Q(x)')))
+
+ print(clausify(lexpr('-(-P(x) & Q(x))')))
+ print(clausify(lexpr('P(x) <-> Q(x)')))
+ print(clausify(lexpr('-(P(x) <-> Q(x))')))
+ print(clausify(lexpr('-(all x.P(x))')))
+ print(clausify(lexpr('-(some x.P(x))')))
+
+ print(clausify(lexpr('some x.P(x)')))
+ print(clausify(lexpr('some x.all y.P(x,y)')))
+ print(clausify(lexpr('all y.some x.P(x,y)')))
+ print(clausify(lexpr('all z.all y.some x.P(x,y,z)')))
+ print(clausify(lexpr('all x.(all y.P(x,y) -> -all y.(Q(x,y) -> R(x,y)))')))
+
+
+def demo():
+ test_clausify()
+ print()
+ testResolutionProver()
+ print()
+
+ p = Expression.fromstring('man(x)')
+ print(ResolutionProverCommand(p, [p]).prove())
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/inference/tableau.py b/nltk/inference/tableau.py
new file mode 100644
index 0000000..2b32d1f
--- /dev/null
+++ b/nltk/inference/tableau.py
@@ -0,0 +1,607 @@
+# Natural Language Toolkit: First-Order Tableau Theorem Prover
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Module for a tableau-based First Order theorem prover.
+"""
+from __future__ import print_function, unicode_literals
+
+from nltk.internals import Counter
+
+from nltk.sem.logic import (VariableExpression, EqualityExpression,
+ ApplicationExpression, Expression,
+ AbstractVariableExpression, AllExpression,
+ NegatedExpression,
+ ExistsExpression, Variable, ImpExpression,
+ AndExpression, unique_variable,
+ LambdaExpression, IffExpression,
+ OrExpression, FunctionVariableExpression)
+
+from nltk.inference.api import Prover, BaseProverCommand
+
+_counter = Counter()
+
+class ProverParseError(Exception): pass
+
+class TableauProver(Prover):
+ _assume_false=False
+
+ def _prove(self, goal=None, assumptions=None, verbose=False):
+ if not assumptions:
+ assumptions = []
+
+ result = None
+ try:
+ agenda = Agenda()
+ if goal:
+ agenda.put(-goal)
+ agenda.put_all(assumptions)
+ debugger = Debug(verbose)
+ result = self._attempt_proof(agenda, set(), set(), debugger)
+ except RuntimeError as e:
+ if self._assume_false and str(e).startswith('maximum recursion depth exceeded'):
+ result = False
+ else:
+ if verbose:
+ print(e)
+ else:
+ raise e
+ return (result, '\n'.join(debugger.lines))
+
+ def _attempt_proof(self, agenda, accessible_vars, atoms, debug):
+ (current, context), category = agenda.pop_first()
+
+ #if there's nothing left in the agenda, and we haven't closed the path
+ if not current:
+ debug.line('AGENDA EMPTY')
+ return False
+
+ proof_method = { Categories.ATOM: self._attempt_proof_atom,
+ Categories.PROP: self._attempt_proof_prop,
+ Categories.N_ATOM: self._attempt_proof_n_atom,
+ Categories.N_PROP: self._attempt_proof_n_prop,
+ Categories.APP: self._attempt_proof_app,
+ Categories.N_APP: self._attempt_proof_n_app,
+ Categories.N_EQ: self._attempt_proof_n_eq,
+ Categories.D_NEG: self._attempt_proof_d_neg,
+ Categories.N_ALL: self._attempt_proof_n_all,
+ Categories.N_EXISTS: self._attempt_proof_n_some,
+ Categories.AND: self._attempt_proof_and,
+ Categories.N_OR: self._attempt_proof_n_or,
+ Categories.N_IMP: self._attempt_proof_n_imp,
+ Categories.OR: self._attempt_proof_or,
+ Categories.IMP: self._attempt_proof_imp,
+ Categories.N_AND: self._attempt_proof_n_and,
+ Categories.IFF: self._attempt_proof_iff,
+ Categories.N_IFF: self._attempt_proof_n_iff,
+ Categories.EQ: self._attempt_proof_eq,
+ Categories.EXISTS: self._attempt_proof_some,
+ Categories.ALL: self._attempt_proof_all,
+ }[category]
+
+ debug.line((current, context))
+ return proof_method(current, context, agenda, accessible_vars, atoms, debug)
+
+ def _attempt_proof_atom(self, current, context, agenda, accessible_vars, atoms, debug):
+ # Check if the branch is closed. Return 'True' if it is
+ if (current, True) in atoms:
+ debug.line('CLOSED', 1)
+ return True
+
+ if context:
+ if isinstance(context.term, NegatedExpression):
+ current = current.negate()
+ agenda.put(context(current).simplify())
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+ else:
+ #mark all AllExpressions as 'not exhausted' into the agenda since we are (potentially) adding new accessible vars
+ agenda.mark_alls_fresh();
+ return self._attempt_proof(agenda, accessible_vars|set(current.args), atoms|set([(current, False)]), debug+1)
+
+ def _attempt_proof_n_atom(self, current, context, agenda, accessible_vars, atoms, debug):
+ # Check if the branch is closed. Return 'True' if it is
+ if (current.term, False) in atoms:
+ debug.line('CLOSED', 1)
+ return True
+
+ if context:
+ if isinstance(context.term, NegatedExpression):
+ current = current.negate()
+ agenda.put(context(current).simplify())
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+ else:
+ #mark all AllExpressions as 'not exhausted' into the agenda since we are (potentially) adding new accessible vars
+ agenda.mark_alls_fresh();
+ return self._attempt_proof(agenda, accessible_vars|set(current.term.args), atoms|set([(current.term, True)]), debug+1)
+
+ def _attempt_proof_prop(self, current, context, agenda, accessible_vars, atoms, debug):
+ # Check if the branch is closed. Return 'True' if it is
+ if (current, True) in atoms:
+ debug.line('CLOSED', 1)
+ return True
+
+ #mark all AllExpressions as 'not exhausted' into the agenda since we are (potentially) adding new accessible vars
+ agenda.mark_alls_fresh();
+ return self._attempt_proof(agenda, accessible_vars, atoms|set([(current, False)]), debug+1)
+
+ def _attempt_proof_n_prop(self, current, context, agenda, accessible_vars, atoms, debug):
+ # Check if the branch is closed. Return 'True' if it is
+ if (current.term, False) in atoms:
+ debug.line('CLOSED', 1)
+ return True
+
+ #mark all AllExpressions as 'not exhausted' into the agenda since we are (potentially) adding new accessible vars
+ agenda.mark_alls_fresh();
+ return self._attempt_proof(agenda, accessible_vars, atoms|set([(current.term, True)]), debug+1)
+
+ def _attempt_proof_app(self, current, context, agenda, accessible_vars, atoms, debug):
+ f, args = current.uncurry()
+ for i, arg in enumerate(args):
+ if not TableauProver.is_atom(arg):
+ ctx = f
+ nv = Variable('X%s' % _counter.get())
+ for j,a in enumerate(args):
+ ctx = (ctx(VariableExpression(nv)) if i == j else ctx(a))
+ if context:
+ ctx = context(ctx).simplify()
+ ctx = LambdaExpression(nv, ctx)
+ agenda.put(arg, ctx)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+ raise Exception('If this method is called, there must be a non-atomic argument')
+
+ def _attempt_proof_n_app(self, current, context, agenda, accessible_vars, atoms, debug):
+ f, args = current.term.uncurry()
+ for i, arg in enumerate(args):
+ if not TableauProver.is_atom(arg):
+ ctx = f
+ nv = Variable('X%s' % _counter.get())
+ for j,a in enumerate(args):
+ ctx = (ctx(VariableExpression(nv)) if i == j else ctx(a))
+ if context:
+ #combine new context with existing
+ ctx = context(ctx).simplify()
+ ctx = LambdaExpression(nv, -ctx)
+ agenda.put(-arg, ctx)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+ raise Exception('If this method is called, there must be a non-atomic argument')
+
+ def _attempt_proof_n_eq(self, current, context, agenda, accessible_vars, atoms, debug):
+ ###########################################################################
+ # Since 'current' is of type '~(a=b)', the path is closed if 'a' == 'b'
+ ###########################################################################
+ if current.term.first == current.term.second:
+ debug.line('CLOSED', 1)
+ return True
+
+ agenda[Categories.N_EQ].add((current,context))
+ current._exhausted = True
+ return self._attempt_proof(agenda, accessible_vars|set([current.term.first, current.term.second]), atoms, debug+1)
+
+ def _attempt_proof_d_neg(self, current, context, agenda, accessible_vars, atoms, debug):
+ agenda.put(current.term.term, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_n_all(self, current, context, agenda, accessible_vars, atoms, debug):
+ agenda[Categories.EXISTS].add((ExistsExpression(current.term.variable, -current.term.term), context))
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_n_some(self, current, context, agenda, accessible_vars, atoms, debug):
+ agenda[Categories.ALL].add((AllExpression(current.term.variable, -current.term.term), context))
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_and(self, current, context, agenda, accessible_vars, atoms, debug):
+ agenda.put(current.first, context)
+ agenda.put(current.second, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_n_or(self, current, context, agenda, accessible_vars, atoms, debug):
+ agenda.put(-current.term.first, context)
+ agenda.put(-current.term.second, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_n_imp(self, current, context, agenda, accessible_vars, atoms, debug):
+ agenda.put(current.term.first, context)
+ agenda.put(-current.term.second, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_or(self, current, context, agenda, accessible_vars, atoms, debug):
+ new_agenda = agenda.clone()
+ agenda.put(current.first, context)
+ new_agenda.put(current.second, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1) and \
+ self._attempt_proof(new_agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_imp(self, current, context, agenda, accessible_vars, atoms, debug):
+ new_agenda = agenda.clone()
+ agenda.put(-current.first, context)
+ new_agenda.put(current.second, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1) and \
+ self._attempt_proof(new_agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_n_and(self, current, context, agenda, accessible_vars, atoms, debug):
+ new_agenda = agenda.clone()
+ agenda.put(-current.term.first, context)
+ new_agenda.put(-current.term.second, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1) and \
+ self._attempt_proof(new_agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_iff(self, current, context, agenda, accessible_vars, atoms, debug):
+ new_agenda = agenda.clone()
+ agenda.put(current.first, context)
+ agenda.put(current.second, context)
+ new_agenda.put(-current.first, context)
+ new_agenda.put(-current.second, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1) and \
+ self._attempt_proof(new_agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_n_iff(self, current, context, agenda, accessible_vars, atoms, debug):
+ new_agenda = agenda.clone()
+ agenda.put(current.term.first, context)
+ agenda.put(-current.term.second, context)
+ new_agenda.put(-current.term.first, context)
+ new_agenda.put(current.term.second, context)
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1) and \
+ self._attempt_proof(new_agenda, accessible_vars, atoms, debug+1)
+
+ def _attempt_proof_eq(self, current, context, agenda, accessible_vars, atoms, debug):
+ #########################################################################
+ # Since 'current' is of the form '(a = b)', replace ALL free instances
+ # of 'a' with 'b'
+ #########################################################################
+ agenda.put_atoms(atoms)
+ agenda.replace_all(current.first, current.second)
+ accessible_vars.discard(current.first)
+ agenda.mark_neqs_fresh();
+ return self._attempt_proof(agenda, accessible_vars, set(), debug+1)
+
+ def _attempt_proof_some(self, current, context, agenda, accessible_vars, atoms, debug):
+ new_unique_variable = VariableExpression(unique_variable())
+ agenda.put(current.term.replace(current.variable, new_unique_variable), context)
+ agenda.mark_alls_fresh()
+ return self._attempt_proof(agenda, accessible_vars|set([new_unique_variable]), atoms, debug+1)
+
+ def _attempt_proof_all(self, current, context, agenda, accessible_vars, atoms, debug):
+ try:
+ current._used_vars
+ except AttributeError:
+ current._used_vars = set()
+
+ #if there are accessible_vars on the path
+ if accessible_vars:
+ # get the set of bound variables that have not be used by this AllExpression
+ bv_available = accessible_vars - current._used_vars
+
+ if bv_available:
+ variable_to_use = list(bv_available)[0]
+ debug.line('--> Using \'%s\'' % variable_to_use, 2)
+ current._used_vars |= set([variable_to_use])
+ agenda.put(current.term.replace(current.variable, variable_to_use), context)
+ agenda[Categories.ALL].add((current,context))
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+
+ else:
+ #no more available variables to substitute
+ debug.line('--> Variables Exhausted', 2)
+ current._exhausted = True
+ agenda[Categories.ALL].add((current,context))
+ return self._attempt_proof(agenda, accessible_vars, atoms, debug+1)
+
+ else:
+ new_unique_variable = VariableExpression(unique_variable())
+ debug.line('--> Using \'%s\'' % new_unique_variable, 2)
+ current._used_vars |= set([new_unique_variable])
+ agenda.put(current.term.replace(current.variable, new_unique_variable), context)
+ agenda[Categories.ALL].add((current,context))
+ agenda.mark_alls_fresh()
+ return self._attempt_proof(agenda, accessible_vars|set([new_unique_variable]), atoms, debug+1)
+
+ @staticmethod
+ def is_atom(e):
+ if isinstance(e, NegatedExpression):
+ e = e.term
+
+ if isinstance(e, ApplicationExpression):
+ for arg in e.args:
+ if not TableauProver.is_atom(arg):
+ return False
+ return True
+ elif isinstance(e, AbstractVariableExpression) or \
+ isinstance(e, LambdaExpression):
+ return True
+ else:
+ return False
+
+
+class TableauProverCommand(BaseProverCommand):
+ def __init__(self, goal=None, assumptions=None, prover=None):
+ """
+ :param goal: Input expression to prove
+ :type goal: sem.Expression
+ :param assumptions: Input expressions to use as assumptions in
+ the proof.
+ :type assumptions: list(sem.Expression)
+ """
+ if prover is not None:
+ assert isinstance(prover, TableauProver)
+ else:
+ prover = TableauProver()
+
+ BaseProverCommand.__init__(self, prover, goal, assumptions)
+
+
+class Agenda(object):
+ def __init__(self):
+ self.sets = tuple(set() for i in range(21))
+
+ def clone(self):
+ new_agenda = Agenda()
+ set_list = [s.copy() for s in self.sets]
+
+ new_allExs = set()
+ for allEx,_ in set_list[Categories.ALL]:
+ new_allEx = AllExpression(allEx.variable, allEx.term)
+ try:
+ new_allEx._used_vars = set(used for used in allEx._used_vars)
+ except AttributeError:
+ new_allEx._used_vars = set()
+ new_allExs.add((new_allEx,None))
+ set_list[Categories.ALL] = new_allExs
+
+ set_list[Categories.N_EQ] = set((NegatedExpression(n_eq.term),ctx)
+ for (n_eq,ctx) in set_list[Categories.N_EQ])
+
+ new_agenda.sets = tuple(set_list)
+ return new_agenda
+
+ def __getitem__(self, index):
+ return self.sets[index]
+
+ def put(self, expression, context=None):
+ if isinstance(expression, AllExpression):
+ ex_to_add = AllExpression(expression.variable, expression.term)
+ try:
+ ex_to_add._used_vars = set(used for used in expression._used_vars)
+ except AttributeError:
+ ex_to_add._used_vars = set()
+ else:
+ ex_to_add = expression
+ self.sets[self._categorize_expression(ex_to_add)].add((ex_to_add, context))
+
+ def put_all(self, expressions):
+ for expression in expressions:
+ self.put(expression)
+
+ def put_atoms(self, atoms):
+ for atom, neg in atoms:
+ if neg:
+ self[Categories.N_ATOM].add((-atom,None))
+ else:
+ self[Categories.ATOM].add((atom,None))
+
+ def pop_first(self):
+ """ Pop the first expression that appears in the agenda """
+ for i,s in enumerate(self.sets):
+ if s:
+ if i in [Categories.N_EQ, Categories.ALL]:
+ for ex in s:
+ try:
+ if not ex[0]._exhausted:
+ s.remove(ex)
+ return (ex, i)
+ except AttributeError:
+ s.remove(ex)
+ return (ex, i)
+ else:
+ return (s.pop(), i)
+ return ((None, None), None)
+
+ def replace_all(self, old, new):
+ for s in self.sets:
+ for ex,ctx in s:
+ ex.replace(old.variable, new)
+ if ctx is not None:
+ ctx.replace(old.variable, new)
+
+ def mark_alls_fresh(self):
+ for u,_ in self.sets[Categories.ALL]:
+ u._exhausted = False
+
+ def mark_neqs_fresh(self):
+ for neq,_ in self.sets[Categories.N_EQ]:
+ neq._exhausted = False
+
+ def _categorize_expression(self, current):
+ if isinstance(current, NegatedExpression):
+ return self._categorize_NegatedExpression(current)
+ elif isinstance(current, FunctionVariableExpression):
+ return Categories.PROP
+ elif TableauProver.is_atom(current):
+ return Categories.ATOM
+ elif isinstance(current, AllExpression):
+ return Categories.ALL
+ elif isinstance(current, AndExpression):
+ return Categories.AND
+ elif isinstance(current, OrExpression):
+ return Categories.OR
+ elif isinstance(current, ImpExpression):
+ return Categories.IMP
+ elif isinstance(current, IffExpression):
+ return Categories.IFF
+ elif isinstance(current, EqualityExpression):
+ return Categories.EQ
+ elif isinstance(current, ExistsExpression):
+ return Categories.EXISTS
+ elif isinstance(current, ApplicationExpression):
+ return Categories.APP
+ else:
+ raise ProverParseError("cannot categorize %s" % \
+ current.__class__.__name__)
+
+ def _categorize_NegatedExpression(self, current):
+ negated = current.term
+
+ if isinstance(negated, NegatedExpression):
+ return Categories.D_NEG
+ elif isinstance(negated, FunctionVariableExpression):
+ return Categories.N_PROP
+ elif TableauProver.is_atom(negated):
+ return Categories.N_ATOM
+ elif isinstance(negated, AllExpression):
+ return Categories.N_ALL
+ elif isinstance(negated, AndExpression):
+ return Categories.N_AND
+ elif isinstance(negated, OrExpression):
+ return Categories.N_OR
+ elif isinstance(negated, ImpExpression):
+ return Categories.N_IMP
+ elif isinstance(negated, IffExpression):
+ return Categories.N_IFF
+ elif isinstance(negated, EqualityExpression):
+ return Categories.N_EQ
+ elif isinstance(negated, ExistsExpression):
+ return Categories.N_EXISTS
+ elif isinstance(negated, ApplicationExpression):
+ return Categories.N_APP
+ else:
+ raise ProverParseError("cannot categorize %s" % \
+ negated.__class__.__name__)
+
+
+class Debug(object):
+ def __init__(self, verbose, indent=0, lines=None):
+ self.verbose = verbose
+ self.indent = indent
+
+ if not lines: lines = []
+ self.lines = lines
+
+ def __add__(self, increment):
+ return Debug(self.verbose, self.indent+1, self.lines)
+
+ def line(self, data, indent=0):
+ if isinstance(data, tuple):
+ ex, ctx = data
+ if ctx:
+ data = '%s, %s' % (ex, ctx)
+ else:
+ data = '%s' % ex
+
+ if isinstance(ex, AllExpression):
+ try:
+ used_vars = "[%s]" % (",".join("%s" % ve.variable.name for ve in ex._used_vars))
+ data += ': %s' % used_vars
+ except AttributeError:
+ data += ': []'
+
+ newline = '%s%s' % (' '*(self.indent+indent), data)
+ self.lines.append(newline)
+
+ if self.verbose:
+ print(newline)
+
+
+class Categories(object):
+ ATOM = 0
+ PROP = 1
+ N_ATOM = 2
+ N_PROP = 3
+ APP = 4
+ N_APP = 5
+ N_EQ = 6
+ D_NEG = 7
+ N_ALL = 8
+ N_EXISTS = 9
+ AND = 10
+ N_OR = 11
+ N_IMP = 12
+ OR = 13
+ IMP = 14
+ N_AND = 15
+ IFF = 16
+ N_IFF = 17
+ EQ = 18
+ EXISTS = 19
+ ALL = 20
+
+
+def testTableauProver():
+ tableau_test('P | -P')
+ tableau_test('P & -P')
+ tableau_test('Q', ['P', '(P -> Q)'])
+ tableau_test('man(x)')
+ tableau_test('(man(x) -> man(x))')
+ tableau_test('(man(x) -> --man(x))')
+ tableau_test('-(man(x) and -man(x))')
+ tableau_test('(man(x) or -man(x))')
+ tableau_test('(man(x) -> man(x))')
+ tableau_test('-(man(x) and -man(x))')
+ tableau_test('(man(x) or -man(x))')
+ tableau_test('(man(x) -> man(x))')
+ tableau_test('(man(x) iff man(x))')
+ tableau_test('-(man(x) iff -man(x))')
+ tableau_test('all x.man(x)')
+ tableau_test('all x.all y.((x = y) -> (y = x))')
+ tableau_test('all x.all y.all z.(((x = y) & (y = z)) -> (x = z))')
+# tableau_test('-all x.some y.F(x,y) & some x.all y.(-F(x,y))')
+# tableau_test('some x.all y.sees(x,y)')
+
+ p1 = 'all x.(man(x) -> mortal(x))'
+ p2 = 'man(Socrates)'
+ c = 'mortal(Socrates)'
+ tableau_test(c, [p1, p2])
+
+ p1 = 'all x.(man(x) -> walks(x))'
+ p2 = 'man(John)'
+ c = 'some y.walks(y)'
+ tableau_test(c, [p1, p2])
+
+ p = '((x = y) & walks(y))'
+ c = 'walks(x)'
+ tableau_test(c, [p])
+
+ p = '((x = y) & ((y = z) & (z = w)))'
+ c = '(x = w)'
+ tableau_test(c, [p])
+
+ p = 'some e1.some e2.(believe(e1,john,e2) & walk(e2,mary))'
+ c = 'some e0.walk(e0,mary)'
+ tableau_test(c, [p])
+
+ c = '(exists x.exists z3.((x = Mary) & ((z3 = John) & sees(z3,x))) <-> exists x.exists z4.((x = John) & ((z4 = Mary) & sees(x,z4))))'
+ tableau_test(c)
+
+# p = 'some e1.some e2.((believe e1 john e2) and (walk e2 mary))'
+# c = 'some x.some e3.some e4.((believe e3 x e4) and (walk e4 mary))'
+# tableau_test(c, [p])
+
+
+def testHigherOrderTableauProver():
+ tableau_test('believe(j, -lie(b))', ['believe(j, -lie(b) & -cheat(b))'])
+ tableau_test('believe(j, lie(b) & cheat(b))', ['believe(j, lie(b))'])
+ tableau_test('believe(j, lie(b))', ['lie(b)']) #how do we capture that John believes all things that are true
+ tableau_test('believe(j, know(b, cheat(b)))', ['believe(j, know(b, lie(b)) & know(b, steals(b) & cheat(b)))'])
+ tableau_test('P(Q(y), R(y) & R(z))', ['P(Q(x) & Q(y), R(y) & R(z))'])
+
+ tableau_test('believe(j, cheat(b) & lie(b))', ['believe(j, lie(b) & cheat(b))'])
+ tableau_test('believe(j, -cheat(b) & -lie(b))', ['believe(j, -lie(b) & -cheat(b))'])
+
+
+def tableau_test(c, ps=None, verbose=False):
+ pc = Expression.fromstring(c)
+ pps = ([Expression.fromstring(p) for p in ps] if ps else [])
+ if not ps:
+ ps = []
+ print('%s |- %s: %s' % (', '.join(ps), pc, TableauProver().prove(pc, pps, verbose=verbose)))
+
+def demo():
+ testTableauProver()
+ testHigherOrderTableauProver()
+
+if __name__ == '__main__':
+ demo()
+
diff --git a/nltk/internals.py b/nltk/internals.py
new file mode 100644
index 0000000..49ac006
--- /dev/null
+++ b/nltk/internals.py
@@ -0,0 +1,889 @@
+# Natural Language Toolkit: Internal utility functions
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# Nitin Madnani <nmadnani at ets.org>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function
+
+import subprocess
+import os
+import re, sre_constants, sre_parse, sre_compile
+import warnings
+import textwrap
+import types
+import sys
+import stat
+
+# Use the c version of ElementTree, which is faster, if possible:
+try:
+ from xml.etree import cElementTree as ElementTree
+except ImportError:
+ from xml.etree import ElementTree
+
+from nltk import __file__
+from nltk import compat
+######################################################################
+# Regular Expression Processing
+######################################################################
+
+def compile_regexp_to_noncapturing(pattern, flags=0):
+ """
+ Compile the regexp pattern after switching all grouping parentheses
+ in the given regexp pattern to non-capturing groups.
+
+ :type pattern: str
+ :rtype: str
+ """
+ def convert_regexp_to_noncapturing_parsed(parsed_pattern):
+ res_data = []
+ for key, value in parsed_pattern.data:
+ if key == sre_constants.SUBPATTERN:
+ index, subpattern = value
+ value = (None, convert_regexp_to_noncapturing_parsed(subpattern))
+ elif key == sre_constants.GROUPREF:
+ raise ValueError('Regular expressions with back-references are not supported: {0}'.format(pattern))
+ res_data.append((key, value))
+ parsed_pattern.data = res_data
+ parsed_pattern.pattern.groups = 1
+ parsed_pattern.pattern.groupdict = {}
+ return parsed_pattern
+
+ return sre_compile.compile(convert_regexp_to_noncapturing_parsed(sre_parse.parse(pattern)), flags=flags)
+
+
+##########################################################################
+# Java Via Command-Line
+##########################################################################
+
+_java_bin = None
+_java_options = []
+# [xx] add classpath option to config_java?
+def config_java(bin=None, options=None, verbose=True):
+ """
+ Configure nltk's java interface, by letting nltk know where it can
+ find the Java binary, and what extra options (if any) should be
+ passed to Java when it is run.
+
+ :param bin: The full path to the Java binary. If not specified,
+ then nltk will search the system for a Java binary; and if
+ one is not found, it will raise a ``LookupError`` exception.
+ :type bin: str
+ :param options: A list of options that should be passed to the
+ Java binary when it is called. A common value is
+ ``'-Xmx512m'``, which tells Java binary to increase
+ the maximum heap size to 512 megabytes. If no options are
+ specified, then do not modify the options list.
+ :type options: list(str)
+ """
+ global _java_bin, _java_options
+ _java_bin = find_binary('java', bin, env_vars=['JAVAHOME', 'JAVA_HOME'], verbose=verbose, binary_names=['java.exe'])
+
+ if options is not None:
+ if isinstance(options, compat.string_types):
+ options = options.split()
+ _java_options = list(options)
+
+def java(cmd, classpath=None, stdin=None, stdout=None, stderr=None,
+ blocking=True):
+ """
+ Execute the given java command, by opening a subprocess that calls
+ Java. If java has not yet been configured, it will be configured
+ by calling ``config_java()`` with no arguments.
+
+ :param cmd: The java command that should be called, formatted as
+ a list of strings. Typically, the first string will be the name
+ of the java class; and the remaining strings will be arguments
+ for that java class.
+ :type cmd: list(str)
+
+ :param classpath: A ``':'`` separated list of directories, JAR
+ archives, and ZIP archives to search for class files.
+ :type classpath: str
+
+ :param stdin, stdout, stderr: Specify the executed programs'
+ standard input, standard output and standard error file
+ handles, respectively. Valid values are ``subprocess.PIPE``,
+ an existing file descriptor (a positive integer), an existing
+ file object, and None. ``subprocess.PIPE`` indicates that a
+ new pipe to the child should be created. With None, no
+ redirection will occur; the child's file handles will be
+ inherited from the parent. Additionally, stderr can be
+ ``subprocess.STDOUT``, which indicates that the stderr data
+ from the applications should be captured into the same file
+ handle as for stdout.
+
+ :param blocking: If ``false``, then return immediately after
+ spawning the subprocess. In this case, the return value is
+ the ``Popen`` object, and not a ``(stdout, stderr)`` tuple.
+
+ :return: If ``blocking=True``, then return a tuple ``(stdout,
+ stderr)``, containing the stdout and stderr outputs generated
+ by the java command if the ``stdout`` and ``stderr`` parameters
+ were set to ``subprocess.PIPE``; or None otherwise. If
+ ``blocking=False``, then return a ``subprocess.Popen`` object.
+
+ :raise OSError: If the java command returns a nonzero return code.
+ """
+ if stdin == 'pipe': stdin = subprocess.PIPE
+ if stdout == 'pipe': stdout = subprocess.PIPE
+ if stderr == 'pipe': stderr = subprocess.PIPE
+ if isinstance(cmd, compat.string_types):
+ raise TypeError('cmd should be a list of strings')
+
+ # Make sure we know where a java binary is.
+ if _java_bin is None:
+ config_java()
+
+ # Set up the classpath.
+ if isinstance(classpath, compat.string_types):
+ classpaths=[classpath]
+ else:
+ classpaths=list(classpath)
+ classpath=os.path.pathsep.join(classpaths)
+
+ # Construct the full command string.
+ cmd = list(cmd)
+ cmd = ['-cp', classpath] + cmd
+ cmd = [_java_bin] + _java_options + cmd
+
+ # Call java via a subprocess
+ p = subprocess.Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr)
+ if not blocking: return p
+ (stdout, stderr) = p.communicate()
+
+ # Check the return code.
+ if p.returncode != 0:
+ print(stderr.decode(sys.stdout.encoding))
+ raise OSError('Java command failed!')
+
+ return (stdout, stderr)
+
+if 0:
+ #config_java(options='-Xmx512m')
+ # Write:
+ #java('weka.classifiers.bayes.NaiveBayes',
+ # ['-d', '/tmp/names.model', '-t', '/tmp/train.arff'],
+ # classpath='/Users/edloper/Desktop/weka/weka.jar')
+ # Read:
+ (a,b) = java(['weka.classifiers.bayes.NaiveBayes',
+ '-l', '/tmp/names.model', '-T', '/tmp/test.arff',
+ '-p', '0'],#, '-distribution'],
+ classpath='/Users/edloper/Desktop/weka/weka.jar')
+
+
+######################################################################
+# Parsing
+######################################################################
+
+class ReadError(ValueError):
+ """
+ Exception raised by read_* functions when they fail.
+ :param position: The index in the input string where an error occurred.
+ :param expected: What was expected when an error occurred.
+ """
+ def __init__(self, expected, position):
+ ValueError.__init__(self, expected, position)
+ self.expected = expected
+ self.position = position
+ def __str__(self):
+ return 'Expected %s at %s' % (self.expected, self.position)
+
+_STRING_START_RE = re.compile(r"[uU]?[rR]?(\"\"\"|\'\'\'|\"|\')")
+def read_str(s, start_position):
+ """
+ If a Python string literal begins at the specified position in the
+ given string, then return a tuple ``(val, end_position)``
+ containing the value of the string literal and the position where
+ it ends. Otherwise, raise a ``ReadError``.
+ """
+ # Read the open quote, and any modifiers.
+ m = _STRING_START_RE.match(s, start_position)
+ if not m: raise ReadError('open quote', start_position)
+ quotemark = m.group(1)
+
+ # Find the close quote.
+ _STRING_END_RE = re.compile(r'\\|%s' % quotemark)
+ position = m.end()
+ while True:
+ match = _STRING_END_RE.search(s, position)
+ if not match: raise ReadError('close quote', position)
+ if match.group(0) == '\\': position = match.end()+1
+ else: break
+
+ # Process it, using eval. Strings with invalid escape sequences
+ # might raise ValueEerror.
+ try:
+ return eval(s[start_position:match.end()]), match.end()
+ except ValueError as e:
+ raise ReadError('invalid string (%s)' % e)
+
+_READ_INT_RE = re.compile(r'-?\d+')
+def read_int(s, start_position):
+ """
+ If an integer begins at the specified position in the given
+ string, then return a tuple ``(val, end_position)`` containing the
+ value of the integer and the position where it ends. Otherwise,
+ raise a ``ReadError``.
+ """
+ m = _READ_INT_RE.match(s, start_position)
+ if not m: raise ReadError('integer', start_position)
+ return int(m.group()), m.end()
+
+_READ_NUMBER_VALUE = re.compile(r'-?(\d*)([.]?\d*)?')
+def read_number(s, start_position):
+ """
+ If an integer or float begins at the specified position in the
+ given string, then return a tuple ``(val, end_position)``
+ containing the value of the number and the position where it ends.
+ Otherwise, raise a ``ReadError``.
+ """
+ m = _READ_NUMBER_VALUE.match(s, start_position)
+ if not m or not (m.group(1) or m.group(2)):
+ raise ReadError('number', start_position)
+ if m.group(2): return float(m.group()), m.end()
+ else: return int(m.group()), m.end()
+
+
+
+######################################################################
+# Check if a method has been overridden
+######################################################################
+
+def overridden(method):
+ """
+ :return: True if ``method`` overrides some method with the same
+ name in a base class. This is typically used when defining
+ abstract base classes or interfaces, to allow subclasses to define
+ either of two related methods:
+
+ >>> class EaterI:
+ ... '''Subclass must define eat() or batch_eat().'''
+ ... def eat(self, food):
+ ... if overridden(self.batch_eat):
+ ... return self.batch_eat([food])[0]
+ ... else:
+ ... raise NotImplementedError()
+ ... def batch_eat(self, foods):
+ ... return [self.eat(food) for food in foods]
+
+ :type method: instance method
+ """
+ # [xx] breaks on classic classes!
+ if isinstance(method, types.MethodType) and compat.get_im_class(method) is not None:
+ name = method.__name__
+ funcs = [cls.__dict__[name]
+ for cls in _mro(compat.get_im_class(method))
+ if name in cls.__dict__]
+ return len(funcs) > 1
+ else:
+ raise TypeError('Expected an instance method.')
+
+def _mro(cls):
+ """
+ Return the method resolution order for ``cls`` -- i.e., a list
+ containing ``cls`` and all its base classes, in the order in which
+ they would be checked by ``getattr``. For new-style classes, this
+ is just cls.__mro__. For classic classes, this can be obtained by
+ a depth-first left-to-right traversal of ``__bases__``.
+ """
+ if isinstance(cls, type):
+ return cls.__mro__
+ else:
+ mro = [cls]
+ for base in cls.__bases__: mro.extend(_mro(base))
+ return mro
+
+######################################################################
+# Deprecation decorator & base class
+######################################################################
+# [xx] dedent msg first if it comes from a docstring.
+
+def _add_epytext_field(obj, field, message):
+ """Add an epytext @field to a given object's docstring."""
+ indent = ''
+ # If we already have a docstring, then add a blank line to separate
+ # it from the new field, and check its indentation.
+ if obj.__doc__:
+ obj.__doc__ = obj.__doc__.rstrip()+'\n\n'
+ indents = re.findall(r'(?<=\n)[ ]+(?!\s)', obj.__doc__.expandtabs())
+ if indents: indent = min(indents)
+ # If we don't have a docstring, add an empty one.
+ else:
+ obj.__doc__ = ''
+
+ obj.__doc__ += textwrap.fill('@%s: %s' % (field, message),
+ initial_indent=indent,
+ subsequent_indent=indent+' ')
+
+def deprecated(message):
+ """
+ A decorator used to mark functions as deprecated. This will cause
+ a warning to be printed the when the function is used. Usage:
+
+ >>> from nltk.internals import deprecated
+ >>> @deprecated('Use foo() instead')
+ ... def bar(x):
+ ... print(x/10)
+
+ """
+
+ def decorator(func):
+ msg = ("Function %s() has been deprecated. %s"
+ % (func.__name__, message))
+ msg = '\n' + textwrap.fill(msg, initial_indent=' ',
+ subsequent_indent=' ')
+ def newFunc(*args, **kwargs):
+ warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+ return func(*args, **kwargs)
+
+ # Copy the old function's name, docstring, & dict
+ newFunc.__dict__.update(func.__dict__)
+ newFunc.__name__ = func.__name__
+ newFunc.__doc__ = func.__doc__
+ newFunc.__deprecated__ = True
+ # Add a @deprecated field to the docstring.
+ _add_epytext_field(newFunc, 'deprecated', message)
+ return newFunc
+ return decorator
+
+class Deprecated(object):
+ """
+ A base class used to mark deprecated classes. A typical usage is to
+ alert users that the name of a class has changed:
+
+ >>> from nltk.internals import Deprecated
+ >>> class NewClassName(object):
+ ... pass # All logic goes here.
+ ...
+ >>> class OldClassName(Deprecated, NewClassName):
+ ... "Use NewClassName instead."
+
+ The docstring of the deprecated class will be used in the
+ deprecation warning message.
+ """
+ def __new__(cls, *args, **kwargs):
+ # Figure out which class is the deprecated one.
+ dep_cls = None
+ for base in _mro(cls):
+ if Deprecated in base.__bases__:
+ dep_cls = base; break
+ assert dep_cls, 'Unable to determine which base is deprecated.'
+
+ # Construct an appropriate warning.
+ doc = dep_cls.__doc__ or ''.strip()
+ # If there's a @deprecated field, strip off the field marker.
+ doc = re.sub(r'\A\s*@deprecated:', r'', doc)
+ # Strip off any indentation.
+ doc = re.sub(r'(?m)^\s*', '', doc)
+ # Construct a 'name' string.
+ name = 'Class %s' % dep_cls.__name__
+ if cls != dep_cls:
+ name += ' (base class for %s)' % cls.__name__
+ # Put it all together.
+ msg = '%s has been deprecated. %s' % (name, doc)
+ # Wrap it.
+ msg = '\n' + textwrap.fill(msg, initial_indent=' ',
+ subsequent_indent=' ')
+ warnings.warn(msg, category=DeprecationWarning, stacklevel=2)
+ # Do the actual work of __new__.
+ return object.__new__(cls)
+
+##########################################################################
+# COUNTER, FOR UNIQUE NAMING
+##########################################################################
+
+class Counter:
+ """
+ A counter that auto-increments each time its value is read.
+ """
+ def __init__(self, initial_value=0):
+ self._value = initial_value
+ def get(self):
+ self._value += 1
+ return self._value
+
+##########################################################################
+# Search for files/binaries
+##########################################################################
+
+def find_file_iter(filename, env_vars=(), searchpath=(),
+ file_names=None, url=None, verbose=True):
+ """
+ Search for a file to be used by nltk.
+
+ :param filename: The name or path of the file.
+ :param env_vars: A list of environment variable names to check.
+ :param file_names: A list of alternative file names to check.
+ :param searchpath: List of directories to search.
+ :param url: URL presented to user for download help.
+ :param verbose: Whether or not to print path when a file is found.
+ """
+ file_names = [filename] + (file_names or [])
+ assert isinstance(filename, compat.string_types)
+ assert not isinstance(file_names, compat.string_types)
+ assert not isinstance(searchpath, compat.string_types)
+ if isinstance(env_vars, compat.string_types):
+ env_vars = env_vars.split()
+ yielded = False
+
+ # File exists, no magic
+ for alternative in file_names:
+ path_to_file = os.path.join(filename, alternative)
+ if os.path.isfile(path_to_file):
+ if verbose:
+ print('[Found %s: %s]' % (filename, path_to_file))
+ yielded = True
+ yield path_to_file
+ # Check the bare alternatives
+ if os.path.isfile(alternative):
+ if verbose:
+ print('[Found %s: %s]' % (filename, alternative))
+ yielded = True
+ yield alternative
+ # Check if the alternative is inside a 'file' directory
+ path_to_file = os.path.join(filename, 'file', alternative)
+ if os.path.isfile(path_to_file):
+ if verbose:
+ print('[Found %s: %s]' % (filename, path_to_file))
+ yielded = True
+ yield path_to_file
+
+ # Check environment variables
+ for env_var in env_vars:
+ if env_var in os.environ:
+ for env_dir in os.environ[env_var].split(os.pathsep):
+ # Check if the environment variable contains a direct path to the bin
+ if os.path.isfile(env_dir):
+ if verbose:
+ print('[Found %s: %s]'%(filename, env_dir))
+ yielded = True
+ yield env_dir
+ # Check if the possible bin names exist inside the environment variable directories
+ for alternative in file_names:
+ path_to_file = os.path.join(env_dir, alternative)
+ if os.path.isfile(path_to_file):
+ if verbose:
+ print('[Found %s: %s]'%(filename, path_to_file))
+ yielded = True
+ yield path_to_file
+ # Check if the alternative is inside a 'file' directory
+ path_to_file = os.path.join(env_dir, 'file', alternative)
+ if os.path.isfile(path_to_file):
+ if verbose:
+ print('[Found %s: %s]' % (filename, path_to_file))
+ yielded = True
+ yield path_to_file
+
+ # Check the path list.
+ for directory in searchpath:
+ for alternative in file_names:
+ path_to_file = os.path.join(directory, alternative)
+ if os.path.isfile(path_to_file):
+ yielded = True
+ yield path_to_file
+
+ # If we're on a POSIX system, then try using the 'which' command
+ # to find the file.
+ if os.name == 'posix':
+ for alternative in file_names:
+ try:
+ p = subprocess.Popen(['which', alternative], stdout=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ path = stdout.decode(sys.stdout.encoding).strip()
+ if path.endswith(alternative) and os.path.exists(path):
+ if verbose:
+ print('[Found %s: %s]' % (filename, path))
+ yielded = True
+ yield path
+ except (KeyboardInterrupt, SystemExit):
+ raise
+ except:
+ pass
+
+ if not yielded:
+ msg = ("NLTK was unable to find the %s file!" "\nUse software specific "
+ "configuration paramaters" % filename)
+ if env_vars: msg += ' or set the %s environment variable' % env_vars[0]
+ msg += '.'
+ if searchpath:
+ msg += '\n\n Searched in:'
+ msg += ''.join('\n - %s' % d for d in searchpath)
+ if url: msg += ('\n\n For more information, on %s, see:\n <%s>' %
+ (filename, url))
+ div = '='*75
+ raise LookupError('\n\n%s\n%s\n%s' % (div, msg, div))
+
+def find_file(filename, env_vars=(), searchpath=(),
+ file_names=None, url=None, verbose=True):
+ return next(find_file_iter(filename, env_vars, searchpath,
+ file_names, url, verbose))
+
+def find_binary_iter(name, path_to_bin=None, env_vars=(), searchpath=(),
+ binary_names=None, url=None, verbose=True):
+ """
+ Search for a file to be used by nltk.
+
+ :param name: The name or path of the file.
+ :param path_to_bin: The user-supplied binary location (deprecated)
+ :param env_vars: A list of environment variable names to check.
+ :param file_names: A list of alternative file names to check.
+ :param searchpath: List of directories to search.
+ :param url: URL presented to user for download help.
+ :param verbose: Whether or not to print path when a file is found.
+ """
+ for file in find_file_iter(path_to_bin or name, env_vars, searchpath, binary_names,
+ url, verbose):
+ yield file
+
+def find_binary(name, path_to_bin=None, env_vars=(), searchpath=(),
+ binary_names=None, url=None, verbose=True):
+ return next(find_binary_iter(name, path_to_bin, env_vars, searchpath,
+ binary_names, url, verbose))
+
+def find_jar_iter(name_pattern, path_to_jar=None, env_vars=(),
+ searchpath=(), url=None, verbose=True, is_regex=False):
+ """
+ Search for a jar that is used by nltk.
+
+ :param name_pattern: The name of the jar file
+ :param path_to_jar: The user-supplied jar location, or None.
+ :param env_vars: A list of environment variable names to check
+ in addition to the CLASSPATH variable which is
+ checked by default.
+ :param searchpath: List of directories to search.
+ :param is_regex: Whether name is a regular expression.
+ """
+
+ assert isinstance(name_pattern, compat.string_types)
+ assert not isinstance(searchpath, compat.string_types)
+ if isinstance(env_vars, compat.string_types):
+ env_vars = env_vars.split()
+ yielded = False
+
+ # Make sure we check the CLASSPATH first
+ env_vars = ['CLASSPATH'] + list(env_vars)
+
+ # If an explicit location was given, then check it, and yield it if
+ # it's present; otherwise, complain.
+ if path_to_jar is not None:
+ if os.path.isfile(path_to_jar):
+ yield path_to_jar
+ raise LookupError('Could not find %s jar file at %s' %
+ (name_pattern, path_to_jar))
+
+ # Check environment variables
+ for env_var in env_vars:
+ if env_var in os.environ:
+ if env_var == 'CLASSPATH':
+ classpath = os.environ['CLASSPATH']
+ for cp in classpath.split(os.path.pathsep):
+ if os.path.isfile(cp):
+ filename=os.path.basename(cp)
+ if is_regex and re.match(name_pattern, filename) or \
+ (not is_regex and filename == name_pattern):
+ if verbose:
+ print('[Found %s: %s]' % (name_pattern, cp))
+ yielded = True
+ yield cp
+ else:
+ jar_env = os.environ[env_var]
+ jar_iter = ((os.path.join(jar_env, path_to_jar) for path_to_jar in os.listdir(jar_env))
+ if os.path.isdir(jar_env) else (jar_env,))
+ for path_to_jar in jar_iter:
+ if os.path.isfile(path_to_jar):
+ filename=os.path.basename(path_to_jar)
+ if is_regex and re.match(name_pattern, filename) or \
+ (not is_regex and filename == name_pattern):
+ if verbose:
+ print('[Found %s: %s]' % (name_pattern, path_to_jar))
+ yielded = True
+ yield path_to_jar
+
+ # Check the path list.
+ for directory in searchpath:
+ if is_regex:
+ for filename in os.listdir(directory):
+ path_to_jar = os.path.join(directory, filename)
+ if os.path.isfile(path_to_jar):
+ if re.match(name_pattern, filename):
+ if verbose:
+ print('[Found %s: %s]' % (filename, path_to_jar))
+ yielded = True
+ yield path_to_jar
+ else:
+ path_to_jar = os.path.join(directory, name_pattern)
+ if os.path.isfile(path_to_jar):
+ if verbose:
+ print('[Found %s: %s]' % (name_pattern, path_to_jar))
+ yielded = True
+ yield path_to_jar
+
+ if not yielded:
+ # If nothing was found, raise an error
+ msg = ("NLTK was unable to find %s!" % name_pattern)
+ if env_vars: msg += ' Set the %s environment variable' % env_vars[0]
+ msg = textwrap.fill(msg+'.', initial_indent=' ',
+ subsequent_indent=' ')
+ if searchpath:
+ msg += '\n\n Searched in:'
+ msg += ''.join('\n - %s' % d for d in searchpath)
+ if url:
+ msg += ('\n\n For more information, on %s, see:\n <%s>' %
+ (name_pattern, url))
+ div = '='*75
+ raise LookupError('\n\n%s\n%s\n%s' % (div, msg, div))
+
+def find_jar(name_pattern, path_to_jar=None, env_vars=(),
+ searchpath=(), url=None, verbose=True, is_regex=False):
+ return next(find_jar_iter(name_pattern, path_to_jar, env_vars,
+ searchpath, url, verbose, is_regex))
+
+##########################################################################
+# Import Stdlib Module
+##########################################################################
+
+def import_from_stdlib(module):
+ """
+ When python is run from within the nltk/ directory tree, the
+ current directory is included at the beginning of the search path.
+ Unfortunately, that means that modules within nltk can sometimes
+ shadow standard library modules. As an example, the stdlib
+ 'inspect' module will attempt to import the stdlib 'tokenize'
+ module, but will instead end up importing NLTK's 'tokenize' module
+ instead (causing the import to fail).
+ """
+ old_path = sys.path
+ sys.path = [d for d in sys.path if d not in ('', '.')]
+ m = __import__(module)
+ sys.path = old_path
+ return m
+
+
+##########################################################################
+# Wrapper for ElementTree Elements
+##########################################################################
+
+ at compat.python_2_unicode_compatible
+class ElementWrapper(object):
+ """
+ A wrapper around ElementTree Element objects whose main purpose is
+ to provide nicer __repr__ and __str__ methods. In addition, any
+ of the wrapped Element's methods that return other Element objects
+ are overridden to wrap those values before returning them.
+
+ This makes Elements more convenient to work with in
+ interactive sessions and doctests, at the expense of some
+ efficiency.
+ """
+
+ # Prevent double-wrapping:
+ def __new__(cls, etree):
+ """
+ Create and return a wrapper around a given Element object.
+ If ``etree`` is an ``ElementWrapper``, then ``etree`` is
+ returned as-is.
+ """
+ if isinstance(etree, ElementWrapper):
+ return etree
+ else:
+ return object.__new__(ElementWrapper)
+
+ def __init__(self, etree):
+ r"""
+ Initialize a new Element wrapper for ``etree``.
+
+ If ``etree`` is a string, then it will be converted to an
+ Element object using ``ElementTree.fromstring()`` first:
+
+ >>> ElementWrapper("<test></test>")
+ <Element "<?xml version='1.0' encoding='utf8'?>\n<test />">
+
+ """
+ if isinstance(etree, compat.string_types):
+ etree = ElementTree.fromstring(etree)
+ self.__dict__['_etree'] = etree
+
+ def unwrap(self):
+ """
+ Return the Element object wrapped by this wrapper.
+ """
+ return self._etree
+
+ ##////////////////////////////////////////////////////////////
+ #{ String Representation
+ ##////////////////////////////////////////////////////////////
+
+ def __repr__(self):
+ s = ElementTree.tostring(self._etree, encoding='utf8').decode('utf8')
+ if len(s) > 60:
+ e = s.rfind('<')
+ if (len(s)-e) > 30: e = -20
+ s = '%s...%s' % (s[:30], s[e:])
+ return '<Element %r>' % s
+
+ def __str__(self):
+ """
+ :return: the result of applying ``ElementTree.tostring()`` to
+ the wrapped Element object.
+ """
+ return ElementTree.tostring(self._etree, encoding='utf8').decode('utf8').rstrip()
+
+ ##////////////////////////////////////////////////////////////
+ #{ Element interface Delegation (pass-through)
+ ##////////////////////////////////////////////////////////////
+
+ def __getattr__(self, attrib):
+ return getattr(self._etree, attrib)
+
+ def __setattr__(self, attr, value):
+ return setattr(self._etree, attr, value)
+
+ def __delattr__(self, attr):
+ return delattr(self._etree, attr)
+
+ def __setitem__(self, index, element):
+ self._etree[index] = element
+
+ def __delitem__(self, index):
+ del self._etree[index]
+
+ def __setslice__(self, start, stop, elements):
+ self._etree[start:stop] = elements
+
+ def __delslice__(self, start, stop):
+ del self._etree[start:stop]
+
+ def __len__(self):
+ return len(self._etree)
+
+ ##////////////////////////////////////////////////////////////
+ #{ Element interface Delegation (wrap result)
+ ##////////////////////////////////////////////////////////////
+
+ def __getitem__(self, index):
+ return ElementWrapper(self._etree[index])
+
+ def __getslice__(self, start, stop):
+ return [ElementWrapper(elt) for elt in self._etree[start:stop]]
+
+ def getchildren(self):
+ return [ElementWrapper(elt) for elt in self._etree]
+
+ def getiterator(self, tag=None):
+ return (ElementWrapper(elt)
+ for elt in self._etree.getiterator(tag))
+
+ def makeelement(self, tag, attrib):
+ return ElementWrapper(self._etree.makeelement(tag, attrib))
+
+ def find(self, path):
+ elt = self._etree.find(path)
+ if elt is None: return elt
+ else: return ElementWrapper(elt)
+
+ def findall(self, path):
+ return [ElementWrapper(elt) for elt in self._etree.findall(path)]
+
+######################################################################
+# Helper for Handling Slicing
+######################################################################
+
+def slice_bounds(sequence, slice_obj, allow_step=False):
+ """
+ Given a slice, return the corresponding (start, stop) bounds,
+ taking into account None indices and negative indices. The
+ following guarantees are made for the returned start and stop values:
+
+ - 0 <= start <= len(sequence)
+ - 0 <= stop <= len(sequence)
+ - start <= stop
+
+ :raise ValueError: If ``slice_obj.step`` is not None.
+ :param allow_step: If true, then the slice object may have a
+ non-None step. If it does, then return a tuple
+ (start, stop, step).
+ """
+ start, stop = (slice_obj.start, slice_obj.stop)
+
+ # If allow_step is true, then include the step in our return
+ # value tuple.
+ if allow_step:
+ step = slice_obj.step
+ if step is None: step = 1
+ # Use a recursive call without allow_step to find the slice
+ # bounds. If step is negative, then the roles of start and
+ # stop (in terms of default values, etc), are swapped.
+ if step < 0:
+ start, stop = slice_bounds(sequence, slice(stop, start))
+ else:
+ start, stop = slice_bounds(sequence, slice(start, stop))
+ return start, stop, step
+
+ # Otherwise, make sure that no non-default step value is used.
+ elif slice_obj.step not in (None, 1):
+ raise ValueError('slices with steps are not supported by %s' %
+ sequence.__class__.__name__)
+
+ # Supply default offsets.
+ if start is None: start = 0
+ if stop is None: stop = len(sequence)
+
+ # Handle negative indices.
+ if start < 0: start = max(0, len(sequence)+start)
+ if stop < 0: stop = max(0, len(sequence)+stop)
+
+ # Make sure stop doesn't go past the end of the list. Note that
+ # we avoid calculating len(sequence) if possible, because for lazy
+ # sequences, calculating the length of a sequence can be expensive.
+ if stop > 0:
+ try: sequence[stop-1]
+ except IndexError: stop = len(sequence)
+
+ # Make sure start isn't past stop.
+ start = min(start, stop)
+
+ # That's all folks!
+ return start, stop
+
+######################################################################
+# Permission Checking
+######################################################################
+
+def is_writable(path):
+ # Ensure that it exists.
+ if not os.path.exists(path):
+ return False
+
+ # If we're on a posix system, check its permissions.
+ if hasattr(os, 'getuid'):
+ statdata = os.stat(path)
+ perm = stat.S_IMODE(statdata.st_mode)
+ # is it world-writable?
+ if (perm & 0o002):
+ return True
+ # do we own it?
+ elif statdata.st_uid == os.getuid() and (perm & 0o200):
+ return True
+ # are we in a group that can write to it?
+ elif (statdata.st_gid in [os.getgid()] + os.getgroups()) \
+ and (perm & 0o020):
+ return True
+ # otherwise, we can't write to it.
+ else:
+ return False
+
+ # Otherwise, we'll assume it's writable.
+ # [xx] should we do other checks on other platforms?
+ return True
+
+######################################################################
+# NLTK Error reporting
+######################################################################
+
+def raise_unorderable_types(ordering, a, b):
+ raise TypeError("unorderable types: %s() %s %s()" % (type(a).__name__, ordering, type(b).__name__))
+
+
diff --git a/nltk/jsontags.py b/nltk/jsontags.py
new file mode 100644
index 0000000..084c649
--- /dev/null
+++ b/nltk/jsontags.py
@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: JSON Encoder/Decoder Helpers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Xu <xxu at student.unimelb.edu.au>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Register JSON tags, so the nltk data loader knows what module and class to look for.
+
+NLTK uses simple '!' tags to mark the types of objects, but the fully-qualified
+"tag:nltk.org,2011:" prefix is also accepted in case anyone ends up
+using it.
+"""
+
+import json
+
+json_tags = {}
+
+TAG_PREFIX = '!'
+
+def register_tag(cls):
+ """
+ Decorates a class to register it's json tag.
+ """
+ json_tags[TAG_PREFIX+getattr(cls, 'json_tag')] = cls
+ return cls
+
+class JSONTaggedEncoder(json.JSONEncoder):
+ def default(self, obj):
+ obj_tag = getattr(obj, 'json_tag', None)
+ if obj_tag is None:
+ return super(JSONTaggedEncoder, self).default(obj)
+ obj_tag = TAG_PREFIX + obj_tag
+ obj = obj.encode_json_obj()
+ return {obj_tag: obj}
+
+class JSONTaggedDecoder(json.JSONDecoder):
+ def decode(self, s):
+ return self.decode_obj(super(JSONTaggedDecoder, self).decode(s))
+
+ @classmethod
+ def decode_obj(cls, obj):
+ #Decode nested objects first.
+ if isinstance(obj, dict):
+ obj=dict((key, cls.decode_obj(val)) for (key, val) in obj.items())
+ elif isinstance(obj, list):
+ obj=list(cls.decode_obj(val) for val in obj)
+ #Check if we have a tagged object.
+ if not isinstance(obj, dict) or len(obj) != 1:
+ return obj
+ obj_tag = next(iter(obj.keys()))
+ if not obj_tag.startswith('!'):
+ return obj
+ if not obj_tag in json_tags:
+ raise ValueError('Unknown tag', obj_tag)
+ obj_cls = json_tags[obj_tag]
+ return obj_cls.decode_json_obj(obj[obj_tag])
+
+__all__ = ['register_tag', 'json_tags',
+ 'JSONTaggedEncoder', 'JSONTaggedDecoder']
diff --git a/nltk/lazyimport.py b/nltk/lazyimport.py
new file mode 100644
index 0000000..2c58878
--- /dev/null
+++ b/nltk/lazyimport.py
@@ -0,0 +1,142 @@
+# This module is from mx/DateTime/LazyModule.py and is
+# distributed under the terms of the eGenix.com Public License Agreement
+# http://www.egenix.com/products/eGenix.com-Public-License-1.1.0.pdf
+
+""" Helper to enable simple lazy module import.
+
+ 'Lazy' means the actual import is deferred until an attribute is
+ requested from the module's namespace. This has the advantage of
+ allowing all imports to be done at the top of a script (in a
+ prominent and visible place) without having a great impact
+ on startup time.
+
+ Copyright (c) 1999-2005, Marc-Andre Lemburg; mailto:mal at lemburg.com
+ See the documentation for further information on copyrights,
+ or contact the author. All Rights Reserved.
+"""
+from __future__ import print_function
+
+### Constants
+
+_debug = 0
+
+###
+
+class LazyModule:
+
+ """ Lazy module class.
+
+ Lazy modules are imported into the given namespaces whenever a
+ non-special attribute (there are some attributes like __doc__
+ that class instances handle without calling __getattr__) is
+ requested. The module is then registered under the given name
+ in locals usually replacing the import wrapper instance. The
+ import itself is done using globals as global namespace.
+
+ Example of creating a lazy load module:
+
+ ISO = LazyModule('ISO',locals(),globals())
+
+ Later, requesting an attribute from ISO will load the module
+ automatically into the locals() namespace, overriding the
+ LazyModule instance:
+
+ t = ISO.Week(1998,1,1)
+
+ """
+ # Flag which inidicates whether the LazyModule is initialized or not
+ __lazymodule_init = 0
+
+ # Name of the module to load
+ __lazymodule_name = ''
+
+ # Flag which indicates whether the module was loaded or not
+ __lazymodule_loaded = 0
+
+ # Locals dictionary where to register the module
+ __lazymodule_locals = None
+
+ # Globals dictionary to use for the module import
+ __lazymodule_globals = None
+
+ def __init__(self, name, locals, globals=None):
+
+ """ Create a LazyModule instance wrapping module name.
+
+ The module will later on be registered in locals under the
+ given module name.
+
+ globals is optional and defaults to locals.
+
+ """
+ self.__lazymodule_locals = locals
+ if globals is None:
+ globals = locals
+ self.__lazymodule_globals = globals
+ mainname = globals.get('__name__', '')
+ if mainname:
+ self.__name__ = mainname + '.' + name
+ self.__lazymodule_name = name
+ else:
+ self.__name__ = self.__lazymodule_name = name
+ self.__lazymodule_init = 1
+
+ def __lazymodule_import(self):
+
+ """ Import the module now.
+ """
+ # Load and register module
+ name = self.__lazymodule_name
+ if self.__lazymodule_loaded:
+ return self.__lazymodule_locals[name]
+ if _debug:
+ print('LazyModule: Loading module %r' % name)
+ self.__lazymodule_locals[name] \
+ = module \
+ = __import__(name,
+ self.__lazymodule_locals,
+ self.__lazymodule_globals,
+ '*')
+
+ # Fill namespace with all symbols from original module to
+ # provide faster access.
+ self.__dict__.update(module.__dict__)
+
+ # Set import flag
+ self.__dict__['__lazymodule_loaded'] = 1
+
+ if _debug:
+ print('LazyModule: Module %r loaded' % name)
+ return module
+
+ def __getattr__(self, name):
+
+ """ Import the module on demand and get the attribute.
+ """
+ if self.__lazymodule_loaded:
+ raise AttributeError(name)
+ if _debug:
+ print('LazyModule: ' \
+ 'Module load triggered by attribute %r read access' % name)
+ module = self.__lazymodule_import()
+ return getattr(module, name)
+
+ def __setattr__(self, name, value):
+
+ """ Import the module on demand and set the attribute.
+ """
+ if not self.__lazymodule_init:
+ self.__dict__[name] = value
+ return
+ if self.__lazymodule_loaded:
+ self.__lazymodule_locals[self.__lazymodule_name] = value
+ self.__dict__[name] = value
+ return
+ if _debug:
+ print('LazyModule: ' \
+ 'Module load triggered by attribute %r write access' % name)
+ module = self.__lazymodule_import()
+ setattr(module, name, value)
+
+ def __repr__(self):
+ return "<LazyModule '%s'>" % self.__name__
diff --git a/nltk/metrics/__init__.py b/nltk/metrics/__init__.py
new file mode 100644
index 0000000..46981d4
--- /dev/null
+++ b/nltk/metrics/__init__.py
@@ -0,0 +1,28 @@
+# Natural Language Toolkit: Metrics
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+"""
+NLTK Metrics
+
+Classes and methods for scoring processing modules.
+"""
+
+from nltk.metrics.scores import (accuracy, precision, recall, f_measure,
+ log_likelihood, approxrand)
+from nltk.metrics.confusionmatrix import ConfusionMatrix
+from nltk.metrics.distance import (edit_distance, binary_distance,
+ jaccard_distance, masi_distance,
+ interval_distance, custom_distance,
+ presence, fractional_presence)
+from nltk.metrics.segmentation import windowdiff, ghd, pk
+from nltk.metrics.agreement import AnnotationTask
+from nltk.metrics.association import (NgramAssocMeasures, BigramAssocMeasures,
+ TrigramAssocMeasures, ContingencyMeasures)
+from nltk.metrics.spearman import (spearman_correlation, ranks_from_sequence,
+ ranks_from_scores)
diff --git a/nltk/metrics/agreement.py b/nltk/metrics/agreement.py
new file mode 100644
index 0000000..5ea7d98
--- /dev/null
+++ b/nltk/metrics/agreement.py
@@ -0,0 +1,401 @@
+# Natural Language Toolkit: Agreement Metrics
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Tom Lippincott <tom at cs.columbia.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+"""
+Implementations of inter-annotator agreement coefficients surveyed by Artstein
+and Poesio (2007), Inter-Coder Agreement for Computational Linguistics.
+
+An agreement coefficient calculates the amount that annotators agreed on label
+assignments beyond what is expected by chance.
+
+In defining the AnnotationTask class, we use naming conventions similar to the
+paper's terminology. There are three types of objects in an annotation task:
+
+ the coders (variables "c" and "C")
+ the items to be annotated (variables "i" and "I")
+ the potential categories to be assigned (variables "k" and "K")
+
+Additionally, it is often the case that we don't want to treat two different
+labels as complete disagreement, and so the AnnotationTask constructor can also
+take a distance metric as a final argument. Distance metrics are simply
+functions that take two arguments, and return a value between 0.0 and 1.0
+indicating the distance between them. If not supplied, the default is binary
+comparison between the arguments.
+
+The simplest way to initialize an AnnotationTask is with a list of triples,
+each containing a coder's assignment for one object in the task:
+
+ task = AnnotationTask(data=[('c1', '1', 'v1'),('c2', '1', 'v1'),...])
+
+Note that the data list needs to contain the same number of triples for each
+individual coder, containing category values for the same set of items.
+
+Alpha (Krippendorff 1980)
+Kappa (Cohen 1960)
+S (Bennet, Albert and Goldstein 1954)
+Pi (Scott 1955)
+
+
+TODO: Describe handling of multiple coders and missing data
+
+Expected results from the Artstein and Poesio survey paper:
+
+ >>> from nltk.metrics.agreement import AnnotationTask
+ >>> import os.path
+ >>> t = AnnotationTask(data=[x.split() for x in open(os.path.join(os.path.dirname(__file__), "artstein_poesio_example.txt"))])
+ >>> t.avg_Ao()
+ 0.88
+ >>> t.pi()
+ 0.7995322418977615...
+ >>> t.S()
+ 0.8199999999999998...
+
+ This would have returned a wrong value (0.0) in @785fb79 as coders are in
+ the wrong order. Subsequently, all values for pi(), S(), and kappa() would
+ have been wrong as they are computed with avg_Ao().
+ >>> t2 = AnnotationTask(data=[('b','1','stat'),('a','1','stat')])
+ >>> t2.avg_Ao()
+ 1.0
+
+ The following, of course, also works.
+ >>> t3 = AnnotationTask(data=[('a','1','othr'),('b','1','othr')])
+ >>> t3.avg_Ao()
+ 1.0
+
+"""
+from __future__ import print_function, unicode_literals
+
+import logging
+from itertools import groupby
+from operator import itemgetter
+
+from nltk.probability import FreqDist, ConditionalFreqDist
+from nltk.internals import deprecated
+from nltk.compat import python_2_unicode_compatible, iteritems
+
+from nltk.metrics.distance import binary_distance
+
+log = logging.getLogger(__file__)
+
+ at python_2_unicode_compatible
+class AnnotationTask(object):
+ """Represents an annotation task, i.e. people assign labels to items.
+
+ Notation tries to match notation in Artstein and Poesio (2007).
+
+ In general, coders and items can be represented as any hashable object.
+ Integers, for example, are fine, though strings are more readable.
+ Labels must support the distance functions applied to them, so e.g.
+ a string-edit-distance makes no sense if your labels are integers,
+ whereas interval distance needs numeric values. A notable case of this
+ is the MASI metric, which requires Python sets.
+ """
+
+ def __init__(self, data=None, distance=binary_distance):
+ """Initialize an empty annotation task.
+
+ """
+ self.distance = distance
+ self.I = set()
+ self.K = set()
+ self.C = set()
+ self.data = []
+ if data is not None:
+ self.load_array(data)
+
+ def __str__(self):
+ return "\r\n".join(map(lambda x:"%s\t%s\t%s" %
+ (x['coder'], x['item'].replace('_', "\t"),
+ ",".join(x['labels'])), self.data))
+
+ def load_array(self, array):
+ """Load the results of annotation.
+
+ The argument is a list of 3-tuples, each representing a coder's labeling of an item:
+ (coder,item,label)
+ """
+ for coder, item, labels in array:
+ self.C.add(coder)
+ self.K.add(labels)
+ self.I.add(item)
+ self.data.append({'coder':coder, 'labels':labels, 'item':item})
+
+ def agr(self, cA, cB, i, data=None):
+ """Agreement between two coders on a given item
+
+ """
+ data = data or self.data
+ # cfedermann: we don't know what combination of coder/item will come
+ # first in x; to avoid StopIteration problems due to assuming an order
+ # cA,cB, we allow either for k1 and then look up the missing as k2.
+ k1 = next((x for x in data if x['coder'] in (cA,cB) and x['item']==i))
+ if k1['coder'] == cA:
+ k2 = next((x for x in data if x['coder']==cB and x['item']==i))
+ else:
+ k2 = next((x for x in data if x['coder']==cA and x['item']==i))
+
+ ret = 1.0 - float(self.distance(k1['labels'], k2['labels']))
+ log.debug("Observed agreement between %s and %s on %s: %f",
+ cA, cB, i, ret)
+ log.debug("Distance between \"%r\" and \"%r\": %f",
+ k1['labels'], k2['labels'], 1.0 - ret)
+ return ret
+
+ def Nk(self, k):
+ return float(sum(1 for x in self.data if x['labels'] == k))
+
+ def Nik(self, i, k):
+ return float(sum(1 for x in self.data if x['item'] == i and x['labels'] == k))
+
+ def Nck(self, c, k):
+ return float(sum(1 for x in self.data if x['coder'] == c and x['labels'] == k))
+
+ @deprecated('Use Nk, Nik or Nck instead')
+ def N(self, k=None, i=None, c=None):
+ """Implements the "n-notation" used in Artstein and Poesio (2007)
+
+ """
+ if k is not None and i is None and c is None:
+ ret = self.Nk(k)
+ elif k is not None and i is not None and c is None:
+ ret = self.Nik(i, k)
+ elif k is not None and c is not None and i is None:
+ ret = self.Nck(c, k)
+ else:
+ raise ValueError("You must pass either i or c, not both! (k=%r,i=%r,c=%r)" % (k, i, c))
+ log.debug("Count on N[%s,%s,%s]: %d", k, i, c, ret)
+ return ret
+
+ def _grouped_data(self, field, data=None):
+ data = data or self.data
+ return groupby(sorted(data, key=itemgetter(field)), itemgetter(field))
+
+ def Ao(self, cA, cB):
+ """Observed agreement between two coders on all items.
+
+ """
+ data = self._grouped_data('item', (x for x in self.data if x['coder'] in (cA, cB)))
+ ret = float(sum(self.agr(cA, cB, item, item_data) for item, item_data in data)) / float(len(self.I))
+ log.debug("Observed agreement between %s and %s: %f", cA, cB, ret)
+ return ret
+
+ def _pairwise_average(self, function):
+ """
+ Calculates the average of function results for each coder pair
+ """
+ total = 0
+ n = 0
+ s = self.C.copy()
+ for cA in self.C:
+ s.remove(cA)
+ for cB in s:
+ total += function(cA, cB)
+ n += 1
+ ret = total / n
+ return ret
+
+ def avg_Ao(self):
+ """Average observed agreement across all coders and items.
+
+ """
+ ret = self._pairwise_average(self.Ao)
+ log.debug("Average observed agreement: %f", ret)
+ return ret
+
+ def Do_alpha(self):
+ """The observed disagreement for the alpha coefficient.
+
+ The alpha coefficient, unlike the other metrics, uses this rather than
+ observed agreement.
+ """
+ total = 0.0
+ for i, itemdata in self._grouped_data('item'):
+ label_freqs = FreqDist(x['labels'] for x in itemdata)
+
+ for j, nj in iteritems(label_freqs):
+ for l, nl in iteritems(label_freqs):
+ total += float(nj * nl) * self.distance(l, j)
+ ret = (1.0 / float((len(self.I) * len(self.C) * (len(self.C) - 1)))) * total
+ log.debug("Observed disagreement: %f", ret)
+ return ret
+
+ def Do_Kw_pairwise(self,cA,cB,max_distance=1.0):
+ """The observed disagreement for the weighted kappa coefficient.
+
+ """
+ total = 0.0
+ data = (x for x in self.data if x['coder'] in (cA, cB))
+ for i, itemdata in self._grouped_data('item', data):
+ # we should have two items; distance doesn't care which comes first
+ total += self.distance(itemdata.next()['labels'],
+ itemdata.next()['labels'])
+
+ ret = total / (len(self.I) * max_distance)
+ log.debug("Observed disagreement between %s and %s: %f", cA, cB, ret)
+ return ret
+
+ def Do_Kw(self, max_distance=1.0):
+ """Averaged over all labelers
+
+ """
+ ret = self._pairwise_average(lambda cA, cB: self.Do_Kw_pairwise(cA, cB, max_distance))
+ log.debug("Observed disagreement: %f", ret)
+ return ret
+
+ # Agreement Coefficients
+ def S(self):
+ """Bennett, Albert and Goldstein 1954
+
+ """
+ Ae = 1.0 / float(len(self.K))
+ ret = (self.avg_Ao() - Ae) / (1.0 - Ae)
+ return ret
+
+ def pi(self):
+ """Scott 1955; here, multi-pi.
+ Equivalent to K from Siegel and Castellan (1988).
+
+ """
+ total = 0.0
+ label_freqs = FreqDist(x['labels'] for x in self.data)
+ for k, f in iteritems(label_freqs):
+ total += f ** 2
+ Ae = total / float((len(self.I) * len(self.C)) ** 2)
+ return (self.avg_Ao() - Ae) / (1 - Ae)
+
+ def Ae_kappa(self, cA, cB):
+ Ae = 0.0
+ nitems = float(len(self.I))
+ label_freqs = ConditionalFreqDist((x['labels'], x['coder']) for x in self.data)
+ for k in label_freqs.conditions():
+ Ae += (label_freqs[k][cA] / nitems) * (label_freqs[k][cB] / nitems)
+ return Ae
+
+ def kappa_pairwise(self, cA, cB):
+ """
+
+ """
+ Ae = self.Ae_kappa(cA, cB)
+ ret = (self.Ao(cA, cB) - Ae) / (1.0 - Ae)
+ log.debug("Expected agreement between %s and %s: %f", cA, cB, Ae)
+ return ret
+
+ def kappa(self):
+ """Cohen 1960
+ Averages naively over kappas for each coder pair.
+
+ """
+ return self._pairwise_average(self.kappa_pairwise)
+
+ def multi_kappa(self):
+ """Davies and Fleiss 1982
+ Averages over observed and expected agreements for each coder pair.
+
+ """
+ Ae = self._pairwise_average(self.Ae_kappa)
+ return (self.avg_Ao() - Ae) / (1.0 - Ae)
+
+ def alpha(self):
+ """Krippendorff 1980
+
+ """
+ De = 0.0
+
+ label_freqs = FreqDist(x['labels'] for x in self.data)
+ for j in self.K:
+ nj = label_freqs[j]
+ for l in self.K:
+ De += float(nj * label_freqs[l]) * self.distance(j, l)
+ De = (1.0 / (len(self.I) * len(self.C) * (len(self.I) * len(self.C) - 1))) * De
+ log.debug("Expected disagreement: %f", De)
+ ret = 1.0 - (self.Do_alpha() / De)
+ return ret
+
+ def weighted_kappa_pairwise(self, cA, cB, max_distance=1.0):
+ """Cohen 1968
+
+ """
+ total = 0.0
+ label_freqs = ConditionalFreqDist((x['coder'], x['labels'])
+ for x in self.data
+ if x['coder'] in (cA, cB))
+ for j in self.K:
+ for l in self.K:
+ total += label_freqs[cA][j] * label_freqs[cB][l] * self.distance(j, l)
+ De = total / (max_distance * pow(len(self.I), 2))
+ log.debug("Expected disagreement between %s and %s: %f", cA, cB, De)
+ Do = self.Do_Kw_pairwise(cA, cB)
+ ret = 1.0 - (Do / De)
+ return ret
+
+ def weighted_kappa(self, max_distance=1.0):
+ """Cohen 1968
+
+ """
+ return self._pairwise_average(lambda cA, cB: self.weighted_kappa_pairwise(cA, cB, max_distance))
+
+
+if __name__ == '__main__':
+
+ import re
+ import optparse
+ from nltk.metrics import distance
+
+ # process command-line arguments
+ parser = optparse.OptionParser()
+ parser.add_option("-d", "--distance", dest="distance", default="binary_distance",
+ help="distance metric to use")
+ parser.add_option("-a", "--agreement", dest="agreement", default="kappa",
+ help="agreement coefficient to calculate")
+ parser.add_option("-e", "--exclude", dest="exclude", action="append",
+ default=[], help="coder names to exclude (may be specified multiple times)")
+ parser.add_option("-i", "--include", dest="include", action="append", default=[],
+ help="coder names to include, same format as exclude")
+ parser.add_option("-f", "--file", dest="file",
+ help="file to read labelings from, each line with three columns: 'labeler item labels'")
+ parser.add_option("-v", "--verbose", dest="verbose", default='0',
+ help="how much debugging to print on stderr (0-4)")
+ parser.add_option("-c", "--columnsep", dest="columnsep", default="\t",
+ help="char/string that separates the three columns in the file, defaults to tab")
+ parser.add_option("-l", "--labelsep", dest="labelsep", default=",",
+ help="char/string that separates labels (if labelers can assign more than one), defaults to comma")
+ parser.add_option("-p", "--presence", dest="presence", default=None,
+ help="convert each labeling into 1 or 0, based on presence of LABEL")
+ parser.add_option("-T", "--thorough", dest="thorough", default=False, action="store_true",
+ help="calculate agreement for every subset of the annotators")
+ (options, remainder) = parser.parse_args()
+
+ if not options.file:
+ parser.print_help()
+ exit()
+
+ logging.basicConfig(level=50 - 10 * int(options.verbose))
+
+ # read in data from the specified file
+ data = []
+ with open(options.file, 'r') as infile:
+ for l in infile:
+ toks = l.split(options.columnsep)
+ coder, object_, labels = toks[0], str(toks[1:-1]), frozenset(toks[-1].strip().split(options.labelsep))
+ if ((options.include == options.exclude) or
+ (len(options.include) > 0 and coder in options.include) or
+ (len(options.exclude) > 0 and coder not in options.exclude)):
+ data.append((coder, object_, labels))
+
+ if options.presence:
+ task = AnnotationTask(data, getattr(distance, options.distance)(options.presence))
+ else:
+ task = AnnotationTask(data, getattr(distance, options.distance))
+
+ if options.thorough:
+ pass
+ else:
+ print(getattr(task, options.agreement)())
+
+ logging.shutdown()
+
diff --git a/nltk/metrics/artstein_poesio_example.txt b/nltk/metrics/artstein_poesio_example.txt
new file mode 100644
index 0000000..cad1b46
--- /dev/null
+++ b/nltk/metrics/artstein_poesio_example.txt
@@ -0,0 +1,200 @@
+a 1 stat
+b 1 stat
+a 2 stat
+b 2 stat
+a 3 stat
+b 3 stat
+a 4 stat
+b 4 stat
+a 5 stat
+b 5 stat
+a 6 stat
+b 6 stat
+a 7 stat
+b 7 stat
+a 8 stat
+b 8 stat
+a 9 stat
+b 9 stat
+a 10 stat
+b 10 stat
+a 11 stat
+b 11 stat
+a 12 stat
+b 12 stat
+a 13 stat
+b 13 stat
+a 14 stat
+b 14 stat
+a 15 stat
+b 15 stat
+a 16 stat
+b 16 stat
+a 17 stat
+b 17 stat
+a 18 stat
+b 18 stat
+a 19 stat
+b 19 stat
+a 20 stat
+b 20 stat
+a 21 stat
+b 21 stat
+a 22 stat
+b 22 stat
+a 23 stat
+b 23 stat
+a 24 stat
+b 24 stat
+a 25 stat
+b 25 stat
+a 26 stat
+b 26 stat
+a 27 stat
+b 27 stat
+a 28 stat
+b 28 stat
+a 29 stat
+b 29 stat
+a 30 stat
+b 30 stat
+a 31 stat
+b 31 stat
+a 32 stat
+b 32 stat
+a 33 stat
+b 33 stat
+a 34 stat
+b 34 stat
+a 35 stat
+b 35 stat
+a 36 stat
+b 36 stat
+a 37 stat
+b 37 stat
+a 38 stat
+b 38 stat
+a 39 stat
+b 39 stat
+a 40 stat
+b 40 stat
+a 41 stat
+b 41 stat
+a 42 stat
+b 42 stat
+a 43 stat
+b 43 stat
+a 44 stat
+b 44 stat
+a 45 stat
+b 45 stat
+a 46 stat
+b 46 stat
+a 47 ireq
+b 47 stat
+a 48 ireq
+b 48 stat
+a 49 ireq
+b 49 stat
+a 50 ireq
+b 50 stat
+a 51 ireq
+b 51 stat
+a 52 ireq
+b 52 stat
+a 53 ireq
+b 53 ireq
+a 54 ireq
+b 54 ireq
+a 55 ireq
+b 55 ireq
+a 56 ireq
+b 56 ireq
+a 57 ireq
+b 57 ireq
+a 58 ireq
+b 58 ireq
+a 59 ireq
+b 59 ireq
+a 60 ireq
+b 60 ireq
+a 61 ireq
+b 61 ireq
+a 62 ireq
+b 62 ireq
+a 63 ireq
+b 63 ireq
+a 64 ireq
+b 64 ireq
+a 65 ireq
+b 65 ireq
+a 66 ireq
+b 66 ireq
+a 67 ireq
+b 67 ireq
+a 68 ireq
+b 68 ireq
+a 69 ireq
+b 69 ireq
+a 70 ireq
+b 70 ireq
+a 71 ireq
+b 71 ireq
+a 72 ireq
+b 72 ireq
+a 73 ireq
+b 73 ireq
+a 74 ireq
+b 74 ireq
+a 75 ireq
+b 75 ireq
+a 76 ireq
+b 76 ireq
+a 77 ireq
+b 77 ireq
+a 78 ireq
+b 78 ireq
+a 79 ireq
+b 79 ireq
+a 80 ireq
+b 80 ireq
+a 81 ireq
+b 81 ireq
+a 82 ireq
+b 82 ireq
+a 83 ireq
+b 83 ireq
+a 84 ireq
+b 84 ireq
+a 85 ireq
+b 85 chck
+a 86 ireq
+b 86 chck
+a 87 ireq
+b 87 chck
+a 88 ireq
+b 88 chck
+a 89 ireq
+b 89 chck
+a 90 ireq
+b 90 chck
+a 91 chck
+b 91 chck
+a 92 chck
+b 92 chck
+a 93 chck
+b 93 chck
+a 94 chck
+b 94 chck
+a 95 chck
+b 95 chck
+a 96 chck
+b 96 chck
+a 97 chck
+b 97 chck
+a 98 chck
+b 98 chck
+a 99 chck
+b 99 chck
+a 100 chck
+b 100 chck
diff --git a/nltk/metrics/association.py b/nltk/metrics/association.py
new file mode 100644
index 0000000..febdb7d
--- /dev/null
+++ b/nltk/metrics/association.py
@@ -0,0 +1,409 @@
+# Natural Language Toolkit: Ngram Association Measures
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Joel Nothman <jnothman at student.usyd.edu.au>
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+
+"""
+Provides scoring functions for a number of association measures through a
+generic, abstract implementation in ``NgramAssocMeasures``, and n-specific
+``BigramAssocMeasures`` and ``TrigramAssocMeasures``.
+"""
+
+import math as _math
+from functools import reduce
+_log2 = lambda x: _math.log(x, 2.0)
+_ln = _math.log
+
+_product = lambda s: reduce(lambda x, y: x * y, s)
+
+_SMALL = 1e-20
+
+try:
+ from scipy.stats import fisher_exact
+except ImportError:
+ def fisher_exact(*_args, **_kwargs):
+ raise NotImplementedError
+
+### Indices to marginals arguments:
+
+NGRAM = 0
+"""Marginals index for the ngram count"""
+
+UNIGRAMS = -2
+"""Marginals index for a tuple of each unigram count"""
+
+TOTAL = -1
+"""Marginals index for the number of words in the data"""
+
+
+class NgramAssocMeasures(object):
+ """
+ An abstract class defining a collection of generic association measures.
+ Each public method returns a score, taking the following arguments::
+
+ score_fn(count_of_ngram,
+ (count_of_n-1gram_1, ..., count_of_n-1gram_j),
+ (count_of_n-2gram_1, ..., count_of_n-2gram_k),
+ ...,
+ (count_of_1gram_1, ..., count_of_1gram_n),
+ count_of_total_words)
+
+ See ``BigramAssocMeasures`` and ``TrigramAssocMeasures``
+
+ Inheriting classes should define a property _n, and a method _contingency
+ which calculates contingency values from marginals in order for all
+ association measures defined here to be usable.
+ """
+
+ _n = 0
+
+ @staticmethod
+ def _contingency(*marginals):
+ """Calculates values of a contingency table from marginal values."""
+ raise NotImplementedError("The contingency table is not available"
+ "in the general ngram case")
+
+ @staticmethod
+ def _marginals(*contingency):
+ """Calculates values of contingency table marginals from its values."""
+ raise NotImplementedError("The contingency table is not available"
+ "in the general ngram case")
+
+ @classmethod
+ def _expected_values(cls, cont):
+ """Calculates expected values for a contingency table."""
+ n_all = sum(cont)
+ bits = [1 << i for i in range(cls._n)]
+
+ # For each contingency table cell
+ for i in range(len(cont)):
+ # Yield the expected value
+ yield (_product(sum(cont[x] for x in range(2 ** cls._n)
+ if (x & j) == (i & j))
+ for j in bits) /
+ float(n_all ** (cls._n - 1)))
+
+ @staticmethod
+ def raw_freq(*marginals):
+ """Scores ngrams by their frequency"""
+ return float(marginals[NGRAM]) / marginals[TOTAL]
+
+ @classmethod
+ def student_t(cls, *marginals):
+ """Scores ngrams using Student's t test with independence hypothesis
+ for unigrams, as in Manning and Schutze 5.3.1.
+ """
+ return ((marginals[NGRAM] -
+ _product(marginals[UNIGRAMS]) /
+ float(marginals[TOTAL] ** (cls._n - 1))) /
+ (marginals[NGRAM] + _SMALL) ** .5)
+
+ @classmethod
+ def chi_sq(cls, *marginals):
+ """Scores ngrams using Pearson's chi-square as in Manning and Schutze
+ 5.3.3.
+ """
+ cont = cls._contingency(*marginals)
+ exps = cls._expected_values(cont)
+ return sum((obs - exp) ** 2 / (exp + _SMALL)
+ for obs, exp in zip(cont, exps))
+
+ @staticmethod
+ def mi_like(*marginals, **kwargs):
+ """Scores ngrams using a variant of mutual information. The keyword
+ argument power sets an exponent (default 3) for the numerator. No
+ logarithm of the result is calculated.
+ """
+ return (marginals[NGRAM] ** kwargs.get('power', 3) /
+ float(_product(marginals[UNIGRAMS])))
+
+ @classmethod
+ def pmi(cls, *marginals):
+ """Scores ngrams by pointwise mutual information, as in Manning and
+ Schutze 5.4.
+ """
+ return (_log2(marginals[NGRAM] * marginals[TOTAL] ** (cls._n - 1)) -
+ _log2(_product(marginals[UNIGRAMS])))
+
+ @classmethod
+ def likelihood_ratio(cls, *marginals):
+ """Scores ngrams using likelihood ratios as in Manning and Schutze 5.3.4.
+ """
+ cont = cls._contingency(*marginals)
+ return (cls._n *
+ sum(obs * _ln(float(obs) / (exp + _SMALL) + _SMALL)
+ for obs, exp in zip(cont, cls._expected_values(cont))))
+
+ @classmethod
+ def poisson_stirling(cls, *marginals):
+ """Scores ngrams using the Poisson-Stirling measure."""
+ exp = (_product(marginals[UNIGRAMS]) /
+ float(marginals[TOTAL] ** (cls._n - 1)))
+ return marginals[NGRAM] * (_log2(marginals[NGRAM] / exp) - 1)
+
+ @classmethod
+ def jaccard(cls, *marginals):
+ """Scores ngrams using the Jaccard index."""
+ cont = cls._contingency(*marginals)
+ return float(cont[0]) / sum(cont[:-1])
+
+
+class BigramAssocMeasures(NgramAssocMeasures):
+ """
+ A collection of bigram association measures. Each association measure
+ is provided as a function with three arguments::
+
+ bigram_score_fn(n_ii, (n_ix, n_xi), n_xx)
+
+ The arguments constitute the marginals of a contingency table, counting
+ the occurrences of particular events in a corpus. The letter i in the
+ suffix refers to the appearance of the word in question, while x indicates
+ the appearance of any word. Thus, for example:
+
+ n_ii counts (w1, w2), i.e. the bigram being scored
+ n_ix counts (w1, *)
+ n_xi counts (*, w2)
+ n_xx counts (*, *), i.e. any bigram
+
+ This may be shown with respect to a contingency table::
+
+ w1 ~w1
+ ------ ------
+ w2 | n_ii | n_oi | = n_xi
+ ------ ------
+ ~w2 | n_io | n_oo |
+ ------ ------
+ = n_ix TOTAL = n_xx
+ """
+
+ _n = 2
+
+ @staticmethod
+ def _contingency(n_ii, n_ix_xi_tuple, n_xx):
+ """Calculates values of a bigram contingency table from marginal values."""
+ (n_ix, n_xi) = n_ix_xi_tuple
+ n_oi = n_xi - n_ii
+ n_io = n_ix - n_ii
+ return (n_ii, n_oi, n_io, n_xx - n_ii - n_oi - n_io)
+
+ @staticmethod
+ def _marginals(n_ii, n_oi, n_io, n_oo):
+ """Calculates values of contingency table marginals from its values."""
+ return (n_ii, (n_oi + n_ii, n_io + n_ii), n_oo + n_oi + n_io + n_ii)
+
+ @staticmethod
+ def _expected_values(cont):
+ """Calculates expected values for a contingency table."""
+ n_xx = sum(cont)
+ # For each contingency table cell
+ for i in range(4):
+ yield (cont[i] + cont[i ^ 1]) * (cont[i] + cont[i ^ 2]) / float(n_xx)
+
+ @classmethod
+ def phi_sq(cls, *marginals):
+ """Scores bigrams using phi-square, the square of the Pearson correlation
+ coefficient.
+ """
+ n_ii, n_io, n_oi, n_oo = cls._contingency(*marginals)
+
+ return (float((n_ii*n_oo - n_io*n_oi)**2) /
+ ((n_ii + n_io) * (n_ii + n_oi) * (n_io + n_oo) * (n_oi + n_oo)))
+
+ @classmethod
+ def chi_sq(cls, n_ii, n_ix_xi_tuple, n_xx):
+ """Scores bigrams using chi-square, i.e. phi-sq multiplied by the number
+ of bigrams, as in Manning and Schutze 5.3.3.
+ """
+ (n_ix, n_xi) = n_ix_xi_tuple
+ return n_xx * cls.phi_sq(n_ii, (n_ix, n_xi), n_xx)
+
+ @classmethod
+ def fisher(cls, *marginals):
+ """Scores bigrams using Fisher's Exact Test (Pedersen 1996). Less
+ sensitive to small counts than PMI or Chi Sq, but also more expensive
+ to compute. Requires scipy.
+ """
+
+ n_ii, n_io, n_oi, n_oo = cls._contingency(*marginals)
+
+ (odds, pvalue) = fisher_exact([[n_ii, n_io], [n_oi, n_oo]], alternative='less')
+ return pvalue
+
+ @staticmethod
+ def dice(n_ii, n_ix_xi_tuple, n_xx):
+ """Scores bigrams using Dice's coefficient."""
+ (n_ix, n_xi) = n_ix_xi_tuple
+ return 2 * float(n_ii) / (n_ix + n_xi)
+
+
+class TrigramAssocMeasures(NgramAssocMeasures):
+ """
+ A collection of trigram association measures. Each association measure
+ is provided as a function with four arguments::
+
+ trigram_score_fn(n_iii,
+ (n_iix, n_ixi, n_xii),
+ (n_ixx, n_xix, n_xxi),
+ n_xxx)
+
+ The arguments constitute the marginals of a contingency table, counting
+ the occurrences of particular events in a corpus. The letter i in the
+ suffix refers to the appearance of the word in question, while x indicates
+ the appearance of any word. Thus, for example:
+ n_iii counts (w1, w2, w3), i.e. the trigram being scored
+ n_ixx counts (w1, *, *)
+ n_xxx counts (*, *, *), i.e. any trigram
+ """
+
+ _n = 3
+
+ @staticmethod
+ def _contingency(n_iii, n_iix_tuple, n_ixx_tuple, n_xxx):
+ """Calculates values of a trigram contingency table (or cube) from
+ marginal values.
+ >>> TrigramAssocMeasures._contingency(1, (1, 1, 1), (1, 73, 1), 2000)
+ (1, 0, 0, 0, 0, 72, 0, 1927)
+ """
+ (n_iix, n_ixi, n_xii) = n_iix_tuple
+ (n_ixx, n_xix, n_xxi) = n_ixx_tuple
+ n_oii = n_xii - n_iii
+ n_ioi = n_ixi - n_iii
+ n_iio = n_iix - n_iii
+ n_ooi = n_xxi - n_iii - n_oii - n_ioi
+ n_oio = n_xix - n_iii - n_oii - n_iio
+ n_ioo = n_ixx - n_iii - n_ioi - n_iio
+ n_ooo = n_xxx - n_iii - n_oii - n_ioi - n_iio - n_ooi - n_oio - n_ioo
+
+ return (n_iii, n_oii, n_ioi, n_ooi,
+ n_iio, n_oio, n_ioo, n_ooo)
+
+ @staticmethod
+ def _marginals(*contingency):
+ """Calculates values of contingency table marginals from its values.
+ >>> TrigramAssocMeasures._marginals(1, 0, 0, 0, 0, 72, 0, 1927)
+ (1, (1, 1, 1), (1, 73, 1), 2000)
+ """
+ n_iii, n_oii, n_ioi, n_ooi, n_iio, n_oio, n_ioo, n_ooo = contingency
+ return (n_iii,
+ (n_iii + n_iio, n_iii + n_ioi, n_iii + n_oii),
+ (n_iii + n_ioi + n_iio + n_ioo,
+ n_iii + n_oii + n_iio + n_oio,
+ n_iii + n_oii + n_ioi + n_ooi),
+ sum(contingency))
+
+
+class QuadgramAssocMeasures(NgramAssocMeasures):
+ """
+ A collection of quadgram association measures. Each association measure
+ is provided as a function with five arguments::
+
+ trigram_score_fn(n_iiii,
+ (n_iiix, n_iixi, n_ixii, n_xiii),
+ (n_iixx, n_ixix, n_ixxi, n_xixi, n_xxii, n_xiix),
+ (n_ixxx, n_xixx, n_xxix, n_xxxi),
+ n_all)
+
+ The arguments constitute the marginals of a contingency table, counting
+ the occurrences of particular events in a corpus. The letter i in the
+ suffix refers to the appearance of the word in question, while x indicates
+ the appearance of any word. Thus, for example:
+ n_iiii counts (w1, w2, w3, w4), i.e. the quadgram being scored
+ n_ixxi counts (w1, *, *, w4)
+ n_xxxx counts (*, *, *, *), i.e. any quadgram
+ """
+
+ _n = 4
+
+ @staticmethod
+ def _contingency(n_iiii, n_iiix_tuple, n_iixx_tuple, n_ixxx_tuple, n_xxxx):
+ """Calculates values of a quadgram contingency table from
+ marginal values.
+ """
+ (n_iiix, n_iixi, n_ixii, n_xiii) = n_iiix_tuple
+ (n_iixx, n_ixix, n_ixxi, n_xixi, n_xxii, n_xiix) = n_iixx_tuple
+ (n_ixxx, n_xixx, n_xxix, n_xxxi) = n_ixxx_tuple
+ n_oiii = n_xiii - n_iiii
+ n_ioii = n_ixii - n_iiii
+ n_iioi = n_iixi - n_iiii
+ n_ooii = n_xxii - n_iiii - n_oiii - n_ioii
+ n_oioi = n_xixi - n_iiii - n_oiii - n_iioi
+ n_iooi = n_ixxi - n_iiii - n_ioii - n_iioi
+ n_oooi = n_xxxi - n_iiii - n_oiii - n_ioii - n_iioi - n_ooii - n_iooi - n_oioi
+ n_iiio = n_iiix - n_iiii
+ n_oiio = n_xiix - n_iiii - n_oiii - n_iiio
+ n_ioio = n_ixix - n_iiii - n_ioii - n_iiio
+ n_ooio = n_xxix - n_iiii - n_oiii - n_ioii - n_iiio - n_ooii - n_ioio - n_oiio
+ n_iioo = n_iixx - n_iiii - n_iioi - n_iiio
+ n_oioo = n_xixx - n_iiii - n_oiii - n_iioi - n_iiio - n_oioi - n_oiio - n_iioo
+ n_iooo = n_ixxx - n_iiii - n_ioii - n_iioi - n_iiio - n_iooi - n_iioo - n_ioio
+ n_oooo = n_xxxx - n_iiii - n_oiii - n_ioii - n_iioi - n_ooii - n_oioi - n_iooi - \
+ n_oooi - n_iiio - n_oiio - n_ioio - n_ooio - n_iioo - n_oioo - n_iooo
+
+ return (n_iiii, n_oiii, n_ioii, n_ooii, n_iioi,
+ n_oioi, n_iooi, n_oooi, n_iiio, n_oiio,
+ n_ioio, n_ooio, n_iioo, n_oioo, n_iooo, n_oooo)
+
+ @staticmethod
+ def _marginals(*contingency):
+ """Calculates values of contingency table marginals from its values.
+ QuadgramAssocMeasures._marginals(1, 0, 2, 46, 552, 825, 2577, 34967, 1, 0, 2, 48, 7250, 9031, 28585, 356653)
+ (1, (2, 553, 3, 1), (7804, 6, 3132, 1378, 49, 2), (38970, 17660, 100, 38970), 440540)
+ """
+ n_iiii, n_oiii, n_ioii, n_ooii, n_iioi, n_oioi, n_iooi, n_oooi, n_iiio, n_oiio, n_ioio, n_ooio, \
+ n_iioo, n_oioo, n_iooo, n_oooo = contingency
+
+ n_iiix = n_iiii + n_iiio
+ n_iixi = n_iiii + n_iioi
+ n_ixii = n_iiii + n_ioii
+ n_xiii = n_iiii + n_oiii
+
+ n_iixx = n_iiii + n_iioi + n_iiio + n_iioo
+ n_ixix = n_iiii + n_ioii + n_iiio + n_ioio
+ n_ixxi = n_iiii + n_ioii + n_iioi + n_iooi
+ n_xixi = n_iiii + n_oiii + n_iioi + n_oioi
+ n_xxii = n_iiii + n_oiii + n_ioii + n_ooii
+ n_xiix = n_iiii + n_oiii + n_iiio + n_oiio
+
+ n_ixxx = n_iiii + n_ioii + n_iioi + n_iiio + n_iooi + n_iioo + n_ioio + n_iooo
+ n_xixx = n_iiii + n_oiii + n_iioi + n_iiio + n_oioi + n_oiio + n_iioo + n_oioo
+ n_xxix = n_iiii + n_oiii + n_ioii + n_iiio + n_ooii + n_ioio + n_oiio + n_ooio
+ n_xxxi = n_iiii + n_oiii + n_ioii + n_iioi + n_ooii + n_iooi + n_oioi + n_oooi
+
+ n_all = sum(contingency)
+
+ return (n_iiii,
+ (n_iiix, n_iixi, n_ixii, n_xiii),
+ (n_iixx, n_ixix, n_ixxi, n_xixi, n_xxii, n_xiix),
+ (n_ixxx, n_xixx, n_xxix, n_xxxi),
+ n_all)
+
+
+class ContingencyMeasures(object):
+ """Wraps NgramAssocMeasures classes such that the arguments of association
+ measures are contingency table values rather than marginals.
+ """
+
+ def __init__(self, measures):
+ """Constructs a ContingencyMeasures given a NgramAssocMeasures class"""
+ self.__class__.__name__ = 'Contingency' + measures.__class__.__name__
+ for k in dir(measures):
+ if k.startswith('__'):
+ continue
+ v = getattr(measures, k)
+ if not k.startswith('_'):
+ v = self._make_contingency_fn(measures, v)
+ setattr(self, k, v)
+
+ @staticmethod
+ def _make_contingency_fn(measures, old_fn):
+ """From an association measure function, produces a new function which
+ accepts contingency table values as its arguments.
+ """
+ def res(*contingency):
+ return old_fn(*measures._marginals(*contingency))
+ res.__doc__ = old_fn.__doc__
+ res.__name__ = old_fn.__name__
+ return res
+
diff --git a/nltk/metrics/confusionmatrix.py b/nltk/metrics/confusionmatrix.py
new file mode 100644
index 0000000..22a6b5f
--- /dev/null
+++ b/nltk/metrics/confusionmatrix.py
@@ -0,0 +1,206 @@
+# Natural Language Toolkit: Confusion Matrices
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+from nltk.probability import FreqDist
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class ConfusionMatrix(object):
+ """
+ The confusion matrix between a list of reference values and a
+ corresponding list of test values. Entry *[r,t]* of this
+ matrix is a count of the number of times that the reference value
+ *r* corresponds to the test value *t*. E.g.:
+
+ >>> from nltk.metrics import ConfusionMatrix
+ >>> ref = 'DET NN VB DET JJ NN NN IN DET NN'.split()
+ >>> test = 'DET VB VB DET NN NN NN IN DET NN'.split()
+ >>> cm = ConfusionMatrix(ref, test)
+ >>> print(cm['NN', 'NN'])
+ 3
+
+ Note that the diagonal entries *Ri=Tj* of this matrix
+ corresponds to correct values; and the off-diagonal entries
+ correspond to incorrect values.
+ """
+
+ def __init__(self, reference, test, sort_by_count=False):
+ """
+ Construct a new confusion matrix from a list of reference
+ values and a corresponding list of test values.
+
+ :type reference: list
+ :param reference: An ordered list of reference values.
+ :type test: list
+ :param test: A list of values to compare against the
+ corresponding reference values.
+ :raise ValueError: If ``reference`` and ``length`` do not have
+ the same length.
+ """
+ if len(reference) != len(test):
+ raise ValueError('Lists must have the same length.')
+
+ # Get a list of all values.
+ if sort_by_count:
+ ref_fdist = FreqDist(reference)
+ test_fdist = FreqDist(test)
+ def key(v): return -(ref_fdist[v]+test_fdist[v])
+ values = sorted(set(reference+test), key=key)
+ else:
+ values = sorted(set(reference+test))
+
+ # Construct a value->index dictionary
+ indices = dict((val,i) for (i,val) in enumerate(values))
+
+ # Make a confusion matrix table.
+ confusion = [[0 for val in values] for val in values]
+ max_conf = 0 # Maximum confusion
+ for w,g in zip(reference, test):
+ confusion[indices[w]][indices[g]] += 1
+ max_conf = max(max_conf, confusion[indices[w]][indices[g]])
+
+ #: A list of all values in ``reference`` or ``test``.
+ self._values = values
+ #: A dictionary mapping values in ``self._values`` to their indices.
+ self._indices = indices
+ #: The confusion matrix itself (as a list of lists of counts).
+ self._confusion = confusion
+ #: The greatest count in ``self._confusion`` (used for printing).
+ self._max_conf = max_conf
+ #: The total number of values in the confusion matrix.
+ self._total = len(reference)
+ #: The number of correct (on-diagonal) values in the matrix.
+ self._correct = sum(confusion[i][i] for i in range(len(values)))
+
+ def __getitem__(self, li_lj_tuple):
+ """
+ :return: The number of times that value ``li`` was expected and
+ value ``lj`` was given.
+ :rtype: int
+ """
+ (li, lj) = li_lj_tuple
+ i = self._indices[li]
+ j = self._indices[lj]
+ return self._confusion[i][j]
+
+ def __repr__(self):
+ return '<ConfusionMatrix: %s/%s correct>' % (self._correct,
+ self._total)
+
+ def __str__(self):
+ return self.pp()
+
+ def pp(self, show_percents=False, values_in_chart=True,
+ truncate=None, sort_by_count=False):
+ """
+ :return: A multi-line string representation of this confusion matrix.
+ :type truncate: int
+ :param truncate: If specified, then only show the specified
+ number of values. Any sorting (e.g., sort_by_count)
+ will be performed before truncation.
+ :param sort_by_count: If true, then sort by the count of each
+ label in the reference data. I.e., labels that occur more
+ frequently in the reference label will be towards the left
+ edge of the matrix, and labels that occur less frequently
+ will be towards the right edge.
+
+ @todo: add marginals?
+ """
+ confusion = self._confusion
+
+ values = self._values
+ if sort_by_count:
+ values = sorted(values, key=lambda v:
+ -sum(self._confusion[self._indices[v]]))
+
+ if truncate:
+ values = values[:truncate]
+
+ if values_in_chart:
+ value_strings = ["%s" % val for val in values]
+ else:
+ value_strings = [str(n+1) for n in range(len(values))]
+
+ # Construct a format string for row values
+ valuelen = max(len(val) for val in value_strings)
+ value_format = '%' + repr(valuelen) + 's | '
+ # Construct a format string for matrix entries
+ if show_percents:
+ entrylen = 6
+ entry_format = '%5.1f%%'
+ zerostr = ' .'
+ else:
+ entrylen = len(repr(self._max_conf))
+ entry_format = '%' + repr(entrylen) + 'd'
+ zerostr = ' '*(entrylen-1) + '.'
+
+ # Write the column values.
+ s = ''
+ for i in range(valuelen):
+ s += (' '*valuelen)+' |'
+ for val in value_strings:
+ if i >= valuelen-len(val):
+ s += val[i-valuelen+len(val)].rjust(entrylen+1)
+ else:
+ s += ' '*(entrylen+1)
+ s += ' |\n'
+
+ # Write a dividing line
+ s += '%s-+-%s+\n' % ('-'*valuelen, '-'*((entrylen+1)*len(values)))
+
+ # Write the entries.
+ for val, li in zip(value_strings, values):
+ i = self._indices[li]
+ s += value_format % val
+ for lj in values:
+ j = self._indices[lj]
+ if confusion[i][j] == 0:
+ s += zerostr
+ elif show_percents:
+ s += entry_format % (100.0*confusion[i][j]/self._total)
+ else:
+ s += entry_format % confusion[i][j]
+ if i == j:
+ prevspace = s.rfind(' ')
+ s = s[:prevspace] + '<' + s[prevspace+1:] + '>'
+ else: s += ' '
+ s += '|\n'
+
+ # Write a dividing line
+ s += '%s-+-%s+\n' % ('-'*valuelen, '-'*((entrylen+1)*len(values)))
+
+ # Write a key
+ s += '(row = reference; col = test)\n'
+ if not values_in_chart:
+ s += 'Value key:\n'
+ for i, value in enumerate(values):
+ s += '%6d: %s\n' % (i+1, value)
+
+ return s
+
+ def key(self):
+ values = self._values
+ str = 'Value key:\n'
+ indexlen = len(repr(len(values)-1))
+ key_format = ' %'+repr(indexlen)+'d: %s\n'
+ for i in range(len(values)):
+ str += key_format % (i, values[i])
+
+ return str
+
+def demo():
+ reference = 'DET NN VB DET JJ NN NN IN DET NN'.split()
+ test = 'DET VB VB DET NN NN NN IN DET NN'.split()
+ print('Reference =', reference)
+ print('Test =', test)
+ print('Confusion matrix:')
+ print(ConfusionMatrix(reference, test))
+ print(ConfusionMatrix(reference, test).pp(sort_by_count=True))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/metrics/distance.py b/nltk/metrics/distance.py
new file mode 100644
index 0000000..0e1c36e
--- /dev/null
+++ b/nltk/metrics/distance.py
@@ -0,0 +1,192 @@
+# Natural Language Toolkit: Distance Metrics
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# Tom Lippincott <tom at cs.columbia.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+"""
+Distance Metrics.
+
+Compute the distance between two items (usually strings).
+As metrics, they must satisfy the following three requirements:
+
+1. d(a, a) = 0
+2. d(a, b) >= 0
+3. d(a, c) <= d(a, b) + d(b, c)
+
+"""
+from __future__ import print_function
+
+
+def _edit_dist_init(len1, len2):
+ lev = []
+ for i in range(len1):
+ lev.append([0] * len2) # initialize 2-D array to zero
+ for i in range(len1):
+ lev[i][0] = i # column 0: 0,1,2,3,4,...
+ for j in range(len2):
+ lev[0][j] = j # row 0: 0,1,2,3,4,...
+ return lev
+
+
+def _edit_dist_step(lev, i, j, s1, s2, transpositions=False):
+ c1 = s1[i - 1]
+ c2 = s2[j - 1]
+
+ # skipping a character in s1
+ a = lev[i - 1][j] + 1
+ # skipping a character in s2
+ b = lev[i][j - 1] + 1
+ # substitution
+ c = lev[i - 1][j - 1] + (c1 != c2)
+
+ # transposition
+ d = c + 1 # never picked by default
+ if transpositions and i > 1 and j > 1:
+ if s1[i - 2] == c2 and s2[j - 2] == c1:
+ d = lev[i - 2][j - 2] + 1
+
+ # pick the cheapest
+ lev[i][j] = min(a, b, c, d)
+
+
+def edit_distance(s1, s2, transpositions=False):
+ """
+ Calculate the Levenshtein edit-distance between two strings.
+ The edit distance is the number of characters that need to be
+ substituted, inserted, or deleted, to transform s1 into s2. For
+ example, transforming "rain" to "shine" requires three steps,
+ consisting of two substitutions and one insertion:
+ "rain" -> "sain" -> "shin" -> "shine". These operations could have
+ been done in other orders, but at least three steps are needed.
+
+ This also optionally allows transposition edits (e.g., "ab" -> "ba"),
+ though this is disabled by default.
+
+ :param s1, s2: The strings to be analysed
+ :param transpositions: Whether to allow transposition edits
+ :type s1: str
+ :type s2: str
+ :type transpositions: bool
+ :rtype int
+ """
+ # set up a 2-D array
+ len1 = len(s1)
+ len2 = len(s2)
+ lev = _edit_dist_init(len1 + 1, len2 + 1)
+
+ # iterate over the array
+ for i in range(len1):
+ for j in range(len2):
+ _edit_dist_step(lev, i + 1, j + 1, s1, s2, transpositions=transpositions)
+ return lev[len1][len2]
+
+
+def binary_distance(label1, label2):
+ """Simple equality test.
+
+ 0.0 if the labels are identical, 1.0 if they are different.
+
+ >>> from nltk.metrics import binary_distance
+ >>> binary_distance(1,1)
+ 0.0
+
+ >>> binary_distance(1,3)
+ 1.0
+ """
+
+ return 0.0 if label1 == label2 else 1.0
+
+
+def jaccard_distance(label1, label2):
+ """Distance metric comparing set-similarity.
+
+ """
+ return (len(label1.union(label2)) - len(label1.intersection(label2)))/float(len(label1.union(label2)))
+
+
+def masi_distance(label1, label2):
+ """Distance metric that takes into account partial agreement when multiple
+ labels are assigned.
+
+ >>> from nltk.metrics import masi_distance
+ >>> masi_distance(set([1,2]), set([1,2,3,4]))
+ 0.665...
+
+ Passonneau 2006, Measuring Agreement on Set-Valued Items (MASI) for Semantic and Pragmatic Annotation.
+ """
+ len_intersection = len(label1.intersection(label2))
+ len_union = len(label1.union(label2))
+ len_label1 = len(label1)
+ len_label2 = len(label2)
+ if len_label1 == len_label2 and len_label1 == len_intersection:
+ m = 1
+ elif len_intersection == min(len_label1, len_label2):
+ m = 0.67
+ elif len_intersection > 0:
+ m = 0.33
+ else:
+ m = 0
+
+ return 1 - (len_intersection / float(len_union)) * m
+
+
+def interval_distance(label1,label2):
+ """Krippendorff'1 interval distance metric
+
+ >>> from nltk.metrics import interval_distance
+ >>> interval_distance(1,10)
+ 81
+
+ Krippendorff 1980, Content Analysis: An Introduction to its Methodology
+ """
+ try:
+ return pow(label1-label2,2)
+# return pow(list(label1)[0]-list(label2)[0],2)
+ except:
+ print("non-numeric labels not supported with interval distance")
+
+
+def presence(label):
+ """Higher-order function to test presence of a given label
+
+ """
+ return lambda x,y: 1.0*((label in x) == (label in y))
+
+
+def fractional_presence(label):
+ return lambda x,y:abs((float(1.0/len(x)) - float(1.0/len(y))))*(label in x and label in y) or 0.0*(label not in x and label not in y) or abs((float(1.0/len(x))))*(label in x and label not in y) or ((float(1.0/len(y))))*(label not in x and label in y)
+
+
+def custom_distance(file):
+ data = {}
+ with open(file, 'r') as infile:
+ for l in infile:
+ labelA, labelB, dist = l.strip().split("\t")
+ labelA = frozenset([labelA])
+ labelB = frozenset([labelB])
+ data[frozenset([labelA,labelB])] = float(dist)
+ return lambda x,y:data[frozenset([x,y])]
+
+
+def demo():
+ edit_distance_examples = [("rain", "shine"), ("abcdef", "acbdef"), ("language", "lnaguaeg"), ("language", "lnaugage"), ("language", "lngauage")]
+ for s1, s2 in edit_distance_examples:
+ print("Edit distance between '%s' and '%s':" % (s1, s2), edit_distance(s1, s2))
+ for s1, s2 in edit_distance_examples:
+ print("Edit distance with transpositions between '%s' and '%s':" % (s1, s2), edit_distance(s1, s2, transpositions=True))
+
+ s1 = set([1, 2, 3, 4])
+ s2 = set([3, 4, 5])
+ print("s1:", s1)
+ print("s2:", s2)
+ print("Binary distance:", binary_distance(s1, s2))
+ print("Jaccard distance:", jaccard_distance(s1, s2))
+ print("MASI distance:", masi_distance(s1, s2))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/metrics/scores.py b/nltk/metrics/scores.py
new file mode 100644
index 0000000..53cf384
--- /dev/null
+++ b/nltk/metrics/scores.py
@@ -0,0 +1,228 @@
+# Natural Language Toolkit: Evaluation
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function
+
+from math import fabs
+import operator
+from random import shuffle
+from functools import reduce
+
+try:
+ from scipy.stats.stats import betai
+except ImportError:
+ betai = None
+
+from nltk.compat import xrange, izip
+from nltk.util import LazyConcatenation, LazyMap
+
+def accuracy(reference, test):
+ """
+ Given a list of reference values and a corresponding list of test
+ values, return the fraction of corresponding values that are
+ equal. In particular, return the fraction of indices
+ ``0<i<=len(test)`` such that ``test[i] == reference[i]``.
+
+ :type reference: list
+ :param reference: An ordered list of reference values.
+ :type test: list
+ :param test: A list of values to compare against the corresponding
+ reference values.
+ :raise ValueError: If ``reference`` and ``length`` do not have the
+ same length.
+ """
+ if len(reference) != len(test):
+ raise ValueError("Lists must have the same length.")
+ return float(sum(x == y for x, y in izip(reference, test))) / len(test)
+
+def precision(reference, test):
+ """
+ Given a set of reference values and a set of test values, return
+ the fraction of test values that appear in the reference set.
+ In particular, return card(``reference`` intersection ``test``)/card(``test``).
+ If ``test`` is empty, then return None.
+
+ :type reference: set
+ :param reference: A set of reference values.
+ :type test: set
+ :param test: A set of values to compare against the reference set.
+ :rtype: float or None
+ """
+ if (not hasattr(reference, 'intersection') or
+ not hasattr(test, 'intersection')):
+ raise TypeError('reference and test should be sets')
+
+ if len(test) == 0:
+ return None
+ else:
+ return float(len(reference.intersection(test)))/len(test)
+
+def recall(reference, test):
+ """
+ Given a set of reference values and a set of test values, return
+ the fraction of reference values that appear in the test set.
+ In particular, return card(``reference`` intersection ``test``)/card(``reference``).
+ If ``reference`` is empty, then return None.
+
+ :type reference: set
+ :param reference: A set of reference values.
+ :type test: set
+ :param test: A set of values to compare against the reference set.
+ :rtype: float or None
+ """
+ if (not hasattr(reference, 'intersection') or
+ not hasattr(test, 'intersection')):
+ raise TypeError('reference and test should be sets')
+
+ if len(reference) == 0:
+ return None
+ else:
+ return float(len(reference.intersection(test)))/len(reference)
+
+def f_measure(reference, test, alpha=0.5):
+ """
+ Given a set of reference values and a set of test values, return
+ the f-measure of the test values, when compared against the
+ reference values. The f-measure is the harmonic mean of the
+ ``precision`` and ``recall``, weighted by ``alpha``. In particular,
+ given the precision *p* and recall *r* defined by:
+
+ - *p* = card(``reference`` intersection ``test``)/card(``test``)
+ - *r* = card(``reference`` intersection ``test``)/card(``reference``)
+
+ The f-measure is:
+
+ - *1/(alpha/p + (1-alpha)/r)*
+
+ If either ``reference`` or ``test`` is empty, then ``f_measure``
+ returns None.
+
+ :type reference: set
+ :param reference: A set of reference values.
+ :type test: set
+ :param test: A set of values to compare against the reference set.
+ :rtype: float or None
+ """
+ p = precision(reference, test)
+ r = recall(reference, test)
+ if p is None or r is None:
+ return None
+ if p == 0 or r == 0:
+ return 0
+ return 1.0/(alpha/p + (1-alpha)/r)
+
+def log_likelihood(reference, test):
+ """
+ Given a list of reference values and a corresponding list of test
+ probability distributions, return the average log likelihood of
+ the reference values, given the probability distributions.
+
+ :param reference: A list of reference values
+ :type reference: list
+ :param test: A list of probability distributions over values to
+ compare against the corresponding reference values.
+ :type test: list(ProbDistI)
+ """
+ if len(reference) != len(test):
+ raise ValueError("Lists must have the same length.")
+
+ # Return the average value of dist.logprob(val).
+ total_likelihood = sum(dist.logprob(val)
+ for (val, dist) in izip(reference, test))
+ return total_likelihood/len(reference)
+
+def approxrand(a, b, **kwargs):
+ """
+ Returns an approximate significance level between two lists of
+ independently generated test values.
+
+ Approximate randomization calculates significance by randomly drawing
+ from a sample of the possible permutations. At the limit of the number
+ of possible permutations, the significance level is exact. The
+ approximate significance level is the sample mean number of times the
+ statistic of the permutated lists varies from the actual statistic of
+ the unpermuted argument lists.
+
+ :return: a tuple containing an approximate significance level, the count
+ of the number of times the pseudo-statistic varied from the
+ actual statistic, and the number of shuffles
+ :rtype: tuple
+ :param a: a list of test values
+ :type a: list
+ :param b: another list of independently generated test values
+ :type b: list
+ """
+ shuffles = kwargs.get('shuffles', 999)
+ # there's no point in trying to shuffle beyond all possible permutations
+ shuffles = \
+ min(shuffles, reduce(operator.mul, xrange(1, len(a) + len(b) + 1)))
+ stat = kwargs.get('statistic', lambda lst: float(sum(lst)) / len(lst))
+ verbose = kwargs.get('verbose', False)
+
+ if verbose:
+ print('shuffles: %d' % shuffles)
+
+ actual_stat = fabs(stat(a) - stat(b))
+
+ if verbose:
+ print('actual statistic: %f' % actual_stat)
+ print('-' * 60)
+
+ c = 1e-100
+ lst = LazyConcatenation([a, b])
+ indices = list(range(len(a) + len(b)))
+
+ for i in xrange(shuffles):
+ if verbose and i % 10 == 0:
+ print('shuffle: %d' % i)
+
+ shuffle(indices)
+
+ pseudo_stat_a = stat(LazyMap(lambda i: lst[i], indices[:len(a)]))
+ pseudo_stat_b = stat(LazyMap(lambda i: lst[i], indices[len(a):]))
+ pseudo_stat = fabs(pseudo_stat_a - pseudo_stat_b)
+
+ if pseudo_stat >= actual_stat:
+ c += 1
+
+ if verbose and i % 10 == 0:
+ print('pseudo-statistic: %f' % pseudo_stat)
+ print('significance: %f' % (float(c + 1) / (i + 1)))
+ print('-' * 60)
+
+ significance = float(c + 1) / (shuffles + 1)
+
+ if verbose:
+ print('significance: %f' % significance)
+ if betai:
+ for phi in [0.01, 0.05, 0.10, 0.15, 0.25, 0.50]:
+ print("prob(phi<=%f): %f" % (phi, betai(c, shuffles, phi)))
+
+ return (significance, c, shuffles)
+
+
+def demo():
+ print('-'*75)
+ reference = 'DET NN VB DET JJ NN NN IN DET NN'.split()
+ test = 'DET VB VB DET NN NN NN IN DET NN'.split()
+ print('Reference =', reference)
+ print('Test =', test)
+ print('Accuracy:', accuracy(reference, test))
+
+ print('-'*75)
+ reference_set = set(reference)
+ test_set = set(test)
+ print('Reference =', reference_set)
+ print('Test = ', test_set)
+ print('Precision:', precision(reference_set, test_set))
+ print(' Recall:', recall(reference_set, test_set))
+ print('F-Measure:', f_measure(reference_set, test_set))
+ print('-'*75)
+
+if __name__ == '__main__':
+ demo()
+
diff --git a/nltk/metrics/segmentation.py b/nltk/metrics/segmentation.py
new file mode 100644
index 0000000..aac4d67
--- /dev/null
+++ b/nltk/metrics/segmentation.py
@@ -0,0 +1,235 @@
+# Natural Language Toolkit: Text Segmentation Metrics
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# David Doukhan <david.doukhan at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+
+
+"""
+Text Segmentation Metrics
+
+1. Windowdiff
+
+Pevzner, L., and Hearst, M., A Critique and Improvement of
+ an Evaluation Metric for Text Segmentation,
+Computational Linguistics 28, 19-36
+
+
+2. Generalized Hamming Distance
+
+Bookstein A., Kulyukin V.A., Raita T.
+Generalized Hamming Distance
+Information Retrieval 5, 2002, pp 353-375
+
+Baseline implementation in C++
+http://digital.cs.usu.edu/~vkulyukin/vkweb/software/ghd/ghd.html
+
+Study describing benefits of Generalized Hamming Distance Versus
+WindowDiff for evaluating text segmentation tasks
+Begsten, Y. Quel indice pour mesurer l'efficacite en segmentation de textes ?
+TALN 2009
+
+
+3. Pk text segmentation metric
+
+Beeferman D., Berger A., Lafferty J. (1999)
+Statistical Models for Text Segmentation
+Machine Learning, 34, 177-210
+"""
+
+try:
+ import numpy as np
+except ImportError:
+ pass
+
+from nltk.compat import xrange
+
+def windowdiff(seg1, seg2, k, boundary="1", weighted=False):
+ """
+ Compute the windowdiff score for a pair of segmentations. A
+ segmentation is any sequence over a vocabulary of two items
+ (e.g. "0", "1"), where the specified boundary value is used to
+ mark the edge of a segmentation.
+
+ >>> s1 = "000100000010"
+ >>> s2 = "000010000100"
+ >>> s3 = "100000010000"
+ >>> '%.2f' % windowdiff(s1, s1, 3)
+ '0.00'
+ >>> '%.2f' % windowdiff(s1, s2, 3)
+ '0.30'
+ >>> '%.2f' % windowdiff(s2, s3, 3)
+ '0.80'
+
+ :param seg1: a segmentation
+ :type seg1: str or list
+ :param seg2: a segmentation
+ :type seg2: str or list
+ :param k: window width
+ :type k: int
+ :param boundary: boundary value
+ :type boundary: str or int or bool
+ :param weighted: use the weighted variant of windowdiff
+ :type weighted: boolean
+ :rtype: float
+ """
+
+ if len(seg1) != len(seg2):
+ raise ValueError("Segmentations have unequal length")
+ if k > len(seg1):
+ raise ValueError("Window width k should be smaller or equal than segmentation lengths")
+ wd = 0
+ for i in range(len(seg1) - k + 1):
+ ndiff = abs(seg1[i:i+k].count(boundary) - seg2[i:i+k].count(boundary))
+ if weighted:
+ wd += ndiff
+ else:
+ wd += min(1, ndiff)
+ return wd / (len(seg1) - k + 1.)
+
+
+
+# Generalized Hamming Distance
+
+def _init_mat(nrows, ncols, ins_cost, del_cost):
+ mat = np.empty((nrows, ncols))
+ mat[0, :] = ins_cost * np.arange(ncols)
+ mat[:, 0] = del_cost * np.arange(nrows)
+ return mat
+
+
+def _ghd_aux(mat, rowv, colv, ins_cost, del_cost, shift_cost_coeff):
+ for i, rowi in enumerate(rowv):
+ for j, colj in enumerate(colv):
+ shift_cost = shift_cost_coeff * abs(rowi - colj) + mat[i, j]
+ if rowi == colj:
+ # boundaries are at the same location, no transformation required
+ tcost = mat[i, j]
+ elif rowi > colj:
+ # boundary match through a deletion
+ tcost = del_cost + mat[i, j + 1]
+ else:
+ # boundary match through an insertion
+ tcost = ins_cost + mat[i + 1, j]
+ mat[i + 1, j + 1] = min(tcost, shift_cost)
+
+
+def ghd(ref, hyp, ins_cost=2.0, del_cost=2.0, shift_cost_coeff=1.0, boundary='1'):
+ """
+ Compute the Generalized Hamming Distance for a reference and a hypothetical
+ segmentation, corresponding to the cost related to the transformation
+ of the hypothetical segmentation into the reference segmentation
+ through boundary insertion, deletion and shift operations.
+
+ A segmentation is any sequence over a vocabulary of two items
+ (e.g. "0", "1"), where the specified boundary value is used to
+ mark the edge of a segmentation.
+
+ Recommended parameter values are a shift_cost_coeff of 2.
+ Associated with a ins_cost, and del_cost equal to the mean segment
+ length in the reference segmentation.
+
+ >>> # Same examples as Kulyukin C++ implementation
+ >>> ghd('1100100000', '1100010000', 1.0, 1.0, 0.5)
+ 0.5
+ >>> ghd('1100100000', '1100000001', 1.0, 1.0, 0.5)
+ 2.0
+ >>> ghd('011', '110', 1.0, 1.0, 0.5)
+ 1.0
+ >>> ghd('1', '0', 1.0, 1.0, 0.5)
+ 1.0
+ >>> ghd('111', '000', 1.0, 1.0, 0.5)
+ 3.0
+ >>> ghd('000', '111', 1.0, 2.0, 0.5)
+ 6.0
+
+ :param ref: the reference segmentation
+ :type ref: str or list
+ :param hyp: the hypothetical segmentation
+ :type hyp: str or list
+ :param ins_cost: insertion cost
+ :type ins_cost: float
+ :param del_cost: deletion cost
+ :type del_cost: float
+ :param shift_cost_coeff: constant used to compute the cost of a shift.
+ shift cost = shift_cost_coeff * |i - j| where i and j are
+ the positions indicating the shift
+ :type shift_cost_coeff: float
+ :param boundary: boundary value
+ :type boundary: str or int or bool
+ :rtype: float
+ """
+
+ ref_idx = [i for (i, val) in enumerate(ref) if val == boundary]
+ hyp_idx = [i for (i, val) in enumerate(hyp) if val == boundary]
+
+ nref_bound = len(ref_idx)
+ nhyp_bound = len(hyp_idx)
+
+ if nref_bound == 0 and nhyp_bound == 0:
+ return 0.0
+ elif nref_bound > 0 and nhyp_bound == 0:
+ return nref_bound * ins_cost
+ elif nref_bound == 0 and nhyp_bound > 0:
+ return nhyp_bound * del_cost
+
+ mat = _init_mat(nhyp_bound + 1, nref_bound + 1, ins_cost, del_cost)
+ _ghd_aux(mat, hyp_idx, ref_idx, ins_cost, del_cost, shift_cost_coeff)
+ return mat[-1, -1]
+
+
+# Beeferman's Pk text segmentation evaluation metric
+
+def pk(ref, hyp, k=None, boundary='1'):
+ """
+ Compute the Pk metric for a pair of segmentations A segmentation
+ is any sequence over a vocabulary of two items (e.g. "0", "1"),
+ where the specified boundary value is used to mark the edge of a
+ segmentation.
+
+ >>> '%.2f' % pk('0100'*100, '1'*400, 2)
+ '0.50'
+ >>> '%.2f' % pk('0100'*100, '0'*400, 2)
+ '0.50'
+ >>> '%.2f' % pk('0100'*100, '0100'*100, 2)
+ '0.00'
+
+ :param ref: the reference segmentation
+ :type ref: str or list
+ :param hyp: the segmentation to evaluate
+ :type hyp: str or list
+ :param k: window size, if None, set to half of the average reference segment length
+ :type boundary: str or int or bool
+ :param boundary: boundary value
+ :type boundary: str or int or bool
+ :rtype: float
+ """
+
+ if k is None:
+ k = int(round(len(ref) / (ref.count(boundary) * 2.)))
+
+ err = 0
+ for i in xrange(len(ref)-k +1):
+ r = ref[i:i+k].count(boundary) > 0
+ h = hyp[i:i+k].count(boundary) > 0
+ if r != h:
+ err += 1
+ return err / (len(ref)-k +1.)
+
+
+# skip doctests if numpy is not installed
+def setup_module(module):
+ from nose import SkipTest
+ try:
+ import numpy
+ except ImportError:
+ raise SkipTest("numpy is required for nltk.metrics.segmentation")
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/metrics/spearman.py b/nltk/metrics/spearman.py
new file mode 100644
index 0000000..cc95d17
--- /dev/null
+++ b/nltk/metrics/spearman.py
@@ -0,0 +1,68 @@
+# Natural Language Toolkit: Spearman Rank Correlation
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Joel Nothman <jnothman at student.usyd.edu.au>
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+
+"""
+Tools for comparing ranked lists.
+"""
+
+def _rank_dists(ranks1, ranks2):
+ """Finds the difference between the values in ranks1 and ranks2 for keys
+ present in both dicts. If the arguments are not dicts, they are converted
+ from (key, rank) sequences.
+ """
+ ranks1 = dict(ranks1)
+ ranks2 = dict(ranks2)
+ for k in ranks1:
+ try:
+ yield k, ranks1[k] - ranks2[k]
+ except KeyError:
+ pass
+
+
+def spearman_correlation(ranks1, ranks2):
+ """Returns the Spearman correlation coefficient for two rankings, which
+ should be dicts or sequences of (key, rank). The coefficient ranges from
+ -1.0 (ranks are opposite) to 1.0 (ranks are identical), and is only
+ calculated for keys in both rankings (for meaningful results, remove keys
+ present in only one list before ranking)."""
+ n = 0
+ res = 0
+ for k, d in _rank_dists(ranks1, ranks2):
+ res += d * d
+ n += 1
+ try:
+ return 1 - (6 * float(res) / (n * (n*n - 1)))
+ except ZeroDivisionError:
+ # Result is undefined if only one item is ranked
+ return 0.0
+
+
+def ranks_from_sequence(seq):
+ """Given a sequence, yields each element with an increasing rank, suitable
+ for use as an argument to ``spearman_correlation``.
+ """
+ return ((k, i) for i, k in enumerate(seq))
+
+
+def ranks_from_scores(scores, rank_gap=1e-15):
+ """Given a sequence of (key, score) tuples, yields each key with an
+ increasing rank, tying with previous key's rank if the difference between
+ their scores is less than rank_gap. Suitable for use as an argument to
+ ``spearman_correlation``.
+ """
+ prev_score = None
+ rank = 0
+ for i, (key, score) in enumerate(scores):
+ try:
+ if abs(score - prev_score) > rank_gap:
+ rank = i
+ except TypeError:
+ pass
+
+ yield key, rank
+ prev_score = score
+
diff --git a/nltk/misc/__init__.py b/nltk/misc/__init__.py
new file mode 100644
index 0000000..bc9210e
--- /dev/null
+++ b/nltk/misc/__init__.py
@@ -0,0 +1,11 @@
+# Natural Language Toolkit: Miscellaneous modules
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk.misc.chomsky import generate_chomsky
+from nltk.misc.wordfinder import word_finder
+from nltk.misc.minimalset import MinimalSet
+from nltk.misc.babelfish import babelize_shell
diff --git a/nltk/misc/babelfish.py b/nltk/misc/babelfish.py
new file mode 100644
index 0000000..481a508
--- /dev/null
+++ b/nltk/misc/babelfish.py
@@ -0,0 +1,10 @@
+"""
+This module previously provided an interface to Babelfish online
+translation service; this service is no longer available; this
+module is kept in NLTK source code in order to provide better error
+messages for people following the NLTK Book 2.0.
+"""
+from __future__ import print_function
+
+def babelize_shell():
+ print("Babelfish online translation service is no longer available.")
diff --git a/nltk/misc/chomsky.py b/nltk/misc/chomsky.py
new file mode 100644
index 0000000..68b4dbf
--- /dev/null
+++ b/nltk/misc/chomsky.py
@@ -0,0 +1,133 @@
+# Chomsky random text generator, version 1.1, Raymond Hettinger, 2005/09/13
+# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/440546
+
+"""
+CHOMSKY is an aid to writing linguistic papers in the style
+of the great master. It is based on selected phrases taken
+from actual books and articles written by Noam Chomsky.
+Upon request, it assembles the phrases in the elegant
+stylistic patterns that Chomsky is noted for.
+To generate n sentences of linguistic wisdom, type
+
+ (CHOMSKY n) -- for example
+ (CHOMSKY 5) generates half a screen of linguistic truth.
+"""
+from __future__ import print_function
+
+leadins = """To characterize a linguistic level L,
+ On the other hand,
+ This suggests that
+ It appears that
+ Furthermore,
+ We will bring evidence in favor of the following thesis:
+ To provide a constituent structure for T(Z,K),
+ From C1, it follows that
+ For any transformation which is sufficiently diversified in \
+application to be of any interest,
+ Analogously,
+ Clearly,
+ Note that
+ Of course,
+ Suppose, for instance, that
+ Thus
+ With this clarification,
+ Conversely,
+ We have already seen that
+ By combining adjunctions and certain deformations,
+ I suggested that these results would follow from the assumption that
+ If the position of the trace in (99c) were only relatively \
+inaccessible to movement,
+ However, this assumption is not correct, since
+ Comparing these examples with their parasitic gap counterparts in \
+(96) and (97), we see that
+ In the discussion of resumptive pronouns following (81),
+ So far,
+ Nevertheless,
+ For one thing,
+ Summarizing, then, we assume that
+ A consequence of the approach just outlined is that
+ Presumably,
+ On our assumptions,
+ It may be, then, that
+ It must be emphasized, once again, that
+ Let us continue to suppose that
+ Notice, incidentally, that """
+# List of LEADINs to buy time.
+
+subjects = """ the notion of level of grammaticalness
+ a case of semigrammaticalness of a different sort
+ most of the methodological work in modern linguistics
+ a subset of English sentences interesting on quite independent grounds
+ the natural general principle that will subsume this case
+ an important property of these three types of EC
+ any associated supporting element
+ the appearance of parasitic gaps in domains relatively inaccessible \
+to ordinary extraction
+ the speaker-hearer's linguistic intuition
+ the descriptive power of the base component
+ the earlier discussion of deviance
+ this analysis of a formative as a pair of sets of features
+ this selectionally introduced contextual feature
+ a descriptively adequate grammar
+ the fundamental error of regarding functional notions as categorial
+ relational information
+ the systematic use of complex symbols
+ the theory of syntactic features developed earlier"""
+# List of SUBJECTs chosen for maximum professorial macho.
+
+verbs = """can be defined in such a way as to impose
+ delimits
+ suffices to account for
+ cannot be arbitrary in
+ is not subject to
+ does not readily tolerate
+ raises serious doubts about
+ is not quite equivalent to
+ does not affect the structure of
+ may remedy and, at the same time, eliminate
+ is not to be considered in determining
+ is to be regarded as
+ is unspecified with respect to
+ is, apparently, determined by
+ is necessary to impose an interpretation on
+ appears to correlate rather closely with
+ is rather different from"""
+#List of VERBs chosen for autorecursive obfuscation.
+
+objects = """ problems of phonemic and morphological analysis.
+ a corpus of utterance tokens upon which conformity has been defined \
+by the paired utterance test.
+ the traditional practice of grammarians.
+ the levels of acceptability from fairly high (e.g. (99a)) to virtual \
+gibberish (e.g. (98d)).
+ a stipulation to place the constructions into these various categories.
+ a descriptive fact.
+ a parasitic gap construction.
+ the extended c-command discussed in connection with (34).
+ the ultimate standard that determines the accuracy of any proposed grammar.
+ the system of base rules exclusive of the lexicon.
+ irrelevant intervening contexts in selectional rules.
+ nondistinctness in the sense of distinctive feature theory.
+ a general convention regarding the forms of the grammar.
+ an abstract underlying order.
+ an important distinction in language use.
+ the requirement that branching is not tolerated within the dominance \
+scope of a complex symbol.
+ the strong generative capacity of the theory."""
+# List of OBJECTs selected for profound sententiousness.
+
+import textwrap, random
+from itertools import chain, islice
+from nltk.compat import izip
+
+def generate_chomsky(times=5, line_length=72):
+ parts = []
+ for part in (leadins, subjects, verbs, objects):
+ phraselist = list(map(str.strip, part.splitlines()))
+ random.shuffle(phraselist)
+ parts.append(phraselist)
+ output = chain(*islice(izip(*parts), 0, times))
+ print(textwrap.fill(" ".join(output), line_length))
+
+if __name__ == '__main__':
+ generate_chomsky()
diff --git a/nltk/misc/minimalset.py b/nltk/misc/minimalset.py
new file mode 100644
index 0000000..42615f4
--- /dev/null
+++ b/nltk/misc/minimalset.py
@@ -0,0 +1,83 @@
+# Natural Language Toolkit: Minimal Sets
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+
+from collections import defaultdict
+
+class MinimalSet(object):
+ """
+ Find contexts where more than one possible target value can
+ appear. E.g. if targets are word-initial letters, and contexts
+ are the remainders of words, then we would like to find cases like
+ "fat" vs "cat", and "training" vs "draining". If targets are
+ parts-of-speech and contexts are words, then we would like to find
+ cases like wind (noun) 'air in rapid motion', vs wind (verb)
+ 'coil, wrap'.
+ """
+ def __init__(self, parameters=None):
+ """
+ Create a new minimal set.
+
+ :param parameters: The (context, target, display) tuples for the item
+ :type parameters: list(tuple(str, str, str))
+ """
+ self._targets = set() # the contrastive information
+ self._contexts = set() # what we are controlling for
+ self._seen = defaultdict(set) # to record what we have seen
+ self._displays = {} # what we will display
+
+ if parameters:
+ for context, target, display in parameters:
+ self.add(context, target, display)
+
+ def add(self, context, target, display):
+ """
+ Add a new item to the minimal set, having the specified
+ context, target, and display form.
+
+ :param context: The context in which the item of interest appears
+ :type context: str
+ :param target: The item of interest
+ :type target: str
+ :param display: The information to be reported for each item
+ :type display: str
+ """
+ # Store the set of targets that occurred in this context
+ self._seen[context].add(target)
+
+ # Keep track of which contexts and targets we have seen
+ self._contexts.add(context)
+ self._targets.add(target)
+
+ # For a given context and target, store the display form
+ self._displays[(context, target)] = display
+
+ def contexts(self, minimum=2):
+ """
+ Determine which contexts occurred with enough distinct targets.
+
+ :param minimum: the minimum number of distinct target forms
+ :type minimum: int
+ :rtype list
+ """
+ return [c for c in self._contexts if len(self._seen[c]) >= minimum]
+
+ def display(self, context, target, default=""):
+ if (context, target) in self._displays:
+ return self._displays[(context, target)]
+ else:
+ return default
+
+ def display_all(self, context):
+ result = []
+ for target in self._targets:
+ x = self.display(context, target)
+ if x: result.append(x)
+ return result
+
+ def targets(self):
+ return self._targets
+
diff --git a/nltk/misc/sort.py b/nltk/misc/sort.py
new file mode 100644
index 0000000..4972a5b
--- /dev/null
+++ b/nltk/misc/sort.py
@@ -0,0 +1,157 @@
+# Natural Language Toolkit: List Sorting
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+This module provides a variety of list sorting algorithms, to
+illustrate the many different algorithms (recipes) for solving a
+problem, and how to analyze algorithms experimentally.
+"""
+from __future__ import print_function, division
+
+# These algorithms are taken from:
+# Levitin (2004) The Design and Analysis of Algorithms
+
+##################################################################
+# Selection Sort
+##################################################################
+
+def selection(a):
+ """
+ Selection Sort: scan the list to find its smallest element, then
+ swap it with the first element. The remainder of the list is one
+ element smaller; apply the same method to this list, and so on.
+ """
+ count = 0
+
+ for i in range(len(a) - 1):
+ min = i
+
+ for j in range(i+1, len(a)):
+ if a[j] < a[min]:
+ min = j
+
+ count += 1
+
+ a[min],a[i] = a[i],a[min]
+
+ return count
+
+##################################################################
+# Bubble Sort
+##################################################################
+
+def bubble(a):
+ """
+ Bubble Sort: compare adjacent elements of the list left-to-right,
+ and swap them if they are out of order. After one pass through
+ the list swapping adjacent items, the largest item will be in
+ the rightmost position. The remainder is one element smaller;
+ apply the same method to this list, and so on.
+ """
+ count = 0
+ for i in range(len(a)-1):
+ for j in range(len(a)-i-1):
+ if a[j+1] < a[j]:
+ a[j],a[j+1] = a[j+1],a[j]
+ count += 1
+ return count
+
+
+##################################################################
+# Merge Sort
+##################################################################
+
+def _merge_lists(b, c):
+ count = 0
+ i = j = 0
+ a = []
+ while (i < len(b) and j < len(c)):
+ count += 1
+ if b[i] <= c[j]:
+ a.append(b[i])
+ i += 1
+ else:
+ a.append(c[j])
+ j += 1
+ if i == len(b):
+ a += c[j:]
+ else:
+ a += b[i:]
+ return a, count
+
+def merge(a):
+ """
+ Merge Sort: split the list in half, and sort each half, then
+ combine the sorted halves.
+ """
+ count = 0
+ if len(a) > 1:
+ midpoint = len(a) // 2
+ b = a[:midpoint]
+ c = a[midpoint:]
+ count_b = merge(b)
+ count_c = merge(c)
+ result, count_a = _merge_lists(b, c)
+ a[:] = result # copy the result back into a.
+ count = count_a + count_b + count_c
+ return count
+
+##################################################################
+# Quick Sort
+##################################################################
+
+def _partition(a, l, r):
+ p = a[l]; i = l; j = r+1
+ count = 0
+ while True:
+ while i < r:
+ i += 1
+ if a[i] >= p: break
+ while j > l:
+ j -= 1
+ if j < l or a[j] <= p: break
+ a[i],a[j] = a[j],a[i] # swap
+ count += 1
+ if i >= j: break
+ a[i],a[j] = a[j],a[i] # undo last swap
+ a[l],a[j] = a[j],a[l]
+ return j, count
+
+def _quick(a, l, r):
+ count = 0
+ if l<r:
+ s, count = _partition(a, l, r)
+ count += _quick(a, l, s-1)
+ count += _quick(a, s+1, r)
+ return count
+
+def quick(a):
+ return _quick(a, 0, len(a)-1)
+
+##################################################################
+# Demonstration
+##################################################################
+
+def demo():
+ from random import shuffle
+
+ for size in (10, 20, 50, 100, 200, 500, 1000):
+ a = list(range(size))
+
+ # various sort methods
+ shuffle(a); count_selection = selection(a)
+ shuffle(a); count_bubble = bubble(a)
+ shuffle(a); count_merge = merge(a)
+ shuffle(a); count_quick = quick(a)
+
+ print((("size=%5d: selection=%8d, bubble=%8d, "
+ "merge=%6d, quick=%6d") %
+ (size, count_selection, count_bubble,
+ count_merge, count_quick)))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/misc/wordfinder.py b/nltk/misc/wordfinder.py
new file mode 100644
index 0000000..a8d45ca
--- /dev/null
+++ b/nltk/misc/wordfinder.py
@@ -0,0 +1,129 @@
+# Natural Language Toolkit: Word Finder
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Simplified from PHP version by Robert Klein <brathna at gmail.com>
+# http://fswordfinder.sourceforge.net/
+from __future__ import print_function
+
+import random
+
+
+# reverse a word with probability 0.5
+def revword(word):
+ if random.randint(1,2) == 1:
+ return word[::-1]
+ return word
+
+# try to insert word at position x,y; direction encoded in xf,yf
+def step(word, x, xf, y, yf, grid):
+ for i in range(len(word)):
+ if grid[xf(i)][yf(i)] != "" and grid[xf(i)][yf(i)] != word[i]:
+ return False
+ for i in range(len(word)):
+ grid[xf(i)][yf(i)] = word[i]
+ return True
+
+# try to insert word at position x,y, in direction dir
+def check(word, dir, x, y, grid, rows, cols):
+ if dir==1:
+ if x-len(word)<0 or y-len(word)<0:
+ return False
+ return step(word, x, lambda i:x-i, y, lambda i:y-i, grid)
+ elif dir==2:
+ if x-len(word)<0:
+ return False
+ return step(word, x, lambda i:x-i, y, lambda i:y, grid)
+ elif dir==3:
+ if x-len(word)<0 or y+(len(word)-1)>=cols:
+ return False
+ return step(word, x, lambda i:x-i, y, lambda i:y+i, grid)
+ elif dir==4:
+ if y-len(word)<0:
+ return False
+ return step(word, x, lambda i:x, y, lambda i:y-i, grid)
+
+def wordfinder(words, rows=20, cols=20, attempts=50,
+ alph='ABCDEFGHIJKLMNOPQRSTUVWXYZ'):
+ """
+ Attempt to arrange words into a letter-grid with the specified
+ number of rows and columns. Try each word in several positions
+ and directions, until it can be fitted into the grid, or the
+ maximum number of allowable attempts is exceeded. Returns a tuple
+ consisting of the grid and the words that were successfully
+ placed.
+
+ :param words: the list of words to be put into the grid
+ :type words: list
+ :param rows: the number of rows in the grid
+ :type rows: int
+ :param cols: the number of columns in the grid
+ :type cols: int
+ :param attempts: the number of times to attempt placing a word
+ :type attempts: int
+ :param alph: the alphabet, to be used for filling blank cells
+ :type alph: list
+ :rtype: tuple
+ """
+
+ # place longer words first
+ words = sorted(words, key=len, reverse=True)
+
+ grid = [] # the letter grid
+ used = [] # the words we used
+
+ # initialize the grid
+ for i in range(rows):
+ grid.append([""] * cols)
+
+ # try to place each word
+ for word in words:
+ word = word.strip().upper() # normalize
+ save = word # keep a record of the word
+ word = revword(word)
+ for attempt in range(attempts):
+ r = random.randint(0, len(word))
+ dir = random.choice([1,2,3,4])
+ x = random.randint(0,rows)
+ y = random.randint(0,cols)
+ if dir==1: x+=r; y+=r
+ elif dir==2: x+=r
+ elif dir==3: x+=r; y-=r
+ elif dir==4: y+=r
+ if 0<=x<rows and 0<=y<cols:
+ if check(word, dir, x, y, grid, rows, cols):
+# used.append((save, dir, x, y, word))
+ used.append(save)
+ break
+
+ # Fill up the remaining spaces
+ for i in range(rows):
+ for j in range(cols):
+ if grid[i][j] == '':
+ grid[i][j] = random.choice(alph)
+
+ return grid, used
+
+def word_finder():
+ from nltk.corpus import words
+ wordlist = words.words()
+ random.shuffle(wordlist)
+ wordlist = wordlist[:200]
+ wordlist = [w for w in wordlist if 3 <= len(w) <= 12]
+ grid, used = wordfinder(wordlist)
+
+ print("Word Finder\n")
+ for i in range(len(grid)):
+ for j in range(len(grid[i])):
+ print(grid[i][j], end=' ')
+ print()
+ print()
+
+ for i in range(len(used)):
+ print("%d:" % (i+1), used[i])
+
+if __name__ == '__main__':
+ word_finder()
diff --git a/nltk/parse/__init__.py b/nltk/parse/__init__.py
new file mode 100644
index 0000000..22bcb88
--- /dev/null
+++ b/nltk/parse/__init__.py
@@ -0,0 +1,77 @@
+# Natural Language Toolkit: Parsers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+"""
+NLTK Parsers
+
+Classes and interfaces for producing tree structures that represent
+the internal organization of a text. This task is known as "parsing"
+the text, and the resulting tree structures are called the text's
+"parses". Typically, the text is a single sentence, and the tree
+structure represents the syntactic structure of the sentence.
+However, parsers can also be used in other domains. For example,
+parsers can be used to derive the morphological structure of the
+morphemes that make up a word, or to derive the discourse structure
+for a set of utterances.
+
+Sometimes, a single piece of text can be represented by more than one
+tree structure. Texts represented by more than one tree structure are
+called "ambiguous" texts. Note that there are actually two ways in
+which a text can be ambiguous:
+
+ - The text has multiple correct parses.
+ - There is not enough information to decide which of several
+ candidate parses is correct.
+
+However, the parser module does *not* distinguish these two types of
+ambiguity.
+
+The parser module defines ``ParserI``, a standard interface for parsing
+texts; and two simple implementations of that interface,
+``ShiftReduceParser`` and ``RecursiveDescentParser``. It also contains
+three sub-modules for specialized kinds of parsing:
+
+ - ``nltk.parser.chart`` defines chart parsing, which uses dynamic
+ programming to efficiently parse texts.
+ - ``nltk.parser.probabilistic`` defines probabilistic parsing, which
+ associates a probability with each parse.
+"""
+
+from nltk.parse.api import ParserI
+from nltk.parse.chart import (ChartParser, SteppingChartParser, TopDownChartParser,
+ BottomUpChartParser, BottomUpLeftCornerChartParser,
+ LeftCornerChartParser)
+from nltk.parse.featurechart import (FeatureChartParser, FeatureTopDownChartParser,
+ FeatureBottomUpChartParser,
+ FeatureBottomUpLeftCornerChartParser)
+from nltk.parse.earleychart import (IncrementalChartParser, EarleyChartParser,
+ IncrementalTopDownChartParser,
+ IncrementalBottomUpChartParser,
+ IncrementalBottomUpLeftCornerChartParser,
+ IncrementalLeftCornerChartParser,
+ FeatureIncrementalChartParser,
+ FeatureEarleyChartParser,
+ FeatureIncrementalTopDownChartParser,
+ FeatureIncrementalBottomUpChartParser,
+ FeatureIncrementalBottomUpLeftCornerChartParser)
+from nltk.parse.pchart import (BottomUpProbabilisticChartParser, InsideChartParser,
+ RandomChartParser, UnsortedChartParser,
+ LongestChartParser)
+from nltk.parse.recursivedescent import (RecursiveDescentParser,
+ SteppingRecursiveDescentParser)
+from nltk.parse.shiftreduce import (ShiftReduceParser, SteppingShiftReduceParser)
+from nltk.parse.util import load_parser, TestGrammar, extract_test_sentences
+from nltk.parse.viterbi import ViterbiParser
+from nltk.parse.dependencygraph import DependencyGraph, nx_graph
+from nltk.parse.projectivedependencyparser import (ProjectiveDependencyParser,
+ ProbabilisticProjectiveDependencyParser)
+from nltk.parse.nonprojectivedependencyparser import (NonprojectiveDependencyParser,
+ NaiveBayesDependencyScorer,
+ ProbabilisticNonprojectiveParser)
+from nltk.parse.malt import MaltParser
diff --git a/nltk/parse/api.py b/nltk/parse/api.py
new file mode 100644
index 0000000..9ec29c8
--- /dev/null
+++ b/nltk/parse/api.py
@@ -0,0 +1,66 @@
+# Natural Language Toolkit: Parser API
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+import itertools
+
+from nltk.internals import overridden
+
+class ParserI(object):
+ """
+ A processing class for deriving trees that represent possible
+ structures for a sequence of tokens. These tree structures are
+ known as "parses". Typically, parsers are used to derive syntax
+ trees for sentences. But parsers can also be used to derive other
+ kinds of tree structure, such as morphological trees and discourse
+ structures.
+
+ Subclasses must define:
+ - at least one of: ``parse()``, ``parse_sents()``.
+
+ Subclasses may define:
+ - ``grammar()``
+ """
+ def grammar(self):
+ """
+ :return: The grammar used by this parser.
+ """
+ raise NotImplementedError()
+
+ def parse(self, sent):
+ """
+ :return: An iterator that generates parse trees for the sentence.
+ When possible this list is sorted from most likely to least likely.
+
+ :param sent: The sentence to be parsed
+ :type sent: list(str)
+ :rtype: iter(Tree)
+ """
+ if overridden(self.parse_sents):
+ return next(self.parse_sents([sent]))
+ elif overridden(self.parse_one):
+ return (tree for tree in [self.parse_one(sent)] if tree is not None)
+ elif overridden(self.parse_all):
+ return iter(self.parse_all(sent))
+ else:
+ raise NotImplementedError()
+
+ def parse_sents(self, sents):
+ """
+ Apply ``self.parse()`` to each element of ``sents``.
+ :rtype: iter(iter(Tree))
+ """
+ return (self.parse(sent) for sent in sents)
+
+ def parse_all(self, sent):
+ """:rtype: list(Tree)"""
+ return list(self.parse(sent))
+
+ def parse_one(self, sent):
+ """:rtype: Tree or None"""
+ return next(self.parse(sent), None)
diff --git a/nltk/parse/broker_test.cfg b/nltk/parse/broker_test.cfg
new file mode 100644
index 0000000..e8ba69a
--- /dev/null
+++ b/nltk/parse/broker_test.cfg
@@ -0,0 +1,10 @@
+%start S
+
+S[sem=<app(?vp, ?subj)>] -> NP[sem=?subj] VP[sem=?vp]
+VP[sem = <app(?v, ?obj)>] -> V[sem = ?v] NP[sem=?obj]
+VP[sem = ?v] -> V[sem = ?v]
+NP[sem = <kim>] -> 'Kim'
+NP[sem = <i>] -> 'I'
+V[sem = <\x y.(like x y)>, tns=pres] -> 'like'
+V[sem = <\x.(sleeps x)>, tns=pres] -> 'sleeps'
+
diff --git a/nltk/parse/chart.py b/nltk/parse/chart.py
new file mode 100644
index 0000000..51be1cc
--- /dev/null
+++ b/nltk/parse/chart.py
@@ -0,0 +1,1681 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: A Chart Parser
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# Jean Mark Gawron <gawron at mail.sdsu.edu>
+# Peter Ljunglöf <peter.ljunglof at heatherleaf.se>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Data classes and parser implementations for "chart parsers", which
+use dynamic programming to efficiently parse a text. A chart
+parser derives parse trees for a text by iteratively adding "edges"
+to a "chart." Each edge represents a hypothesis about the tree
+structure for a subsequence of the text. The chart is a
+"blackboard" for composing and combining these hypotheses.
+
+When a chart parser begins parsing a text, it creates a new (empty)
+chart, spanning the text. It then incrementally adds new edges to the
+chart. A set of "chart rules" specifies the conditions under which
+new edges should be added to the chart. Once the chart reaches a
+stage where none of the chart rules adds any new edges, parsing is
+complete.
+
+Charts are encoded with the ``Chart`` class, and edges are encoded with
+the ``TreeEdge`` and ``LeafEdge`` classes. The chart parser module
+defines three chart parsers:
+
+ - ``ChartParser`` is a simple and flexible chart parser. Given a
+ set of chart rules, it will apply those rules to the chart until
+ no more edges are added.
+
+ - ``SteppingChartParser`` is a subclass of ``ChartParser`` that can
+ be used to step through the parsing process.
+"""
+from __future__ import print_function, division, unicode_literals
+
+import itertools
+import re
+import warnings
+
+from nltk import compat
+from nltk.tree import Tree
+from nltk.grammar import PCFG, is_nonterminal, is_terminal
+from nltk.util import OrderedDict
+from nltk.internals import raise_unorderable_types
+from nltk.compat import (total_ordering, python_2_unicode_compatible,
+ unicode_repr)
+
+from nltk.parse.api import ParserI
+
+
+########################################################################
+## Edges
+########################################################################
+
+ at total_ordering
+class EdgeI(object):
+ """
+ A hypothesis about the structure of part of a sentence.
+ Each edge records the fact that a structure is (partially)
+ consistent with the sentence. An edge contains:
+
+ - A span, indicating what part of the sentence is
+ consistent with the hypothesized structure.
+ - A left-hand side, specifying what kind of structure is
+ hypothesized.
+ - A right-hand side, specifying the contents of the
+ hypothesized structure.
+ - A dot position, indicating how much of the hypothesized
+ structure is consistent with the sentence.
+
+ Every edge is either complete or incomplete:
+
+ - An edge is complete if its structure is fully consistent
+ with the sentence.
+ - An edge is incomplete if its structure is partially
+ consistent with the sentence. For every incomplete edge, the
+ span specifies a possible prefix for the edge's structure.
+
+ There are two kinds of edge:
+
+ - A ``TreeEdge`` records which trees have been found to
+ be (partially) consistent with the text.
+ - A ``LeafEdge`` records the tokens occurring in the text.
+
+ The ``EdgeI`` interface provides a common interface to both types
+ of edge, allowing chart parsers to treat them in a uniform manner.
+ """
+ def __init__(self):
+ if self.__class__ == EdgeI:
+ raise TypeError('Edge is an abstract interface')
+
+ #////////////////////////////////////////////////////////////
+ # Span
+ #////////////////////////////////////////////////////////////
+
+ def span(self):
+ """
+ Return a tuple ``(s, e)``, where ``tokens[s:e]`` is the
+ portion of the sentence that is consistent with this
+ edge's structure.
+
+ :rtype: tuple(int, int)
+ """
+ raise NotImplementedError()
+
+ def start(self):
+ """
+ Return the start index of this edge's span.
+
+ :rtype: int
+ """
+ raise NotImplementedError()
+
+ def end(self):
+ """
+ Return the end index of this edge's span.
+
+ :rtype: int
+ """
+ raise NotImplementedError()
+
+ def length(self):
+ """
+ Return the length of this edge's span.
+
+ :rtype: int
+ """
+ raise NotImplementedError()
+
+ #////////////////////////////////////////////////////////////
+ # Left Hand Side
+ #////////////////////////////////////////////////////////////
+
+ def lhs(self):
+ """
+ Return this edge's left-hand side, which specifies what kind
+ of structure is hypothesized by this edge.
+
+ :see: ``TreeEdge`` and ``LeafEdge`` for a description of
+ the left-hand side values for each edge type.
+ """
+ raise NotImplementedError()
+
+ #////////////////////////////////////////////////////////////
+ # Right Hand Side
+ #////////////////////////////////////////////////////////////
+
+ def rhs(self):
+ """
+ Return this edge's right-hand side, which specifies
+ the content of the structure hypothesized by this edge.
+
+ :see: ``TreeEdge`` and ``LeafEdge`` for a description of
+ the right-hand side values for each edge type.
+ """
+ raise NotImplementedError()
+
+ def dot(self):
+ """
+ Return this edge's dot position, which indicates how much of
+ the hypothesized structure is consistent with the
+ sentence. In particular, ``self.rhs[:dot]`` is consistent
+ with ``tokens[self.start():self.end()]``.
+
+ :rtype: int
+ """
+ raise NotImplementedError()
+
+ def nextsym(self):
+ """
+ Return the element of this edge's right-hand side that
+ immediately follows its dot.
+
+ :rtype: Nonterminal or terminal or None
+ """
+ raise NotImplementedError()
+
+ def is_complete(self):
+ """
+ Return True if this edge's structure is fully consistent
+ with the text.
+
+ :rtype: bool
+ """
+ raise NotImplementedError()
+
+ def is_incomplete(self):
+ """
+ Return True if this edge's structure is partially consistent
+ with the text.
+
+ :rtype: bool
+ """
+ raise NotImplementedError()
+
+ #////////////////////////////////////////////////////////////
+ # Comparisons & hashing
+ #////////////////////////////////////////////////////////////
+
+ def __eq__(self, other):
+ return (self.__class__ is other.__class__ and
+ self._comparison_key == other._comparison_key)
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, EdgeI):
+ raise_unorderable_types("<", self, other)
+ if self.__class__ is other.__class__:
+ return self._comparison_key < other._comparison_key
+ else:
+ return self.__class__.__name__ < other.__class__.__name__
+
+ def __hash__(self):
+ try:
+ return self._hash
+ except AttributeError:
+ self._hash = hash(self._comparison_key)
+ return self._hash
+
+
+ at python_2_unicode_compatible
+class TreeEdge(EdgeI):
+ """
+ An edge that records the fact that a tree is (partially)
+ consistent with the sentence. A tree edge consists of:
+
+ - A span, indicating what part of the sentence is
+ consistent with the hypothesized tree.
+ - A left-hand side, specifying the hypothesized tree's node
+ value.
+ - A right-hand side, specifying the hypothesized tree's
+ children. Each element of the right-hand side is either a
+ terminal, specifying a token with that terminal as its leaf
+ value; or a nonterminal, specifying a subtree with that
+ nonterminal's symbol as its node value.
+ - A dot position, indicating which children are consistent
+ with part of the sentence. In particular, if ``dot`` is the
+ dot position, ``rhs`` is the right-hand size, ``(start,end)``
+ is the span, and ``sentence`` is the list of tokens in the
+ sentence, then ``tokens[start:end]`` can be spanned by the
+ children specified by ``rhs[:dot]``.
+
+ For more information about edges, see the ``EdgeI`` interface.
+ """
+ def __init__(self, span, lhs, rhs, dot=0):
+ """
+ Construct a new ``TreeEdge``.
+
+ :type span: tuple(int, int)
+ :param span: A tuple ``(s, e)``, where ``tokens[s:e]`` is the
+ portion of the sentence that is consistent with the new
+ edge's structure.
+ :type lhs: Nonterminal
+ :param lhs: The new edge's left-hand side, specifying the
+ hypothesized tree's node value.
+ :type rhs: list(Nonterminal and str)
+ :param rhs: The new edge's right-hand side, specifying the
+ hypothesized tree's children.
+ :type dot: int
+ :param dot: The position of the new edge's dot. This position
+ specifies what prefix of the production's right hand side
+ is consistent with the text. In particular, if
+ ``sentence`` is the list of tokens in the sentence, then
+ ``okens[span[0]:span[1]]`` can be spanned by the
+ children specified by ``rhs[:dot]``.
+ """
+ self._span = span
+ self._lhs = lhs
+ rhs = tuple(rhs)
+ self._rhs = rhs
+ self._dot = dot
+ self._comparison_key = (span, lhs, rhs, dot)
+
+ @staticmethod
+ def from_production(production, index):
+ """
+ Return a new ``TreeEdge`` formed from the given production.
+ The new edge's left-hand side and right-hand side will
+ be taken from ``production``; its span will be
+ ``(index,index)``; and its dot position will be ``0``.
+
+ :rtype: TreeEdge
+ """
+ return TreeEdge(span=(index, index), lhs=production.lhs(),
+ rhs=production.rhs(), dot=0)
+
+ def move_dot_forward(self, new_end):
+ """
+ Return a new ``TreeEdge`` formed from this edge.
+ The new edge's dot position is increased by ``1``,
+ and its end index will be replaced by ``new_end``.
+
+ :param new_end: The new end index.
+ :type new_end: int
+ :rtype: TreeEdge
+ """
+ return TreeEdge(span=(self._span[0], new_end),
+ lhs=self._lhs, rhs=self._rhs,
+ dot=self._dot+1)
+
+ # Accessors
+ def lhs(self): return self._lhs
+ def span(self): return self._span
+ def start(self): return self._span[0]
+ def end(self): return self._span[1]
+ def length(self): return self._span[1] - self._span[0]
+ def rhs(self): return self._rhs
+ def dot(self): return self._dot
+ def is_complete(self): return self._dot == len(self._rhs)
+ def is_incomplete(self): return self._dot != len(self._rhs)
+ def nextsym(self):
+ if self._dot >= len(self._rhs): return None
+ else: return self._rhs[self._dot]
+
+ # String representation
+ def __str__(self):
+ str = '[%s:%s] ' % (self._span[0], self._span[1])
+ str += '%-2r ->' % (self._lhs,)
+
+ for i in range(len(self._rhs)):
+ if i == self._dot: str += ' *'
+ str += ' %s' % unicode_repr(self._rhs[i])
+ if len(self._rhs) == self._dot: str += ' *'
+ return str
+
+ def __repr__(self):
+ return '[Edge: %s]' % self
+
+
+ at python_2_unicode_compatible
+class LeafEdge(EdgeI):
+ """
+ An edge that records the fact that a leaf value is consistent with
+ a word in the sentence. A leaf edge consists of:
+
+ - An index, indicating the position of the word.
+ - A leaf, specifying the word's content.
+
+ A leaf edge's left-hand side is its leaf value, and its right hand
+ side is ``()``. Its span is ``[index, index+1]``, and its dot
+ position is ``0``.
+ """
+ def __init__(self, leaf, index):
+ """
+ Construct a new ``LeafEdge``.
+
+ :param leaf: The new edge's leaf value, specifying the word
+ that is recorded by this edge.
+ :param index: The new edge's index, specifying the position of
+ the word that is recorded by this edge.
+ """
+ self._leaf = leaf
+ self._index = index
+ self._comparison_key = (leaf, index)
+
+ # Accessors
+ def lhs(self): return self._leaf
+ def span(self): return (self._index, self._index+1)
+ def start(self): return self._index
+ def end(self): return self._index+1
+ def length(self): return 1
+ def rhs(self): return ()
+ def dot(self): return 0
+ def is_complete(self): return True
+ def is_incomplete(self): return False
+ def nextsym(self): return None
+
+ # String representations
+ def __str__(self):
+ return '[%s:%s] %s' % (self._index, self._index+1, unicode_repr(self._leaf))
+ def __repr__(self):
+ return '[Edge: %s]' % (self)
+
+########################################################################
+## Chart
+########################################################################
+
+class Chart(object):
+ """
+ A blackboard for hypotheses about the syntactic constituents of a
+ sentence. A chart contains a set of edges, and each edge encodes
+ a single hypothesis about the structure of some portion of the
+ sentence.
+
+ The ``select`` method can be used to select a specific collection
+ of edges. For example ``chart.select(is_complete=True, start=0)``
+ yields all complete edges whose start indices are 0. To ensure
+ the efficiency of these selection operations, ``Chart`` dynamically
+ creates and maintains an index for each set of attributes that
+ have been selected on.
+
+ In order to reconstruct the trees that are represented by an edge,
+ the chart associates each edge with a set of child pointer lists.
+ A child pointer list is a list of the edges that license an
+ edge's right-hand side.
+
+ :ivar _tokens: The sentence that the chart covers.
+ :ivar _num_leaves: The number of tokens.
+ :ivar _edges: A list of the edges in the chart
+ :ivar _edge_to_cpls: A dictionary mapping each edge to a set
+ of child pointer lists that are associated with that edge.
+ :ivar _indexes: A dictionary mapping tuples of edge attributes
+ to indices, where each index maps the corresponding edge
+ attribute values to lists of edges.
+ """
+ def __init__(self, tokens):
+ """
+ Construct a new chart. The chart is initialized with the
+ leaf edges corresponding to the terminal leaves.
+
+ :type tokens: list
+ :param tokens: The sentence that this chart will be used to parse.
+ """
+ # Record the sentence token and the sentence length.
+ self._tokens = tuple(tokens)
+ self._num_leaves = len(self._tokens)
+
+ # Initialise the chart.
+ self.initialize()
+
+ def initialize(self):
+ """
+ Clear the chart.
+ """
+ # A list of edges contained in this chart.
+ self._edges = []
+
+ # The set of child pointer lists associated with each edge.
+ self._edge_to_cpls = {}
+
+ # Indexes mapping attribute values to lists of edges
+ # (used by select()).
+ self._indexes = {}
+
+ #////////////////////////////////////////////////////////////
+ # Sentence Access
+ #////////////////////////////////////////////////////////////
+
+ def num_leaves(self):
+ """
+ Return the number of words in this chart's sentence.
+
+ :rtype: int
+ """
+ return self._num_leaves
+
+ def leaf(self, index):
+ """
+ Return the leaf value of the word at the given index.
+
+ :rtype: str
+ """
+ return self._tokens[index]
+
+ def leaves(self):
+ """
+ Return a list of the leaf values of each word in the
+ chart's sentence.
+
+ :rtype: list(str)
+ """
+ return self._tokens
+
+ #////////////////////////////////////////////////////////////
+ # Edge access
+ #////////////////////////////////////////////////////////////
+
+ def edges(self):
+ """
+ Return a list of all edges in this chart. New edges
+ that are added to the chart after the call to edges()
+ will *not* be contained in this list.
+
+ :rtype: list(EdgeI)
+ :see: ``iteredges``, ``select``
+ """
+ return self._edges[:]
+
+ def iteredges(self):
+ """
+ Return an iterator over the edges in this chart. It is
+ not guaranteed that new edges which are added to the
+ chart before the iterator is exhausted will also be generated.
+
+ :rtype: iter(EdgeI)
+ :see: ``edges``, ``select``
+ """
+ return iter(self._edges)
+
+ # Iterating over the chart yields its edges.
+ __iter__ = iteredges
+
+ def num_edges(self):
+ """
+ Return the number of edges contained in this chart.
+
+ :rtype: int
+ """
+ return len(self._edge_to_cpls)
+
+ def select(self, **restrictions):
+ """
+ Return an iterator over the edges in this chart. Any
+ new edges that are added to the chart before the iterator
+ is exahusted will also be generated. ``restrictions``
+ can be used to restrict the set of edges that will be
+ generated.
+
+ :param span: Only generate edges ``e`` where ``e.span()==span``
+ :param start: Only generate edges ``e`` where ``e.start()==start``
+ :param end: Only generate edges ``e`` where ``e.end()==end``
+ :param length: Only generate edges ``e`` where ``e.length()==length``
+ :param lhs: Only generate edges ``e`` where ``e.lhs()==lhs``
+ :param rhs: Only generate edges ``e`` where ``e.rhs()==rhs``
+ :param nextsym: Only generate edges ``e`` where
+ ``e.nextsym()==nextsym``
+ :param dot: Only generate edges ``e`` where ``e.dot()==dot``
+ :param is_complete: Only generate edges ``e`` where
+ ``e.is_complete()==is_complete``
+ :param is_incomplete: Only generate edges ``e`` where
+ ``e.is_incomplete()==is_incomplete``
+ :rtype: iter(EdgeI)
+ """
+ # If there are no restrictions, then return all edges.
+ if restrictions=={}: return iter(self._edges)
+
+ # Find the index corresponding to the given restrictions.
+ restr_keys = sorted(restrictions.keys())
+ restr_keys = tuple(restr_keys)
+
+ # If it doesn't exist, then create it.
+ if restr_keys not in self._indexes:
+ self._add_index(restr_keys)
+
+ vals = tuple(restrictions[key] for key in restr_keys)
+ return iter(self._indexes[restr_keys].get(vals, []))
+
+ def _add_index(self, restr_keys):
+ """
+ A helper function for ``select``, which creates a new index for
+ a given set of attributes (aka restriction keys).
+ """
+ # Make sure it's a valid index.
+ for key in restr_keys:
+ if not hasattr(EdgeI, key):
+ raise ValueError('Bad restriction: %s' % key)
+
+ # Create the index.
+ index = self._indexes[restr_keys] = {}
+
+ # Add all existing edges to the index.
+ for edge in self._edges:
+ vals = tuple(getattr(edge, key)() for key in restr_keys)
+ index.setdefault(vals, []).append(edge)
+
+ def _register_with_indexes(self, edge):
+ """
+ A helper function for ``insert``, which registers the new
+ edge with all existing indexes.
+ """
+ for (restr_keys, index) in self._indexes.items():
+ vals = tuple(getattr(edge, key)() for key in restr_keys)
+ index.setdefault(vals, []).append(edge)
+
+ #////////////////////////////////////////////////////////////
+ # Edge Insertion
+ #////////////////////////////////////////////////////////////
+
+ def insert_with_backpointer(self, new_edge, previous_edge, child_edge):
+ """
+ Add a new edge to the chart, using a pointer to the previous edge.
+ """
+ cpls = self.child_pointer_lists(previous_edge)
+ new_cpls = [cpl+(child_edge,) for cpl in cpls]
+ return self.insert(new_edge, *new_cpls)
+
+ def insert(self, edge, *child_pointer_lists):
+ """
+ Add a new edge to the chart, and return True if this operation
+ modified the chart. In particular, return true iff the chart
+ did not already contain ``edge``, or if it did not already associate
+ ``child_pointer_lists`` with ``edge``.
+
+ :type edge: EdgeI
+ :param edge: The new edge
+ :type child_pointer_lists: sequence of tuple(EdgeI)
+ :param child_pointer_lists: A sequence of lists of the edges that
+ were used to form this edge. This list is used to reconstruct
+ the trees (or partial trees) that are associated with ``edge``.
+ :rtype: bool
+ """
+ # Is it a new edge?
+ if edge not in self._edge_to_cpls:
+ # Add it to the list of edges.
+ self._append_edge(edge)
+ # Register with indexes.
+ self._register_with_indexes(edge)
+
+ # Get the set of child pointer lists for this edge.
+ cpls = self._edge_to_cpls.setdefault(edge, OrderedDict())
+ chart_was_modified = False
+ for child_pointer_list in child_pointer_lists:
+ child_pointer_list = tuple(child_pointer_list)
+ if child_pointer_list not in cpls:
+ # It's a new CPL; register it, and return true.
+ cpls[child_pointer_list] = True
+ chart_was_modified = True
+ return chart_was_modified
+
+ def _append_edge(self, edge):
+ self._edges.append(edge)
+
+ #////////////////////////////////////////////////////////////
+ # Tree extraction & child pointer lists
+ #////////////////////////////////////////////////////////////
+
+ def parses(self, root, tree_class=Tree):
+ """
+ Return an iterator of the complete tree structures that span
+ the entire chart, and whose root node is ``root``.
+ """
+ for edge in self.select(start=0, end=self._num_leaves, lhs=root):
+ for tree in self.trees(edge, tree_class=tree_class, complete=True):
+ yield tree
+
+ def trees(self, edge, tree_class=Tree, complete=False):
+ """
+ Return an iterator of the tree structures that are associated
+ with ``edge``.
+
+ If ``edge`` is incomplete, then the unexpanded children will be
+ encoded as childless subtrees, whose node value is the
+ corresponding terminal or nonterminal.
+
+ :rtype: list(Tree)
+ :note: If two trees share a common subtree, then the same
+ Tree may be used to encode that subtree in
+ both trees. If you need to eliminate this subtree
+ sharing, then create a deep copy of each tree.
+ """
+ return iter(self._trees(edge, complete, memo={}, tree_class=tree_class))
+
+ def _trees(self, edge, complete, memo, tree_class):
+ """
+ A helper function for ``trees``.
+
+ :param memo: A dictionary used to record the trees that we've
+ generated for each edge, so that when we see an edge more
+ than once, we can reuse the same trees.
+ """
+ # If we've seen this edge before, then reuse our old answer.
+ if edge in memo:
+ return memo[edge]
+
+ # when we're reading trees off the chart, don't use incomplete edges
+ if complete and edge.is_incomplete():
+ return []
+
+ # Leaf edges.
+ if isinstance(edge, LeafEdge):
+ leaf = self._tokens[edge.start()]
+ memo[edge] = [leaf]
+ return [leaf]
+
+ # Until we're done computing the trees for edge, set
+ # memo[edge] to be empty. This has the effect of filtering
+ # out any cyclic trees (i.e., trees that contain themselves as
+ # descendants), because if we reach this edge via a cycle,
+ # then it will appear that the edge doesn't generate any trees.
+ memo[edge] = []
+ trees = []
+ lhs = edge.lhs().symbol()
+
+ # Each child pointer list can be used to form trees.
+ for cpl in self.child_pointer_lists(edge):
+ # Get the set of child choices for each child pointer.
+ # child_choices[i] is the set of choices for the tree's
+ # ith child.
+ child_choices = [self._trees(cp, complete, memo, tree_class)
+ for cp in cpl]
+
+ # For each combination of children, add a tree.
+ for children in itertools.product(*child_choices):
+ trees.append(tree_class(lhs, children))
+
+ # If the edge is incomplete, then extend it with "partial trees":
+ if edge.is_incomplete():
+ unexpanded = [tree_class(elt,[])
+ for elt in edge.rhs()[edge.dot():]]
+ for tree in trees:
+ tree.extend(unexpanded)
+
+ # Update the memoization dictionary.
+ memo[edge] = trees
+
+ # Return the list of trees.
+ return trees
+
+ def child_pointer_lists(self, edge):
+ """
+ Return the set of child pointer lists for the given edge.
+ Each child pointer list is a list of edges that have
+ been used to form this edge.
+
+ :rtype: list(list(EdgeI))
+ """
+ # Make a copy, in case they modify it.
+ return self._edge_to_cpls.get(edge, {}).keys()
+
+ #////////////////////////////////////////////////////////////
+ # Display
+ #////////////////////////////////////////////////////////////
+ def pp_edge(self, edge, width=None):
+ """
+ Return a pretty-printed string representation of a given edge
+ in this chart.
+
+ :rtype: str
+ :param width: The number of characters allotted to each
+ index in the sentence.
+ """
+ if width is None: width = 50 // (self.num_leaves()+1)
+ (start, end) = (edge.start(), edge.end())
+
+ str = '|' + ('.'+' '*(width-1))*start
+
+ # Zero-width edges are "#" if complete, ">" if incomplete
+ if start == end:
+ if edge.is_complete(): str += '#'
+ else: str += '>'
+
+ # Spanning complete edges are "[===]"; Other edges are
+ # "[---]" if complete, "[--->" if incomplete
+ elif edge.is_complete() and edge.span() == (0,self._num_leaves):
+ str += '['+('='*width)*(end-start-1) + '='*(width-1)+']'
+ elif edge.is_complete():
+ str += '['+('-'*width)*(end-start-1) + '-'*(width-1)+']'
+ else:
+ str += '['+('-'*width)*(end-start-1) + '-'*(width-1)+'>'
+
+ str += (' '*(width-1)+'.')*(self._num_leaves-end)
+ return str + '| %s' % edge
+
+ def pp_leaves(self, width=None):
+ """
+ Return a pretty-printed string representation of this
+ chart's leaves. This string can be used as a header
+ for calls to ``pp_edge``.
+ """
+ if width is None: width = 50 // (self.num_leaves()+1)
+
+ if self._tokens is not None and width>1:
+ header = '|.'
+ for tok in self._tokens:
+ header += tok[:width-1].center(width-1)+'.'
+ header += '|'
+ else:
+ header = ''
+
+ return header
+
+ def pp(self, width=None):
+ """
+ Return a pretty-printed string representation of this chart.
+
+ :param width: The number of characters allotted to each
+ index in the sentence.
+ :rtype: str
+ """
+ if width is None: width = 50 // (self.num_leaves()+1)
+ # sort edges: primary key=length, secondary key=start index.
+ # (and filter out the token edges)
+ edges = sorted([(e.length(), e.start(), e) for e in self])
+ edges = [e for (_,_,e) in edges]
+
+ return (self.pp_leaves(width) + '\n' +
+ '\n'.join(self.pp_edge(edge, width) for edge in edges))
+
+ #////////////////////////////////////////////////////////////
+ # Display: Dot (AT&T Graphviz)
+ #////////////////////////////////////////////////////////////
+
+ def dot_digraph(self):
+ # Header
+ s = 'digraph nltk_chart {\n'
+ #s += ' size="5,5";\n'
+ s += ' rankdir=LR;\n'
+ s += ' node [height=0.1,width=0.1];\n'
+ s += ' node [style=filled, color="lightgray"];\n'
+
+ # Set up the nodes
+ for y in range(self.num_edges(), -1, -1):
+ if y == 0:
+ s += ' node [style=filled, color="black"];\n'
+ for x in range(self.num_leaves()+1):
+ if y == 0 or (x <= self._edges[y-1].start() or
+ x >= self._edges[y-1].end()):
+ s += ' %04d.%04d [label=""];\n' % (x,y)
+
+ # Add a spacer
+ s += ' x [style=invis]; x->0000.0000 [style=invis];\n'
+
+ # Declare ranks.
+ for x in range(self.num_leaves()+1):
+ s += ' {rank=same;'
+ for y in range(self.num_edges()+1):
+ if y == 0 or (x <= self._edges[y-1].start() or
+ x >= self._edges[y-1].end()):
+ s += ' %04d.%04d' % (x,y)
+ s += '}\n'
+
+ # Add the leaves
+ s += ' edge [style=invis, weight=100];\n'
+ s += ' node [shape=plaintext]\n'
+ s += ' 0000.0000'
+ for x in range(self.num_leaves()):
+ s += '->%s->%04d.0000' % (self.leaf(x), x+1)
+ s += ';\n\n'
+
+ # Add the edges
+ s += ' edge [style=solid, weight=1];\n'
+ for y, edge in enumerate(self):
+ for x in range(edge.start()):
+ s += (' %04d.%04d -> %04d.%04d [style="invis"];\n' %
+ (x, y+1, x+1, y+1))
+ s += (' %04d.%04d -> %04d.%04d [label="%s"];\n' %
+ (edge.start(), y+1, edge.end(), y+1, edge))
+ for x in range(edge.end(), self.num_leaves()):
+ s += (' %04d.%04d -> %04d.%04d [style="invis"];\n' %
+ (x, y+1, x+1, y+1))
+ s += '}\n'
+ return s
+
+########################################################################
+## Chart Rules
+########################################################################
+
+class ChartRuleI(object):
+ """
+ A rule that specifies what new edges are licensed by any given set
+ of existing edges. Each chart rule expects a fixed number of
+ edges, as indicated by the class variable ``NUM_EDGES``. In
+ particular:
+
+ - A chart rule with ``NUM_EDGES=0`` specifies what new edges are
+ licensed, regardless of existing edges.
+ - A chart rule with ``NUM_EDGES=1`` specifies what new edges are
+ licensed by a single existing edge.
+ - A chart rule with ``NUM_EDGES=2`` specifies what new edges are
+ licensed by a pair of existing edges.
+
+ :type NUM_EDGES: int
+ :cvar NUM_EDGES: The number of existing edges that this rule uses
+ to license new edges. Typically, this number ranges from zero
+ to two.
+ """
+ def apply(self, chart, grammar, *edges):
+ """
+ Return a generator that will add edges licensed by this rule
+ and the given edges to the chart, one at a time. Each
+ time the generator is resumed, it will either add a new
+ edge and yield that edge; or return.
+
+ :type edges: list(EdgeI)
+ :param edges: A set of existing edges. The number of edges
+ that should be passed to ``apply()`` is specified by the
+ ``NUM_EDGES`` class variable.
+ :rtype: iter(EdgeI)
+ """
+ raise NotImplementedError()
+
+ def apply_everywhere(self, chart, grammar):
+ """
+ Return a generator that will add all edges licensed by
+ this rule, given the edges that are currently in the
+ chart, one at a time. Each time the generator is resumed,
+ it will either add a new edge and yield that edge; or return.
+
+ :rtype: iter(EdgeI)
+ """
+ raise NotImplementedError()
+
+
+ at python_2_unicode_compatible
+class AbstractChartRule(ChartRuleI):
+ """
+ An abstract base class for chart rules. ``AbstractChartRule``
+ provides:
+
+ - A default implementation for ``apply``.
+ - A default implementation for ``apply_everywhere``,
+ (Currently, this implementation assumes that ``NUM_EDGES``<=3.)
+ - A default implementation for ``__str__``, which returns a
+ name based on the rule's class name.
+ """
+
+ # Subclasses must define apply.
+ def apply(self, chart, grammar, *edges):
+ raise NotImplementedError()
+
+ # Default: loop through the given number of edges, and call
+ # self.apply() for each set of edges.
+ def apply_everywhere(self, chart, grammar):
+ if self.NUM_EDGES == 0:
+ for new_edge in self.apply(chart, grammar):
+ yield new_edge
+
+ elif self.NUM_EDGES == 1:
+ for e1 in chart:
+ for new_edge in self.apply(chart, grammar, e1):
+ yield new_edge
+
+ elif self.NUM_EDGES == 2:
+ for e1 in chart:
+ for e2 in chart:
+ for new_edge in self.apply(chart, grammar, e1, e2):
+ yield new_edge
+
+ elif self.NUM_EDGES == 3:
+ for e1 in chart:
+ for e2 in chart:
+ for e3 in chart:
+ for new_edge in self.apply(chart,grammar,e1,e2,e3):
+ yield new_edge
+
+ else:
+ raise AssertionError('NUM_EDGES>3 is not currently supported')
+
+ # Default: return a name based on the class name.
+ def __str__(self):
+ # Add spaces between InitialCapsWords.
+ return re.sub('([a-z])([A-Z])', r'\1 \2', self.__class__.__name__)
+
+#////////////////////////////////////////////////////////////
+# Fundamental Rule
+#////////////////////////////////////////////////////////////
+
+class FundamentalRule(AbstractChartRule):
+ """
+ A rule that joins two adjacent edges to form a single combined
+ edge. In particular, this rule specifies that any pair of edges
+
+ - ``[A -> alpha \* B beta][i:j]``
+ - ``[B -> gamma \*][j:k]``
+
+ licenses the edge:
+
+ - ``[A -> alpha B * beta][i:j]``
+ """
+ NUM_EDGES = 2
+ def apply(self, chart, grammar, left_edge, right_edge):
+ # Make sure the rule is applicable.
+ if not (left_edge.is_incomplete() and
+ right_edge.is_complete() and
+ left_edge.end() == right_edge.start() and
+ left_edge.nextsym() == right_edge.lhs()):
+ return
+
+ # Construct the new edge.
+ new_edge = left_edge.move_dot_forward(right_edge.end())
+
+ # Insert it into the chart.
+ if chart.insert_with_backpointer(new_edge, left_edge, right_edge):
+ yield new_edge
+
+class SingleEdgeFundamentalRule(FundamentalRule):
+ """
+ A rule that joins a given edge with adjacent edges in the chart,
+ to form combined edges. In particular, this rule specifies that
+ either of the edges:
+
+ - ``[A -> alpha \* B beta][i:j]``
+ - ``[B -> gamma \*][j:k]``
+
+ licenses the edge:
+
+ - ``[A -> alpha B * beta][i:j]``
+
+ if the other edge is already in the chart.
+
+ :note: This is basically ``FundamentalRule``, with one edge left
+ unspecified.
+ """
+ NUM_EDGES = 1
+
+ def apply(self, chart, grammar, edge):
+ if edge.is_incomplete():
+ for new_edge in self._apply_incomplete(chart, grammar, edge):
+ yield new_edge
+ else:
+ for new_edge in self._apply_complete(chart, grammar, edge):
+ yield new_edge
+
+ def _apply_complete(self, chart, grammar, right_edge):
+ for left_edge in chart.select(end=right_edge.start(),
+ is_complete=False,
+ nextsym=right_edge.lhs()):
+ new_edge = left_edge.move_dot_forward(right_edge.end())
+ if chart.insert_with_backpointer(new_edge, left_edge, right_edge):
+ yield new_edge
+
+ def _apply_incomplete(self, chart, grammar, left_edge):
+ for right_edge in chart.select(start=left_edge.end(),
+ is_complete=True,
+ lhs=left_edge.nextsym()):
+ new_edge = left_edge.move_dot_forward(right_edge.end())
+ if chart.insert_with_backpointer(new_edge, left_edge, right_edge):
+ yield new_edge
+
+#////////////////////////////////////////////////////////////
+# Inserting Terminal Leafs
+#////////////////////////////////////////////////////////////
+
+class LeafInitRule(AbstractChartRule):
+ NUM_EDGES=0
+ def apply(self, chart, grammar):
+ for index in range(chart.num_leaves()):
+ new_edge = LeafEdge(chart.leaf(index), index)
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+#////////////////////////////////////////////////////////////
+# Top-Down Prediction
+#////////////////////////////////////////////////////////////
+
+class TopDownInitRule(AbstractChartRule):
+ """
+ A rule licensing edges corresponding to the grammar productions for
+ the grammar's start symbol. In particular, this rule specifies that
+ ``[S -> \* alpha][0:i]`` is licensed for each grammar production
+ ``S -> alpha``, where ``S`` is the grammar's start symbol.
+ """
+ NUM_EDGES = 0
+ def apply(self, chart, grammar):
+ for prod in grammar.productions(lhs=grammar.start()):
+ new_edge = TreeEdge.from_production(prod, 0)
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+class TopDownPredictRule(AbstractChartRule):
+ """
+ A rule licensing edges corresponding to the grammar productions
+ for the nonterminal following an incomplete edge's dot. In
+ particular, this rule specifies that
+ ``[A -> alpha \* B beta][i:j]`` licenses the edge
+ ``[B -> \* gamma][j:j]`` for each grammar production ``B -> gamma``.
+
+ :note: This rule corresponds to the Predictor Rule in Earley parsing.
+ """
+ NUM_EDGES = 1
+ def apply(self, chart, grammar, edge):
+ if edge.is_complete(): return
+ for prod in grammar.productions(lhs=edge.nextsym()):
+ new_edge = TreeEdge.from_production(prod, edge.end())
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+class CachedTopDownPredictRule(TopDownPredictRule):
+ """
+ A cached version of ``TopDownPredictRule``. After the first time
+ this rule is applied to an edge with a given ``end`` and ``next``,
+ it will not generate any more edges for edges with that ``end`` and
+ ``next``.
+
+ If ``chart`` or ``grammar`` are changed, then the cache is flushed.
+ """
+ def __init__(self):
+ TopDownPredictRule.__init__(self)
+ self._done = {}
+
+ def apply(self, chart, grammar, edge):
+ if edge.is_complete(): return
+ nextsym, index = edge.nextsym(), edge.end()
+ if not is_nonterminal(nextsym): return
+
+ # If we've already applied this rule to an edge with the same
+ # next & end, and the chart & grammar have not changed, then
+ # just return (no new edges to add).
+ done = self._done.get((nextsym, index), (None,None))
+ if done[0] is chart and done[1] is grammar: return
+
+ # Add all the edges indicated by the top down expand rule.
+ for prod in grammar.productions(lhs=nextsym):
+ # If the left corner in the predicted production is
+ # leaf, it must match with the input.
+ if prod.rhs():
+ first = prod.rhs()[0]
+ if is_terminal(first):
+ if index >= chart.num_leaves() or first != chart.leaf(index): continue
+
+ new_edge = TreeEdge.from_production(prod, index)
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+ # Record the fact that we've applied this rule.
+ self._done[nextsym, index] = (chart, grammar)
+
+#////////////////////////////////////////////////////////////
+# Bottom-Up Prediction
+#////////////////////////////////////////////////////////////
+
+class BottomUpPredictRule(AbstractChartRule):
+ """
+ A rule licensing any edge corresponding to a production whose
+ right-hand side begins with a complete edge's left-hand side. In
+ particular, this rule specifies that ``[A -> alpha \*]`` licenses
+ the edge ``[B -> \* A beta]`` for each grammar production ``B -> A beta``.
+ """
+ NUM_EDGES = 1
+ def apply(self, chart, grammar, edge):
+ if edge.is_incomplete(): return
+ for prod in grammar.productions(rhs=edge.lhs()):
+ new_edge = TreeEdge.from_production(prod, edge.start())
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+class BottomUpPredictCombineRule(BottomUpPredictRule):
+ """
+ A rule licensing any edge corresponding to a production whose
+ right-hand side begins with a complete edge's left-hand side. In
+ particular, this rule specifies that ``[A -> alpha \*]``
+ licenses the edge ``[B -> A \* beta]`` for each grammar
+ production ``B -> A beta``.
+
+ :note: This is like ``BottomUpPredictRule``, but it also applies
+ the ``FundamentalRule`` to the resulting edge.
+ """
+ NUM_EDGES = 1
+ def apply(self, chart, grammar, edge):
+ if edge.is_incomplete(): return
+ for prod in grammar.productions(rhs=edge.lhs()):
+ new_edge = TreeEdge(edge.span(), prod.lhs(), prod.rhs(), 1)
+ if chart.insert(new_edge, (edge,)):
+ yield new_edge
+
+class EmptyPredictRule(AbstractChartRule):
+ """
+ A rule that inserts all empty productions as passive edges,
+ in every position in the chart.
+ """
+ NUM_EDGES = 0
+ def apply(self, chart, grammar):
+ for prod in grammar.productions(empty=True):
+ for index in compat.xrange(chart.num_leaves() + 1):
+ new_edge = TreeEdge.from_production(prod, index)
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+
+########################################################################
+## Filtered Bottom Up
+########################################################################
+
+class FilteredSingleEdgeFundamentalRule(SingleEdgeFundamentalRule):
+ def _apply_complete(self, chart, grammar, right_edge):
+ end = right_edge.end()
+ nexttoken = end < chart.num_leaves() and chart.leaf(end)
+ for left_edge in chart.select(end=right_edge.start(),
+ is_complete=False,
+ nextsym=right_edge.lhs()):
+ if _bottomup_filter(grammar, nexttoken, left_edge.rhs(), left_edge.dot()):
+ new_edge = left_edge.move_dot_forward(right_edge.end())
+ if chart.insert_with_backpointer(new_edge, left_edge, right_edge):
+ yield new_edge
+
+ def _apply_incomplete(self, chart, grammar, left_edge):
+ for right_edge in chart.select(start=left_edge.end(),
+ is_complete=True,
+ lhs=left_edge.nextsym()):
+ end = right_edge.end()
+ nexttoken = end < chart.num_leaves() and chart.leaf(end)
+ if _bottomup_filter(grammar, nexttoken, left_edge.rhs(), left_edge.dot()):
+ new_edge = left_edge.move_dot_forward(right_edge.end())
+ if chart.insert_with_backpointer(new_edge, left_edge, right_edge):
+ yield new_edge
+
+class FilteredBottomUpPredictCombineRule(BottomUpPredictCombineRule):
+ def apply(self, chart, grammar, edge):
+ if edge.is_incomplete():
+ return
+
+ end = edge.end()
+ nexttoken = end < chart.num_leaves() and chart.leaf(end)
+ for prod in grammar.productions(rhs=edge.lhs()):
+ if _bottomup_filter(grammar, nexttoken, prod.rhs()):
+ new_edge = TreeEdge(edge.span(), prod.lhs(), prod.rhs(), 1)
+ if chart.insert(new_edge, (edge,)):
+ yield new_edge
+
+def _bottomup_filter(grammar, nexttoken, rhs, dot=0):
+ if len(rhs) <= dot + 1:
+ return True
+ _next = rhs[dot + 1]
+ if is_terminal(_next):
+ return nexttoken == _next
+ else:
+ return grammar.is_leftcorner(_next, nexttoken)
+
+
+########################################################################
+## Generic Chart Parser
+########################################################################
+
+TD_STRATEGY = [LeafInitRule(),
+ TopDownInitRule(),
+ CachedTopDownPredictRule(),
+ SingleEdgeFundamentalRule()]
+BU_STRATEGY = [LeafInitRule(),
+ EmptyPredictRule(),
+ BottomUpPredictRule(),
+ SingleEdgeFundamentalRule()]
+BU_LC_STRATEGY = [LeafInitRule(),
+ EmptyPredictRule(),
+ BottomUpPredictCombineRule(),
+ SingleEdgeFundamentalRule()]
+
+LC_STRATEGY = [LeafInitRule(),
+ FilteredBottomUpPredictCombineRule(),
+ FilteredSingleEdgeFundamentalRule()]
+
+class ChartParser(ParserI):
+ """
+ A generic chart parser. A "strategy", or list of
+ ``ChartRuleI`` instances, is used to decide what edges to add to
+ the chart. In particular, ``ChartParser`` uses the following
+ algorithm to parse texts:
+
+ | Until no new edges are added:
+ | For each *rule* in *strategy*:
+ | Apply *rule* to any applicable edges in the chart.
+ | Return any complete parses in the chart
+ """
+ def __init__(self, grammar, strategy=BU_LC_STRATEGY, trace=0,
+ trace_chart_width=50, use_agenda=True, chart_class=Chart):
+ """
+ Create a new chart parser, that uses ``grammar`` to parse
+ texts.
+
+ :type grammar: CFG
+ :param grammar: The grammar used to parse texts.
+ :type strategy: list(ChartRuleI)
+ :param strategy: A list of rules that should be used to decide
+ what edges to add to the chart (top-down strategy by default).
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ and higher numbers will produce more verbose tracing
+ output.
+ :type trace_chart_width: int
+ :param trace_chart_width: The default total width reserved for
+ the chart in trace output. The remainder of each line will
+ be used to display edges.
+ :type use_agenda: bool
+ :param use_agenda: Use an optimized agenda-based algorithm,
+ if possible.
+ :param chart_class: The class that should be used to create
+ the parse charts.
+ """
+ self._grammar = grammar
+ self._strategy = strategy
+ self._trace = trace
+ self._trace_chart_width = trace_chart_width
+ # If the strategy only consists of axioms (NUM_EDGES==0) and
+ # inference rules (NUM_EDGES==1), we can use an agenda-based algorithm:
+ self._use_agenda = use_agenda
+ self._chart_class = chart_class
+
+ self._axioms = []
+ self._inference_rules = []
+ for rule in strategy:
+ if rule.NUM_EDGES == 0:
+ self._axioms.append(rule)
+ elif rule.NUM_EDGES == 1:
+ self._inference_rules.append(rule)
+ else:
+ self._use_agenda = False
+
+ def grammar(self):
+ return self._grammar
+
+ def _trace_new_edges(self, chart, rule, new_edges, trace, edge_width):
+ if not trace: return
+ print_rule_header = trace > 1
+ for edge in new_edges:
+ if print_rule_header:
+ print('%s:' % rule)
+ print_rule_header = False
+ print(chart.pp_edge(edge, edge_width))
+
+ def chart_parse(self, tokens, trace=None):
+ """
+ Return the final parse ``Chart`` from which all possible
+ parse trees can be extracted.
+
+ :param tokens: The sentence to be parsed
+ :type tokens: list(str)
+ :rtype: Chart
+ """
+ if trace is None: trace = self._trace
+ trace_new_edges = self._trace_new_edges
+
+ tokens = list(tokens)
+ self._grammar.check_coverage(tokens)
+ chart = self._chart_class(tokens)
+ grammar = self._grammar
+
+ # Width, for printing trace edges.
+ trace_edge_width = self._trace_chart_width // (chart.num_leaves() + 1)
+ if trace: print(chart.pp_leaves(trace_edge_width))
+
+ if self._use_agenda:
+ # Use an agenda-based algorithm.
+ for axiom in self._axioms:
+ new_edges = list(axiom.apply(chart, grammar))
+ trace_new_edges(chart, axiom, new_edges, trace, trace_edge_width)
+
+ inference_rules = self._inference_rules
+ agenda = chart.edges()
+ # We reverse the initial agenda, since it is a stack
+ # but chart.edges() functions as a queue.
+ agenda.reverse()
+ while agenda:
+ edge = agenda.pop()
+ for rule in inference_rules:
+ new_edges = list(rule.apply(chart, grammar, edge))
+ if trace:
+ trace_new_edges(chart, rule, new_edges, trace, trace_edge_width)
+ agenda += new_edges
+
+ else:
+ # Do not use an agenda-based algorithm.
+ edges_added = True
+ while edges_added:
+ edges_added = False
+ for rule in self._strategy:
+ new_edges = list(rule.apply_everywhere(chart, grammar))
+ edges_added = len(new_edges)
+ trace_new_edges(chart, rule, new_edges, trace, trace_edge_width)
+
+ # Return the final chart.
+ return chart
+
+ def parse_all(self, tokens, tree_class=Tree):
+ chart = self.chart_parse(tokens)
+ return chart.parses(self._grammar.start(), tree_class=tree_class)
+
+class TopDownChartParser(ChartParser):
+ """
+ A ``ChartParser`` using a top-down parsing strategy.
+ See ``ChartParser`` for more information.
+ """
+ def __init__(self, grammar, **parser_args):
+ ChartParser.__init__(self, grammar, TD_STRATEGY, **parser_args)
+
+class BottomUpChartParser(ChartParser):
+ """
+ A ``ChartParser`` using a bottom-up parsing strategy.
+ See ``ChartParser`` for more information.
+ """
+ def __init__(self, grammar, **parser_args):
+ if isinstance(grammar, PCFG):
+ warnings.warn("BottomUpChartParser only works for CFG, "
+ "use BottomUpProbabilisticChartParser instead",
+ category=DeprecationWarning)
+ ChartParser.__init__(self, grammar, BU_STRATEGY, **parser_args)
+
+class BottomUpLeftCornerChartParser(ChartParser):
+ """
+ A ``ChartParser`` using a bottom-up left-corner parsing strategy.
+ This strategy is often more efficient than standard bottom-up.
+ See ``ChartParser`` for more information.
+ """
+ def __init__(self, grammar, **parser_args):
+ ChartParser.__init__(self, grammar, BU_LC_STRATEGY, **parser_args)
+
+class LeftCornerChartParser(ChartParser):
+ def __init__(self, grammar, **parser_args):
+ if not grammar.is_nonempty():
+ raise ValueError("LeftCornerParser only works for grammars "
+ "without empty productions.")
+ ChartParser.__init__(self, grammar, LC_STRATEGY, **parser_args)
+
+########################################################################
+## Stepping Chart Parser
+########################################################################
+
+class SteppingChartParser(ChartParser):
+ """
+ A ``ChartParser`` that allows you to step through the parsing
+ process, adding a single edge at a time. It also allows you to
+ change the parser's strategy or grammar midway through parsing a
+ text.
+
+ The ``initialize`` method is used to start parsing a text. ``step``
+ adds a single edge to the chart. ``set_strategy`` changes the
+ strategy used by the chart parser. ``parses`` returns the set of
+ parses that has been found by the chart parser.
+
+ :ivar _restart: Records whether the parser's strategy, grammar,
+ or chart has been changed. If so, then ``step`` must restart
+ the parsing algorithm.
+ """
+ def __init__(self, grammar, strategy=[], trace=0):
+ self._chart = None
+ self._current_chartrule = None
+ self._restart = False
+ ChartParser.__init__(self, grammar, strategy, trace)
+
+ #////////////////////////////////////////////////////////////
+ # Initialization
+ #////////////////////////////////////////////////////////////
+
+ def initialize(self, tokens):
+ "Begin parsing the given tokens."
+ self._chart = Chart(list(tokens))
+ self._restart = True
+
+ #////////////////////////////////////////////////////////////
+ # Stepping
+ #////////////////////////////////////////////////////////////
+
+ def step(self):
+ """
+ Return a generator that adds edges to the chart, one at a
+ time. Each time the generator is resumed, it adds a single
+ edge and yields that edge. If no more edges can be added,
+ then it yields None.
+
+ If the parser's strategy, grammar, or chart is changed, then
+ the generator will continue adding edges using the new
+ strategy, grammar, or chart.
+
+ Note that this generator never terminates, since the grammar
+ or strategy might be changed to values that would add new
+ edges. Instead, it yields None when no more edges can be
+ added with the current strategy and grammar.
+ """
+ if self._chart is None:
+ raise ValueError('Parser must be initialized first')
+ while True:
+ self._restart = False
+ w = 50 // (self._chart.num_leaves()+1)
+
+ for e in self._parse():
+ if self._trace > 1: print(self._current_chartrule)
+ if self._trace > 0: print(self._chart.pp_edge(e,w))
+ yield e
+ if self._restart: break
+ else:
+ yield None # No more edges.
+
+ def _parse(self):
+ """
+ A generator that implements the actual parsing algorithm.
+ ``step`` iterates through this generator, and restarts it
+ whenever the parser's strategy, grammar, or chart is modified.
+ """
+ chart = self._chart
+ grammar = self._grammar
+ edges_added = 1
+ while edges_added > 0:
+ edges_added = 0
+ for rule in self._strategy:
+ self._current_chartrule = rule
+ for e in rule.apply_everywhere(chart, grammar):
+ edges_added += 1
+ yield e
+
+ #////////////////////////////////////////////////////////////
+ # Accessors
+ #////////////////////////////////////////////////////////////
+
+ def strategy(self):
+ "Return the strategy used by this parser."
+ return self._strategy
+
+ def grammar(self):
+ "Return the grammar used by this parser."
+ return self._grammar
+
+ def chart(self):
+ "Return the chart that is used by this parser."
+ return self._chart
+
+ def current_chartrule(self):
+ "Return the chart rule used to generate the most recent edge."
+ return self._current_chartrule
+
+ def parses(self, tree_class=Tree):
+ "Return the parse trees currently contained in the chart."
+ return self._chart.parses(self._grammar.start(), tree_class)
+
+ #////////////////////////////////////////////////////////////
+ # Parser modification
+ #////////////////////////////////////////////////////////////
+
+ def set_strategy(self, strategy):
+ """
+ Change the strategy that the parser uses to decide which edges
+ to add to the chart.
+
+ :type strategy: list(ChartRuleI)
+ :param strategy: A list of rules that should be used to decide
+ what edges to add to the chart.
+ """
+ if strategy == self._strategy: return
+ self._strategy = strategy[:] # Make a copy.
+ self._restart = True
+
+ def set_grammar(self, grammar):
+ "Change the grammar used by the parser."
+ if grammar is self._grammar: return
+ self._grammar = grammar
+ self._restart = True
+
+ def set_chart(self, chart):
+ "Load a given chart into the chart parser."
+ if chart is self._chart: return
+ self._chart = chart
+ self._restart = True
+
+ #////////////////////////////////////////////////////////////
+ # Standard parser methods
+ #////////////////////////////////////////////////////////////
+
+ def parse(self, tokens, tree_class=Tree):
+ tokens = list(tokens)
+ self._grammar.check_coverage(tokens)
+
+ # Initialize ourselves.
+ self.initialize(tokens)
+
+ # Step until no more edges are generated.
+ for e in self.step():
+ if e is None: break
+
+ # Return an iterator of complete parses.
+ return self.parses(tree_class=tree_class)
+
+########################################################################
+## Demo Code
+########################################################################
+
+def demo_grammar():
+ from nltk.grammar import CFG
+ return CFG.fromstring("""
+S -> NP VP
+PP -> "with" NP
+NP -> NP PP
+VP -> VP PP
+VP -> Verb NP
+VP -> Verb
+NP -> Det Noun
+NP -> "John"
+NP -> "I"
+Det -> "the"
+Det -> "my"
+Det -> "a"
+Noun -> "dog"
+Noun -> "cookie"
+Verb -> "ate"
+Verb -> "saw"
+Prep -> "with"
+Prep -> "under"
+""")
+
+def demo(choice=None,
+ print_times=True, print_grammar=False,
+ print_trees=True, trace=2,
+ sent='I saw John with a dog with my cookie', numparses=5):
+ """
+ A demonstration of the chart parsers.
+ """
+ import sys, time
+ from nltk import nonterminals, Production, CFG
+
+ # The grammar for ChartParser and SteppingChartParser:
+ grammar = demo_grammar()
+ if print_grammar:
+ print("* Grammar")
+ print(grammar)
+
+ # Tokenize the sample sentence.
+ print("* Sentence:")
+ print(sent)
+ tokens = sent.split()
+ print(tokens)
+ print()
+
+ # Ask the user which parser to test,
+ # if the parser wasn't provided as an argument
+ if choice is None:
+ print(' 1: Top-down chart parser')
+ print(' 2: Bottom-up chart parser')
+ print(' 3: Bottom-up left-corner chart parser')
+ print(' 4: Left-corner chart parser with bottom-up filter')
+ print(' 5: Stepping chart parser (alternating top-down & bottom-up)')
+ print(' 6: All parsers')
+ print('\nWhich parser (1-6)? ', end=' ')
+ choice = sys.stdin.readline().strip()
+ print()
+
+ choice = str(choice)
+ if choice not in "123456":
+ print('Bad parser number')
+ return
+
+ # Keep track of how long each parser takes.
+ times = {}
+
+ strategies = {'1': ('Top-down', TD_STRATEGY),
+ '2': ('Bottom-up', BU_STRATEGY),
+ '3': ('Bottom-up left-corner', BU_LC_STRATEGY),
+ '4': ('Filtered left-corner', LC_STRATEGY)}
+ choices = []
+ if choice in strategies: choices = [choice]
+ if choice=='6': choices = "1234"
+
+ # Run the requested chart parser(s), except the stepping parser.
+ for strategy in choices:
+ print("* Strategy: " + strategies[strategy][0])
+ print()
+ cp = ChartParser(grammar, strategies[strategy][1], trace=trace)
+ t = time.time()
+ # parses = cp.parse_all(tokens)
+ chart = cp.chart_parse(tokens)
+ parses = list(chart.parses(grammar.start()))
+ times[strategies[strategy][0]] = time.time()-t
+ print("Nr edges in chart:", len(chart.edges()))
+ if numparses:
+ assert len(parses)==numparses, 'Not all parses found'
+ if print_trees:
+ for tree in parses: print(tree)
+ else:
+ print("Nr trees:", len(parses))
+ print()
+
+ # Run the stepping parser, if requested.
+ if choice in "56":
+ print("* Strategy: Stepping (top-down vs bottom-up)")
+ print()
+ t = time.time()
+ cp = SteppingChartParser(grammar, trace=trace)
+ cp.initialize(tokens)
+ for i in range(5):
+ print('*** SWITCH TO TOP DOWN')
+ cp.set_strategy(TD_STRATEGY)
+ for j, e in enumerate(cp.step()):
+ if j>20 or e is None: break
+ print('*** SWITCH TO BOTTOM UP')
+ cp.set_strategy(BU_STRATEGY)
+ for j, e in enumerate(cp.step()):
+ if j>20 or e is None: break
+ times['Stepping'] = time.time()-t
+ print("Nr edges in chart:", len(cp.chart().edges()))
+ if numparses:
+ assert len(list(cp.parses()))==numparses, 'Not all parses found'
+ if print_trees:
+ for tree in cp.parses(): print(tree)
+ else:
+ print("Nr trees:", len(list(cp.parses())))
+ print()
+
+ # Print the times of all parsers:
+ if not (print_times and times): return
+ print("* Parsing times")
+ print()
+ maxlen = max(len(key) for key in times)
+ format = '%' + repr(maxlen) + 's parser: %6.3fsec'
+ times_items = times.items()
+ for (parser, t) in sorted(times_items, key=lambda a:a[1]):
+ print(format % (parser, t))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/dependencygraph.py b/nltk/parse/dependencygraph.py
new file mode 100644
index 0000000..918753e
--- /dev/null
+++ b/nltk/parse/dependencygraph.py
@@ -0,0 +1,516 @@
+# Natural Language Toolkit: Dependency Grammars
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Jason Narad <jason.narad at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (modifications)
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+
+"""
+Tools for reading and writing dependency trees.
+The input is assumed to be in Malt-TAB format
+(http://stp.lingfil.uu.se/~nivre/research/MaltXML.html).
+"""
+from __future__ import print_function, unicode_literals
+
+import re
+from pprint import pformat
+
+from nltk.tree import Tree
+from nltk.compat import python_2_unicode_compatible
+
+#################################################################
+# DependencyGraph Class
+#################################################################
+
+ at python_2_unicode_compatible
+class DependencyGraph(object):
+ """
+ A container for the nodes and labelled edges of a dependency structure.
+ """
+ def __init__(self, tree_str=None):
+ """
+ We place a dummy 'top' node in the first position
+ in the nodelist, since the root node is often assigned '0'
+ as its head. This also means that the indexing of the nodelist
+ corresponds directly to the Malt-TAB format, which starts at 1.
+ """
+ top = {'word': None, 'lemma': None, 'ctag': 'TOP', 'tag': 'TOP', 'feats': None, 'rel': 'TOP', 'deps': [], 'address': 0}
+ self.nodelist = [top]
+ self.root = None
+ self.stream = None
+ if tree_str:
+ self._parse(tree_str)
+
+ def remove_by_address(self, address):
+ """
+ Removes the node with the given address. References
+ to this node in others will still exist.
+ """
+ node_index = len(self.nodelist) - 1
+ while(node_index >= 0):
+ node = self.nodelist[node_index]
+ if node['address'] == address:
+ self.nodelist.pop(node_index)
+ node_index -= 1
+
+ def redirect_arcs(self, originals, redirect):
+ """
+ Redirects arcs to any of the nodes in the originals list
+ to the redirect node address.
+ """
+ for node in self.nodelist:
+ new_deps = []
+ for dep in node['deps']:
+ if dep in originals:
+ new_deps.append(redirect)
+ else:
+ new_deps.append(dep)
+ node['deps'] = new_deps
+
+ def add_arc(self, head_address, mod_address):
+ """
+ Adds an arc from the node specified by head_address to the
+ node specified by the mod address.
+ """
+ for node in self.nodelist:
+ if node['address'] == head_address and (mod_address not in node['deps']):
+ node['deps'].append(mod_address)
+
+ def connect_graph(self):
+ """
+ Fully connects all non-root nodes. All nodes are set to be dependents
+ of the root node.
+ """
+ for node1 in self.nodelist:
+ for node2 in self.nodelist:
+ if node1['address'] != node2['address'] and node2['rel'] != 'TOP':
+ node1['deps'].append(node2['address'])
+
+ # fix error and return
+ def get_by_address(self, node_address):
+ """
+ Returns the node with the given address.
+ """
+ for node in self.nodelist:
+ if node['address'] == node_address:
+ return node
+ print('THROW ERROR: address not found in -get_by_address-')
+ return -1
+
+ def contains_address(self, node_address):
+ """
+ Returns true if the graph contains a node with the given node
+ address, false otherwise.
+ """
+ for node in self.nodelist:
+ if node['address'] == node_address:
+ return True
+ return False
+
+ def __str__(self):
+ return pformat(self.nodelist)
+
+ def __repr__(self):
+ return "<DependencyGraph with %d nodes>" % len(self.nodelist)
+
+ @staticmethod
+ def load(file):
+ """
+ :param file: a file in Malt-TAB format
+ :return: a list of DependencyGraphs
+ """
+ with open(file) as infile:
+ return [DependencyGraph(tree_str) for tree_str in
+ infile.read().split('\n\n')]
+
+ @staticmethod
+ def _normalize(line):
+ """
+ Deal with lines in which spaces are used rather than tabs.
+ """
+ SPC = re.compile(' +')
+ return re.sub(SPC, '\t', line).strip()
+
+ def left_children(self, node_index):
+ """
+ Returns the number of left children under the node specified
+ by the given address.
+ """
+ children = self.nodelist[node_index]['deps']
+ index = self.nodelist[node_index]['address']
+ return sum(1 for c in children if c < index)
+
+ def right_children(self, node_index):
+ """
+ Returns the number of right children under the node specified
+ by the given address.
+ """
+ children = self.nodelist[node_index]['deps']
+ index = self.nodelist[node_index]['address']
+ return sum(1 for c in children if c > index)
+
+ def add_node(self, node):
+ if not self.contains_address(node['address']):
+ self.nodelist.append(node)
+
+ def _parse(self, input):
+ lines = [DependencyGraph._normalize(line) for line in input.split('\n') if line.strip()]
+ temp = []
+ for index, line in enumerate(lines):
+# print line
+ try:
+ cells = line.split('\t')
+ nrCells = len(cells)
+ if nrCells == 3:
+ word, tag, head = cells
+ lemma, ctag, feats, rel = word, tag, '', ''
+ elif nrCells == 4:
+ word, tag, head, rel = cells
+ lemma, ctag, feats = word, tag, ''
+ elif nrCells == 10:
+ _, word, lemma, ctag, tag, feats, head, rel, _, _ = cells
+ else:
+ raise ValueError('Number of tab-delimited fields (%d) not supported by CoNLL(10) or Malt-Tab(4) format' % (nrCells))
+
+ head = int(head)
+ self.nodelist.append({'address': index+1, 'word': word, 'lemma': lemma, 'ctag': ctag, 'tag': tag, 'feats': feats, 'head': head, 'rel': rel,
+ 'deps': [d for (d,h) in temp if h == index+1]})
+
+ try:
+ self.nodelist[head]['deps'].append(index+1)
+ except IndexError:
+ temp.append((index+1, head))
+
+ except ValueError:
+ break
+
+ root_address = self.nodelist[0]['deps'][0]
+ self.root = self.nodelist[root_address]
+
+ def _word(self, node, filter=True):
+ w = node['word']
+ if filter:
+ if w != ',': return w
+ return w
+
+ def _tree(self, i):
+ """
+ Recursive function for turning dependency graphs into
+ NLTK trees.
+ :type i: int
+ :param i: index of a node in ``nodelist``
+ :return: either a word (if the indexed node
+ is a leaf) or a ``Tree``.
+ """
+ node = self.get_by_address(i)
+ word = node['word']
+ deps = node['deps']
+
+ return (Tree(word, [self._tree(j) for j in deps]) if len(deps) != 0 else word)
+
+ def tree(self):
+ """
+ Starting with the ``root`` node, build a dependency tree using the NLTK
+ ``Tree`` constructor. Dependency labels are omitted.
+ """
+ node = self.root
+ word = node['word']
+ deps = node['deps']
+ return Tree(word, [self._tree(i) for i in deps])
+
+ def _hd(self, i):
+ try:
+ return self.nodelist[i]['head']
+ except IndexError:
+ return None
+
+ def _rel(self, i):
+ try:
+ return self.nodelist[i]['rel']
+ except IndexError:
+ return None
+
+ # what's the return type? Boolean or list?
+ def contains_cycle(self):
+ distances = {}
+ for node in self.nodelist:
+ for dep in node['deps']:
+ key = tuple([node['address'], dep]) #'%d -> %d' % (node['address'], dep)
+ distances[key] = 1
+ for n in range(len(self.nodelist)):
+ new_entries = {}
+ for pair1 in distances:
+ for pair2 in distances:
+ if pair1[1] == pair2[0]:
+ key = tuple([pair1[0], pair2[1]])
+ new_entries[key] = distances[pair1] + distances[pair2]
+ for pair in new_entries:
+ distances[pair] = new_entries[pair]
+ if pair[0] == pair[1]:
+ print(pair[0])
+ path = self.get_cycle_path(self.get_by_address(pair[0]), pair[0]) #self.nodelist[pair[0]], pair[0])
+ return path
+ return False # return []?
+
+
+ def get_cycle_path(self, curr_node, goal_node_index):
+ for dep in curr_node['deps']:
+ if dep == goal_node_index:
+ return [curr_node['address']]
+ for dep in curr_node['deps']:
+ path = self.get_cycle_path(self.get_by_address(dep), goal_node_index)#self.nodelist[dep], goal_node_index)
+ if len(path) > 0:
+ path.insert(0, curr_node['address'])
+ return path
+ return []
+
+ def to_conll(self, style):
+ """
+ The dependency graph in CoNLL format.
+
+ :param style: the style to use for the format (3, 4, 10 columns)
+ :type style: int
+ :rtype: str
+ """
+
+ lines = []
+ for i, node in enumerate(self.nodelist[1:]):
+ word, lemma, ctag, tag, feats, head, rel = node['word'], node['lemma'], node['ctag'], node['tag'], node['feats'], node['head'], node['rel']
+ if style == 3:
+ lines.append('%s\t%s\t%s\n' % (word, tag, head))
+ elif style == 4:
+ lines.append('%s\t%s\t%s\t%s\n' % (word, tag, head, rel))
+ elif style == 10:
+ lines.append('%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t_\t_\n' % (i+1, word, lemma, ctag, tag, feats, head, rel))
+ else:
+ raise ValueError('Number of tab-delimited fields (%d) not supported by CoNLL(10) or Malt-Tab(4) format' % (style))
+ return ''.join(lines)
+
+
+def nx_graph(self):
+ """
+ Convert the data in a ``nodelist`` into a networkx
+ labeled directed graph.
+ :rtype: XDigraph
+ """
+ nx_nodelist = list(range(1, len(self.nodelist)))
+ nx_edgelist = [(n, self._hd(n), self._rel(n))
+ for n in nx_nodelist if self._hd(n)]
+ self.nx_labels = {}
+ for n in nx_nodelist:
+ self.nx_labels[n] = self.nodelist[n]['word']
+
+ g = NX.XDiGraph()
+ g.add_nodes_from(nx_nodelist)
+ g.add_edges_from(nx_edgelist)
+
+ return g
+
+def demo():
+ malt_demo()
+ conll_demo()
+ conll_file_demo()
+ cycle_finding_demo()
+
+def malt_demo(nx=False):
+ """
+ A demonstration of the result of reading a dependency
+ version of the first sentence of the Penn Treebank.
+ """
+ dg = DependencyGraph("""Pierre NNP 2 NMOD
+Vinken NNP 8 SUB
+, , 2 P
+61 CD 5 NMOD
+years NNS 6 AMOD
+old JJ 2 NMOD
+, , 2 P
+will MD 0 ROOT
+join VB 8 VC
+the DT 11 NMOD
+board NN 9 OBJ
+as IN 9 VMOD
+a DT 15 NMOD
+nonexecutive JJ 15 NMOD
+director NN 12 PMOD
+Nov. NNP 9 VMOD
+29 CD 16 NMOD
+. . 9 VMOD
+""")
+ tree = dg.tree()
+ print(tree.pprint())
+ if nx:
+ #currently doesn't work
+ import networkx as NX
+ import pylab as P
+
+ g = dg.nx_graph()
+ g.info()
+ pos = NX.spring_layout(g, dim=1)
+ NX.draw_networkx_nodes(g, pos, node_size=50)
+ #NX.draw_networkx_edges(g, pos, edge_color='k', width=8)
+ NX.draw_networkx_labels(g, pos, dg.nx_labels)
+ P.xticks([])
+ P.yticks([])
+ P.savefig('tree.png')
+ P.show()
+
+
+def conll_demo():
+ """
+ A demonstration of how to read a string representation of
+ a CoNLL format dependency tree.
+ """
+ dg = DependencyGraph(conll_data1)
+ tree = dg.tree()
+ print(tree.pprint())
+ print(dg)
+ print(dg.to_conll(4))
+
+def conll_file_demo():
+ print('Mass conll_read demo...')
+ graphs = [DependencyGraph(entry)
+ for entry in conll_data2.split('\n\n') if entry]
+ for graph in graphs:
+ tree = graph.tree()
+ print('\n' + tree.pprint())
+
+def cycle_finding_demo():
+ dg = DependencyGraph(treebank_data)
+ print(dg.contains_cycle())
+ cyclic_dg = DependencyGraph()
+ top = {'word':None, 'deps':[1], 'rel': 'TOP', 'address': 0}
+ child1 = {'word':None, 'deps':[2], 'rel': 'NTOP', 'address': 1}
+ child2 = {'word':None, 'deps':[4], 'rel': 'NTOP', 'address': 2}
+ child3 = {'word':None, 'deps':[1], 'rel': 'NTOP', 'address': 3}
+ child4 = {'word':None, 'deps':[3], 'rel': 'NTOP', 'address': 4}
+ cyclic_dg.nodelist = [top, child1, child2, child3, child4]
+ cyclic_dg.root = top
+ print(cyclic_dg.contains_cycle())
+
+treebank_data = """Pierre NNP 2 NMOD
+Vinken NNP 8 SUB
+, , 2 P
+61 CD 5 NMOD
+years NNS 6 AMOD
+old JJ 2 NMOD
+, , 2 P
+will MD 0 ROOT
+join VB 8 VC
+the DT 11 NMOD
+board NN 9 OBJ
+as IN 9 VMOD
+a DT 15 NMOD
+nonexecutive JJ 15 NMOD
+director NN 12 PMOD
+Nov. NNP 9 VMOD
+29 CD 16 NMOD
+. . 9 VMOD
+"""
+
+conll_data1 = """
+1 Ze ze Pron Pron per|3|evofmv|nom 2 su _ _
+2 had heb V V trans|ovt|1of2of3|ev 0 ROOT _ _
+3 met met Prep Prep voor 8 mod _ _
+4 haar haar Pron Pron bez|3|ev|neut|attr 5 det _ _
+5 moeder moeder N N soort|ev|neut 3 obj1 _ _
+6 kunnen kan V V hulp|ott|1of2of3|mv 2 vc _ _
+7 gaan ga V V hulp|inf 6 vc _ _
+8 winkelen winkel V V intrans|inf 11 cnj _ _
+9 , , Punc Punc komma 8 punct _ _
+10 zwemmen zwem V V intrans|inf 11 cnj _ _
+11 of of Conj Conj neven 7 vc _ _
+12 terrassen terras N N soort|mv|neut 11 cnj _ _
+13 . . Punc Punc punt 12 punct _ _
+"""
+
+conll_data2 = """1 Cathy Cathy N N eigen|ev|neut 2 su _ _
+2 zag zie V V trans|ovt|1of2of3|ev 0 ROOT _ _
+3 hen hen Pron Pron per|3|mv|datofacc 2 obj1 _ _
+4 wild wild Adj Adj attr|stell|onverv 5 mod _ _
+5 zwaaien zwaai N N soort|mv|neut 2 vc _ _
+6 . . Punc Punc punt 5 punct _ _
+
+1 Ze ze Pron Pron per|3|evofmv|nom 2 su _ _
+2 had heb V V trans|ovt|1of2of3|ev 0 ROOT _ _
+3 met met Prep Prep voor 8 mod _ _
+4 haar haar Pron Pron bez|3|ev|neut|attr 5 det _ _
+5 moeder moeder N N soort|ev|neut 3 obj1 _ _
+6 kunnen kan V V hulp|ott|1of2of3|mv 2 vc _ _
+7 gaan ga V V hulp|inf 6 vc _ _
+8 winkelen winkel V V intrans|inf 11 cnj _ _
+9 , , Punc Punc komma 8 punct _ _
+10 zwemmen zwem V V intrans|inf 11 cnj _ _
+11 of of Conj Conj neven 7 vc _ _
+12 terrassen terras N N soort|mv|neut 11 cnj _ _
+13 . . Punc Punc punt 12 punct _ _
+
+1 Dat dat Pron Pron aanw|neut|attr 2 det _ _
+2 werkwoord werkwoord N N soort|ev|neut 6 obj1 _ _
+3 had heb V V hulp|ovt|1of2of3|ev 0 ROOT _ _
+4 ze ze Pron Pron per|3|evofmv|nom 6 su _ _
+5 zelf zelf Pron Pron aanw|neut|attr|wzelf 3 predm _ _
+6 uitgevonden vind V V trans|verldw|onverv 3 vc _ _
+7 . . Punc Punc punt 6 punct _ _
+
+1 Het het Pron Pron onbep|neut|zelfst 2 su _ _
+2 hoorde hoor V V trans|ovt|1of2of3|ev 0 ROOT _ _
+3 bij bij Prep Prep voor 2 ld _ _
+4 de de Art Art bep|zijdofmv|neut 6 det _ _
+5 warme warm Adj Adj attr|stell|vervneut 6 mod _ _
+6 zomerdag zomerdag N N soort|ev|neut 3 obj1 _ _
+7 die die Pron Pron betr|neut|zelfst 6 mod _ _
+8 ze ze Pron Pron per|3|evofmv|nom 12 su _ _
+9 ginds ginds Adv Adv gew|aanw 12 mod _ _
+10 achter achter Adv Adv gew|geenfunc|stell|onverv 12 svp _ _
+11 had heb V V hulp|ovt|1of2of3|ev 7 body _ _
+12 gelaten laat V V trans|verldw|onverv 11 vc _ _
+13 . . Punc Punc punt 12 punct _ _
+
+1 Ze ze Pron Pron per|3|evofmv|nom 2 su _ _
+2 hadden heb V V trans|ovt|1of2of3|mv 0 ROOT _ _
+3 languit languit Adv Adv gew|geenfunc|stell|onverv 11 mod _ _
+4 naast naast Prep Prep voor 11 mod _ _
+5 elkaar elkaar Pron Pron rec|neut 4 obj1 _ _
+6 op op Prep Prep voor 11 ld _ _
+7 de de Art Art bep|zijdofmv|neut 8 det _ _
+8 strandstoelen strandstoel N N soort|mv|neut 6 obj1 _ _
+9 kunnen kan V V hulp|inf 2 vc _ _
+10 gaan ga V V hulp|inf 9 vc _ _
+11 liggen lig V V intrans|inf 10 vc _ _
+12 . . Punc Punc punt 11 punct _ _
+
+1 Zij zij Pron Pron per|3|evofmv|nom 2 su _ _
+2 zou zal V V hulp|ovt|1of2of3|ev 7 cnj _ _
+3 mams mams N N soort|ev|neut 4 det _ _
+4 rug rug N N soort|ev|neut 5 obj1 _ _
+5 ingewreven wrijf V V trans|verldw|onverv 6 vc _ _
+6 hebben heb V V hulp|inf 2 vc _ _
+7 en en Conj Conj neven 0 ROOT _ _
+8 mam mam V V trans|ovt|1of2of3|ev 7 cnj _ _
+9 de de Art Art bep|zijdofmv|neut 10 det _ _
+10 hare hare Pron Pron bez|3|ev|neut|attr 8 obj1 _ _
+11 . . Punc Punc punt 10 punct _ _
+
+1 Of of Conj Conj onder|metfin 0 ROOT _ _
+2 ze ze Pron Pron per|3|evofmv|nom 3 su _ _
+3 had heb V V hulp|ovt|1of2of3|ev 0 ROOT _ _
+4 gewoon gewoon Adj Adj adv|stell|onverv 10 mod _ _
+5 met met Prep Prep voor 10 mod _ _
+6 haar haar Pron Pron bez|3|ev|neut|attr 7 det _ _
+7 vriendinnen vriendin N N soort|mv|neut 5 obj1 _ _
+8 rond rond Adv Adv deelv 10 svp _ _
+9 kunnen kan V V hulp|inf 3 vc _ _
+10 slenteren slenter V V intrans|inf 9 vc _ _
+11 in in Prep Prep voor 10 mod _ _
+12 de de Art Art bep|zijdofmv|neut 13 det _ _
+13 buurt buurt N N soort|ev|neut 11 obj1 _ _
+14 van van Prep Prep voor 13 mod _ _
+15 Trafalgar_Square Trafalgar_Square MWU N_N eigen|ev|neut_eigen|ev|neut 14 obj1 _ _
+16 . . Punc Punc punt 15 punct _ _
+"""
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/earleychart.py b/nltk/parse/earleychart.py
new file mode 100644
index 0000000..1a4b81c
--- /dev/null
+++ b/nltk/parse/earleychart.py
@@ -0,0 +1,451 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: An Incremental Earley Chart Parser
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Peter Ljunglöf <peter.ljunglof at heatherleaf.se>
+# Rob Speer <rspeer at mit.edu>
+# Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# Jean Mark Gawron <gawron at mail.sdsu.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Data classes and parser implementations for *incremental* chart
+parsers, which use dynamic programming to efficiently parse a text.
+A "chart parser" derives parse trees for a text by iteratively adding
+\"edges\" to a \"chart\". Each "edge" represents a hypothesis about the tree
+structure for a subsequence of the text. The "chart" is a
+\"blackboard\" for composing and combining these hypotheses.
+
+A parser is "incremental", if it guarantees that for all i, j where i < j,
+all edges ending at i are built before any edges ending at j.
+This is appealing for, say, speech recognizer hypothesis filtering.
+
+The main parser class is ``EarleyChartParser``, which is a top-down
+algorithm, originally formulated by Jay Earley (1970).
+"""
+from __future__ import print_function, division
+
+from nltk.compat import xrange
+from nltk.parse.chart import (Chart, ChartParser, EdgeI, LeafEdge, LeafInitRule,
+ BottomUpPredictRule, BottomUpPredictCombineRule,
+ TopDownInitRule, SingleEdgeFundamentalRule,
+ EmptyPredictRule,
+ CachedTopDownPredictRule,
+ FilteredSingleEdgeFundamentalRule,
+ FilteredBottomUpPredictCombineRule)
+from nltk.parse.featurechart import (FeatureChart, FeatureChartParser,
+ FeatureTopDownInitRule,
+ FeatureTopDownPredictRule,
+ FeatureEmptyPredictRule,
+ FeatureBottomUpPredictRule,
+ FeatureBottomUpPredictCombineRule,
+ FeatureSingleEdgeFundamentalRule)
+
+#////////////////////////////////////////////////////////////
+# Incremental Chart
+#////////////////////////////////////////////////////////////
+
+class IncrementalChart(Chart):
+ def initialize(self):
+ # A sequence of edge lists contained in this chart.
+ self._edgelists = tuple([] for x in self._positions())
+
+ # The set of child pointer lists associated with each edge.
+ self._edge_to_cpls = {}
+
+ # Indexes mapping attribute values to lists of edges
+ # (used by select()).
+ self._indexes = {}
+
+ def edges(self):
+ return list(self.iteredges())
+
+ def iteredges(self):
+ return (edge for edgelist in self._edgelists for edge in edgelist)
+
+ def select(self, end, **restrictions):
+ edgelist = self._edgelists[end]
+
+ # If there are no restrictions, then return all edges.
+ if restrictions=={}: return iter(edgelist)
+
+ # Find the index corresponding to the given restrictions.
+ restr_keys = sorted(restrictions.keys())
+ restr_keys = tuple(restr_keys)
+
+ # If it doesn't exist, then create it.
+ if restr_keys not in self._indexes:
+ self._add_index(restr_keys)
+
+ vals = tuple(restrictions[key] for key in restr_keys)
+ return iter(self._indexes[restr_keys][end].get(vals, []))
+
+ def _add_index(self, restr_keys):
+ # Make sure it's a valid index.
+ for key in restr_keys:
+ if not hasattr(EdgeI, key):
+ raise ValueError('Bad restriction: %s' % key)
+
+ # Create the index.
+ index = self._indexes[restr_keys] = tuple({} for x in self._positions())
+
+ # Add all existing edges to the index.
+ for end, edgelist in enumerate(self._edgelists):
+ this_index = index[end]
+ for edge in edgelist:
+ vals = tuple(getattr(edge, key)() for key in restr_keys)
+ this_index.setdefault(vals, []).append(edge)
+
+ def _register_with_indexes(self, edge):
+ end = edge.end()
+ for (restr_keys, index) in self._indexes.items():
+ vals = tuple(getattr(edge, key)() for key in restr_keys)
+ index[end].setdefault(vals, []).append(edge)
+
+ def _append_edge(self, edge):
+ self._edgelists[edge.end()].append(edge)
+
+ def _positions(self):
+ return xrange(self.num_leaves() + 1)
+
+
+class FeatureIncrementalChart(IncrementalChart, FeatureChart):
+ def select(self, end, **restrictions):
+ edgelist = self._edgelists[end]
+
+ # If there are no restrictions, then return all edges.
+ if restrictions=={}: return iter(edgelist)
+
+ # Find the index corresponding to the given restrictions.
+ restr_keys = sorted(restrictions.keys())
+ restr_keys = tuple(restr_keys)
+
+ # If it doesn't exist, then create it.
+ if restr_keys not in self._indexes:
+ self._add_index(restr_keys)
+
+ vals = tuple(self._get_type_if_possible(restrictions[key])
+ for key in restr_keys)
+ return iter(self._indexes[restr_keys][end].get(vals, []))
+
+ def _add_index(self, restr_keys):
+ # Make sure it's a valid index.
+ for key in restr_keys:
+ if not hasattr(EdgeI, key):
+ raise ValueError('Bad restriction: %s' % key)
+
+ # Create the index.
+ index = self._indexes[restr_keys] = tuple({} for x in self._positions())
+
+ # Add all existing edges to the index.
+ for end, edgelist in enumerate(self._edgelists):
+ this_index = index[end]
+ for edge in edgelist:
+ vals = tuple(self._get_type_if_possible(getattr(edge, key)())
+ for key in restr_keys)
+ this_index.setdefault(vals, []).append(edge)
+
+ def _register_with_indexes(self, edge):
+ end = edge.end()
+ for (restr_keys, index) in self._indexes.items():
+ vals = tuple(self._get_type_if_possible(getattr(edge, key)())
+ for key in restr_keys)
+ index[end].setdefault(vals, []).append(edge)
+
+#////////////////////////////////////////////////////////////
+# Incremental CFG Rules
+#////////////////////////////////////////////////////////////
+
+class CompleteFundamentalRule(SingleEdgeFundamentalRule):
+ def _apply_incomplete(self, chart, grammar, left_edge):
+ end = left_edge.end()
+ # When the chart is incremental, we only have to look for
+ # empty complete edges here.
+ for right_edge in chart.select(start=end, end=end,
+ is_complete=True,
+ lhs=left_edge.nextsym()):
+ new_edge = left_edge.move_dot_forward(right_edge.end())
+ if chart.insert_with_backpointer(new_edge, left_edge, right_edge):
+ yield new_edge
+
+class CompleterRule(CompleteFundamentalRule):
+ _fundamental_rule = CompleteFundamentalRule()
+ def apply(self, chart, grammar, edge):
+ if not isinstance(edge, LeafEdge):
+ for new_edge in self._fundamental_rule.apply(chart, grammar, edge):
+ yield new_edge
+
+class ScannerRule(CompleteFundamentalRule):
+ _fundamental_rule = CompleteFundamentalRule()
+ def apply(self, chart, grammar, edge):
+ if isinstance(edge, LeafEdge):
+ for new_edge in self._fundamental_rule.apply(chart, grammar, edge):
+ yield new_edge
+
+class PredictorRule(CachedTopDownPredictRule):
+ pass
+
+class FilteredCompleteFundamentalRule(FilteredSingleEdgeFundamentalRule):
+ def apply(self, chart, grammar, edge):
+ # Since the Filtered rule only works for grammars without empty productions,
+ # we only have to bother with complete edges here.
+ if edge.is_complete():
+ for new_edge in self._apply_complete(chart, grammar, edge):
+ yield new_edge
+
+#////////////////////////////////////////////////////////////
+# Incremental FCFG Rules
+#////////////////////////////////////////////////////////////
+
+class FeatureCompleteFundamentalRule(FeatureSingleEdgeFundamentalRule):
+ def _apply_incomplete(self, chart, grammar, left_edge):
+ fr = self._fundamental_rule
+ end = left_edge.end()
+ # When the chart is incremental, we only have to look for
+ # empty complete edges here.
+ for right_edge in chart.select(start=end, end=end,
+ is_complete=True,
+ lhs=left_edge.nextsym()):
+ for new_edge in fr.apply(chart, grammar, left_edge, right_edge):
+ yield new_edge
+
+class FeatureCompleterRule(CompleterRule):
+ _fundamental_rule = FeatureCompleteFundamentalRule()
+
+class FeatureScannerRule(ScannerRule):
+ _fundamental_rule = FeatureCompleteFundamentalRule()
+
+class FeaturePredictorRule(FeatureTopDownPredictRule):
+ pass
+
+#////////////////////////////////////////////////////////////
+# Incremental CFG Chart Parsers
+#////////////////////////////////////////////////////////////
+
+EARLEY_STRATEGY = [LeafInitRule(),
+ TopDownInitRule(),
+ CompleterRule(),
+ ScannerRule(),
+ PredictorRule()]
+TD_INCREMENTAL_STRATEGY = [LeafInitRule(),
+ TopDownInitRule(),
+ CachedTopDownPredictRule(),
+ CompleteFundamentalRule()]
+BU_INCREMENTAL_STRATEGY = [LeafInitRule(),
+ EmptyPredictRule(),
+ BottomUpPredictRule(),
+ CompleteFundamentalRule()]
+BU_LC_INCREMENTAL_STRATEGY = [LeafInitRule(),
+ EmptyPredictRule(),
+ BottomUpPredictCombineRule(),
+ CompleteFundamentalRule()]
+
+LC_INCREMENTAL_STRATEGY = [LeafInitRule(),
+ FilteredBottomUpPredictCombineRule(),
+ FilteredCompleteFundamentalRule()]
+
+class IncrementalChartParser(ChartParser):
+ """
+ An *incremental* chart parser implementing Jay Earley's
+ parsing algorithm:
+
+ | For each index end in [0, 1, ..., N]:
+ | For each edge such that edge.end = end:
+ | If edge is incomplete and edge.next is not a part of speech:
+ | Apply PredictorRule to edge
+ | If edge is incomplete and edge.next is a part of speech:
+ | Apply ScannerRule to edge
+ | If edge is complete:
+ | Apply CompleterRule to edge
+ | Return any complete parses in the chart
+ """
+ def __init__(self, grammar, strategy=BU_LC_INCREMENTAL_STRATEGY,
+ trace=0, trace_chart_width=50,
+ chart_class=IncrementalChart):
+ """
+ Create a new Earley chart parser, that uses ``grammar`` to
+ parse texts.
+
+ :type grammar: CFG
+ :param grammar: The grammar used to parse texts.
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ and higher numbers will produce more verbose tracing
+ output.
+ :type trace_chart_width: int
+ :param trace_chart_width: The default total width reserved for
+ the chart in trace output. The remainder of each line will
+ be used to display edges.
+ :param chart_class: The class that should be used to create
+ the charts used by this parser.
+ """
+ self._grammar = grammar
+ self._trace = trace
+ self._trace_chart_width = trace_chart_width
+ self._chart_class = chart_class
+
+ self._axioms = []
+ self._inference_rules = []
+ for rule in strategy:
+ if rule.NUM_EDGES == 0:
+ self._axioms.append(rule)
+ elif rule.NUM_EDGES == 1:
+ self._inference_rules.append(rule)
+ else:
+ raise ValueError("Incremental inference rules must have "
+ "NUM_EDGES == 0 or 1")
+
+ def chart_parse(self, tokens, trace=None):
+ if trace is None: trace = self._trace
+ trace_new_edges = self._trace_new_edges
+
+ tokens = list(tokens)
+ self._grammar.check_coverage(tokens)
+ chart = self._chart_class(tokens)
+ grammar = self._grammar
+
+ # Width, for printing trace edges.
+ trace_edge_width = self._trace_chart_width // (chart.num_leaves() + 1)
+ if trace: print(chart.pp_leaves(trace_edge_width))
+
+ for axiom in self._axioms:
+ new_edges = list(axiom.apply(chart, grammar))
+ trace_new_edges(chart, axiom, new_edges, trace, trace_edge_width)
+
+ inference_rules = self._inference_rules
+ for end in range(chart.num_leaves()+1):
+ if trace > 1: print("\n* Processing queue:", end, "\n")
+ agenda = list(chart.select(end=end))
+ while agenda:
+ edge = agenda.pop()
+ for rule in inference_rules:
+ new_edges = list(rule.apply(chart, grammar, edge))
+ trace_new_edges(chart, rule, new_edges, trace, trace_edge_width)
+ for new_edge in new_edges:
+ if new_edge.end()==end:
+ agenda.append(new_edge)
+
+ return chart
+
+class EarleyChartParser(IncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ IncrementalChartParser.__init__(self, grammar, EARLEY_STRATEGY, **parser_args)
+ pass
+
+class IncrementalTopDownChartParser(IncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ IncrementalChartParser.__init__(self, grammar, TD_INCREMENTAL_STRATEGY, **parser_args)
+
+class IncrementalBottomUpChartParser(IncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ IncrementalChartParser.__init__(self, grammar, BU_INCREMENTAL_STRATEGY, **parser_args)
+
+class IncrementalBottomUpLeftCornerChartParser(IncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ IncrementalChartParser.__init__(self, grammar, BU_LC_INCREMENTAL_STRATEGY, **parser_args)
+
+class IncrementalLeftCornerChartParser(IncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ if not grammar.is_nonempty():
+ raise ValueError("IncrementalLeftCornerParser only works for grammars "
+ "without empty productions.")
+ IncrementalChartParser.__init__(self, grammar, LC_INCREMENTAL_STRATEGY, **parser_args)
+
+#////////////////////////////////////////////////////////////
+# Incremental FCFG Chart Parsers
+#////////////////////////////////////////////////////////////
+
+EARLEY_FEATURE_STRATEGY = [LeafInitRule(),
+ FeatureTopDownInitRule(),
+ FeatureCompleterRule(),
+ FeatureScannerRule(),
+ FeaturePredictorRule()]
+TD_INCREMENTAL_FEATURE_STRATEGY = [LeafInitRule(),
+ FeatureTopDownInitRule(),
+ FeatureTopDownPredictRule(),
+ FeatureCompleteFundamentalRule()]
+BU_INCREMENTAL_FEATURE_STRATEGY = [LeafInitRule(),
+ FeatureEmptyPredictRule(),
+ FeatureBottomUpPredictRule(),
+ FeatureCompleteFundamentalRule()]
+BU_LC_INCREMENTAL_FEATURE_STRATEGY = [LeafInitRule(),
+ FeatureEmptyPredictRule(),
+ FeatureBottomUpPredictCombineRule(),
+ FeatureCompleteFundamentalRule()]
+
+class FeatureIncrementalChartParser(IncrementalChartParser, FeatureChartParser):
+ def __init__(self, grammar,
+ strategy=BU_LC_INCREMENTAL_FEATURE_STRATEGY,
+ trace_chart_width=20,
+ chart_class=FeatureIncrementalChart,
+ **parser_args):
+ IncrementalChartParser.__init__(self, grammar,
+ strategy=strategy,
+ trace_chart_width=trace_chart_width,
+ chart_class=chart_class,
+ **parser_args)
+
+class FeatureEarleyChartParser(FeatureIncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ FeatureIncrementalChartParser.__init__(self, grammar, EARLEY_FEATURE_STRATEGY, **parser_args)
+
+class FeatureIncrementalTopDownChartParser(FeatureIncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ FeatureIncrementalChartParser.__init__(self, grammar, TD_INCREMENTAL_FEATURE_STRATEGY, **parser_args)
+
+class FeatureIncrementalBottomUpChartParser(FeatureIncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ FeatureIncrementalChartParser.__init__(self, grammar, BU_INCREMENTAL_FEATURE_STRATEGY, **parser_args)
+
+class FeatureIncrementalBottomUpLeftCornerChartParser(FeatureIncrementalChartParser):
+ def __init__(self, grammar, **parser_args):
+ FeatureIncrementalChartParser.__init__(self, grammar, BU_LC_INCREMENTAL_FEATURE_STRATEGY, **parser_args)
+
+
+#////////////////////////////////////////////////////////////
+# Demonstration
+#////////////////////////////////////////////////////////////
+
+def demo(print_times=True, print_grammar=False,
+ print_trees=True, trace=2,
+ sent='I saw John with a dog with my cookie', numparses=5):
+ """
+ A demonstration of the Earley parsers.
+ """
+ import sys, time
+ from nltk.parse.chart import demo_grammar
+
+ # The grammar for ChartParser and SteppingChartParser:
+ grammar = demo_grammar()
+ if print_grammar:
+ print("* Grammar")
+ print(grammar)
+
+ # Tokenize the sample sentence.
+ print("* Sentence:")
+ print(sent)
+ tokens = sent.split()
+ print(tokens)
+ print()
+
+ # Do the parsing.
+ earley = EarleyChartParser(grammar, trace=trace)
+ t = time.clock()
+ chart = earley.chart_parse(tokens)
+ parses = list(chart.parses(grammar.start()))
+ t = time.clock()-t
+
+ # Print results.
+ if numparses:
+ assert len(parses)==numparses, 'Not all parses found'
+ if print_trees:
+ for tree in parses: print(tree)
+ else:
+ print("Nr trees:", len(parses))
+ if print_times:
+ print("Time:", t)
+
+if __name__ == '__main__': demo()
diff --git a/nltk/parse/featurechart.py b/nltk/parse/featurechart.py
new file mode 100644
index 0000000..90ff716
--- /dev/null
+++ b/nltk/parse/featurechart.py
@@ -0,0 +1,576 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Chart Parser for Feature-Based Grammars
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Rob Speer <rspeer at mit.edu>
+# Peter Ljunglöf <peter.ljunglof at heatherleaf.se>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Extension of chart parsing implementation to handle grammars with
+feature structures as nodes.
+"""
+from __future__ import print_function, unicode_literals
+
+from nltk.compat import xrange, python_2_unicode_compatible
+from nltk.featstruct import FeatStruct, unify, TYPE, find_variables
+from nltk.sem import logic
+from nltk.tree import Tree
+from nltk.grammar import (Nonterminal, Production, CFG,
+ FeatStructNonterminal, is_nonterminal,
+ is_terminal)
+from nltk.parse.chart import (TreeEdge, Chart, ChartParser, EdgeI,
+ FundamentalRule, LeafInitRule,
+ EmptyPredictRule, BottomUpPredictRule,
+ SingleEdgeFundamentalRule,
+ BottomUpPredictCombineRule,
+ CachedTopDownPredictRule,
+ TopDownInitRule)
+
+#////////////////////////////////////////////////////////////
+# Tree Edge
+#////////////////////////////////////////////////////////////
+
+ at python_2_unicode_compatible
+class FeatureTreeEdge(TreeEdge):
+ """
+ A specialized tree edge that allows shared variable bindings
+ between nonterminals on the left-hand side and right-hand side.
+
+ Each ``FeatureTreeEdge`` contains a set of ``bindings``, i.e., a
+ dictionary mapping from variables to values. If the edge is not
+ complete, then these bindings are simply stored. However, if the
+ edge is complete, then the constructor applies these bindings to
+ every nonterminal in the edge whose symbol implements the
+ interface ``SubstituteBindingsI``.
+ """
+ def __init__(self, span, lhs, rhs, dot=0, bindings=None):
+ """
+ Construct a new edge. If the edge is incomplete (i.e., if
+ ``dot<len(rhs)``), then store the bindings as-is. If the edge
+ is complete (i.e., if ``dot==len(rhs)``), then apply the
+ bindings to all nonterminals in ``lhs`` and ``rhs``, and then
+ clear the bindings. See ``TreeEdge`` for a description of
+ the other arguments.
+ """
+ if bindings is None: bindings = {}
+
+ # If the edge is complete, then substitute in the bindings,
+ # and then throw them away. (If we didn't throw them away, we
+ # might think that 2 complete edges are different just because
+ # they have different bindings, even though all bindings have
+ # already been applied.)
+ if dot == len(rhs) and bindings:
+ lhs = self._bind(lhs, bindings)
+ rhs = [self._bind(elt, bindings) for elt in rhs]
+ bindings = {}
+
+ # Initialize the edge.
+ TreeEdge.__init__(self, span, lhs, rhs, dot)
+ self._bindings = bindings
+ self._comparison_key = (self._comparison_key, tuple(sorted(bindings.items())))
+
+ @staticmethod
+ def from_production(production, index):
+ """
+ :return: A new ``TreeEdge`` formed from the given production.
+ The new edge's left-hand side and right-hand side will
+ be taken from ``production``; its span will be
+ ``(index,index)``; and its dot position will be ``0``.
+ :rtype: TreeEdge
+ """
+ return FeatureTreeEdge(span=(index, index), lhs=production.lhs(),
+ rhs=production.rhs(), dot=0)
+
+ def move_dot_forward(self, new_end, bindings=None):
+ """
+ :return: A new ``FeatureTreeEdge`` formed from this edge.
+ The new edge's dot position is increased by ``1``,
+ and its end index will be replaced by ``new_end``.
+ :rtype: FeatureTreeEdge
+ :param new_end: The new end index.
+ :type new_end: int
+ :param bindings: Bindings for the new edge.
+ :type bindings: dict
+ """
+ return FeatureTreeEdge(span=(self._span[0], new_end),
+ lhs=self._lhs, rhs=self._rhs,
+ dot=self._dot+1, bindings=bindings)
+
+ def _bind(self, nt, bindings):
+ if not isinstance(nt, FeatStructNonterminal): return nt
+ return nt.substitute_bindings(bindings)
+
+ def next_with_bindings(self):
+ return self._bind(self.nextsym(), self._bindings)
+
+ def bindings(self):
+ """
+ Return a copy of this edge's bindings dictionary.
+ """
+ return self._bindings.copy()
+
+ def variables(self):
+ """
+ :return: The set of variables used by this edge.
+ :rtype: set(Variable)
+ """
+ return find_variables([self._lhs] + list(self._rhs) +
+ list(self._bindings.keys()) +
+ list(self._bindings.values()),
+ fs_class=FeatStruct)
+
+ def __str__(self):
+ if self.is_complete():
+ return TreeEdge.__unicode__(self)
+ else:
+ bindings = '{%s}' % ', '.join('%s: %r' % item for item in
+ sorted(self._bindings.items()))
+ return '%s %s' % (TreeEdge.__unicode__(self), bindings)
+
+
+#////////////////////////////////////////////////////////////
+# A specialized Chart for feature grammars
+#////////////////////////////////////////////////////////////
+
+# TODO: subsumes check when adding new edges
+
+class FeatureChart(Chart):
+ """
+ A Chart for feature grammars.
+ :see: ``Chart`` for more information.
+ """
+
+ def select(self, **restrictions):
+ """
+ Returns an iterator over the edges in this chart.
+ See ``Chart.select`` for more information about the
+ ``restrictions`` on the edges.
+ """
+ # If there are no restrictions, then return all edges.
+ if restrictions=={}: return iter(self._edges)
+
+ # Find the index corresponding to the given restrictions.
+ restr_keys = sorted(restrictions.keys())
+ restr_keys = tuple(restr_keys)
+
+ # If it doesn't exist, then create it.
+ if restr_keys not in self._indexes:
+ self._add_index(restr_keys)
+
+ vals = tuple(self._get_type_if_possible(restrictions[key])
+ for key in restr_keys)
+ return iter(self._indexes[restr_keys].get(vals, []))
+
+ def _add_index(self, restr_keys):
+ """
+ A helper function for ``select``, which creates a new index for
+ a given set of attributes (aka restriction keys).
+ """
+ # Make sure it's a valid index.
+ for key in restr_keys:
+ if not hasattr(EdgeI, key):
+ raise ValueError('Bad restriction: %s' % key)
+
+ # Create the index.
+ index = self._indexes[restr_keys] = {}
+
+ # Add all existing edges to the index.
+ for edge in self._edges:
+ vals = tuple(self._get_type_if_possible(getattr(edge, key)())
+ for key in restr_keys)
+ index.setdefault(vals, []).append(edge)
+
+ def _register_with_indexes(self, edge):
+ """
+ A helper function for ``insert``, which registers the new
+ edge with all existing indexes.
+ """
+ for (restr_keys, index) in self._indexes.items():
+ vals = tuple(self._get_type_if_possible(getattr(edge, key)())
+ for key in restr_keys)
+ index.setdefault(vals, []).append(edge)
+
+ def _get_type_if_possible(self, item):
+ """
+ Helper function which returns the ``TYPE`` feature of the ``item``,
+ if it exists, otherwise it returns the ``item`` itself
+ """
+ if isinstance(item, dict) and TYPE in item:
+ return item[TYPE]
+ else:
+ return item
+
+ def parses(self, start, tree_class=Tree):
+ for edge in self.select(start=0, end=self._num_leaves):
+ if ((isinstance(edge, FeatureTreeEdge)) and
+ (edge.lhs()[TYPE] == start[TYPE]) and
+ (unify(edge.lhs(), start, rename_vars=True))
+ ):
+ for tree in self.trees(edge, complete=True, tree_class=tree_class):
+ yield tree
+
+
+#////////////////////////////////////////////////////////////
+# Fundamental Rule
+#////////////////////////////////////////////////////////////
+
+class FeatureFundamentalRule(FundamentalRule):
+ """
+ A specialized version of the fundamental rule that operates on
+ nonterminals whose symbols are ``FeatStructNonterminal``s. Rather
+ tha simply comparing the nonterminals for equality, they are
+ unified. Variable bindings from these unifications are collected
+ and stored in the chart using a ``FeatureTreeEdge``. When a
+ complete edge is generated, these bindings are applied to all
+ nonterminals in the edge.
+
+ The fundamental rule states that:
+
+ - ``[A -> alpha \* B1 beta][i:j]``
+ - ``[B2 -> gamma \*][j:k]``
+
+ licenses the edge:
+
+ - ``[A -> alpha B3 \* beta][i:j]``
+
+ assuming that B1 and B2 can be unified to generate B3.
+ """
+ def apply(self, chart, grammar, left_edge, right_edge):
+ # Make sure the rule is applicable.
+ if not (left_edge.end() == right_edge.start() and
+ left_edge.is_incomplete() and
+ right_edge.is_complete() and
+ isinstance(left_edge, FeatureTreeEdge)):
+ return
+ found = right_edge.lhs()
+ nextsym = left_edge.nextsym()
+ if isinstance(right_edge, FeatureTreeEdge):
+ if not is_nonterminal(nextsym): return
+ if left_edge.nextsym()[TYPE] != right_edge.lhs()[TYPE]: return
+ # Create a copy of the bindings.
+ bindings = left_edge.bindings()
+ # We rename vars here, because we don't want variables
+ # from the two different productions to match.
+ found = found.rename_variables(used_vars=left_edge.variables())
+ # Unify B1 (left_edge.nextsym) with B2 (right_edge.lhs) to
+ # generate B3 (result).
+ result = unify(nextsym, found, bindings, rename_vars=False)
+ if result is None: return
+ else:
+ if nextsym != found: return
+ # Create a copy of the bindings.
+ bindings = left_edge.bindings()
+
+ # Construct the new edge.
+ new_edge = left_edge.move_dot_forward(right_edge.end(), bindings)
+
+ # Add it to the chart, with appropriate child pointers.
+ if chart.insert_with_backpointer(new_edge, left_edge, right_edge):
+ yield new_edge
+
+class FeatureSingleEdgeFundamentalRule(SingleEdgeFundamentalRule):
+ """
+ A specialized version of the completer / single edge fundamental rule
+ that operates on nonterminals whose symbols are ``FeatStructNonterminal``s.
+ Rather than simply comparing the nonterminals for equality, they are
+ unified.
+ """
+ _fundamental_rule = FeatureFundamentalRule()
+
+ def _apply_complete(self, chart, grammar, right_edge):
+ fr = self._fundamental_rule
+ for left_edge in chart.select(end=right_edge.start(),
+ is_complete=False,
+ nextsym=right_edge.lhs()):
+ for new_edge in fr.apply(chart, grammar, left_edge, right_edge):
+ yield new_edge
+
+ def _apply_incomplete(self, chart, grammar, left_edge):
+ fr = self._fundamental_rule
+ for right_edge in chart.select(start=left_edge.end(),
+ is_complete=True,
+ lhs=left_edge.nextsym()):
+ for new_edge in fr.apply(chart, grammar, left_edge, right_edge):
+ yield new_edge
+
+
+#////////////////////////////////////////////////////////////
+# Top-Down Prediction
+#////////////////////////////////////////////////////////////
+
+class FeatureTopDownInitRule(TopDownInitRule):
+ def apply(self, chart, grammar):
+ for prod in grammar.productions(lhs=grammar.start()):
+ new_edge = FeatureTreeEdge.from_production(prod, 0)
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+class FeatureTopDownPredictRule(CachedTopDownPredictRule):
+ """
+ A specialized version of the (cached) top down predict rule that operates
+ on nonterminals whose symbols are ``FeatStructNonterminal``s. Rather
+ than simply comparing the nonterminals for equality, they are
+ unified.
+
+ The top down expand rule states that:
+
+ - ``[A -> alpha \* B1 beta][i:j]``
+
+ licenses the edge:
+
+ - ``[B2 -> \* gamma][j:j]``
+
+ for each grammar production ``B2 -> gamma``, assuming that B1
+ and B2 can be unified.
+ """
+ def apply(self, chart, grammar, edge):
+ if edge.is_complete(): return
+ nextsym, index = edge.nextsym(), edge.end()
+ if not is_nonterminal(nextsym): return
+
+ # If we've already applied this rule to an edge with the same
+ # next & end, and the chart & grammar have not changed, then
+ # just return (no new edges to add).
+ done = self._done.get((nextsym, index), (None,None))
+ if done[0] is chart and done[1] is grammar: return
+
+ for prod in grammar.productions(lhs=edge.nextsym()):
+ # If the left corner in the predicted production is
+ # leaf, it must match with the input.
+ if prod.rhs():
+ first = prod.rhs()[0]
+ if is_terminal(first):
+ if index >= chart.num_leaves(): continue
+ if first != chart.leaf(index): continue
+
+ # We rename vars here, because we don't want variables
+ # from the two different productions to match.
+ if unify(prod.lhs(), edge.next_with_bindings(), rename_vars=True):
+ new_edge = FeatureTreeEdge.from_production(prod, edge.end())
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+ # Record the fact that we've applied this rule.
+ self._done[nextsym, index] = (chart, grammar)
+
+
+#////////////////////////////////////////////////////////////
+# Bottom-Up Prediction
+#////////////////////////////////////////////////////////////
+
+class FeatureBottomUpPredictRule(BottomUpPredictRule):
+ def apply(self, chart, grammar, edge):
+ if edge.is_incomplete(): return
+ for prod in grammar.productions(rhs=edge.lhs()):
+ if isinstance(edge, FeatureTreeEdge):
+ _next = prod.rhs()[0]
+ if not is_nonterminal(_next): continue
+
+ new_edge = FeatureTreeEdge.from_production(prod, edge.start())
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+class FeatureBottomUpPredictCombineRule(BottomUpPredictCombineRule):
+ def apply(self, chart, grammar, edge):
+ if edge.is_incomplete(): return
+ found = edge.lhs()
+ for prod in grammar.productions(rhs=found):
+ bindings = {}
+ if isinstance(edge, FeatureTreeEdge):
+ _next = prod.rhs()[0]
+ if not is_nonterminal(_next): continue
+
+ # We rename vars here, because we don't want variables
+ # from the two different productions to match.
+ used_vars = find_variables((prod.lhs(),) + prod.rhs(),
+ fs_class=FeatStruct)
+ found = found.rename_variables(used_vars=used_vars)
+
+ result = unify(_next, found, bindings, rename_vars=False)
+ if result is None: continue
+
+ new_edge = (FeatureTreeEdge.from_production(prod, edge.start())
+ .move_dot_forward(edge.end(), bindings))
+ if chart.insert(new_edge, (edge,)):
+ yield new_edge
+
+class FeatureEmptyPredictRule(EmptyPredictRule):
+ def apply(self, chart, grammar):
+ for prod in grammar.productions(empty=True):
+ for index in xrange(chart.num_leaves() + 1):
+ new_edge = FeatureTreeEdge.from_production(prod, index)
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+
+#////////////////////////////////////////////////////////////
+# Feature Chart Parser
+#////////////////////////////////////////////////////////////
+
+TD_FEATURE_STRATEGY = [LeafInitRule(),
+ FeatureTopDownInitRule(),
+ FeatureTopDownPredictRule(),
+ FeatureSingleEdgeFundamentalRule()]
+BU_FEATURE_STRATEGY = [LeafInitRule(),
+ FeatureEmptyPredictRule(),
+ FeatureBottomUpPredictRule(),
+ FeatureSingleEdgeFundamentalRule()]
+BU_LC_FEATURE_STRATEGY = [LeafInitRule(),
+ FeatureEmptyPredictRule(),
+ FeatureBottomUpPredictCombineRule(),
+ FeatureSingleEdgeFundamentalRule()]
+
+class FeatureChartParser(ChartParser):
+ def __init__(self, grammar,
+ strategy=BU_LC_FEATURE_STRATEGY,
+ trace_chart_width=20,
+ chart_class=FeatureChart,
+ **parser_args):
+ ChartParser.__init__(self, grammar,
+ strategy=strategy,
+ trace_chart_width=trace_chart_width,
+ chart_class=chart_class,
+ **parser_args)
+
+class FeatureTopDownChartParser(FeatureChartParser):
+ def __init__(self, grammar, **parser_args):
+ FeatureChartParser.__init__(self, grammar, TD_FEATURE_STRATEGY, **parser_args)
+
+class FeatureBottomUpChartParser(FeatureChartParser):
+ def __init__(self, grammar, **parser_args):
+ FeatureChartParser.__init__(self, grammar, BU_FEATURE_STRATEGY, **parser_args)
+
+class FeatureBottomUpLeftCornerChartParser(FeatureChartParser):
+ def __init__(self, grammar, **parser_args):
+ FeatureChartParser.__init__(self, grammar, BU_LC_FEATURE_STRATEGY, **parser_args)
+
+
+#////////////////////////////////////////////////////////////
+# Instantiate Variable Chart
+#////////////////////////////////////////////////////////////
+
+class InstantiateVarsChart(FeatureChart):
+ """
+ A specialized chart that 'instantiates' variables whose names
+ start with '@', by replacing them with unique new variables.
+ In particular, whenever a complete edge is added to the chart, any
+ variables in the edge's ``lhs`` whose names start with '@' will be
+ replaced by unique new ``Variable``s.
+ """
+ def __init__(self, tokens):
+ FeatureChart.__init__(self, tokens)
+
+ def initialize(self):
+ self._instantiated = set()
+ FeatureChart.initialize(self)
+
+ def insert(self, edge, child_pointer_list):
+ if edge in self._instantiated: return False
+ self.instantiate_edge(edge)
+ return FeatureChart.insert(self, edge, child_pointer_list)
+
+ def instantiate_edge(self, edge):
+ """
+ If the edge is a ``FeatureTreeEdge``, and it is complete,
+ then instantiate all variables whose names start with '@',
+ by replacing them with unique new variables.
+
+ Note that instantiation is done in-place, since the
+ parsing algorithms might already hold a reference to
+ the edge for future use.
+ """
+ # If the edge is a leaf, or is not complete, or is
+ # already in the chart, then just return it as-is.
+ if not isinstance(edge, FeatureTreeEdge): return
+ if not edge.is_complete(): return
+ if edge in self._edge_to_cpls: return
+
+ # Get a list of variables that need to be instantiated.
+ # If there are none, then return as-is.
+ inst_vars = self.inst_vars(edge)
+ if not inst_vars: return
+
+ # Instantiate the edge!
+ self._instantiated.add(edge)
+ edge._lhs = edge.lhs().substitute_bindings(inst_vars)
+
+ def inst_vars(self, edge):
+ return dict((var, logic.unique_variable())
+ for var in edge.lhs().variables()
+ if var.name.startswith('@'))
+
+
+#////////////////////////////////////////////////////////////
+# Demo
+#////////////////////////////////////////////////////////////
+
+def demo_grammar():
+ from nltk.grammar import FeatureGrammar
+ return FeatureGrammar.fromstring("""
+S -> NP VP
+PP -> Prep NP
+NP -> NP PP
+VP -> VP PP
+VP -> Verb NP
+VP -> Verb
+NP -> Det[pl=?x] Noun[pl=?x]
+NP -> "John"
+NP -> "I"
+Det -> "the"
+Det -> "my"
+Det[-pl] -> "a"
+Noun[-pl] -> "dog"
+Noun[-pl] -> "cookie"
+Verb -> "ate"
+Verb -> "saw"
+Prep -> "with"
+Prep -> "under"
+""")
+
+def demo(print_times=True, print_grammar=True,
+ print_trees=True, print_sentence=True,
+ trace=1,
+ parser=FeatureChartParser,
+ sent='I saw John with a dog with my cookie'):
+ import sys, time
+ print()
+ grammar = demo_grammar()
+ if print_grammar:
+ print(grammar)
+ print()
+ print("*", parser.__name__)
+ if print_sentence:
+ print("Sentence:", sent)
+ tokens = sent.split()
+ t = time.clock()
+ cp = parser(grammar, trace=trace)
+ chart = cp.chart_parse(tokens)
+ trees = list(chart.parses(grammar.start()))
+ if print_times:
+ print("Time: %s" % (time.clock() - t))
+ if print_trees:
+ for tree in trees: print(tree)
+ else:
+ print("Nr trees:", len(trees))
+
+def run_profile():
+ import profile
+ profile.run('for i in range(1): demo()', '/tmp/profile.out')
+ import pstats
+ p = pstats.Stats('/tmp/profile.out')
+ p.strip_dirs().sort_stats('time', 'cum').print_stats(60)
+ p.strip_dirs().sort_stats('cum', 'time').print_stats(60)
+
+if __name__ == '__main__':
+ from nltk.data import load
+ demo()
+ print()
+ grammar = load('grammars/book_grammars/feat0.fcfg')
+ cp = FeatureChartParser(grammar, trace=2)
+ sent = 'Kim likes children'
+ tokens = sent.split()
+ trees = cp.parse(tokens)
+ for tree in trees:
+ print(tree)
diff --git a/nltk/parse/generate.py b/nltk/parse/generate.py
new file mode 100644
index 0000000..ab5ae1f
--- /dev/null
+++ b/nltk/parse/generate.py
@@ -0,0 +1,76 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Generating from a CFG
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Peter Ljunglöf <peter.ljunglof at heatherleaf.se>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+from __future__ import print_function
+
+import itertools
+import sys
+from nltk.grammar import Nonterminal
+
+
+def generate(grammar, start=None, depth=None, n=None):
+ """
+ Generates an iterator of all sentences from a CFG.
+
+ :param grammar: The Grammar used to generate sentences.
+ :param start: The Nonterminal from which to start generate sentences.
+ :param depth: The maximal depth of the generated tree.
+ :param n: The maximum number of sentences to return.
+ :return: An iterator of lists of terminal tokens.
+ """
+ if not start:
+ start = grammar.start()
+ if depth is None:
+ depth = sys.maxsize
+
+ iter = _generate_all(grammar, [start], depth)
+
+ if n:
+ iter = itertools.islice(iter, n)
+
+ return iter
+
+def _generate_all(grammar, items, depth):
+ if items:
+ for frag1 in _generate_one(grammar, items[0], depth):
+ for frag2 in _generate_all(grammar, items[1:], depth):
+ yield frag1 + frag2
+ else:
+ yield []
+
+def _generate_one(grammar, item, depth):
+ if depth > 0:
+ if isinstance(item, Nonterminal):
+ for prod in grammar.productions(lhs=item):
+ for frag in _generate_all(grammar, prod.rhs(), depth-1):
+ yield frag
+ else:
+ yield [item]
+
+demo_grammar = """
+ S -> NP VP
+ NP -> Det N
+ PP -> P NP
+ VP -> 'slept' | 'saw' NP | 'walked' PP
+ Det -> 'the' | 'a'
+ N -> 'man' | 'park' | 'dog'
+ P -> 'in' | 'with'
+"""
+
+def demo(N=23):
+ from nltk.grammar import CFG
+
+ print('Generating the first %d sentences for demo grammar:' % (N,))
+ print(demo_grammar)
+ grammar = CFG.fromstring(demo_grammar)
+ for n, sent in enumerate(generate(grammar, n=N), 1):
+ print('%3d. %s' % (n, ' '.join(sent)))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/malt.py b/nltk/parse/malt.py
new file mode 100644
index 0000000..73ef5bc
--- /dev/null
+++ b/nltk/parse/malt.py
@@ -0,0 +1,283 @@
+# Natural Language Toolkit: Interface to MaltParser
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function
+
+import os
+import tempfile
+import glob
+from operator import add
+from functools import reduce
+import subprocess
+
+from nltk.data import ZipFilePathPointer
+from nltk.tag import RegexpTagger
+from nltk.tokenize import word_tokenize
+from nltk.internals import find_binary
+
+from nltk.parse.api import ParserI
+from nltk.parse.dependencygraph import DependencyGraph
+
+class MaltParser(ParserI):
+
+ def __init__(self, tagger=None, mco=None, working_dir=None, additional_java_args=None):
+ """
+ An interface for parsing with the Malt Parser.
+
+ :param mco: The name of the pre-trained model. If provided, training
+ will not be required, and MaltParser will use the model file in
+ ${working_dir}/${mco}.mco.
+ :type mco: str
+ """
+ self.config_malt()
+ self.mco = 'malt_temp' if mco is None else mco
+ self.working_dir = tempfile.gettempdir() if working_dir is None\
+ else working_dir
+ self.additional_java_args = [] if additional_java_args is None else additional_java_args
+ self._trained = mco is not None
+
+ if tagger is not None:
+ self.tagger = tagger
+ else:
+ self.tagger = RegexpTagger(
+ [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
+ (r'(The|the|A|a|An|an)$', 'AT'), # articles
+ (r'.*able$', 'JJ'), # adjectives
+ (r'.*ness$', 'NN'), # nouns formed from adjectives
+ (r'.*ly$', 'RB'), # adverbs
+ (r'.*s$', 'NNS'), # plural nouns
+ (r'.*ing$', 'VBG'), # gerunds
+ (r'.*ed$', 'VBD'), # past tense verbs
+ (r'.*', 'NN') # nouns (default)
+ ])
+
+ def config_malt(self, bin=None, verbose=False):
+ """
+ Configure NLTK's interface to the ``malt`` package. This
+ searches for a directory containing the malt jar
+
+ :param bin: The full path to the ``malt`` binary. If not
+ specified, then nltk will search the system for a ``malt``
+ binary; and if one is not found, it will raise a
+ ``LookupError`` exception.
+ :type bin: str
+ """
+ #: A list of directories that should be searched for the malt
+ #: executables. This list is used by ``config_malt`` when searching
+ #: for the malt executables.
+ _malt_path = ['.',
+ '/usr/lib/malt-1*',
+ '/usr/share/malt-1*',
+ '/usr/local/bin',
+ '/usr/local/malt-1*',
+ '/usr/local/bin/malt-1*',
+ '/usr/local/malt-1*',
+ '/usr/local/share/malt-1*']
+
+ # Expand wildcards in _malt_path:
+ malt_path = reduce(add, map(glob.glob, _malt_path))
+
+ # Find the malt binary.
+ self._malt_bin = find_binary('malt.jar', bin,
+ searchpath=malt_path, env_vars=['MALTPARSERHOME'],
+ url='http://www.maltparser.org/',
+ verbose=verbose)
+
+ def parse_all(self, sentence, verbose=False):
+ """
+ Use MaltParser to parse a sentence. Takes a sentence as a list of
+ words; it will be automatically tagged with this MaltParser instance's
+ tagger.
+
+ :param sentence: Input sentence to parse
+ :type sentence: list(str)
+ :return: list(DependencyGraph)
+ """
+ return self.parse_sents([sentence], verbose)
+
+ def parse_sents(self, sentences, verbose=False):
+ """
+ Use MaltParser to parse multiple sentence. Takes multiple sentences as a
+ list where each sentence is a list of words.
+ Each sentence will be automatically tagged with this MaltParser instance's
+ tagger.
+
+ :param sentences: Input sentences to parse
+ :type sentence: list(list(str))
+ :return: list(DependencyGraph)
+ """
+ tagged_sentences = [self.tagger.tag(sentence) for sentence in sentences]
+ return self.tagged_parse_sents(tagged_sentences, verbose)
+
+ def parse(self, sentence, verbose=False):
+ """
+ Use MaltParser to parse a sentence. Takes a sentence as a list of words.
+ The sentence will be automatically tagged with this MaltParser instance's
+ tagger.
+
+ :param sentence: Input sentence to parse
+ :type sentence: list(str)
+ :return: ``DependencyGraph`` the dependency graph representation of the sentence
+ """
+ return self.parse_sents([sentence], verbose)[0]
+
+ def raw_parse(self, sentence, verbose=False):
+ """
+ Use MaltParser to parse a sentence. Takes a sentence as a string;
+ before parsing, it will be automatically tokenized and tagged with this
+ MaltParser instance's tagger.
+
+ :param sentence: Input sentence to parse
+ :type sentence: str
+ :return: list(DependencyGraph)
+ """
+ words = word_tokenize(sentence)
+ return self.parse(words, verbose)
+
+ def tagged_parse(self, sentence, verbose=False):
+ """
+ Use MaltParser to parse a sentence. Takes a sentence as a list of
+ (word, tag) tuples; the sentence must have already been tokenized and
+ tagged.
+
+ :param sentence: Input sentence to parse
+ :type sentence: list(tuple(str, str))
+ :return: ``DependencyGraph`` the dependency graph representation of the sentence
+ """
+ return self.tagged_parse_sents([sentence], verbose)[0]
+
+ def tagged_parse_sents(self, sentences, verbose=False):
+ """
+ Use MaltParser to parse multiple sentences. Takes multiple sentences
+ where each sentence is a list of (word, tag) tuples.
+ The sentences must have already been tokenized and tagged.
+
+ :param sentences: Input sentences to parse
+ :type sentence: list(list(tuple(str, str)))
+ :return: list(``DependencyGraph``) the dependency graph representation
+ of each sentence
+ """
+
+ if not self._malt_bin:
+ raise Exception("MaltParser location is not configured. Call config_malt() first.")
+ if not self._trained:
+ raise Exception("Parser has not been trained. Call train() first.")
+
+ input_file = tempfile.NamedTemporaryFile(prefix='malt_input.conll',
+ dir=self.working_dir,
+ delete=False)
+ output_file = tempfile.NamedTemporaryFile(prefix='malt_output.conll',
+ dir=self.working_dir,
+ delete=False)
+
+ try:
+ for sentence in sentences:
+ for (i, (word, tag)) in enumerate(sentence, start=1):
+ input_str = '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' %\
+ (i, word, '_', tag, tag, '_', '0', 'a', '_', '_')
+ input_file.write(input_str.encode("utf8"))
+ input_file.write(b'\n\n')
+ input_file.close()
+
+ cmd = ['java'] + self.additional_java_args + ['-jar', self._malt_bin,
+ '-w', self.working_dir,
+ '-c', self.mco, '-i', input_file.name,
+ '-o', output_file.name, '-m', 'parse']
+
+ ret = self._execute(cmd, verbose)
+ if ret != 0:
+ raise Exception("MaltParser parsing (%s) failed with exit "
+ "code %d" % (' '.join(cmd), ret))
+
+ return DependencyGraph.load(output_file.name)
+ finally:
+ input_file.close()
+ os.remove(input_file.name)
+ output_file.close()
+ os.remove(output_file.name)
+
+ def train(self, depgraphs, verbose=False):
+ """
+ Train MaltParser from a list of ``DependencyGraph`` objects
+
+ :param depgraphs: list of ``DependencyGraph`` objects for training input data
+ """
+ input_file = tempfile.NamedTemporaryFile(prefix='malt_train.conll',
+ dir=self.working_dir,
+ delete=False)
+ try:
+ input_str = ('\n'.join(dg.to_conll(10) for dg in depgraphs))
+ input_file.write(input_str.encode("utf8"))
+ input_file.close()
+ self.train_from_file(input_file.name, verbose=verbose)
+ finally:
+ input_file.close()
+ os.remove(input_file.name)
+
+ def train_from_file(self, conll_file, verbose=False):
+ """
+ Train MaltParser from a file
+
+ :param conll_file: str for the filename of the training input data
+ """
+ if not self._malt_bin:
+ raise Exception("MaltParser location is not configured. Call config_malt() first.")
+
+ # If conll_file is a ZipFilePathPointer, then we need to do some extra
+ # massaging
+ if isinstance(conll_file, ZipFilePathPointer):
+ input_file = tempfile.NamedTemporaryFile(prefix='malt_train.conll',
+ dir=self.working_dir,
+ delete=False)
+ try:
+ conll_str = conll_file.open().read()
+ conll_file.close()
+ input_file.write(conll_str)
+ input_file.close()
+ return self.train_from_file(input_file.name, verbose=verbose)
+ finally:
+ input_file.close()
+ os.remove(input_file.name)
+
+ cmd = ['java', '-jar', self._malt_bin, '-w', self.working_dir,
+ '-c', self.mco, '-i', conll_file, '-m', 'learn']
+
+ ret = self._execute(cmd, verbose)
+ if ret != 0:
+ raise Exception("MaltParser training (%s) "
+ "failed with exit code %d" %
+ (' '.join(cmd), ret))
+
+ self._trained = True
+
+ @staticmethod
+ def _execute(cmd, verbose=False):
+ output = None if verbose else subprocess.PIPE
+ p = subprocess.Popen(cmd, stdout=output, stderr=output)
+ return p.wait()
+
+
+def demo():
+ dg1 = DependencyGraph("""1 John _ NNP _ _ 2 SUBJ _ _
+ 2 sees _ VB _ _ 0 ROOT _ _
+ 3 a _ DT _ _ 4 SPEC _ _
+ 4 dog _ NN _ _ 2 OBJ _ _
+ """)
+ dg2 = DependencyGraph("""1 John _ NNP _ _ 2 SUBJ _ _
+ 2 walks _ VB _ _ 0 ROOT _ _
+ """)
+
+ verbose = False
+
+ maltParser = MaltParser()
+ maltParser.train([dg1,dg2], verbose=verbose)
+
+ print(maltParser.raw_parse('John sees Mary', verbose=verbose).tree().pprint())
+ print(maltParser.raw_parse('a man runs', verbose=verbose).tree().pprint())
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/nonprojectivedependencyparser.py b/nltk/parse/nonprojectivedependencyparser.py
new file mode 100644
index 0000000..1186b2d
--- /dev/null
+++ b/nltk/parse/nonprojectivedependencyparser.py
@@ -0,0 +1,640 @@
+# Natural Language Toolkit: Dependency Grammars
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Jason Narad <jason.narad at gmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+from __future__ import print_function
+
+import math
+
+from nltk.compat import xrange
+
+from nltk.parse.dependencygraph import DependencyGraph, conll_data2
+
+#################################################################
+# DependencyScorerI - Interface for Graph-Edge Weight Calculation
+#################################################################
+
+class DependencyScorerI(object):
+ """
+ A scorer for calculated the weights on the edges of a weighted
+ dependency graph. This is used by a
+ ``ProbabilisticNonprojectiveParser`` to initialize the edge
+ weights of a ``DependencyGraph``. While typically this would be done
+ by training a binary classifier, any class that can return a
+ multidimensional list representation of the edge weights can
+ implement this interface. As such, it has no necessary
+ fields.
+ """
+
+ def __init__(self):
+ if self.__class__ == DependencyScorerI:
+ raise TypeError('DependencyScorerI is an abstract interface')
+
+ def train(self, graphs):
+ """
+ :type graphs: list(DependencyGraph)
+ :param graphs: A list of dependency graphs to train the scorer.
+ Typically the edges present in the graphs can be used as
+ positive training examples, and the edges not present as negative
+ examples.
+ """
+ raise NotImplementedError()
+
+ def score(self, graph):
+ """
+ :type graph: DependencyGraph
+ :param graph: A dependency graph whose set of edges need to be
+ scored.
+ :rtype: A three-dimensional list of numbers.
+ :return: The score is returned in a multidimensional(3) list, such
+ that the outer-dimension refers to the head, and the
+ inner-dimension refers to the dependencies. For instance,
+ scores[0][1] would reference the list of scores corresponding to
+ arcs from node 0 to node 1. The node's 'address' field can be used
+ to determine its number identification.
+
+ For further illustration, a score list corresponding to Fig.2 of
+ Keith Hall's 'K-best Spanning Tree Parsing' paper:
+ scores = [[[], [5], [1], [1]],
+ [[], [], [11], [4]],
+ [[], [10], [], [5]],
+ [[], [8], [8], []]]
+ When used in conjunction with a MaxEntClassifier, each score would
+ correspond to the confidence of a particular edge being classified
+ with the positive training examples.
+ """
+ raise NotImplementedError()
+
+
+
+#################################################################
+# NaiveBayesDependencyScorer
+#################################################################
+
+class NaiveBayesDependencyScorer(DependencyScorerI):
+ """
+ A dependency scorer built around a MaxEnt classifier. In this
+ particular class that classifier is a ``NaiveBayesClassifier``.
+ It uses head-word, head-tag, child-word, and child-tag features
+ for classification.
+ """
+
+ def __init__(self):
+ pass # Do nothing without throwing error
+
+ def train(self, graphs):
+ """
+ Trains a ``NaiveBayesClassifier`` using the edges present in
+ graphs list as positive examples, the edges not present as
+ negative examples. Uses a feature vector of head-word,
+ head-tag, child-word, and child-tag.
+
+ :type graphs: list(DependencyGraph)
+ :param graphs: A list of dependency graphs to train the scorer.
+ """
+
+ # Create training labeled training examples
+ labeled_examples = []
+ for graph in graphs:
+ for head_node in graph.nodelist:
+ for child_index in range(len(graph.nodelist)):
+ child_node = graph.get_by_address(child_index)
+ if child_index in head_node['deps']:
+ label = "T"
+ else:
+ label = "F"
+ labeled_examples.append((dict(a=head_node['word'],b=head_node['tag'],c=child_node['word'],d=child_node['tag']), label))
+ # Train the classifier
+ import nltk
+ nltk.usage(nltk.ClassifierI)
+ self.classifier = nltk.classify.NaiveBayesClassifier.train(labeled_examples)
+
+ def score(self, graph):
+ """
+ Converts the graph into a feature-based representation of
+ each edge, and then assigns a score to each based on the
+ confidence of the classifier in assigning it to the
+ positive label. Scores are returned in a multidimensional list.
+
+ :type graph: DependencyGraph
+ :param graph: A dependency graph to score.
+ :rtype: 3 dimensional list
+ :return: Edge scores for the graph parameter.
+ """
+ # Convert graph to feature representation
+ edges = []
+ for i in range(len(graph.nodelist)):
+ for j in range(len(graph.nodelist)):
+ head_node = graph.get_by_address(i)
+ child_node = graph.get_by_address(j)
+ print(head_node)
+ print(child_node)
+ edges.append((dict(a=head_node['word'],b=head_node['tag'],c=child_node['word'],d=child_node['tag'])))
+ # Score edges
+ edge_scores = []
+ row = []
+ count = 0
+ for pdist in self.classifier.prob_classify_many(edges):
+ print('%.4f %.4f' % (pdist.prob('T'), pdist.prob('F')))
+ row.append([math.log(pdist.prob("T"))])
+ count += 1
+ if count == len(graph.nodelist):
+ edge_scores.append(row)
+ row = []
+ count = 0
+ return edge_scores
+
+
+#################################################################
+# A Scorer for Demo Purposes
+#################################################################
+# A short class necessary to show parsing example from paper
+class DemoScorer(DependencyScorerI):
+ def train(self, graphs):
+ print('Training...')
+
+ def score(self, graph):
+ # scores for Keith Hall 'K-best Spanning Tree Parsing' paper
+ return [[[], [5], [1], [1]],
+ [[], [], [11], [4]],
+ [[], [10], [], [5]],
+ [[], [8], [8], []]]
+
+#################################################################
+# Non-Projective Probabilistic Parsing
+#################################################################
+
+class ProbabilisticNonprojectiveParser(object):
+ """
+ A probabilistic non-projective dependency parser. Nonprojective
+ dependencies allows for "crossing branches" in the parse tree
+ which is necessary for representing particular linguistic
+ phenomena, or even typical parses in some languages. This parser
+ follows the MST parsing algorithm, outlined in McDonald(2005),
+ which likens the search for the best non-projective parse to
+ finding the maximum spanning tree in a weighted directed graph.
+ """
+ def __init__(self):
+ """
+ Creates a new non-projective parser.
+ """
+ print('initializing prob. nonprojective...')
+
+ def train(self, graphs, dependency_scorer):
+ """
+ Trains a ``DependencyScorerI`` from a set of ``DependencyGraph`` objects,
+ and establishes this as the parser's scorer. This is used to
+ initialize the scores on a ``DependencyGraph`` during the parsing
+ procedure.
+
+ :type graphs: list(DependencyGraph)
+ :param graphs: A list of dependency graphs to train the scorer.
+ :type dependency_scorer: DependencyScorerI
+ :param dependency_scorer: A scorer which implements the
+ ``DependencyScorerI`` interface.
+ """
+ self._scorer = dependency_scorer
+ self._scorer.train(graphs)
+
+ def initialize_edge_scores(self, graph):
+ """
+ Assigns a score to every edge in the ``DependencyGraph`` graph.
+ These scores are generated via the parser's scorer which
+ was assigned during the training process.
+
+ :type graph: DependencyGraph
+ :param graph: A dependency graph to assign scores to.
+ """
+ self.scores = self._scorer.score(graph)
+
+ def collapse_nodes(self, new_node, cycle_path, g_graph, b_graph, c_graph):
+ """
+ Takes a list of nodes that have been identified to belong to a cycle,
+ and collapses them into on larger node. The arcs of all nodes in
+ the graph must be updated to account for this.
+
+ :type new_node: Node.
+ :param new_node: A Node (Dictionary) to collapse the cycle nodes into.
+ :type cycle_path: A list of integers.
+ :param cycle_path: A list of node addresses, each of which is in the cycle.
+ :type g_graph, b_graph, c_graph: DependencyGraph
+ :param g_graph, b_graph, c_graph: Graphs which need to be updated.
+ """
+ print('Collapsing nodes...')
+ # Collapse all cycle nodes into v_n+1 in G_Graph
+ for cycle_node_index in cycle_path:
+ g_graph.remove_by_address(cycle_node_index)
+ g_graph.nodelist.append(new_node)
+ g_graph.redirect_arcs(cycle_path, new_node['address'])
+
+ def update_edge_scores(self, new_node, cycle_path):
+ """
+ Updates the edge scores to reflect a collapse operation into
+ new_node.
+
+ :type new_node: A Node.
+ :param new_node: The node which cycle nodes are collapsed into.
+ :type cycle_path: A list of integers.
+ :param cycle_path: A list of node addresses that belong to the cycle.
+ """
+ print('cycle', cycle_path)
+ cycle_path = self.compute_original_indexes(cycle_path)
+ print('old cycle ', cycle_path)
+ print('Prior to update:\n', self.scores)
+ for i, row in enumerate(self.scores):
+ for j, column in enumerate(self.scores[i]):
+ print(self.scores[i][j])
+ if j in cycle_path and not i in cycle_path and len(self.scores[i][j]) > 0:
+ new_vals = []
+ subtract_val = self.compute_max_subtract_score(j, cycle_path)
+ print(self.scores[i][j], ' - ', subtract_val)
+ for cur_val in self.scores[i][j]:
+ new_vals.append(cur_val - subtract_val)
+ self.scores[i][j] = new_vals
+ for i, row in enumerate(self.scores):
+ for j, cell in enumerate(self.scores[i]):
+ if i in cycle_path and j in cycle_path:
+ self.scores[i][j] = []
+ print('After update:\n', self.scores)
+
+ def compute_original_indexes(self, new_indexes):
+ """
+ As nodes are collapsed into others, they are replaced
+ by the new node in the graph, but it's still necessary
+ to keep track of what these original nodes were. This
+ takes a list of node addresses and replaces any collapsed
+ node addresses with their original addresses.
+
+ :type new_indexes: A list of integers.
+ :param new_indexes: A list of node addresses to check for
+ subsumed nodes.
+ """
+ swapped = True
+ while swapped:
+ originals = []
+ swapped = False
+ for new_index in new_indexes:
+ if new_index in self.inner_nodes:
+ for old_val in self.inner_nodes[new_index]:
+ if not old_val in originals:
+ originals.append(old_val)
+ swapped = True
+ else:
+ originals.append(new_index)
+ new_indexes = originals
+ return new_indexes
+
+ def compute_max_subtract_score(self, column_index, cycle_indexes):
+ """
+ When updating scores the score of the highest-weighted incoming
+ arc is subtracted upon collapse. This returns the correct
+ amount to subtract from that edge.
+
+ :type column_index: integer.
+ :param column_index: A index representing the column of incoming arcs
+ to a particular node being updated
+ :type cycle_indexes: A list of integers.
+ :param cycle_indexes: Only arcs from cycle nodes are considered. This
+ is a list of such nodes addresses.
+ """
+ max_score = -100000
+ for row_index in cycle_indexes:
+ for subtract_val in self.scores[row_index][column_index]:
+ if subtract_val > max_score:
+ max_score = subtract_val
+ return max_score
+
+
+ def best_incoming_arc(self, node_index):
+ """
+ Returns the source of the best incoming arc to the
+ node with address: node_index
+
+ :type node_index: integer.
+ :param node_index: The address of the 'destination' node,
+ the node that is arced to.
+ """
+ originals = self.compute_original_indexes([node_index])
+ print('originals:', originals)
+ max_arc = None
+ max_score = None
+ for row_index in range(len(self.scores)):
+ for col_index in range(len(self.scores[row_index])):
+# print self.scores[row_index][col_index]
+ if col_index in originals and (max_score is None or self.scores[row_index][col_index] > max_score):
+ max_score = self.scores[row_index][col_index]
+ max_arc = row_index
+ print(row_index, ',', col_index)
+ print(max_score)
+ for key in self.inner_nodes:
+ replaced_nodes = self.inner_nodes[key]
+ if max_arc in replaced_nodes:
+ return key
+ return max_arc
+
+ def original_best_arc(self, node_index):
+ """
+ ???
+ """
+ originals = self.compute_original_indexes([node_index])
+ max_arc = None
+ max_score = None
+ max_orig = None
+ for row_index in range(len(self.scores)):
+ for col_index in range(len(self.scores[row_index])):
+ if col_index in originals and (max_score is None or self.scores[row_index][col_index] > max_score):
+ max_score = self.scores[row_index][col_index]
+ max_arc = row_index
+ max_orig = col_index
+ return [max_arc, max_orig]
+
+
+ def parse(self, tokens, tags):
+ """
+ Parses a list of tokens in accordance to the MST parsing algorithm
+ for non-projective dependency parses. Assumes that the tokens to
+ be parsed have already been tagged and those tags are provided. Various
+ scoring methods can be used by implementing the ``DependencyScorerI``
+ interface and passing it to the training algorithm.
+
+ :type tokens: list(str)
+ :param tokens: A list of words or punctuation to be parsed.
+ :type tags: list(str)
+ :param tags: A list of tags corresponding by index to the words in the tokens list.
+ :return: An iterator of non-projective parses.
+ :rtype: iter(DependencyGraph)
+ """
+ self.inner_nodes = {}
+ # Initialize g_graph
+ g_graph = DependencyGraph()
+ for index, token in enumerate(tokens):
+ g_graph.nodelist.append({'word':token, 'tag':tags[index], 'deps':[], 'rel':'NTOP', 'address':index+1})
+ # Fully connect non-root nodes in g_graph
+ g_graph.connect_graph()
+ original_graph = DependencyGraph()
+ for index, token in enumerate(tokens):
+ original_graph.nodelist.append({'word':token, 'tag':tags[index], 'deps':[], 'rel':'NTOP', 'address':index+1})
+
+ # Initialize b_graph
+ b_graph = DependencyGraph()
+ b_graph.nodelist = []
+ # Initialize c_graph
+ c_graph = DependencyGraph()
+ c_graph.nodelist = [{'word':token, 'tag':tags[index], 'deps':[],
+ 'rel':'NTOP', 'address':index+1}
+ for index, token in enumerate(tokens)]
+ # Assign initial scores to g_graph edges
+ self.initialize_edge_scores(g_graph)
+ print(self.scores)
+ # Initialize a list of unvisited vertices (by node address)
+ unvisited_vertices = [vertex['address'] for vertex in c_graph.nodelist]
+ # Iterate over unvisited vertices
+ nr_vertices = len(tokens)
+ betas = {}
+ while len(unvisited_vertices) > 0:
+ # Mark current node as visited
+ current_vertex = unvisited_vertices.pop(0)
+ print('current_vertex:', current_vertex)
+ # Get corresponding node n_i to vertex v_i
+ current_node = g_graph.get_by_address(current_vertex)
+ print('current_node:', current_node)
+ # Get best in-edge node b for current node
+ best_in_edge = self.best_incoming_arc(current_vertex)
+ betas[current_vertex] = self.original_best_arc(current_vertex)
+ print('best in arc: ', best_in_edge, ' --> ', current_vertex)
+ # b_graph = Union(b_graph, b)
+ for new_vertex in [current_vertex, best_in_edge]:
+ b_graph.add_node({'word':'TEMP', 'deps':[], 'rel': 'NTOP', 'address': new_vertex})
+ b_graph.add_arc(best_in_edge, current_vertex)
+ # Beta(current node) = b - stored for parse recovery
+ # If b_graph contains a cycle, collapse it
+ cycle_path = b_graph.contains_cycle()
+ if cycle_path:
+ # Create a new node v_n+1 with address = len(nodes) + 1
+ new_node = {'word': 'NONE', 'deps':[], 'rel': 'NTOP', 'address': nr_vertices + 1}
+ # c_graph = Union(c_graph, v_n+1)
+ c_graph.add_node(new_node)
+ # Collapse all nodes in cycle C into v_n+1
+ self.update_edge_scores(new_node, cycle_path)
+ self.collapse_nodes(new_node, cycle_path, g_graph, b_graph, c_graph)
+ for cycle_index in cycle_path:
+ c_graph.add_arc(new_node['address'], cycle_index)
+# self.replaced_by[cycle_index] = new_node['address']
+
+ self.inner_nodes[new_node['address']] = cycle_path
+
+ # Add v_n+1 to list of unvisited vertices
+ unvisited_vertices.insert(0, nr_vertices + 1)
+ # increment # of nodes counter
+ nr_vertices += 1
+ # Remove cycle nodes from b_graph; B = B - cycle c
+ for cycle_node_address in cycle_path:
+ b_graph.remove_by_address(cycle_node_address)
+ print('g_graph:\n', g_graph)
+ print()
+ print('b_graph:\n', b_graph)
+ print()
+ print('c_graph:\n', c_graph)
+ print()
+ print('Betas:\n', betas)
+ print('replaced nodes', self.inner_nodes)
+ print()
+ #Recover parse tree
+ print('Final scores:\n', self.scores)
+ print('Recovering parse...')
+ for i in range(len(tokens) + 1, nr_vertices + 1):
+ betas[betas[i][1]] = betas[i]
+ print('Betas: ', betas)
+ for node in original_graph.nodelist:
+ node['deps'] = []
+ for i in range(1, len(tokens) + 1):
+# print i, betas[i]
+ original_graph.add_arc(betas[i][0], betas[i][1])
+# print original_graph
+ print('Done.')
+ yield original_graph
+
+
+
+#################################################################
+# Rule-based Non-Projective Parser
+#################################################################
+
+class NonprojectiveDependencyParser(object):
+ """
+ A non-projective, rule-based, dependency parser. This parser
+ will return the set of all possible non-projective parses based on
+ the word-to-word relations defined in the parser's dependency
+ grammar, and will allow the branches of the parse tree to cross
+ in order to capture a variety of linguistic phenomena that a
+ projective parser will not.
+ """
+
+ def __init__(self, dependency_grammar):
+ """
+ Creates a new ``NonprojectiveDependencyParser``.
+
+ :param dependency_grammar: a grammar of word-to-word relations.
+ :type dependency_grammar: DependencyGrammar
+ """
+ self._grammar = dependency_grammar
+
+ def parse(self, tokens):
+ """
+ Parses the input tokens with respect to the parser's grammar. Parsing
+ is accomplished by representing the search-space of possible parses as
+ a fully-connected directed graph. Arcs that would lead to ungrammatical
+ parses are removed and a lattice is constructed of length n, where n is
+ the number of input tokens, to represent all possible grammatical
+ traversals. All possible paths through the lattice are then enumerated
+ to produce the set of non-projective parses.
+
+ param tokens: A list of tokens to parse.
+ type tokens: list(str)
+ return: An iterator of non-projective parses.
+ rtype: iter(DependencyGraph)
+ """
+ # Create graph representation of tokens
+ self._graph = DependencyGraph()
+ self._graph.nodelist = [] # Remove the default root
+ for index, token in enumerate(tokens):
+ self._graph.nodelist.append({'word':token, 'deps':[], 'rel':'NTOP', 'address':index})
+ for head_node in self._graph.nodelist:
+ deps = []
+ for dep_node in self._graph.nodelist:
+ if self._grammar.contains(head_node['word'], dep_node['word']) and not head_node['word'] == dep_node['word']:
+ deps.append(dep_node['address'])
+ head_node['deps'] = deps
+ # Create lattice of possible heads
+ roots = []
+ possible_heads = []
+ for i, word in enumerate(tokens):
+ heads = []
+ for j, head in enumerate(tokens):
+ if (i != j) and self._grammar.contains(head, word):
+ heads.append(j)
+ if len(heads) == 0:
+ roots.append(i)
+ possible_heads.append(heads)
+
+ # Set roots to attempt
+ if len(roots) < 2:
+ if len(roots) == 0:
+ for i in range(len(tokens)):
+ roots.append(i)
+
+ # Traverse lattice
+ analyses = []
+ for root in roots:
+ stack = []
+ analysis = [[] for i in range(len(possible_heads))]
+ i = 0
+ forward = True
+ while i >= 0:
+ if forward:
+ if len(possible_heads[i]) == 1:
+ analysis[i] = possible_heads[i][0]
+ elif len(possible_heads[i]) == 0:
+ analysis[i] = -1
+ else:
+ head = possible_heads[i].pop()
+ analysis[i] = head
+ stack.append([i, head])
+ if not forward:
+ index_on_stack = False
+ for stack_item in stack:
+# print stack_item
+ if stack_item[0] == i:
+ index_on_stack = True
+ orig_length = len(possible_heads[i])
+# print len(possible_heads[i])
+ if index_on_stack and orig_length == 0:
+ for j in xrange(len(stack) -1, -1, -1):
+ stack_item = stack[j]
+ if stack_item[0] == i:
+ possible_heads[i].append(stack.pop(j)[1])
+# print stack
+ elif index_on_stack and orig_length > 0:
+ head = possible_heads[i].pop()
+ analysis[i] = head
+ stack.append([i, head])
+ forward = True
+
+# print 'Index on stack:', i, index_on_stack
+ if i + 1 == len(possible_heads):
+ analyses.append(analysis[:])
+ forward = False
+ if forward:
+ i += 1
+ else:
+ i -= 1
+
+ # Filter parses
+ # ensure 1 root, every thing has 1 head
+ for analysis in analyses:
+ root_count = 0
+ root = []
+ for i, cell in enumerate(analysis):
+ if cell == -1:
+ root_count += 1
+ root = i
+ if root_count == 1:
+ graph = DependencyGraph()
+ graph.nodelist[0]['deps'] = root + 1
+ for i in range(len(tokens)):
+ node = {'word': tokens[i], 'address': i+1}
+ node['deps'] = [j+1 for j in range(len(tokens)) if analysis[j] == i]
+ graph.nodelist.append(node)
+# cycle = graph.contains_cycle()
+# if not cycle:
+ yield graph
+
+
+#################################################################
+# Demos
+#################################################################
+
+def demo():
+# hall_demo()
+ nonprojective_conll_parse_demo()
+ rule_based_demo()
+
+
+def hall_demo():
+ npp = ProbabilisticNonprojectiveParser()
+ npp.train([], DemoScorer())
+ for parse_graph in npp.parse(['v1', 'v2', 'v3'], [None, None, None]):
+ print(parse_graph)
+
+def nonprojective_conll_parse_demo():
+ graphs = [DependencyGraph(entry)
+ for entry in conll_data2.split('\n\n') if entry]
+ npp = ProbabilisticNonprojectiveParser()
+ npp.train(graphs, NaiveBayesDependencyScorer())
+ for parse_graph in npp.parse(['Cathy', 'zag', 'hen', 'zwaaien', '.'], ['N', 'V', 'Pron', 'Adj', 'N', 'Punc']):
+ print(parse_graph)
+
+def rule_based_demo():
+ from nltk.grammar import DependencyGrammar
+
+ grammar = DependencyGrammar.fromstring("""
+ 'taught' -> 'play' | 'man'
+ 'man' -> 'the' | 'in'
+ 'in' -> 'corner'
+ 'corner' -> 'the'
+ 'play' -> 'golf' | 'dachshund' | 'to'
+ 'dachshund' -> 'his'
+ """)
+ print(grammar)
+ ndp = NonprojectiveDependencyParser(grammar)
+ graphs = ndp.parse(['the', 'man', 'in', 'the', 'corner', 'taught', 'his', 'dachshund', 'to', 'play', 'golf'])
+ print('Graphs:')
+ for graph in graphs:
+ print(graph)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/pchart.py b/nltk/parse/pchart.py
new file mode 100644
index 0000000..cacfd3f
--- /dev/null
+++ b/nltk/parse/pchart.py
@@ -0,0 +1,485 @@
+# Natural Language Toolkit: Probabilistic Chart Parsers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classes and interfaces for associating probabilities with tree
+structures that represent the internal organization of a text. The
+probabilistic parser module defines ``BottomUpProbabilisticChartParser``.
+
+``BottomUpProbabilisticChartParser`` is an abstract class that implements
+a bottom-up chart parser for ``PCFG`` grammars. It maintains a queue of edges,
+and adds them to the chart one at a time. The ordering of this queue
+is based on the probabilities associated with the edges, allowing the
+parser to expand more likely edges before less likely ones. Each
+subclass implements a different queue ordering, producing different
+search strategies. Currently the following subclasses are defined:
+
+ - ``InsideChartParser`` searches edges in decreasing order of
+ their trees' inside probabilities.
+ - ``RandomChartParser`` searches edges in random order.
+ - ``LongestChartParser`` searches edges in decreasing order of their
+ location's length.
+
+The ``BottomUpProbabilisticChartParser`` constructor has an optional
+argument beam_size. If non-zero, this controls the size of the beam
+(aka the edge queue). This option is most useful with InsideChartParser.
+"""
+from __future__ import print_function, unicode_literals
+
+##//////////////////////////////////////////////////////
+## Bottom-Up PCFG Chart Parser
+##//////////////////////////////////////////////////////
+
+# [XX] This might not be implemented quite right -- it would be better
+# to associate probabilities with child pointer lists.
+
+from functools import reduce
+from nltk.tree import Tree, ProbabilisticTree
+from nltk.grammar import Nonterminal, PCFG
+
+from nltk.parse.api import ParserI
+from nltk.parse.chart import Chart, LeafEdge, TreeEdge, AbstractChartRule
+from nltk.compat import python_2_unicode_compatible
+
+# Probabilistic edges
+class ProbabilisticLeafEdge(LeafEdge):
+ def prob(self): return 1.0
+
+class ProbabilisticTreeEdge(TreeEdge):
+ def __init__(self, prob, *args, **kwargs):
+ TreeEdge.__init__(self, *args, **kwargs)
+ self._prob = prob
+ # two edges with different probabilities are not equal.
+ self._comparison_key = (self._comparison_key, prob)
+
+ def prob(self): return self._prob
+
+ @staticmethod
+ def from_production(production, index, p):
+ return ProbabilisticTreeEdge(p, (index, index), production.lhs(),
+ production.rhs(), 0)
+
+# Rules using probabilistic edges
+class ProbabilisticBottomUpInitRule(AbstractChartRule):
+ NUM_EDGES=0
+ def apply(self, chart, grammar):
+ for index in range(chart.num_leaves()):
+ new_edge = ProbabilisticLeafEdge(chart.leaf(index), index)
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+class ProbabilisticBottomUpPredictRule(AbstractChartRule):
+ NUM_EDGES=1
+ def apply(self, chart, grammar, edge):
+ if edge.is_incomplete(): return
+ for prod in grammar.productions():
+ if edge.lhs() == prod.rhs()[0]:
+ new_edge = ProbabilisticTreeEdge.from_production(prod, edge.start(), prod.prob())
+ if chart.insert(new_edge, ()):
+ yield new_edge
+
+class ProbabilisticFundamentalRule(AbstractChartRule):
+ NUM_EDGES=2
+ def apply(self, chart, grammar, left_edge, right_edge):
+ # Make sure the rule is applicable.
+ if not (left_edge.end() == right_edge.start() and
+ left_edge.nextsym() == right_edge.lhs() and
+ left_edge.is_incomplete() and right_edge.is_complete()):
+ return
+
+ # Construct the new edge.
+ p = left_edge.prob() * right_edge.prob()
+ new_edge = ProbabilisticTreeEdge(p,
+ span=(left_edge.start(), right_edge.end()),
+ lhs=left_edge.lhs(), rhs=left_edge.rhs(),
+ dot=left_edge.dot()+1)
+
+ # Add it to the chart, with appropriate child pointers.
+ changed_chart = False
+ for cpl1 in chart.child_pointer_lists(left_edge):
+ if chart.insert(new_edge, cpl1+(right_edge,)):
+ changed_chart = True
+
+ # If we changed the chart, then generate the edge.
+ if changed_chart: yield new_edge
+
+ at python_2_unicode_compatible
+class SingleEdgeProbabilisticFundamentalRule(AbstractChartRule):
+ NUM_EDGES=1
+
+ _fundamental_rule = ProbabilisticFundamentalRule()
+
+ def apply(self, chart, grammar, edge1):
+ fr = self._fundamental_rule
+ if edge1.is_incomplete():
+ # edge1 = left_edge; edge2 = right_edge
+ for edge2 in chart.select(start=edge1.end(), is_complete=True,
+ lhs=edge1.nextsym()):
+ for new_edge in fr.apply(chart, grammar, edge1, edge2):
+ yield new_edge
+ else:
+ # edge2 = left_edge; edge1 = right_edge
+ for edge2 in chart.select(end=edge1.start(), is_complete=False,
+ nextsym=edge1.lhs()):
+ for new_edge in fr.apply(chart, grammar, edge2, edge1):
+ yield new_edge
+
+ def __str__(self):
+ return 'Fundamental Rule'
+
+class BottomUpProbabilisticChartParser(ParserI):
+ """
+ An abstract bottom-up parser for ``PCFG`` grammars that uses a ``Chart`` to
+ record partial results. ``BottomUpProbabilisticChartParser`` maintains
+ a queue of edges that can be added to the chart. This queue is
+ initialized with edges for each token in the text that is being
+ parsed. ``BottomUpProbabilisticChartParser`` inserts these edges into
+ the chart one at a time, starting with the most likely edges, and
+ proceeding to less likely edges. For each edge that is added to
+ the chart, it may become possible to insert additional edges into
+ the chart; these are added to the queue. This process continues
+ until enough complete parses have been generated, or until the
+ queue is empty.
+
+ The sorting order for the queue is not specified by
+ ``BottomUpProbabilisticChartParser``. Different sorting orders will
+ result in different search strategies. The sorting order for the
+ queue is defined by the method ``sort_queue``; subclasses are required
+ to provide a definition for this method.
+
+ :type _grammar: PCFG
+ :ivar _grammar: The grammar used to parse sentences.
+ :type _trace: int
+ :ivar _trace: The level of tracing output that should be generated
+ when parsing a text.
+ """
+ def __init__(self, grammar, beam_size=0, trace=0):
+ """
+ Create a new ``BottomUpProbabilisticChartParser``, that uses
+ ``grammar`` to parse texts.
+
+ :type grammar: PCFG
+ :param grammar: The grammar used to parse texts.
+ :type beam_size: int
+ :param beam_size: The maximum length for the parser's edge queue.
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ and higher numbers will produce more verbose tracing
+ output.
+ """
+ if not isinstance(grammar, PCFG):
+ raise ValueError("The grammar must be probabilistic PCFG")
+ self._grammar = grammar
+ self.beam_size = beam_size
+ self._trace = trace
+
+ def grammar(self):
+ return self._grammar
+
+ def trace(self, trace=2):
+ """
+ Set the level of tracing output that should be generated when
+ parsing a text.
+
+ :type trace: int
+ :param trace: The trace level. A trace level of ``0`` will
+ generate no tracing output; and higher trace levels will
+ produce more verbose tracing output.
+ :rtype: None
+ """
+ self._trace = trace
+
+ # TODO: change this to conform more with the standard ChartParser
+ def parse(self, tokens):
+ self._grammar.check_coverage(tokens)
+ chart = Chart(list(tokens))
+ grammar = self._grammar
+
+ # Chart parser rules.
+ bu_init = ProbabilisticBottomUpInitRule()
+ bu = ProbabilisticBottomUpPredictRule()
+ fr = SingleEdgeProbabilisticFundamentalRule()
+
+ # Our queue
+ queue = []
+
+ # Initialize the chart.
+ for edge in bu_init.apply(chart, grammar):
+ if self._trace > 1:
+ print(' %-50s [%s]' % (chart.pp_edge(edge,width=2),
+ edge.prob()))
+ queue.append(edge)
+
+ while len(queue) > 0:
+ # Re-sort the queue.
+ self.sort_queue(queue, chart)
+
+ # Prune the queue to the correct size if a beam was defined
+ if self.beam_size:
+ self._prune(queue, chart)
+
+ # Get the best edge.
+ edge = queue.pop()
+ if self._trace > 0:
+ print(' %-50s [%s]' % (chart.pp_edge(edge,width=2),
+ edge.prob()))
+
+ # Apply BU & FR to it.
+ queue.extend(bu.apply(chart, grammar, edge))
+ queue.extend(fr.apply(chart, grammar, edge))
+
+ # Get a list of complete parses.
+ parses = list(chart.parses(grammar.start(), ProbabilisticTree))
+
+ # Assign probabilities to the trees.
+ prod_probs = {}
+ for prod in grammar.productions():
+ prod_probs[prod.lhs(), prod.rhs()] = prod.prob()
+ for parse in parses:
+ self._setprob(parse, prod_probs)
+
+ # Sort by probability
+ parses.sort(reverse=True, key=lambda tree: tree.prob())
+
+ return iter(parses)
+
+ def _setprob(self, tree, prod_probs):
+ if tree.prob() is not None: return
+
+ # Get the prob of the CFG production.
+ lhs = Nonterminal(tree.label())
+ rhs = []
+ for child in tree:
+ if isinstance(child, Tree):
+ rhs.append(Nonterminal(child.label()))
+ else:
+ rhs.append(child)
+ prob = prod_probs[lhs, tuple(rhs)]
+
+ # Get the probs of children.
+ for child in tree:
+ if isinstance(child, Tree):
+ self._setprob(child, prod_probs)
+ prob *= child.prob()
+
+ tree.set_prob(prob)
+
+ def sort_queue(self, queue, chart):
+ """
+ Sort the given queue of ``Edge`` objects, placing the edge that should
+ be tried first at the beginning of the queue. This method
+ will be called after each ``Edge`` is added to the queue.
+
+ :param queue: The queue of ``Edge`` objects to sort. Each edge in
+ this queue is an edge that could be added to the chart by
+ the fundamental rule; but that has not yet been added.
+ :type queue: list(Edge)
+ :param chart: The chart being used to parse the text. This
+ chart can be used to provide extra information for sorting
+ the queue.
+ :type chart: Chart
+ :rtype: None
+ """
+ raise NotImplementedError()
+
+ def _prune(self, queue, chart):
+ """ Discard items in the queue if the queue is longer than the beam."""
+ if len(queue) > self.beam_size:
+ split = len(queue)-self.beam_size
+ if self._trace > 2:
+ for edge in queue[:split]:
+ print(' %-50s [DISCARDED]' % chart.pp_edge(edge,2))
+ del queue[:split]
+
+class InsideChartParser(BottomUpProbabilisticChartParser):
+ """
+ A bottom-up parser for ``PCFG`` grammars that tries edges in descending
+ order of the inside probabilities of their trees. The "inside
+ probability" of a tree is simply the
+ probability of the entire tree, ignoring its context. In
+ particular, the inside probability of a tree generated by
+ production *p* with children *c[1], c[2], ..., c[n]* is
+ *P(p)P(c[1])P(c[2])...P(c[n])*; and the inside
+ probability of a token is 1 if it is present in the text, and 0 if
+ it is absent.
+
+ This sorting order results in a type of lowest-cost-first search
+ strategy.
+ """
+ # Inherit constructor.
+ def sort_queue(self, queue, chart):
+ """
+ Sort the given queue of edges, in descending order of the
+ inside probabilities of the edges' trees.
+
+ :param queue: The queue of ``Edge`` objects to sort. Each edge in
+ this queue is an edge that could be added to the chart by
+ the fundamental rule; but that has not yet been added.
+ :type queue: list(Edge)
+ :param chart: The chart being used to parse the text. This
+ chart can be used to provide extra information for sorting
+ the queue.
+ :type chart: Chart
+ :rtype: None
+ """
+ queue.sort(key=lambda edge: edge.prob())
+
+# Eventually, this will become some sort of inside-outside parser:
+# class InsideOutsideParser(BottomUpProbabilisticChartParser):
+# def __init__(self, grammar, trace=0):
+# # Inherit docs.
+# BottomUpProbabilisticChartParser.__init__(self, grammar, trace)
+#
+# # Find the best path from S to each nonterminal
+# bestp = {}
+# for production in grammar.productions(): bestp[production.lhs()]=0
+# bestp[grammar.start()] = 1.0
+#
+# for i in range(len(grammar.productions())):
+# for production in grammar.productions():
+# lhs = production.lhs()
+# for elt in production.rhs():
+# bestp[elt] = max(bestp[lhs]*production.prob(),
+# bestp.get(elt,0))
+#
+# self._bestp = bestp
+# for (k,v) in self._bestp.items(): print k,v
+#
+# def _sortkey(self, edge):
+# return edge.structure()[PROB] * self._bestp[edge.lhs()]
+#
+# def sort_queue(self, queue, chart):
+# queue.sort(key=self._sortkey)
+
+import random
+class RandomChartParser(BottomUpProbabilisticChartParser):
+ """
+ A bottom-up parser for ``PCFG`` grammars that tries edges in random order.
+ This sorting order results in a random search strategy.
+ """
+ # Inherit constructor
+ def sort_queue(self, queue, chart):
+ i = random.randint(0, len(queue)-1)
+ (queue[-1], queue[i]) = (queue[i], queue[-1])
+
+class UnsortedChartParser(BottomUpProbabilisticChartParser):
+ """
+ A bottom-up parser for ``PCFG`` grammars that tries edges in whatever order.
+ """
+ # Inherit constructor
+ def sort_queue(self, queue, chart): return
+
+class LongestChartParser(BottomUpProbabilisticChartParser):
+ """
+ A bottom-up parser for ``PCFG`` grammars that tries longer edges before
+ shorter ones. This sorting order results in a type of best-first
+ search strategy.
+ """
+ # Inherit constructor
+ def sort_queue(self, queue, chart):
+ queue.sort(key=lambda edge: edge.length())
+
+##//////////////////////////////////////////////////////
+## Test Code
+##//////////////////////////////////////////////////////
+
+def demo(choice=None, draw_parses=None, print_parses=None):
+ """
+ A demonstration of the probabilistic parsers. The user is
+ prompted to select which demo to run, and how many parses should
+ be found; and then each parser is run on the same demo, and a
+ summary of the results are displayed.
+ """
+ import sys, time
+ from nltk import tokenize, toy_pcfg1, toy_pcfg2
+ from nltk.parse import pchart
+
+ # Define two demos. Each demo has a sentence and a grammar.
+ demos = [('I saw John with my telescope', toy_pcfg1),
+ ('the boy saw Jack with Bob under the table with a telescope',
+ toy_pcfg2)]
+
+ if choice is None:
+ # Ask the user which demo they want to use.
+ print()
+ for i in range(len(demos)):
+ print('%3s: %s' % (i+1, demos[i][0]))
+ print(' %r' % demos[i][1])
+ print()
+ print('Which demo (%d-%d)? ' % (1, len(demos)), end=' ')
+ choice = int(sys.stdin.readline().strip())-1
+ try:
+ sent, grammar = demos[choice]
+ except:
+ print('Bad sentence number')
+ return
+
+ # Tokenize the sentence.
+ tokens = sent.split()
+
+ # Define a list of parsers. We'll use all parsers.
+ parsers = [
+ pchart.InsideChartParser(grammar),
+ pchart.RandomChartParser(grammar),
+ pchart.UnsortedChartParser(grammar),
+ pchart.LongestChartParser(grammar),
+ pchart.InsideChartParser(grammar, beam_size = len(tokens)+1) # was BeamParser
+ ]
+
+ # Run the parsers on the tokenized sentence.
+ times = []
+ average_p = []
+ num_parses = []
+ all_parses = {}
+ for parser in parsers:
+ print('\ns: %s\nparser: %s\ngrammar: %s' % (sent,parser,grammar))
+ parser.trace(3)
+ t = time.time()
+ parses = list(parser.parse(tokens))
+ times.append(time.time()-t)
+ p = (reduce(lambda a,b:a+b.prob(), parses, 0)/len(parses) if parses else 0)
+ average_p.append(p)
+ num_parses.append(len(parses))
+ for p in parses: all_parses[p.freeze()] = 1
+
+ # Print some summary statistics
+ print()
+ print(' Parser Beam | Time (secs) # Parses Average P(parse)')
+ print('------------------------+------------------------------------------')
+ for i in range(len(parsers)):
+ print('%18s %4d |%11.4f%11d%19.14f' % (parsers[i].__class__.__name__,
+ parsers[i].beam_size,
+ times[i],num_parses[i],average_p[i]))
+ parses = all_parses.keys()
+ if parses: p = reduce(lambda a,b:a+b.prob(), parses, 0)/len(parses)
+ else: p = 0
+ print('------------------------+------------------------------------------')
+ print('%18s |%11s%11d%19.14f' % ('(All Parses)', 'n/a', len(parses), p))
+
+ if draw_parses is None:
+ # Ask the user if we should draw the parses.
+ print()
+ print('Draw parses (y/n)? ', end=' ')
+ draw_parses = sys.stdin.readline().strip().lower().startswith('y')
+ if draw_parses:
+ from nltk.draw.tree import draw_trees
+ print(' please wait...')
+ draw_trees(*parses)
+
+ if print_parses is None:
+ # Ask the user if we should print the parses.
+ print()
+ print('Print parses (y/n)? ', end=' ')
+ print_parses = sys.stdin.readline().strip().lower().startswith('y')
+ if print_parses:
+ for parse in parses:
+ print(parse)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/projectivedependencyparser.py b/nltk/parse/projectivedependencyparser.py
new file mode 100644
index 0000000..3398cd7
--- /dev/null
+++ b/nltk/parse/projectivedependencyparser.py
@@ -0,0 +1,540 @@
+# Natural Language Toolkit: Dependency Grammars
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Jason Narad <jason.narad at gmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+#
+from __future__ import print_function, unicode_literals
+
+from collections import defaultdict
+
+from nltk.grammar import (DependencyProduction, DependencyGrammar,
+ ProbabilisticDependencyGrammar)
+from nltk.parse.dependencygraph import DependencyGraph, conll_data2
+from nltk.internals import raise_unorderable_types
+from nltk.compat import total_ordering, python_2_unicode_compatible
+
+#################################################################
+# Dependency Span
+#################################################################
+
+ at total_ordering
+ at python_2_unicode_compatible
+class DependencySpan(object):
+ """
+ A contiguous span over some part of the input string representing
+ dependency (head -> modifier) relationships amongst words. An atomic
+ span corresponds to only one word so it isn't a 'span' in the conventional
+ sense, as its _start_index = _end_index = _head_index for concatenation
+ purposes. All other spans are assumed to have arcs between all nodes
+ within the start and end indexes of the span, and one head index corresponding
+ to the head word for the entire span. This is the same as the root node if
+ the dependency structure were depicted as a graph.
+ """
+ def __init__(self, start_index, end_index, head_index, arcs, tags):
+ self._start_index = start_index
+ self._end_index = end_index
+ self._head_index = head_index
+ self._arcs = arcs
+ self._tags = tags
+ self._comparison_key = (start_index, end_index, head_index, tuple(arcs))
+ self._hash = hash(self._comparison_key)
+
+ def head_index(self):
+ """
+ :return: An value indexing the head of the entire ``DependencySpan``.
+ :rtype: int
+ """
+ return self._head_index
+
+ def __repr__(self):
+ """
+ :return: A concise string representatino of the ``DependencySpan``.
+ :rtype: str.
+ """
+ return 'Span %d-%d; Head Index: %d' % (self._start_index, self._end_index, self._head_index)
+
+ def __str__(self):
+ """
+ :return: A verbose string representation of the ``DependencySpan``.
+ :rtype: str
+ """
+ str = 'Span %d-%d; Head Index: %d' % (self._start_index, self._end_index, self._head_index)
+ for i in range(len(self._arcs)):
+ str += '\n%d <- %d, %s' % (i, self._arcs[i], self._tags[i])
+ return str
+
+ def __eq__(self, other):
+ return (type(self) == type(other) and
+ self._comparison_key == other._comparison_key)
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, DependencySpan):
+ raise_unorderable_types("<", self, other)
+ return self._comparison_key < other._comparison_key
+
+ def __hash__(self):
+ """
+ :return: The hash value of this ``DependencySpan``.
+ """
+ return self._hash
+
+#################################################################
+# Chart Cell
+#################################################################
+
+ at python_2_unicode_compatible
+class ChartCell(object):
+ """
+ A cell from the parse chart formed when performing the CYK algorithm.
+ Each cell keeps track of its x and y coordinates (though this will probably
+ be discarded), and a list of spans serving as the cell's entries.
+ """
+ def __init__(self, x, y):
+ """
+ :param x: This cell's x coordinate.
+ :type x: int.
+ :param y: This cell's y coordinate.
+ :type y: int.
+ """
+ self._x = x
+ self._y = y
+ self._entries = set([])
+
+ def add(self, span):
+ """
+ Appends the given span to the list of spans
+ representing the chart cell's entries.
+
+ :param span: The span to add.
+ :type span: DependencySpan
+ """
+ self._entries.add(span)
+
+ def __str__(self):
+ """
+ :return: A verbose string representation of this ``ChartCell``.
+ :rtype: str.
+ """
+ return 'CC[%d,%d]: %s' % (self._x, self._y, self._entries)
+
+ def __repr__(self):
+ """
+ :return: A concise string representation of this ``ChartCell``.
+ :rtype: str.
+ """
+ return '%s' % self
+
+
+#################################################################
+# Parsing with Dependency Grammars
+#################################################################
+
+
+class ProjectiveDependencyParser(object):
+ """
+ A projective, rule-based, dependency parser. A ProjectiveDependencyParser
+ is created with a DependencyGrammar, a set of productions specifying
+ word-to-word dependency relations. The parse() method will then
+ return the set of all parses, in tree representation, for a given input
+ sequence of tokens. Each parse must meet the requirements of the both
+ the grammar and the projectivity constraint which specifies that the
+ branches of the dependency tree are not allowed to cross. Alternatively,
+ this can be understood as stating that each parent node and its children
+ in the parse tree form a continuous substring of the input sequence.
+ """
+
+ def __init__(self, dependency_grammar):
+ """
+ Create a new ProjectiveDependencyParser, from a word-to-word
+ dependency grammar ``DependencyGrammar``.
+
+ :param dependency_grammar: A word-to-word relation dependencygrammar.
+ :type dependency_grammar: DependencyGrammar
+ """
+ self._grammar = dependency_grammar
+
+ def parse(self, tokens):
+ """
+ Performs a projective dependency parse on the list of tokens using
+ a chart-based, span-concatenation algorithm similar to Eisner (1996).
+
+ :param tokens: The list of input tokens.
+ :type tokens: list(str)
+ :return: An iterator over parse trees.
+ :rtype: iter(Tree)
+ """
+ self._tokens = list(tokens)
+ chart = []
+ for i in range(0, len(self._tokens) + 1):
+ chart.append([])
+ for j in range(0, len(self._tokens) + 1):
+ chart[i].append(ChartCell(i,j))
+ if i==j+1:
+ chart[i][j].add(DependencySpan(i-1,i,i-1,[-1], ['null']))
+
+ for i in range(1,len(self._tokens)+1):
+ for j in range(i-2,-1,-1):
+ for k in range(i-1,j,-1):
+ for span1 in chart[k][j]._entries:
+ for span2 in chart[i][k]._entries:
+ for newspan in self.concatenate(span1, span2):
+ chart[i][j].add(newspan)
+
+ for parse in chart[len(self._tokens)][0]._entries:
+ conll_format = ""
+# malt_format = ""
+ for i in range(len(tokens)):
+# malt_format += '%s\t%s\t%d\t%s\n' % (tokens[i], 'null', parse._arcs[i] + 1, 'null')
+ conll_format += '\t%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s\t%s\t%s\n' % (i+1, tokens[i], tokens[i], 'null', 'null', 'null', parse._arcs[i] + 1, 'null', '-', '-')
+ dg = DependencyGraph(conll_format)
+# if self.meets_arity(dg):
+ yield dg.tree()
+
+
+ def concatenate(self, span1, span2):
+ """
+ Concatenates the two spans in whichever way possible. This
+ includes rightward concatenation (from the leftmost word of the
+ leftmost span to the rightmost word of the rightmost span) and
+ leftward concatenation (vice-versa) between adjacent spans. Unlike
+ Eisner's presentation of span concatenation, these spans do not
+ share or pivot on a particular word/word-index.
+
+ :return: A list of new spans formed through concatenation.
+ :rtype: list(DependencySpan)
+ """
+ spans = []
+ if span1._start_index == span2._start_index:
+ print('Error: Mismatched spans - replace this with thrown error')
+ if span1._start_index > span2._start_index:
+ temp_span = span1
+ span1 = span2
+ span2 = temp_span
+ # adjacent rightward covered concatenation
+ new_arcs = span1._arcs + span2._arcs
+ new_tags = span1._tags + span2._tags
+ if self._grammar.contains(self._tokens[span1._head_index], self._tokens[span2._head_index]):
+# print 'Performing rightward cover %d to %d' % (span1._head_index, span2._head_index)
+ new_arcs[span2._head_index - span1._start_index] = span1._head_index
+ spans.append(DependencySpan(span1._start_index, span2._end_index, span1._head_index, new_arcs, new_tags))
+ # adjacent leftward covered concatenation
+ new_arcs = span1._arcs + span2._arcs
+ if self._grammar.contains(self._tokens[span2._head_index], self._tokens[span1._head_index]):
+# print 'performing leftward cover %d to %d' % (span2._head_index, span1._head_index)
+ new_arcs[span1._head_index - span1._start_index] = span2._head_index
+ spans.append(DependencySpan(span1._start_index, span2._end_index, span2._head_index, new_arcs, new_tags))
+ return spans
+
+
+
+#################################################################
+# Parsing with Probabilistic Dependency Grammars
+#################################################################
+
+class ProbabilisticProjectiveDependencyParser(object):
+ """
+ A probabilistic, projective dependency parser. This parser returns
+ the most probable projective parse derived from the probabilistic
+ dependency grammar derived from the train() method. The probabilistic
+ model is an implementation of Eisner's (1996) Model C, which conditions
+ on head-word, head-tag, child-word, and child-tag. The decoding
+ uses a bottom-up chart-based span concatenation algorithm that's
+ identical to the one utilized by the rule-based projective parser.
+ """
+
+ def __init__(self):
+ """
+ Create a new probabilistic dependency parser. No additional
+ operations are necessary.
+ """
+ print('')
+
+ def parse(self, tokens):
+ """
+ Parses the list of tokens subject to the projectivity constraint
+ and the productions in the parser's grammar. This uses a method
+ similar to the span-concatenation algorithm defined in Eisner (1996).
+ It returns the most probable parse derived from the parser's
+ probabilistic dependency grammar.
+ """
+ self._tokens = list(tokens)
+ chart = []
+ for i in range(0, len(self._tokens) + 1):
+ chart.append([])
+ for j in range(0, len(self._tokens) + 1):
+ chart[i].append(ChartCell(i,j))
+ if i==j+1:
+ if tokens[i-1] in self._grammar._tags:
+ for tag in self._grammar._tags[tokens[i-1]]:
+ chart[i][j].add(DependencySpan(i-1,i,i-1,[-1], [tag]))
+ else:
+ print('No tag found for input token \'%s\', parse is impossible.' % tokens[i-1])
+ return []
+ for i in range(1,len(self._tokens)+1):
+ for j in range(i-2,-1,-1):
+ for k in range(i-1,j,-1):
+ for span1 in chart[k][j]._entries:
+ for span2 in chart[i][k]._entries:
+ for newspan in self.concatenate(span1, span2):
+ chart[i][j].add(newspan)
+ trees = []
+ max_parse = None
+ max_score = 0
+ for parse in chart[len(self._tokens)][0]._entries:
+ conll_format = ""
+ malt_format = ""
+ for i in range(len(tokens)):
+ malt_format += '%s\t%s\t%d\t%s\n' % (tokens[i], 'null', parse._arcs[i] + 1, 'null')
+ conll_format += '\t%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s\t%s\t%s\n' % (i+1, tokens[i], tokens[i], parse._tags[i], parse._tags[i], 'null', parse._arcs[i] + 1, 'null', '-', '-')
+ dg = DependencyGraph(conll_format)
+ score = self.compute_prob(dg)
+ trees.append((score, dg.tree()))
+ trees.sort()
+ return (tree for (score, tree) in trees)
+
+
+ def concatenate(self, span1, span2):
+ """
+ Concatenates the two spans in whichever way possible. This
+ includes rightward concatenation (from the leftmost word of the
+ leftmost span to the rightmost word of the rightmost span) and
+ leftward concatenation (vice-versa) between adjacent spans. Unlike
+ Eisner's presentation of span concatenation, these spans do not
+ share or pivot on a particular word/word-index.
+
+ :return: A list of new spans formed through concatenation.
+ :rtype: list(DependencySpan)
+ """
+ spans = []
+ if span1._start_index == span2._start_index:
+ print('Error: Mismatched spans - replace this with thrown error')
+ if span1._start_index > span2._start_index:
+ temp_span = span1
+ span1 = span2
+ span2 = temp_span
+ # adjacent rightward covered concatenation
+ new_arcs = span1._arcs + span2._arcs
+ new_tags = span1._tags + span2._tags
+ if self._grammar.contains(self._tokens[span1._head_index], self._tokens[span2._head_index]):
+ new_arcs[span2._head_index - span1._start_index] = span1._head_index
+ spans.append(DependencySpan(span1._start_index, span2._end_index, span1._head_index, new_arcs, new_tags))
+ # adjacent leftward covered concatenation
+ new_arcs = span1._arcs + span2._arcs
+ new_tags = span1._tags + span2._tags
+ if self._grammar.contains(self._tokens[span2._head_index], self._tokens[span1._head_index]):
+ new_arcs[span1._head_index - span1._start_index] = span2._head_index
+ spans.append(DependencySpan(span1._start_index, span2._end_index, span2._head_index, new_arcs, new_tags))
+ return spans
+
+ def train(self, graphs):
+ """
+ Trains a ProbabilisticDependencyGrammar based on the list of input
+ DependencyGraphs. This model is an implementation of Eisner's (1996)
+ Model C, which derives its statistics from head-word, head-tag,
+ child-word, and child-tag relationships.
+
+ :param graphs: A list of dependency graphs to train from.
+ :type: list(DependencyGraph)
+ """
+ productions = []
+ events = defaultdict(int)
+ tags = {}
+ for dg in graphs:
+ for node_index in range(1,len(dg.nodelist)):
+ children = dg.nodelist[node_index]['deps']
+ nr_left_children = dg.left_children(node_index)
+ nr_right_children = dg.right_children(node_index)
+ nr_children = nr_left_children + nr_right_children
+ for child_index in range(0 - (nr_left_children + 1), nr_right_children + 2):
+ head_word = dg.nodelist[node_index]['word']
+ head_tag = dg.nodelist[node_index]['tag']
+ if head_word in tags:
+ tags[head_word].add(head_tag)
+ else:
+ tags[head_word] = set([head_tag])
+ child = 'STOP'
+ child_tag = 'STOP'
+ prev_word = 'START'
+ prev_tag = 'START'
+ if child_index < 0:
+ array_index = child_index + nr_left_children
+ if array_index >= 0:
+ child = dg.nodelist[children[array_index]]['word']
+ child_tag = dg.nodelist[children[array_index]]['tag']
+ if child_index != -1:
+ prev_word = dg.nodelist[children[array_index + 1]]['word']
+ prev_tag = dg.nodelist[children[array_index + 1]]['tag']
+ if child != 'STOP':
+ productions.append(DependencyProduction(head_word, [child]))
+ head_event = '(head (%s %s) (mods (%s, %s, %s) left))' % (child, child_tag, prev_tag, head_word, head_tag)
+ mod_event = '(mods (%s, %s, %s) left))' % (prev_tag, head_word, head_tag)
+ events[head_event] += 1
+ events[mod_event] += 1
+ elif child_index > 0:
+ array_index = child_index + nr_left_children - 1
+ if array_index < nr_children:
+ child = dg.nodelist[children[array_index]]['word']
+ child_tag = dg.nodelist[children[array_index]]['tag']
+ if child_index != 1:
+ prev_word = dg.nodelist[children[array_index - 1]]['word']
+ prev_tag = dg.nodelist[children[array_index - 1]]['tag']
+ if child != 'STOP':
+ productions.append(DependencyProduction(head_word, [child]))
+ head_event = '(head (%s %s) (mods (%s, %s, %s) right))' % (child, child_tag, prev_tag, head_word, head_tag)
+ mod_event = '(mods (%s, %s, %s) right))' % (prev_tag, head_word, head_tag)
+ events[head_event] += 1
+ events[mod_event] += 1
+ self._grammar = ProbabilisticDependencyGrammar(productions, events, tags)
+# print self._grammar
+
+ def compute_prob(self, dg):
+ """
+ Computes the probability of a dependency graph based
+ on the parser's probability model (defined by the parser's
+ statistical dependency grammar).
+
+ :param dg: A dependency graph to score.
+ :type dg: DependencyGraph
+ :return: The probability of the dependency graph.
+ :rtype: int
+ """
+ prob = 1.0
+ for node_index in range(1,len(dg.nodelist)):
+ children = dg.nodelist[node_index]['deps']
+ nr_left_children = dg.left_children(node_index)
+ nr_right_children = dg.right_children(node_index)
+ nr_children = nr_left_children + nr_right_children
+ for child_index in range(0 - (nr_left_children + 1), nr_right_children + 2):
+ head_word = dg.nodelist[node_index]['word']
+ head_tag = dg.nodelist[node_index]['tag']
+ child = 'STOP'
+ child_tag = 'STOP'
+ prev_word = 'START'
+ prev_tag = 'START'
+ if child_index < 0:
+ array_index = child_index + nr_left_children
+ if array_index >= 0:
+ child = dg.nodelist[children[array_index]]['word']
+ child_tag = dg.nodelist[children[array_index]]['tag']
+ if child_index != -1:
+ prev_word = dg.nodelist[children[array_index + 1]]['word']
+ prev_tag = dg.nodelist[children[array_index + 1]]['tag']
+ head_event = '(head (%s %s) (mods (%s, %s, %s) left))' % (child, child_tag, prev_tag, head_word, head_tag)
+ mod_event = '(mods (%s, %s, %s) left))' % (prev_tag, head_word, head_tag)
+ h_count = self._grammar._events[head_event]
+ m_count = self._grammar._events[mod_event]
+ prob *= (h_count / m_count)
+ elif child_index > 0:
+ array_index = child_index + nr_left_children - 1
+ if array_index < nr_children:
+ child = dg.nodelist[children[array_index]]['word']
+ child_tag = dg.nodelist[children[array_index]]['tag']
+ if child_index != 1:
+ prev_word = dg.nodelist[children[array_index - 1]]['word']
+ prev_tag = dg.nodelist[children[array_index - 1]]['tag']
+ head_event = '(head (%s %s) (mods (%s, %s, %s) right))' % (child, child_tag, prev_tag, head_word, head_tag)
+ mod_event = '(mods (%s, %s, %s) right))' % (prev_tag, head_word, head_tag)
+ h_count = self._grammar._events[head_event]
+ m_count = self._grammar._events[mod_event]
+ prob *= (h_count / m_count)
+ return prob
+
+
+#################################################################
+# Demos
+#################################################################
+
+def demo():
+ projective_rule_parse_demo()
+# arity_parse_demo()
+ projective_prob_parse_demo()
+
+
+def projective_rule_parse_demo():
+ """
+ A demonstration showing the creation and use of a
+ ``DependencyGrammar`` to perform a projective dependency
+ parse.
+ """
+ grammar = DependencyGrammar.fromstring("""
+ 'scratch' -> 'cats' | 'walls'
+ 'walls' -> 'the'
+ 'cats' -> 'the'
+ """)
+ print(grammar)
+ pdp = ProjectiveDependencyParser(grammar)
+ trees = pdp.parse(['the', 'cats', 'scratch', 'the', 'walls'])
+ for tree in trees:
+ print(tree)
+
+def arity_parse_demo():
+ """
+ A demonstration showing the creation of a ``DependencyGrammar``
+ in which a specific number of modifiers is listed for a given
+ head. This can further constrain the number of possible parses
+ created by a ``ProjectiveDependencyParser``.
+ """
+ print()
+ print('A grammar with no arity constraints. Each DependencyProduction')
+ print('specifies a relationship between one head word and only one')
+ print('modifier word.')
+ grammar = DependencyGrammar.fromstring("""
+ 'fell' -> 'price' | 'stock'
+ 'price' -> 'of' | 'the'
+ 'of' -> 'stock'
+ 'stock' -> 'the'
+ """)
+ print(grammar)
+
+ print()
+ print('For the sentence \'The price of the stock fell\', this grammar')
+ print('will produce the following three parses:')
+ pdp = ProjectiveDependencyParser(grammar)
+ trees = pdp.parse(['the', 'price', 'of', 'the', 'stock', 'fell'])
+ for tree in trees:
+ print(tree)
+
+ print()
+ print('By contrast, the following grammar contains a ')
+ print('DependencyProduction that specifies a relationship')
+ print('between a single head word, \'price\', and two modifier')
+ print('words, \'of\' and \'the\'.')
+ grammar = DependencyGrammar.fromstring("""
+ 'fell' -> 'price' | 'stock'
+ 'price' -> 'of' 'the'
+ 'of' -> 'stock'
+ 'stock' -> 'the'
+ """)
+ print(grammar)
+
+ print()
+ print('This constrains the number of possible parses to just one:') # unimplemented, soon to replace
+ pdp = ProjectiveDependencyParser(grammar)
+ trees = pdp.parse(['the', 'price', 'of', 'the', 'stock', 'fell'])
+ for tree in trees:
+ print(tree)
+
+def projective_prob_parse_demo():
+ """
+ A demo showing the training and use of a projective
+ dependency parser.
+ """
+ graphs = [DependencyGraph(entry)
+ for entry in conll_data2.split('\n\n') if entry]
+ ppdp = ProbabilisticProjectiveDependencyParser()
+ print('Training Probabilistic Projective Dependency Parser...')
+ ppdp.train(graphs)
+ sent = ['Cathy', 'zag', 'hen', 'wild', 'zwaaien', '.']
+ print('Parsing \'', " ".join(sent), '\'...')
+ print('Parse:')
+ for tree in ppdp.parse(sent):
+ print(tree)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/recursivedescent.py b/nltk/parse/recursivedescent.py
new file mode 100644
index 0000000..540dfde
--- /dev/null
+++ b/nltk/parse/recursivedescent.py
@@ -0,0 +1,656 @@
+# Natural Language Toolkit: Recursive Descent Parser
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+from nltk.grammar import Nonterminal
+from nltk.tree import Tree, ImmutableTree
+from nltk.compat import unicode_repr
+
+from nltk.parse.api import ParserI
+
+##//////////////////////////////////////////////////////
+## Recursive Descent Parser
+##//////////////////////////////////////////////////////
+class RecursiveDescentParser(ParserI):
+ """
+ A simple top-down CFG parser that parses texts by recursively
+ expanding the fringe of a Tree, and matching it against a
+ text.
+
+ ``RecursiveDescentParser`` uses a list of tree locations called a
+ "frontier" to remember which subtrees have not yet been expanded
+ and which leaves have not yet been matched against the text. Each
+ tree location consists of a list of child indices specifying the
+ path from the root of the tree to a subtree or a leaf; see the
+ reference documentation for Tree for more information
+ about tree locations.
+
+ When the parser begins parsing a text, it constructs a tree
+ containing only the start symbol, and a frontier containing the
+ location of the tree's root node. It then extends the tree to
+ cover the text, using the following recursive procedure:
+
+ - If the frontier is empty, and the text is covered by the tree,
+ then return the tree as a possible parse.
+ - If the frontier is empty, and the text is not covered by the
+ tree, then return no parses.
+ - If the first element of the frontier is a subtree, then
+ use CFG productions to "expand" it. For each applicable
+ production, add the expanded subtree's children to the
+ frontier, and recursively find all parses that can be
+ generated by the new tree and frontier.
+ - If the first element of the frontier is a token, then "match"
+ it against the next token from the text. Remove the token
+ from the frontier, and recursively find all parses that can be
+ generated by the new tree and frontier.
+
+ :see: ``nltk.grammar``
+ """
+ def __init__(self, grammar, trace=0):
+ """
+ Create a new ``RecursiveDescentParser``, that uses ``grammar``
+ to parse texts.
+
+ :type grammar: CFG
+ :param grammar: The grammar used to parse texts.
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ and higher numbers will produce more verbose tracing
+ output.
+ """
+ self._grammar = grammar
+ self._trace = trace
+
+ def grammar(self):
+ return self._grammar
+
+ def parse(self, tokens):
+ # Inherit docs from ParserI
+
+ tokens = list(tokens)
+ self._grammar.check_coverage(tokens)
+
+ # Start a recursive descent parse, with an initial tree
+ # containing just the start symbol.
+ start = self._grammar.start().symbol()
+ initial_tree = Tree(start, [])
+ frontier = [()]
+ if self._trace:
+ self._trace_start(initial_tree, frontier, tokens)
+ return self._parse(tokens, initial_tree, frontier)
+
+ def _parse(self, remaining_text, tree, frontier):
+ """
+ Recursively expand and match each elements of ``tree``
+ specified by ``frontier``, to cover ``remaining_text``. Return
+ a list of all parses found.
+
+ :return: An iterator of all parses that can be generated by
+ matching and expanding the elements of ``tree``
+ specified by ``frontier``.
+ :rtype: iter(Tree)
+ :type tree: Tree
+ :param tree: A partial structure for the text that is
+ currently being parsed. The elements of ``tree``
+ that are specified by ``frontier`` have not yet been
+ expanded or matched.
+ :type remaining_text: list(str)
+ :param remaining_text: The portion of the text that is not yet
+ covered by ``tree``.
+ :type frontier: list(tuple(int))
+ :param frontier: A list of the locations within ``tree`` of
+ all subtrees that have not yet been expanded, and all
+ leaves that have not yet been matched. This list sorted
+ in left-to-right order of location within the tree.
+ """
+
+ # If the tree covers the text, and there's nothing left to
+ # expand, then we've found a complete parse; return it.
+ if len(remaining_text) == 0 and len(frontier) == 0:
+ if self._trace:
+ self._trace_succeed(tree, frontier)
+ yield tree
+
+ # If there's still text, but nothing left to expand, we failed.
+ elif len(frontier) == 0:
+ if self._trace:
+ self._trace_backtrack(tree, frontier)
+
+ # If the next element on the frontier is a tree, expand it.
+ elif isinstance(tree[frontier[0]], Tree):
+ for result in self._expand(remaining_text, tree, frontier):
+ yield result
+
+ # If the next element on the frontier is a token, match it.
+ else:
+ for result in self._match(remaining_text, tree, frontier):
+ yield result
+
+ def _match(self, rtext, tree, frontier):
+ """
+ :rtype: iter(Tree)
+ :return: an iterator of all parses that can be generated by
+ matching the first element of ``frontier`` against the
+ first token in ``rtext``. In particular, if the first
+ element of ``frontier`` has the same type as the first
+ token in ``rtext``, then substitute the token into
+ ``tree``; and return all parses that can be generated by
+ matching and expanding the remaining elements of
+ ``frontier``. If the first element of ``frontier`` does not
+ have the same type as the first token in ``rtext``, then
+ return empty list.
+
+ :type tree: Tree
+ :param tree: A partial structure for the text that is
+ currently being parsed. The elements of ``tree``
+ that are specified by ``frontier`` have not yet been
+ expanded or matched.
+ :type rtext: list(str)
+ :param rtext: The portion of the text that is not yet
+ covered by ``tree``.
+ :type frontier: list of tuple of int
+ :param frontier: A list of the locations within ``tree`` of
+ all subtrees that have not yet been expanded, and all
+ leaves that have not yet been matched.
+ """
+
+ tree_leaf = tree[frontier[0]]
+ if (len(rtext) > 0 and tree_leaf == rtext[0]):
+ # If it's a terminal that matches rtext[0], then substitute
+ # in the token, and continue parsing.
+ newtree = tree.copy(deep=True)
+ newtree[frontier[0]] = rtext[0]
+ if self._trace:
+ self._trace_match(newtree, frontier[1:], rtext[0])
+ for result in self._parse(rtext[1:], newtree, frontier[1:]):
+ yield result
+ else:
+ # If it's a non-matching terminal, fail.
+ if self._trace:
+ self._trace_backtrack(tree, frontier, rtext[:1])
+
+ def _expand(self, remaining_text, tree, frontier, production=None):
+ """
+ :rtype: iter(Tree)
+ :return: An iterator of all parses that can be generated by
+ expanding the first element of ``frontier`` with
+ ``production``. In particular, if the first element of
+ ``frontier`` is a subtree whose node type is equal to
+ ``production``'s left hand side, then add a child to that
+ subtree for each element of ``production``'s right hand
+ side; and return all parses that can be generated by
+ matching and expanding the remaining elements of
+ ``frontier``. If the first element of ``frontier`` is not a
+ subtree whose node type is equal to ``production``'s left
+ hand side, then return an empty list. If ``production`` is
+ not specified, then return a list of all parses that can
+ be generated by expanding the first element of ``frontier``
+ with *any* CFG production.
+
+ :type tree: Tree
+ :param tree: A partial structure for the text that is
+ currently being parsed. The elements of ``tree``
+ that are specified by ``frontier`` have not yet been
+ expanded or matched.
+ :type remaining_text: list(str)
+ :param remaining_text: The portion of the text that is not yet
+ covered by ``tree``.
+ :type frontier: list(tuple(int))
+ :param frontier: A list of the locations within ``tree`` of
+ all subtrees that have not yet been expanded, and all
+ leaves that have not yet been matched.
+ """
+
+ if production is None: productions = self._grammar.productions()
+ else: productions = [production]
+
+ for production in productions:
+ lhs = production.lhs().symbol()
+ if lhs == tree[frontier[0]].label():
+ subtree = self._production_to_tree(production)
+ if frontier[0] == ():
+ newtree = subtree
+ else:
+ newtree = tree.copy(deep=True)
+ newtree[frontier[0]] = subtree
+ new_frontier = [frontier[0]+(i,) for i in
+ range(len(production.rhs()))]
+ if self._trace:
+ self._trace_expand(newtree, new_frontier, production)
+ for result in self._parse(remaining_text, newtree,
+ new_frontier + frontier[1:]):
+ yield result
+
+ def _production_to_tree(self, production):
+ """
+ :rtype: Tree
+ :return: The Tree that is licensed by ``production``.
+ In particular, given the production ``[lhs -> elt[1] ... elt[n]]``
+ return a tree that has a node ``lhs.symbol``, and
+ ``n`` children. For each nonterminal element
+ ``elt[i]`` in the production, the tree token has a
+ childless subtree with node value ``elt[i].symbol``; and
+ for each terminal element ``elt[j]``, the tree token has
+ a leaf token with type ``elt[j]``.
+
+ :param production: The CFG production that licenses the tree
+ token that should be returned.
+ :type production: Production
+ """
+ children = []
+ for elt in production.rhs():
+ if isinstance(elt, Nonterminal):
+ children.append(Tree(elt.symbol(), []))
+ else:
+ # This will be matched.
+ children.append(elt)
+ return Tree(production.lhs().symbol(), children)
+
+ def trace(self, trace=2):
+ """
+ Set the level of tracing output that should be generated when
+ parsing a text.
+
+ :type trace: int
+ :param trace: The trace level. A trace level of ``0`` will
+ generate no tracing output; and higher trace levels will
+ produce more verbose tracing output.
+ :rtype: None
+ """
+ self._trace = trace
+
+ def _trace_fringe(self, tree, treeloc=None):
+ """
+ Print trace output displaying the fringe of ``tree``. The
+ fringe of ``tree`` consists of all of its leaves and all of
+ its childless subtrees.
+
+ :rtype: None
+ """
+
+ if treeloc == (): print("*", end=' ')
+ if isinstance(tree, Tree):
+ if len(tree) == 0:
+ print(unicode_repr(Nonterminal(tree.label())), end=' ')
+ for i in range(len(tree)):
+ if treeloc is not None and i == treeloc[0]:
+ self._trace_fringe(tree[i], treeloc[1:])
+ else:
+ self._trace_fringe(tree[i])
+ else:
+ print(unicode_repr(tree), end=' ')
+
+ def _trace_tree(self, tree, frontier, operation):
+ """
+ Print trace output displaying the parser's current state.
+
+ :param operation: A character identifying the operation that
+ generated the current state.
+ :rtype: None
+ """
+ if self._trace == 2: print(' %c [' % operation, end=' ')
+ else: print(' [', end=' ')
+ if len(frontier) > 0: self._trace_fringe(tree, frontier[0])
+ else: self._trace_fringe(tree)
+ print(']')
+
+ def _trace_start(self, tree, frontier, text):
+ print('Parsing %r' % " ".join(text))
+ if self._trace > 2: print('Start:')
+ if self._trace > 1: self._trace_tree(tree, frontier, ' ')
+
+ def _trace_expand(self, tree, frontier, production):
+ if self._trace > 2: print('Expand: %s' % production)
+ if self._trace > 1: self._trace_tree(tree, frontier, 'E')
+
+ def _trace_match(self, tree, frontier, tok):
+ if self._trace > 2: print('Match: %r' % tok)
+ if self._trace > 1: self._trace_tree(tree, frontier, 'M')
+
+ def _trace_succeed(self, tree, frontier):
+ if self._trace > 2: print('GOOD PARSE:')
+ if self._trace == 1: print('Found a parse:\n%s' % tree)
+ if self._trace > 1: self._trace_tree(tree, frontier, '+')
+
+ def _trace_backtrack(self, tree, frontier, toks=None):
+ if self._trace > 2:
+ if toks: print('Backtrack: %r match failed' % toks[0])
+ else: print('Backtrack')
+
+##//////////////////////////////////////////////////////
+## Stepping Recursive Descent Parser
+##//////////////////////////////////////////////////////
+class SteppingRecursiveDescentParser(RecursiveDescentParser):
+ """
+ A ``RecursiveDescentParser`` that allows you to step through the
+ parsing process, performing a single operation at a time.
+
+ The ``initialize`` method is used to start parsing a text.
+ ``expand`` expands the first element on the frontier using a single
+ CFG production, and ``match`` matches the first element on the
+ frontier against the next text token. ``backtrack`` undoes the most
+ recent expand or match operation. ``step`` performs a single
+ expand, match, or backtrack operation. ``parses`` returns the set
+ of parses that have been found by the parser.
+
+ :ivar _history: A list of ``(rtext, tree, frontier)`` tripples,
+ containing the previous states of the parser. This history is
+ used to implement the ``backtrack`` operation.
+ :ivar _tried_e: A record of all productions that have been tried
+ for a given tree. This record is used by ``expand`` to perform
+ the next untried production.
+ :ivar _tried_m: A record of what tokens have been matched for a
+ given tree. This record is used by ``step`` to decide whether
+ or not to match a token.
+ :see: ``nltk.grammar``
+ """
+ def __init__(self, grammar, trace=0):
+ self._grammar = grammar
+ self._trace = trace
+ self._rtext = None
+ self._tree = None
+ self._frontier = [()]
+ self._tried_e = {}
+ self._tried_m = {}
+ self._history = []
+ self._parses = []
+
+ # [XX] TEMPORARY HACK WARNING! This should be replaced with
+ # something nicer when we get the chance.
+ def _freeze(self, tree):
+ c = tree.copy()
+# for pos in c.treepositions('leaves'):
+# c[pos] = c[pos].freeze()
+ return ImmutableTree.convert(c)
+
+ def parse(self, tokens):
+ tokens = list(tokens)
+ self.initialize(tokens)
+ while self.step() is not None:
+ pass
+ return self.parses()
+
+ def initialize(self, tokens):
+ """
+ Start parsing a given text. This sets the parser's tree to
+ the start symbol, its frontier to the root node, and its
+ remaining text to ``token['SUBTOKENS']``.
+ """
+
+ self._rtext = tokens
+ start = self._grammar.start().symbol()
+ self._tree = Tree(start, [])
+ self._frontier = [()]
+ self._tried_e = {}
+ self._tried_m = {}
+ self._history = []
+ self._parses = []
+ if self._trace:
+ self._trace_start(self._tree, self._frontier, self._rtext)
+
+ def remaining_text(self):
+ """
+ :return: The portion of the text that is not yet covered by the
+ tree.
+ :rtype: list(str)
+ """
+ return self._rtext
+
+ def frontier(self):
+ """
+ :return: A list of the tree locations of all subtrees that
+ have not yet been expanded, and all leaves that have not
+ yet been matched.
+ :rtype: list(tuple(int))
+ """
+ return self._frontier
+
+ def tree(self):
+ """
+ :return: A partial structure for the text that is
+ currently being parsed. The elements specified by the
+ frontier have not yet been expanded or matched.
+ :rtype: Tree
+ """
+ return self._tree
+
+ def step(self):
+ """
+ Perform a single parsing operation. If an untried match is
+ possible, then perform the match, and return the matched
+ token. If an untried expansion is possible, then perform the
+ expansion, and return the production that it is based on. If
+ backtracking is possible, then backtrack, and return True.
+ Otherwise, return None.
+
+ :return: None if no operation was performed; a token if a match
+ was performed; a production if an expansion was performed;
+ and True if a backtrack operation was performed.
+ :rtype: Production or String or bool
+ """
+ # Try matching (if we haven't already)
+ if self.untried_match():
+ token = self.match()
+ if token is not None: return token
+
+ # Try expanding.
+ production = self.expand()
+ if production is not None: return production
+
+ # Try backtracking
+ if self.backtrack():
+ self._trace_backtrack(self._tree, self._frontier)
+ return True
+
+ # Nothing left to do.
+ return None
+
+ def expand(self, production=None):
+ """
+ Expand the first element of the frontier. In particular, if
+ the first element of the frontier is a subtree whose node type
+ is equal to ``production``'s left hand side, then add a child
+ to that subtree for each element of ``production``'s right hand
+ side. If ``production`` is not specified, then use the first
+ untried expandable production. If all expandable productions
+ have been tried, do nothing.
+
+ :return: The production used to expand the frontier, if an
+ expansion was performed. If no expansion was performed,
+ return None.
+ :rtype: Production or None
+ """
+
+ # Make sure we *can* expand.
+ if len(self._frontier) == 0:
+ return None
+ if not isinstance(self._tree[self._frontier[0]], Tree):
+ return None
+
+ # If they didn't specify a production, check all untried ones.
+ if production is None:
+ productions = self.untried_expandable_productions()
+ else: productions = [production]
+
+ parses = []
+ for prod in productions:
+ # Record that we've tried this production now.
+ self._tried_e.setdefault(self._freeze(self._tree), []).append(prod)
+
+ # Try expanding.
+ for _result in self._expand(self._rtext, self._tree, self._frontier, prod):
+ return prod
+
+ # We didn't expand anything.
+ return None
+
+ def match(self):
+ """
+ Match the first element of the frontier. In particular, if
+ the first element of the frontier has the same type as the
+ next text token, then substitute the text token into the tree.
+
+ :return: The token matched, if a match operation was
+ performed. If no match was performed, return None
+ :rtype: str or None
+ """
+
+ # Record that we've tried matching this token.
+ tok = self._rtext[0]
+ self._tried_m.setdefault(self._freeze(self._tree), []).append(tok)
+
+ # Make sure we *can* match.
+ if len(self._frontier) == 0:
+ return None
+ if isinstance(self._tree[self._frontier[0]], Tree):
+ return None
+
+ for _result in self._match(self._rtext, self._tree, self._frontier):
+ # Return the token we just matched.
+ return self._history[-1][0][0]
+ return None
+
+ def backtrack(self):
+ """
+ Return the parser to its state before the most recent
+ match or expand operation. Calling ``undo`` repeatedly return
+ the parser to successively earlier states. If no match or
+ expand operations have been performed, ``undo`` will make no
+ changes.
+
+ :return: true if an operation was successfully undone.
+ :rtype: bool
+ """
+ if len(self._history) == 0: return False
+ (self._rtext, self._tree, self._frontier) = self._history.pop()
+ return True
+
+ def expandable_productions(self):
+ """
+ :return: A list of all the productions for which expansions
+ are available for the current parser state.
+ :rtype: list(Production)
+ """
+ # Make sure we *can* expand.
+ if len(self._frontier) == 0: return []
+ frontier_child = self._tree[self._frontier[0]]
+ if (len(self._frontier) == 0 or
+ not isinstance(frontier_child, Tree)):
+ return []
+
+ return [p for p in self._grammar.productions()
+ if p.lhs().symbol() == frontier_child.label()]
+
+ def untried_expandable_productions(self):
+ """
+ :return: A list of all the untried productions for which
+ expansions are available for the current parser state.
+ :rtype: list(Production)
+ """
+
+ tried_expansions = self._tried_e.get(self._freeze(self._tree), [])
+ return [p for p in self.expandable_productions()
+ if p not in tried_expansions]
+
+ def untried_match(self):
+ """
+ :return: Whether the first element of the frontier is a token
+ that has not yet been matched.
+ :rtype: bool
+ """
+
+ if len(self._rtext) == 0: return False
+ tried_matches = self._tried_m.get(self._freeze(self._tree), [])
+ return (self._rtext[0] not in tried_matches)
+
+ def currently_complete(self):
+ """
+ :return: Whether the parser's current state represents a
+ complete parse.
+ :rtype: bool
+ """
+ return (len(self._frontier) == 0 and len(self._rtext) == 0)
+
+ def _parse(self, remaining_text, tree, frontier):
+ """
+ A stub version of ``_parse`` that sets the parsers current
+ state to the given arguments. In ``RecursiveDescentParser``,
+ the ``_parse`` method is used to recursively continue parsing a
+ text. ``SteppingRecursiveDescentParser`` overrides it to
+ capture these recursive calls. It records the parser's old
+ state in the history (to allow for backtracking), and updates
+ the parser's new state using the given arguments. Finally, it
+ returns ``[1]``, which is used by ``match`` and ``expand`` to
+ detect whether their operations were successful.
+
+ :return: ``[1]``
+ :rtype: list of int
+ """
+ self._history.append( (self._rtext, self._tree, self._frontier) )
+ self._rtext = remaining_text
+ self._tree = tree
+ self._frontier = frontier
+
+ # Is it a good parse? If so, record it.
+ if (len(frontier) == 0 and len(remaining_text) == 0):
+ self._parses.append(tree)
+ self._trace_succeed(self._tree, self._frontier)
+
+ return [1]
+
+ def parses(self):
+ """
+ :return: An iterator of the parses that have been found by this
+ parser so far.
+ :rtype: list of Tree
+ """
+ return iter(self._parses)
+
+ def set_grammar(self, grammar):
+ """
+ Change the grammar used to parse texts.
+
+ :param grammar: The new grammar.
+ :type grammar: CFG
+ """
+ self._grammar = grammar
+
+##//////////////////////////////////////////////////////
+## Demonstration Code
+##//////////////////////////////////////////////////////
+
+def demo():
+ """
+ A demonstration of the recursive descent parser.
+ """
+
+ from nltk import parse, CFG
+
+ grammar = CFG.fromstring("""
+ S -> NP VP
+ NP -> Det N | Det N PP
+ VP -> V NP | V NP PP
+ PP -> P NP
+ NP -> 'I'
+ N -> 'man' | 'park' | 'telescope' | 'dog'
+ Det -> 'the' | 'a'
+ P -> 'in' | 'with'
+ V -> 'saw'
+ """)
+
+ for prod in grammar.productions():
+ print(prod)
+
+ sent = 'I saw a man in the park'.split()
+ parser = parse.RecursiveDescentParser(grammar, trace=2)
+ for p in parser.parse(sent):
+ print(p)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/shiftreduce.py b/nltk/parse/shiftreduce.py
new file mode 100644
index 0000000..f1f7ecb
--- /dev/null
+++ b/nltk/parse/shiftreduce.py
@@ -0,0 +1,459 @@
+# Natural Language Toolkit: Shift-Reduce Parser
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+from nltk.grammar import Nonterminal
+from nltk.tree import Tree
+from nltk.compat import unicode_repr
+
+from nltk.parse.api import ParserI
+
+##//////////////////////////////////////////////////////
+## Shift/Reduce Parser
+##//////////////////////////////////////////////////////
+class ShiftReduceParser(ParserI):
+ """
+ A simple bottom-up CFG parser that uses two operations, "shift"
+ and "reduce", to find a single parse for a text.
+
+ ``ShiftReduceParser`` maintains a stack, which records the
+ structure of a portion of the text. This stack is a list of
+ strings and Trees that collectively cover a portion of
+ the text. For example, while parsing the sentence "the dog saw
+ the man" with a typical grammar, ``ShiftReduceParser`` will produce
+ the following stack, which covers "the dog saw"::
+
+ [(NP: (Det: 'the') (N: 'dog')), (V: 'saw')]
+
+ ``ShiftReduceParser`` attempts to extend the stack to cover the
+ entire text, and to combine the stack elements into a single tree,
+ producing a complete parse for the sentence.
+
+ Initially, the stack is empty. It is extended to cover the text,
+ from left to right, by repeatedly applying two operations:
+
+ - "shift" moves a token from the beginning of the text to the
+ end of the stack.
+ - "reduce" uses a CFG production to combine the rightmost stack
+ elements into a single Tree.
+
+ Often, more than one operation can be performed on a given stack.
+ In this case, ``ShiftReduceParser`` uses the following heuristics
+ to decide which operation to perform:
+
+ - Only shift if no reductions are available.
+ - If multiple reductions are available, then apply the reduction
+ whose CFG production is listed earliest in the grammar.
+
+ Note that these heuristics are not guaranteed to choose an
+ operation that leads to a parse of the text. Also, if multiple
+ parses exists, ``ShiftReduceParser`` will return at most one of
+ them.
+
+ :see: ``nltk.grammar``
+ """
+ def __init__(self, grammar, trace=0):
+ """
+ Create a new ``ShiftReduceParser``, that uses ``grammar`` to
+ parse texts.
+
+ :type grammar: Grammar
+ :param grammar: The grammar used to parse texts.
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ and higher numbers will produce more verbose tracing
+ output.
+ """
+ self._grammar = grammar
+ self._trace = trace
+ self._check_grammar()
+
+ def grammar(self):
+ return self._grammar
+
+ def parse(self, tokens):
+ tokens = list(tokens)
+ self._grammar.check_coverage(tokens)
+
+ # initialize the stack.
+ stack = []
+ remaining_text = tokens
+
+ # Trace output.
+ if self._trace:
+ print('Parsing %r' % " ".join(tokens))
+ self._trace_stack(stack, remaining_text)
+
+ # iterate through the text, pushing the token onto
+ # the stack, then reducing the stack.
+ while len(remaining_text) > 0:
+ self._shift(stack, remaining_text)
+ while self._reduce(stack, remaining_text): pass
+
+ # Did we reduce everything?
+ if len(stack) == 1:
+ # Did we end up with the right category?
+ if stack[0].label() == self._grammar.start().symbol():
+ yield stack[0]
+
+ def _shift(self, stack, remaining_text):
+ """
+ Move a token from the beginning of ``remaining_text`` to the
+ end of ``stack``.
+
+ :type stack: list(str and Tree)
+ :param stack: A list of strings and Trees, encoding
+ the structure of the text that has been parsed so far.
+ :type remaining_text: list(str)
+ :param remaining_text: The portion of the text that is not yet
+ covered by ``stack``.
+ :rtype: None
+ """
+ stack.append(remaining_text[0])
+ remaining_text.remove(remaining_text[0])
+ if self._trace: self._trace_shift(stack, remaining_text)
+
+ def _match_rhs(self, rhs, rightmost_stack):
+ """
+ :rtype: bool
+ :return: true if the right hand side of a CFG production
+ matches the rightmost elements of the stack. ``rhs``
+ matches ``rightmost_stack`` if they are the same length,
+ and each element of ``rhs`` matches the corresponding
+ element of ``rightmost_stack``. A nonterminal element of
+ ``rhs`` matches any Tree whose node value is equal
+ to the nonterminal's symbol. A terminal element of ``rhs``
+ matches any string whose type is equal to the terminal.
+ :type rhs: list(terminal and Nonterminal)
+ :param rhs: The right hand side of a CFG production.
+ :type rightmost_stack: list(string and Tree)
+ :param rightmost_stack: The rightmost elements of the parser's
+ stack.
+ """
+
+ if len(rightmost_stack) != len(rhs): return False
+ for i in range(len(rightmost_stack)):
+ if isinstance(rightmost_stack[i], Tree):
+ if not isinstance(rhs[i], Nonterminal): return False
+ if rightmost_stack[i].label() != rhs[i].symbol(): return False
+ else:
+ if isinstance(rhs[i], Nonterminal): return False
+ if rightmost_stack[i] != rhs[i]: return False
+ return True
+
+ def _reduce(self, stack, remaining_text, production=None):
+ """
+ Find a CFG production whose right hand side matches the
+ rightmost stack elements; and combine those stack elements
+ into a single Tree, with the node specified by the
+ production's left-hand side. If more than one CFG production
+ matches the stack, then use the production that is listed
+ earliest in the grammar. The new Tree replaces the
+ elements in the stack.
+
+ :rtype: Production or None
+ :return: If a reduction is performed, then return the CFG
+ production that the reduction is based on; otherwise,
+ return false.
+ :type stack: list(string and Tree)
+ :param stack: A list of strings and Trees, encoding
+ the structure of the text that has been parsed so far.
+ :type remaining_text: list(str)
+ :param remaining_text: The portion of the text that is not yet
+ covered by ``stack``.
+ """
+ if production is None:
+ productions = self._grammar.productions()
+ else:
+ productions = [production]
+
+ # Try each production, in order.
+ for production in productions:
+ rhslen = len(production.rhs())
+
+ # check if the RHS of a production matches the top of the stack
+ if self._match_rhs(production.rhs(), stack[-rhslen:]):
+
+ # combine the tree to reflect the reduction
+ tree = Tree(production.lhs().symbol(), stack[-rhslen:])
+ stack[-rhslen:] = [tree]
+
+ # We reduced something
+ if self._trace:
+ self._trace_reduce(stack, production, remaining_text)
+ return production
+
+ # We didn't reduce anything
+ return None
+
+ def trace(self, trace=2):
+ """
+ Set the level of tracing output that should be generated when
+ parsing a text.
+
+ :type trace: int
+ :param trace: The trace level. A trace level of ``0`` will
+ generate no tracing output; and higher trace levels will
+ produce more verbose tracing output.
+ :rtype: None
+ """
+ # 1: just show shifts.
+ # 2: show shifts & reduces
+ # 3: display which tokens & productions are shifed/reduced
+ self._trace = trace
+
+ def _trace_stack(self, stack, remaining_text, marker=' '):
+ """
+ Print trace output displaying the given stack and text.
+
+ :rtype: None
+ :param marker: A character that is printed to the left of the
+ stack. This is used with trace level 2 to print 'S'
+ before shifted stacks and 'R' before reduced stacks.
+ """
+ s = ' '+marker+' [ '
+ for elt in stack:
+ if isinstance(elt, Tree):
+ s += unicode_repr(Nonterminal(elt.label())) + ' '
+ else:
+ s += unicode_repr(elt) + ' '
+ s += '* ' + ' '.join(remaining_text) + ']'
+ print(s)
+
+ def _trace_shift(self, stack, remaining_text):
+ """
+ Print trace output displaying that a token has been shifted.
+
+ :rtype: None
+ """
+ if self._trace > 2: print('Shift %r:' % stack[-1])
+ if self._trace == 2: self._trace_stack(stack, remaining_text, 'S')
+ elif self._trace > 0: self._trace_stack(stack, remaining_text)
+
+ def _trace_reduce(self, stack, production, remaining_text):
+ """
+ Print trace output displaying that ``production`` was used to
+ reduce ``stack``.
+
+ :rtype: None
+ """
+ if self._trace > 2:
+ rhs = " ".join(production.rhs())
+ print('Reduce %r <- %s' % (production.lhs(), rhs))
+ if self._trace == 2: self._trace_stack(stack, remaining_text, 'R')
+ elif self._trace > 1: self._trace_stack(stack, remaining_text)
+
+ def _check_grammar(self):
+ """
+ Check to make sure that all of the CFG productions are
+ potentially useful. If any productions can never be used,
+ then print a warning.
+
+ :rtype: None
+ """
+ productions = self._grammar.productions()
+
+ # Any production whose RHS is an extension of another production's RHS
+ # will never be used.
+ for i in range(len(productions)):
+ for j in range(i+1, len(productions)):
+ rhs1 = productions[i].rhs()
+ rhs2 = productions[j].rhs()
+ if rhs1[:len(rhs2)] == rhs2:
+ print('Warning: %r will never be used' % productions[i])
+
+##//////////////////////////////////////////////////////
+## Stepping Shift/Reduce Parser
+##//////////////////////////////////////////////////////
+class SteppingShiftReduceParser(ShiftReduceParser):
+ """
+ A ``ShiftReduceParser`` that allows you to setp through the parsing
+ process, performing a single operation at a time. It also allows
+ you to change the parser's grammar midway through parsing a text.
+
+ The ``initialize`` method is used to start parsing a text.
+ ``shift`` performs a single shift operation, and ``reduce`` performs
+ a single reduce operation. ``step`` will perform a single reduce
+ operation if possible; otherwise, it will perform a single shift
+ operation. ``parses`` returns the set of parses that have been
+ found by the parser.
+
+ :ivar _history: A list of ``(stack, remaining_text)`` pairs,
+ containing all of the previous states of the parser. This
+ history is used to implement the ``undo`` operation.
+ :see: ``nltk.grammar``
+ """
+ def __init__(self, grammar, trace=0):
+ self._grammar = grammar
+ self._trace = trace
+ self._stack = None
+ self._remaining_text = None
+ self._history = []
+
+ def parse(self, tokens):
+ tokens = list(tokens)
+ self.initialize(tokens)
+ while self.step():
+ pass
+ return self.parses()
+
+ def stack(self):
+ """
+ :return: The parser's stack.
+ :rtype: list(str and Tree)
+ """
+ return self._stack
+
+ def remaining_text(self):
+ """
+ :return: The portion of the text that is not yet covered by the
+ stack.
+ :rtype: list(str)
+ """
+ return self._remaining_text
+
+ def initialize(self, tokens):
+ """
+ Start parsing a given text. This sets the parser's stack to
+ ``[]`` and sets its remaining text to ``tokens``.
+ """
+ self._stack = []
+ self._remaining_text = tokens
+ self._history = []
+
+ def step(self):
+ """
+ Perform a single parsing operation. If a reduction is
+ possible, then perform that reduction, and return the
+ production that it is based on. Otherwise, if a shift is
+ possible, then perform it, and return True. Otherwise,
+ return False.
+
+ :return: False if no operation was performed; True if a shift was
+ performed; and the CFG production used to reduce if a
+ reduction was performed.
+ :rtype: Production or bool
+ """
+ return self.reduce() or self.shift()
+
+ def shift(self):
+ """
+ Move a token from the beginning of the remaining text to the
+ end of the stack. If there are no more tokens in the
+ remaining text, then do nothing.
+
+ :return: True if the shift operation was successful.
+ :rtype: bool
+ """
+ if len(self._remaining_text) == 0: return False
+ self._history.append( (self._stack[:], self._remaining_text[:]) )
+ self._shift(self._stack, self._remaining_text)
+ return True
+
+ def reduce(self, production=None):
+ """
+ Use ``production`` to combine the rightmost stack elements into
+ a single Tree. If ``production`` does not match the
+ rightmost stack elements, then do nothing.
+
+ :return: The production used to reduce the stack, if a
+ reduction was performed. If no reduction was performed,
+ return None.
+
+ :rtype: Production or None
+ """
+ self._history.append( (self._stack[:], self._remaining_text[:]) )
+ return_val = self._reduce(self._stack, self._remaining_text,
+ production)
+
+ if not return_val: self._history.pop()
+ return return_val
+
+ def undo(self):
+ """
+ Return the parser to its state before the most recent
+ shift or reduce operation. Calling ``undo`` repeatedly return
+ the parser to successively earlier states. If no shift or
+ reduce operations have been performed, ``undo`` will make no
+ changes.
+
+ :return: true if an operation was successfully undone.
+ :rtype: bool
+ """
+ if len(self._history) == 0: return False
+ (self._stack, self._remaining_text) = self._history.pop()
+ return True
+
+ def reducible_productions(self):
+ """
+ :return: A list of the productions for which reductions are
+ available for the current parser state.
+ :rtype: list(Production)
+ """
+ productions = []
+ for production in self._grammar.productions():
+ rhslen = len(production.rhs())
+ if self._match_rhs(production.rhs(), self._stack[-rhslen:]):
+ productions.append(production)
+ return productions
+
+ def parses(self):
+ """
+ :return: An iterator of the parses that have been found by this
+ parser so far.
+ :rtype: iter(Tree)
+ """
+ if (len(self._remaining_text) == 0 and
+ len(self._stack) == 1 and
+ self._stack[0].label() == self._grammar.start().symbol()
+ ):
+ yield self._stack[0]
+
+# copied from nltk.parser
+
+ def set_grammar(self, grammar):
+ """
+ Change the grammar used to parse texts.
+
+ :param grammar: The new grammar.
+ :type grammar: CFG
+ """
+ self._grammar = grammar
+
+##//////////////////////////////////////////////////////
+## Demonstration Code
+##//////////////////////////////////////////////////////
+
+def demo():
+ """
+ A demonstration of the shift-reduce parser.
+ """
+
+ from nltk import parse, CFG
+
+ grammar = CFG.fromstring("""
+ S -> NP VP
+ NP -> Det N | Det N PP
+ VP -> V NP | V NP PP
+ PP -> P NP
+ NP -> 'I'
+ N -> 'man' | 'park' | 'telescope' | 'dog'
+ Det -> 'the' | 'a'
+ P -> 'in' | 'with'
+ V -> 'saw'
+ """)
+
+ sent = 'I saw a man in the park'.split()
+
+ parser = parse.ShiftReduceParser(grammar, trace=2)
+ for p in parser.parse(sent):
+ print(p)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/parse/stanford.py b/nltk/parse/stanford.py
new file mode 100644
index 0000000..ddd6259
--- /dev/null
+++ b/nltk/parse/stanford.py
@@ -0,0 +1,255 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the Stanford Parser
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Xu <xxu at student.unimelb.edu.au>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import unicode_literals
+
+import tempfile
+import os
+import re
+from subprocess import PIPE
+
+from nltk import compat
+from nltk.internals import find_jar, find_jar_iter, config_java, java, _java_options
+
+from nltk.parse.api import ParserI
+from nltk.tree import Tree
+
+_stanford_url = 'http://nlp.stanford.edu/software/lex-parser.shtml'
+
+class StanfordParser(ParserI):
+ r"""
+ Interface to the Stanford Parser
+
+ >>> parser=StanfordParser(
+ ... model_path="edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz"
+ ... )
+ >>> parser.raw_parse_sents((
+ ... "the quick brown fox jumps over the lazy dog",
+ ... "the quick grey wolf jumps over the lazy fox"
+ ... ))
+ [Tree('ROOT', [Tree('NP', [Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['quick']), Tree('JJ', ['brown']),
+ Tree('NN', ['fox'])]), Tree('NP', [Tree('NP', [Tree('NNS', ['jumps'])]), Tree('PP', [Tree('IN', ['over']),
+ Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['lazy']), Tree('NN', ['dog'])])])])])]), Tree('ROOT', [Tree('NP',
+ [Tree('NP', [Tree('DT', ['the']), Tree('JJ', ['quick']), Tree('JJ', ['grey']), Tree('NN', ['wolf'])]), Tree('NP',
+ [Tree('NP', [Tree('NNS', ['jumps'])]), Tree('PP', [Tree('IN', ['over']), Tree('NP', [Tree('DT', ['the']),
+ Tree('JJ', ['lazy']), Tree('NN', ['fox'])])])])])])]
+
+ >>> parser.parse_sents((
+ ... "I 'm a dog".split(),
+ ... "This is my friends ' cat ( the tabby )".split(),
+ ... ))
+ [Tree('ROOT', [Tree('S', [Tree('NP', [Tree('PRP', ['I'])]), Tree('VP', [Tree('VBP', ["'m"]),
+ Tree('NP', [Tree('DT', ['a']), Tree('NN', ['dog'])])])])]), Tree('ROOT', [Tree('S', [Tree('NP',
+ [Tree('DT', ['This'])]), Tree('VP', [Tree('VBZ', ['is']), Tree('NP', [Tree('NP', [Tree('NP', [Tree('PRP$', ['my']),
+ Tree('NNS', ['friends']), Tree('POS', ["'"])]), Tree('NN', ['cat'])]), Tree('PRN', [Tree('-LRB-', ['-LRB-']),
+ Tree('NP', [Tree('DT', ['the']), Tree('NN', ['tabby'])]), Tree('-RRB-', ['-RRB-'])])])])])])]
+
+ >>> parser.tagged_parse_sents((
+ ... (
+ ... ("The", "DT"),
+ ... ("quick", "JJ"),
+ ... ("brown", "JJ"),
+ ... ("fox", "NN"),
+ ... ("jumped", "VBD"),
+ ... ("over", "IN"),
+ ... ("the", "DT"),
+ ... ("lazy", "JJ"),
+ ... ("dog", "NN"),
+ ... (".", "."),
+ ... ),
+ ... ))
+ [Tree('ROOT', [Tree('S', [Tree('NP', [Tree('DT', ['The']), Tree('JJ', ['quick']), Tree('JJ', ['brown']),
+ Tree('NN', ['fox'])]), Tree('VP', [Tree('VBD', ['jumped']), Tree('PP', [Tree('IN', ['over']), Tree('NP',
+ [Tree('DT', ['the']), Tree('JJ', ['lazy']), Tree('NN', ['dog'])])])]), Tree('.', ['.'])])])]
+ """
+ _MODEL_JAR_PATTERN = r'stanford-parser-(\d+)\.(\d+)\.(\d+)-models\.jar'
+ _JAR = 'stanford-parser.jar'
+
+ def __init__(self, path_to_jar=None, path_to_models_jar=None,
+ model_path='edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz',
+ encoding='UTF-8', verbose=False, java_options='-mx1000m'):
+
+ self._stanford_jar = find_jar(
+ self._JAR, path_to_jar,
+ env_vars=('STANFORD_PARSER',),
+ searchpath=(), url=_stanford_url,
+ verbose=verbose
+ )
+
+ # find the most recent model
+ self._model_jar=max(
+ find_jar_iter(
+ self._MODEL_JAR_PATTERN, path_to_models_jar,
+ env_vars=('STANFORD_MODELS',),
+ searchpath=(), url=_stanford_url,
+ verbose=verbose, is_regex=True
+ ),
+ key=lambda model_name: re.match(self._MODEL_JAR_PATTERN, model_name)
+ )
+
+ self.model_path = model_path
+ self._encoding = encoding
+ self.java_options = java_options
+
+ @staticmethod
+ def _parse_trees_output(output_):
+ res = []
+ cur_lines = []
+ for line in output_.splitlines(False):
+ if line == '':
+ res.append(Tree.fromstring('\n'.join(cur_lines)))
+ cur_lines = []
+ else:
+ cur_lines.append(line)
+ return res
+
+ def parse_all(self, sentence, verbose=False):
+ """
+ Use StanfordParser to parse a sentence. Takes a sentence as a list of
+ words; it will be automatically tagged with this StanfordParser instance's
+ tagger.
+
+ :param sentence: Input sentence to parse
+ :type sentence: list(str)
+ :rtype: Tree
+ """
+ return self.parse_sents([sentence], verbose)
+
+ def parse_sents(self, sentences, verbose=False):
+ """
+ Use StanfordParser to parse multiple sentences. Takes multiple sentences as a
+ list where each sentence is a list of words.
+ Each sentence will be automatically tagged with this StanfordParser instance's
+ tagger.
+ If whitespaces exists inside a token, then the token will be treated as
+ separate tokens.
+
+ :param sentences: Input sentences to parse
+ :type sentences: list(list(str))
+ :rtype: list(Tree)
+ """
+ cmd = [
+ 'edu.stanford.nlp.parser.lexparser.LexicalizedParser',
+ '-model', self.model_path,
+ '-sentences', 'newline',
+ '-outputFormat', 'penn',
+ '-tokenized',
+ '-escaper', 'edu.stanford.nlp.process.PTBEscapingProcessor',
+ ]
+ return self._parse_trees_output(self._execute(
+ cmd, '\n'.join(' '.join(sentence) for sentence in sentences), verbose))
+
+ def raw_parse(self, sentence, verbose=False):
+ """
+ Use StanfordParser to parse a sentence. Takes a sentence as a string;
+ before parsing, it will be automatically tokenized and tagged by
+ the Stanford Parser.
+
+ :param sentence: Input sentence to parse
+ :type sentence: str
+ :rtype: Tree
+ """
+ return self.raw_parse_sents((sentence,), verbose)
+
+ def raw_parse_sents(self, sentences, verbose=False):
+ """
+ Use StanfordParser to parse multiple sentences. Takes multiple sentences as a
+ list of strings.
+ Each sentence will be automatically tokenized and tagged by the Stanford Parser.
+
+ :param sentences: Input sentences to parse
+ :type sentences: list(str)
+ :rtype: list(Tree)
+ """
+ cmd = [
+ 'edu.stanford.nlp.parser.lexparser.LexicalizedParser',
+ '-model', self.model_path,
+ '-sentences', 'newline',
+ '-outputFormat', 'penn',
+ ]
+ return self._parse_trees_output(self._execute(cmd, '\n'.join(sentences), verbose))
+
+ def tagged_parse(self, sentence, verbose=False):
+ """
+ Use StanfordParser to parse a sentence. Takes a sentence as a list of
+ (word, tag) tuples; the sentence must have already been tokenized and
+ tagged.
+
+ :param sentence: Input sentence to parse
+ :type sentence: list(tuple(str, str))
+ :rtype: Tree
+ """
+ return self.tagged_parse_sents([sentence], verbose)[0]
+
+ def tagged_parse_sents(self, sentences, verbose=False):
+ """
+ Use StanfordParser to parse multiple sentences. Takes multiple sentences
+ where each sentence is a list of (word, tag) tuples.
+ The sentences must have already been tokenized and tagged.
+
+ :param sentences: Input sentences to parse
+ :type sentences: list(list(tuple(str, str)))
+ :rtype: Tree
+ """
+ tag_separator = '/'
+ cmd = [
+ 'edu.stanford.nlp.parser.lexparser.LexicalizedParser',
+ '-model', self.model_path,
+ '-sentences', 'newline',
+ '-outputFormat', 'penn',
+ '-tokenized',
+ '-tagSeparator', tag_separator,
+ '-tokenizerFactory', 'edu.stanford.nlp.process.WhitespaceTokenizer',
+ '-tokenizerMethod', 'newCoreLabelTokenizerFactory',
+ ]
+ # We don't need to escape slashes as "splitting is done on the last instance of the character in the token"
+ return self._parse_trees_output(self._execute(
+ cmd, '\n'.join(' '.join(tag_separator.join(tagged) for tagged in sentence) for sentence in sentences), verbose))
+
+ def _execute(self, cmd, input_, verbose=False):
+ encoding = self._encoding
+ cmd.extend(['-encoding', encoding])
+
+ default_options = ' '.join(_java_options)
+
+ # Configure java.
+ config_java(options=self.java_options, verbose=verbose)
+
+ # Windows is incompatible with NamedTemporaryFile() without passing in delete=False.
+ with tempfile.NamedTemporaryFile(mode='wb', delete=False) as input_file:
+ # Write the actual sentences to the temporary input file
+ if isinstance(input_, compat.text_type) and encoding:
+ input_ = input_.encode(encoding)
+ input_file.write(input_)
+ input_file.flush()
+
+ cmd.append(input_file.name)
+
+ # Run the tagger and get the output.
+ stdout, stderr = java(cmd, classpath=(self._stanford_jar, self._model_jar),
+ stdout=PIPE, stderr=PIPE)
+ stdout = stdout.decode(encoding)
+
+ os.unlink(input_file.name)
+
+ # Return java configurations to their default values.
+ config_java(options=default_options, verbose=False)
+
+ return stdout
+
+
+def setup_module(module):
+ from nose import SkipTest
+
+ try:
+ StanfordParser(
+ model_path='edu/stanford/nlp/models/lexparser/englishPCFG.ser.gz'
+ )
+ except LookupError:
+ raise SkipTest('doctests from nltk.parse.stanford are skipped because the stanford parser jar doesn\'t exist')
diff --git a/nltk/parse/test.cfg b/nltk/parse/test.cfg
new file mode 100644
index 0000000..e8ba69a
--- /dev/null
+++ b/nltk/parse/test.cfg
@@ -0,0 +1,10 @@
+%start S
+
+S[sem=<app(?vp, ?subj)>] -> NP[sem=?subj] VP[sem=?vp]
+VP[sem = <app(?v, ?obj)>] -> V[sem = ?v] NP[sem=?obj]
+VP[sem = ?v] -> V[sem = ?v]
+NP[sem = <kim>] -> 'Kim'
+NP[sem = <i>] -> 'I'
+V[sem = <\x y.(like x y)>, tns=pres] -> 'like'
+V[sem = <\x.(sleeps x)>, tns=pres] -> 'sleeps'
+
diff --git a/nltk/parse/util.py b/nltk/parse/util.py
new file mode 100644
index 0000000..b670cd9
--- /dev/null
+++ b/nltk/parse/util.py
@@ -0,0 +1,168 @@
+# Natural Language Toolkit: Parser Utility Functions
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+
+"""
+Utility functions for parsers.
+"""
+from __future__ import print_function
+
+from nltk.grammar import CFG, FeatureGrammar, PCFG
+from nltk.data import load
+
+from nltk.parse.chart import Chart, ChartParser
+from nltk.parse.pchart import InsideChartParser
+from nltk.parse.featurechart import FeatureChart, FeatureChartParser
+
+def load_parser(grammar_url, trace=0,
+ parser=None, chart_class=None,
+ beam_size=0, **load_args):
+ """
+ Load a grammar from a file, and build a parser based on that grammar.
+ The parser depends on the grammar format, and might also depend
+ on properties of the grammar itself.
+
+ The following grammar formats are currently supported:
+ - ``'cfg'`` (CFGs: ``CFG``)
+ - ``'pcfg'`` (probabilistic CFGs: ``PCFG``)
+ - ``'fcfg'`` (feature-based CFGs: ``CFG``)
+
+ :type grammar_url: str
+ :param grammar_url: A URL specifying where the grammar is located.
+ The default protocol is ``"nltk:"``, which searches for the file
+ in the the NLTK data package.
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ and higher numbers will produce more verbose tracing output.
+ :param parser: The class used for parsing; should be ``ChartParser``
+ or a subclass.
+ If None, the class depends on the grammar format.
+ :param chart_class: The class used for storing the chart;
+ should be ``Chart`` or a subclass.
+ Only used for CFGs and feature CFGs.
+ If None, the chart class depends on the grammar format.
+ :type beam_size: int
+ :param beam_size: The maximum length for the parser's edge queue.
+ Only used for probabilistic CFGs.
+ :param load_args: Keyword parameters used when loading the grammar.
+ See ``data.load`` for more information.
+ """
+ grammar = load(grammar_url, **load_args)
+ if not isinstance(grammar, CFG):
+ raise ValueError("The grammar must be a CFG, "
+ "or a subclass thereof.")
+ if isinstance(grammar, PCFG):
+ if parser is None:
+ parser = InsideChartParser
+ return parser(grammar, trace=trace, beam_size=beam_size)
+
+ elif isinstance(grammar, FeatureGrammar):
+ if parser is None:
+ parser = FeatureChartParser
+ if chart_class is None:
+ chart_class = FeatureChart
+ return parser(grammar, trace=trace, chart_class=chart_class)
+
+ else: # Plain CFG.
+ if parser is None:
+ parser = ChartParser
+ if chart_class is None:
+ chart_class = Chart
+ return parser(grammar, trace=trace, chart_class=chart_class)
+
+
+######################################################################
+#{ Test Suites
+######################################################################
+
+class TestGrammar(object):
+ """
+ Unit tests for CFG.
+ """
+ def __init__(self, grammar, suite, accept=None, reject=None):
+ self.test_grammar = grammar
+
+ self.cp = load_parser(grammar, trace=0)
+ self.suite = suite
+ self._accept = accept
+ self._reject = reject
+
+
+ def run(self, show_trees=False):
+ """
+ Sentences in the test suite are divided into two classes:
+ - grammatical (``accept``) and
+ - ungrammatical (``reject``).
+ If a sentence should parse accordng to the grammar, the value of
+ ``trees`` will be a non-empty list. If a sentence should be rejected
+ according to the grammar, then the value of ``trees`` will be None.
+ """
+ for test in self.suite:
+ print(test['doc'] + ":", end=' ')
+ for key in ['accept', 'reject']:
+ for sent in test[key]:
+ tokens = sent.split()
+ trees = list(self.cp.parse(tokens))
+ if show_trees and trees:
+ print()
+ print(sent)
+ for tree in trees:
+ print(tree)
+ if key == 'accept':
+ if trees == []:
+ raise ValueError("Sentence '%s' failed to parse'" % sent)
+ else:
+ accepted = True
+ else:
+ if trees:
+ raise ValueError("Sentence '%s' received a parse'" % sent)
+ else:
+ rejected = True
+ if accepted and rejected:
+ print("All tests passed!")
+
+def extract_test_sentences(string, comment_chars="#%;", encoding=None):
+ """
+ Parses a string with one test sentence per line.
+ Lines can optionally begin with:
+ - a bool, saying if the sentence is grammatical or not, or
+ - an int, giving the number of parse trees is should have,
+ The result information is followed by a colon, and then the sentence.
+ Empty lines and lines beginning with a comment char are ignored.
+
+ :return: a list of tuple of sentences and expected results,
+ where a sentence is a list of str,
+ and a result is None, or bool, or int
+
+ :param comment_chars: ``str`` of possible comment characters.
+ :param encoding: the encoding of the string, if it is binary
+ """
+ if encoding is not None:
+ string = string.decode(encoding)
+ sentences = []
+ for sentence in string.split('\n'):
+ if sentence == '' or sentence[0] in comment_chars:
+ continue
+ split_info = sentence.split(':', 1)
+ result = None
+ if len(split_info) == 2:
+ if split_info[0] in ['True','true','False','false']:
+ result = split_info[0] in ['True','true']
+ sentence = split_info[1]
+ else:
+ result = int(split_info[0])
+ sentence = split_info[1]
+ tokens = sentence.split()
+ if tokens == []:
+ continue
+ sentences += [(tokens, result)]
+ return sentences
+
+# nose thinks it is a test
+extract_test_sentences.__test__ = False
diff --git a/nltk/parse/viterbi.py b/nltk/parse/viterbi.py
new file mode 100644
index 0000000..9e7c102
--- /dev/null
+++ b/nltk/parse/viterbi.py
@@ -0,0 +1,401 @@
+# Natural Language Toolkit: Viterbi Probabilistic Parser
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+from functools import reduce
+from nltk.tree import Tree, ProbabilisticTree
+from nltk.compat import python_2_unicode_compatible
+
+from nltk.parse.api import ParserI
+
+##//////////////////////////////////////////////////////
+## Viterbi PCFG Parser
+##//////////////////////////////////////////////////////
+
+ at python_2_unicode_compatible
+class ViterbiParser(ParserI):
+ """
+ A bottom-up ``PCFG`` parser that uses dynamic programming to find
+ the single most likely parse for a text. The ``ViterbiParser`` parser
+ parses texts by filling in a "most likely constituent table".
+ This table records the most probable tree representation for any
+ given span and node value. In particular, it has an entry for
+ every start index, end index, and node value, recording the most
+ likely subtree that spans from the start index to the end index,
+ and has the given node value.
+
+ The ``ViterbiParser`` parser fills in this table incrementally. It starts
+ by filling in all entries for constituents that span one element
+ of text (i.e., entries where the end index is one greater than the
+ start index). After it has filled in all table entries for
+ constituents that span one element of text, it fills in the
+ entries for constitutants that span two elements of text. It
+ continues filling in the entries for constituents spanning larger
+ and larger portions of the text, until the entire table has been
+ filled. Finally, it returns the table entry for a constituent
+ spanning the entire text, whose node value is the grammar's start
+ symbol.
+
+ In order to find the most likely constituent with a given span and
+ node value, the ``ViterbiParser`` parser considers all productions that
+ could produce that node value. For each production, it finds all
+ children that collectively cover the span and have the node values
+ specified by the production's right hand side. If the probability
+ of the tree formed by applying the production to the children is
+ greater than the probability of the current entry in the table,
+ then the table is updated with this new tree.
+
+ A pseudo-code description of the algorithm used by
+ ``ViterbiParser`` is:
+
+ | Create an empty most likely constituent table, *MLC*.
+ | For width in 1...len(text):
+ | For start in 1...len(text)-width:
+ | For prod in grammar.productions:
+ | For each sequence of subtrees [t[1], t[2], ..., t[n]] in MLC,
+ | where t[i].label()==prod.rhs[i],
+ | and the sequence covers [start:start+width]:
+ | old_p = MLC[start, start+width, prod.lhs]
+ | new_p = P(t[1])P(t[1])...P(t[n])P(prod)
+ | if new_p > old_p:
+ | new_tree = Tree(prod.lhs, t[1], t[2], ..., t[n])
+ | MLC[start, start+width, prod.lhs] = new_tree
+ | Return MLC[0, len(text), start_symbol]
+
+ :type _grammar: PCFG
+ :ivar _grammar: The grammar used to parse sentences.
+ :type _trace: int
+ :ivar _trace: The level of tracing output that should be generated
+ when parsing a text.
+ """
+ def __init__(self, grammar, trace=0):
+ """
+ Create a new ``ViterbiParser`` parser, that uses ``grammar`` to
+ parse texts.
+
+ :type grammar: PCFG
+ :param grammar: The grammar used to parse texts.
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ and higher numbers will produce more verbose tracing
+ output.
+ """
+ self._grammar = grammar
+ self._trace = trace
+
+ def grammar(self):
+ return self._grammar
+
+ def trace(self, trace=2):
+ """
+ Set the level of tracing output that should be generated when
+ parsing a text.
+
+ :type trace: int
+ :param trace: The trace level. A trace level of ``0`` will
+ generate no tracing output; and higher trace levels will
+ produce more verbose tracing output.
+ :rtype: None
+ """
+ self._trace = trace
+
+ def parse(self, tokens):
+ # Inherit docs from ParserI
+
+ tokens = list(tokens)
+ self._grammar.check_coverage(tokens)
+
+ # The most likely constituent table. This table specifies the
+ # most likely constituent for a given span and type.
+ # Constituents can be either Trees or tokens. For Trees,
+ # the "type" is the Nonterminal for the tree's root node
+ # value. For Tokens, the "type" is the token's type.
+ # The table is stored as a dictionary, since it is sparse.
+ constituents = {}
+
+ # Initialize the constituents dictionary with the words from
+ # the text.
+ if self._trace: print(('Inserting tokens into the most likely'+
+ ' constituents table...'))
+ for index in range(len(tokens)):
+ token = tokens[index]
+ constituents[index,index+1,token] = token
+ if self._trace > 1:
+ self._trace_lexical_insertion(token, index, len(tokens))
+
+ # Consider each span of length 1, 2, ..., n; and add any trees
+ # that might cover that span to the constituents dictionary.
+ for length in range(1, len(tokens)+1):
+ if self._trace:
+ print(('Finding the most likely constituents'+
+ ' spanning %d text elements...' % length))
+ for start in range(len(tokens)-length+1):
+ span = (start, start+length)
+ self._add_constituents_spanning(span, constituents,
+ tokens)
+
+ # Return the tree that spans the entire text & have the right cat
+ tree = constituents.get((0, len(tokens), self._grammar.start()))
+ if tree is not None:
+ yield tree
+
+ def _add_constituents_spanning(self, span, constituents, tokens):
+ """
+ Find any constituents that might cover ``span``, and add them
+ to the most likely constituents table.
+
+ :rtype: None
+ :type span: tuple(int, int)
+ :param span: The section of the text for which we are
+ trying to find possible constituents. The span is
+ specified as a pair of integers, where the first integer
+ is the index of the first token that should be included in
+ the constituent; and the second integer is the index of
+ the first token that should not be included in the
+ constituent. I.e., the constituent should cover
+ ``text[span[0]:span[1]]``, where ``text`` is the text
+ that we are parsing.
+
+ :type constituents: dict(tuple(int,int,Nonterminal) -> ProbabilisticToken or ProbabilisticTree)
+ :param constituents: The most likely constituents table. This
+ table records the most probable tree representation for
+ any given span and node value. In particular,
+ ``constituents(s,e,nv)`` is the most likely
+ ``ProbabilisticTree`` that covers ``text[s:e]``
+ and has a node value ``nv.symbol()``, where ``text``
+ is the text that we are parsing. When
+ ``_add_constituents_spanning`` is called, ``constituents``
+ should contain all possible constituents that are shorter
+ than ``span``.
+
+ :type tokens: list of tokens
+ :param tokens: The text we are parsing. This is only used for
+ trace output.
+ """
+ # Since some of the grammar productions may be unary, we need to
+ # repeatedly try all of the productions until none of them add any
+ # new constituents.
+ changed = True
+ while changed:
+ changed = False
+
+ # Find all ways instantiations of the grammar productions that
+ # cover the span.
+ instantiations = self._find_instantiations(span, constituents)
+
+ # For each production instantiation, add a new
+ # ProbabilisticTree whose probability is the product
+ # of the childrens' probabilities and the production's
+ # probability.
+ for (production, children) in instantiations:
+ subtrees = [c for c in children if isinstance(c, Tree)]
+ p = reduce(lambda pr,t:pr*t.prob(),
+ subtrees, production.prob())
+ node = production.lhs().symbol()
+ tree = ProbabilisticTree(node, children, prob=p)
+
+ # If it's new a constituent, then add it to the
+ # constituents dictionary.
+ c = constituents.get((span[0], span[1], production.lhs()))
+ if self._trace > 1:
+ if c is None or c != tree:
+ if c is None or c.prob() < tree.prob():
+ print(' Insert:', end=' ')
+ else:
+ print(' Discard:', end=' ')
+ self._trace_production(production, p, span, len(tokens))
+ if c is None or c.prob() < tree.prob():
+ constituents[span[0], span[1], production.lhs()] = tree
+ changed = True
+
+ def _find_instantiations(self, span, constituents):
+ """
+ :return: a list of the production instantiations that cover a
+ given span of the text. A "production instantiation" is
+ a tuple containing a production and a list of children,
+ where the production's right hand side matches the list of
+ children; and the children cover ``span``. :rtype: list
+ of ``pair`` of ``Production``, (list of
+ (``ProbabilisticTree`` or token.
+
+ :type span: tuple(int, int)
+ :param span: The section of the text for which we are
+ trying to find production instantiations. The span is
+ specified as a pair of integers, where the first integer
+ is the index of the first token that should be covered by
+ the production instantiation; and the second integer is
+ the index of the first token that should not be covered by
+ the production instantiation.
+ :type constituents: dict(tuple(int,int,Nonterminal) -> ProbabilisticToken or ProbabilisticTree)
+ :param constituents: The most likely constituents table. This
+ table records the most probable tree representation for
+ any given span and node value. See the module
+ documentation for more information.
+ """
+ rv = []
+ for production in self._grammar.productions():
+ childlists = self._match_rhs(production.rhs(), span, constituents)
+
+ for childlist in childlists:
+ rv.append( (production, childlist) )
+ return rv
+
+ def _match_rhs(self, rhs, span, constituents):
+ """
+ :return: a set of all the lists of children that cover ``span``
+ and that match ``rhs``.
+ :rtype: list(list(ProbabilisticTree or token)
+
+ :type rhs: list(Nonterminal or any)
+ :param rhs: The list specifying what kinds of children need to
+ cover ``span``. Each nonterminal in ``rhs`` specifies
+ that the corresponding child should be a tree whose node
+ value is that nonterminal's symbol. Each terminal in ``rhs``
+ specifies that the corresponding child should be a token
+ whose type is that terminal.
+ :type span: tuple(int, int)
+ :param span: The section of the text for which we are
+ trying to find child lists. The span is specified as a
+ pair of integers, where the first integer is the index of
+ the first token that should be covered by the child list;
+ and the second integer is the index of the first token
+ that should not be covered by the child list.
+ :type constituents: dict(tuple(int,int,Nonterminal) -> ProbabilisticToken or ProbabilisticTree)
+ :param constituents: The most likely constituents table. This
+ table records the most probable tree representation for
+ any given span and node value. See the module
+ documentation for more information.
+ """
+ (start, end) = span
+
+ # Base case
+ if start >= end and rhs == (): return [[]]
+ if start >= end or rhs == (): return []
+
+ # Find everything that matches the 1st symbol of the RHS
+ childlists = []
+ for split in range(start, end+1):
+ l=constituents.get((start,split,rhs[0]))
+ if l is not None:
+ rights = self._match_rhs(rhs[1:], (split,end), constituents)
+ childlists += [[l]+r for r in rights]
+
+ return childlists
+
+ def _trace_production(self, production, p, span, width):
+ """
+ Print trace output indicating that a given production has been
+ applied at a given location.
+
+ :param production: The production that has been applied
+ :type production: Production
+ :param p: The probability of the tree produced by the production.
+ :type p: float
+ :param span: The span of the production
+ :type span: tuple
+ :rtype: None
+ """
+
+ str = '|' + '.' * span[0]
+ str += '=' * (span[1] - span[0])
+ str += '.' * (width - span[1]) + '| '
+ str += '%s' % production
+ if self._trace > 2: str = '%-40s %12.10f ' % (str, p)
+
+ print(str)
+
+ def _trace_lexical_insertion(self, token, index, width):
+ str = ' Insert: |' + '.' * index + '=' + '.' * (width-index-1) + '| '
+ str += '%s' % (token,)
+ print(str)
+
+ def __repr__(self):
+ return '<ViterbiParser for %r>' % self._grammar
+
+
+##//////////////////////////////////////////////////////
+## Test Code
+##//////////////////////////////////////////////////////
+
+def demo():
+ """
+ A demonstration of the probabilistic parsers. The user is
+ prompted to select which demo to run, and how many parses should
+ be found; and then each parser is run on the same demo, and a
+ summary of the results are displayed.
+ """
+ import sys, time
+ from nltk import tokenize
+ from nltk.parse import ViterbiParser
+ from nltk.grammar import toy_pcfg1, toy_pcfg2
+
+ # Define two demos. Each demo has a sentence and a grammar.
+ demos = [('I saw the man with my telescope', toy_pcfg1),
+ ('the boy saw Jack with Bob under the table with a telescope', toy_pcfg2)]
+
+ # Ask the user which demo they want to use.
+ print()
+ for i in range(len(demos)):
+ print('%3s: %s' % (i+1, demos[i][0]))
+ print(' %r' % demos[i][1])
+ print()
+ print('Which demo (%d-%d)? ' % (1, len(demos)), end=' ')
+ try:
+ snum = int(sys.stdin.readline().strip())-1
+ sent, grammar = demos[snum]
+ except:
+ print('Bad sentence number')
+ return
+
+ # Tokenize the sentence.
+ tokens = sent.split()
+
+ parser = ViterbiParser(grammar)
+ all_parses = {}
+
+ print('\nsent: %s\nparser: %s\ngrammar: %s' % (sent,parser,grammar))
+ parser.trace(3)
+ t = time.time()
+ parses = parser.parse_all(tokens)
+ time = time.time()-t
+ average = (reduce(lambda a,b:a+b.prob(), parses, 0)/len(parses)
+ if parses else 0)
+ num_parses = len(parses)
+ for p in parses:
+ all_parses[p.freeze()] = 1
+
+ # Print some summary statistics
+ print()
+ print('Time (secs) # Parses Average P(parse)')
+ print('-----------------------------------------')
+ print('%11.4f%11d%19.14f' % (time, num_parses, average))
+ parses = all_parses.keys()
+ if parses:
+ p = reduce(lambda a,b:a+b.prob(), parses, 0)/len(parses)
+ else: p = 0
+ print('------------------------------------------')
+ print('%11s%11d%19.14f' % ('n/a', len(parses), p))
+
+ # Ask the user if we should draw the parses.
+ print()
+ print('Draw parses (y/n)? ', end=' ')
+ if sys.stdin.readline().strip().lower().startswith('y'):
+ from nltk.draw.tree import draw_trees
+ print(' please wait...')
+ draw_trees(*parses)
+
+ # Ask the user if we should print the parses.
+ print()
+ print('Print parses (y/n)? ', end=' ')
+ if sys.stdin.readline().strip().lower().startswith('y'):
+ for parse in parses:
+ print(parse)
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/probability.py b/nltk/probability.py
new file mode 100644
index 0000000..605d8e8
--- /dev/null
+++ b/nltk/probability.py
@@ -0,0 +1,2225 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Probability and Statistics
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (additions)
+# Trevor Cohn <tacohn at cs.mu.oz.au> (additions)
+# Peter Ljunglöf <peter.ljunglof at heatherleaf.se> (additions)
+# Liang Dong <ldong at clemson.edu> (additions)
+# Geoffrey Sampson <sampson at cantab.net> (additions)
+# Ilia Kurenkov <ilia.kurenkov at gmail.com> (additions)
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classes for representing and processing probabilistic information.
+
+The ``FreqDist`` class is used to encode "frequency distributions",
+which count the number of times that each outcome of an experiment
+occurs.
+
+The ``ProbDistI`` class defines a standard interface for "probability
+distributions", which encode the probability of each outcome for an
+experiment. There are two types of probability distribution:
+
+ - "derived probability distributions" are created from frequency
+ distributions. They attempt to model the probability distribution
+ that generated the frequency distribution.
+ - "analytic probability distributions" are created directly from
+ parameters (such as variance).
+
+The ``ConditionalFreqDist`` class and ``ConditionalProbDistI`` interface
+are used to encode conditional distributions. Conditional probability
+distributions can be derived or analytic; but currently the only
+implementation of the ``ConditionalProbDistI`` interface is
+``ConditionalProbDist``, a derived distribution.
+
+"""
+from __future__ import print_function, unicode_literals
+
+import math
+import random
+import warnings
+import array
+from operator import itemgetter
+from itertools import islice
+from collections import defaultdict
+from functools import reduce
+from nltk import compat
+from nltk.compat import Counter
+
+from nltk.internals import raise_unorderable_types
+
+_NINF = float('-1e300')
+
+##//////////////////////////////////////////////////////
+## Frequency Distributions
+##//////////////////////////////////////////////////////
+
+ at compat.python_2_unicode_compatible
+class FreqDist(Counter):
+ """
+ A frequency distribution for the outcomes of an experiment. A
+ frequency distribution records the number of times each outcome of
+ an experiment has occurred. For example, a frequency distribution
+ could be used to record the frequency of each word type in a
+ document. Formally, a frequency distribution can be defined as a
+ function mapping from each sample to the number of times that
+ sample occurred as an outcome.
+
+ Frequency distributions are generally constructed by running a
+ number of experiments, and incrementing the count for a sample
+ every time it is an outcome of an experiment. For example, the
+ following code will produce a frequency distribution that encodes
+ how often each word occurs in a text:
+
+ >>> from nltk.tokenize import word_tokenize
+ >>> from nltk.probability import FreqDist
+ >>> sent = 'This is an example sentence'
+ >>> fdist = FreqDist()
+ >>> for word in word_tokenize(sent):
+ ... fdist[word.lower()] += 1
+
+ An equivalent way to do this is with the initializer:
+
+ >>> fdist = FreqDist(word.lower() for word in word_tokenize(sent))
+
+ """
+
+ def __init__(self, samples=None):
+ """
+ Construct a new frequency distribution. If ``samples`` is
+ given, then the frequency distribution will be initialized
+ with the count of each object in ``samples``; otherwise, it
+ will be initialized to be empty.
+
+ In particular, ``FreqDist()`` returns an empty frequency
+ distribution; and ``FreqDist(samples)`` first creates an empty
+ frequency distribution, and then calls ``update`` with the
+ list ``samples``.
+
+ :param samples: The samples to initialize the frequency
+ distribution with.
+ :type samples: Sequence
+ """
+ Counter.__init__(self, samples)
+
+ def N(self):
+ """
+ Return the total number of sample outcomes that have been
+ recorded by this FreqDist. For the number of unique
+ sample values (or bins) with counts greater than zero, use
+ ``FreqDist.B()``.
+
+ :rtype: int
+ """
+ return sum(self.values())
+
+ def B(self):
+ """
+ Return the total number of sample values (or "bins") that
+ have counts greater than zero. For the total
+ number of sample outcomes recorded, use ``FreqDist.N()``.
+ (FreqDist.B() is the same as len(FreqDist).)
+
+ :rtype: int
+ """
+ return len(self)
+
+ def hapaxes(self):
+ """
+ Return a list of all samples that occur once (hapax legomena)
+
+ :rtype: list
+ """
+ return [item for item in self if self[item] == 1]
+
+
+ def Nr(self, r, bins=None):
+ return self.r_Nr(bins)[r]
+
+ def r_Nr(self, bins=None):
+ """
+ Return the dictionary mapping r to Nr, the number of samples with frequency r, where Nr > 0.
+
+ :type bins: int
+ :param bins: The number of possible sample outcomes. ``bins``
+ is used to calculate Nr(0). In particular, Nr(0) is
+ ``bins-self.B()``. If ``bins`` is not specified, it
+ defaults to ``self.B()`` (so Nr(0) will be 0).
+ :rtype: int
+ """
+
+ _r_Nr = defaultdict(int)
+ for count in self.values():
+ _r_Nr[count] += 1
+
+ # Special case for Nr[0]:
+ _r_Nr[0] = bins - self.B() if bins is not None else 0
+
+ return _r_Nr
+
+ def _cumulative_frequencies(self, samples):
+ """
+ Return the cumulative frequencies of the specified samples.
+ If no samples are specified, all counts are returned, starting
+ with the largest.
+
+ :param samples: the samples whose frequencies should be returned.
+ :type samples: any
+ :rtype: list(float)
+ """
+ cf = 0.0
+ for sample in samples:
+ cf += self[sample]
+ yield cf
+
+ # slightly odd nomenclature freq() if FreqDist does counts and ProbDist does probs,
+ # here, freq() does probs
+ def freq(self, sample):
+ """
+ Return the frequency of a given sample. The frequency of a
+ sample is defined as the count of that sample divided by the
+ total number of sample outcomes that have been recorded by
+ this FreqDist. The count of a sample is defined as the
+ number of times that sample outcome was recorded by this
+ FreqDist. Frequencies are always real numbers in the range
+ [0, 1].
+
+ :param sample: the sample whose frequency
+ should be returned.
+ :type sample: any
+ :rtype: float
+ """
+ if self.N() == 0:
+ return 0
+ return float(self[sample]) / self.N()
+
+ def max(self):
+ """
+ Return the sample with the greatest number of outcomes in this
+ frequency distribution. If two or more samples have the same
+ number of outcomes, return one of them; which sample is
+ returned is undefined. If no outcomes have occurred in this
+ frequency distribution, return None.
+
+ :return: The sample with the maximum number of outcomes in this
+ frequency distribution.
+ :rtype: any or None
+ """
+ if len(self) == 0:
+ raise ValueError('A FreqDist must have at least one sample before max is defined.')
+ return self.most_common(1)[0][0]
+
+ def plot(self, *args, **kwargs):
+ """
+ Plot samples from the frequency distribution
+ displaying the most frequent sample first. If an integer
+ parameter is supplied, stop after this many samples have been
+ plotted. If two integer parameters m, n are supplied, plot a
+ subset of the samples, beginning with m and stopping at n-1.
+ For a cumulative plot, specify cumulative=True.
+ (Requires Matplotlib to be installed.)
+
+ :param title: The title for the graph
+ :type title: str
+ :param cumulative: A flag to specify whether the plot is cumulative (default = False)
+ :type title: bool
+ """
+ try:
+ import pylab
+ except ImportError:
+ raise ValueError('The plot function requires the matplotlib package (aka pylab). '
+ 'See http://matplotlib.sourceforge.net/')
+
+ if len(args) == 0:
+ args = [len(self)]
+ samples = list(islice(self, *args))
+
+ cumulative = _get_kwarg(kwargs, 'cumulative', False)
+ if cumulative:
+ freqs = list(self._cumulative_frequencies(samples))
+ ylabel = "Cumulative Counts"
+ else:
+ freqs = [self[sample] for sample in samples]
+ ylabel = "Counts"
+ # percents = [f * 100 for f in freqs] only in ProbDist?
+
+ pylab.grid(True, color="silver")
+ if not "linewidth" in kwargs:
+ kwargs["linewidth"] = 2
+ if "title" in kwargs:
+ pylab.title(kwargs["title"])
+ del kwargs["title"]
+ pylab.plot(freqs, **kwargs)
+ pylab.xticks(range(len(samples)), [compat.text_type(s) for s in samples], rotation=90)
+ pylab.xlabel("Samples")
+ pylab.ylabel(ylabel)
+ pylab.show()
+
+ def tabulate(self, *args, **kwargs):
+ """
+ Tabulate the given samples from the frequency distribution (cumulative),
+ displaying the most frequent sample first. If an integer
+ parameter is supplied, stop after this many samples have been
+ plotted. If two integer parameters m, n are supplied, plot a
+ subset of the samples, beginning with m and stopping at n-1.
+ (Requires Matplotlib to be installed.)
+
+ :param samples: The samples to plot (default is all samples)
+ :type samples: list
+ """
+ if len(args) == 0:
+ args = [len(self)]
+ samples = list(islice(self, *args))
+
+ cumulative = _get_kwarg(kwargs, 'cumulative', False)
+ if cumulative:
+ freqs = list(self._cumulative_frequencies(samples))
+ else:
+ freqs = [self[sample] for sample in samples]
+ # percents = [f * 100 for f in freqs] only in ProbDist?
+
+ for i in range(len(samples)):
+ print("%4s" % samples[i], end=' ')
+ print()
+ for i in range(len(samples)):
+ print("%4d" % freqs[i], end=' ')
+ print()
+
+ def copy(self):
+ """
+ Create a copy of this frequency distribution.
+
+ :rtype: FreqDist
+ """
+ return self.__class__(self)
+
+ def __le__(self, other):
+ if not isinstance(other, FreqDist):
+ raise_unorderable_types("<=", self, other)
+ return set(self).issubset(other) and all(self[key] <= other[key] for key in self)
+
+ # @total_ordering doesn't work here, since the class inherits from a builtin class
+ __ge__ = lambda self, other: not self <= other or self == other
+ __lt__ = lambda self, other: self <= other and not self == other
+ __gt__ = lambda self, other: not self <= other
+
+ def __repr__(self):
+ """
+ Return a string representation of this FreqDist.
+
+ :rtype: string
+ """
+ return '<FreqDist with %d samples and %d outcomes>' % (len(self), self.N())
+
+ def pprint(self, maxlen=10):
+ """
+ Return a string representation of this FreqDist.
+
+ :param maxlen: The maximum number of items to display
+ :type maxlen: int
+ :rtype: string
+ """
+ items = ['{0!r}: {1!r}'.format(*item) for item in self.most_common(maxlen)]
+ if len(self) > maxlen:
+ items.append('...')
+ return 'FreqDist({{{0}}})'.format(', '.join(items))
+
+ def __str__(self):
+ """
+ Return a string representation of this FreqDist.
+
+ :rtype: string
+ """
+ return self.pprint()
+
+
+##//////////////////////////////////////////////////////
+## Probability Distributions
+##//////////////////////////////////////////////////////
+
+class ProbDistI(object):
+ """
+ A probability distribution for the outcomes of an experiment. A
+ probability distribution specifies how likely it is that an
+ experiment will have any given outcome. For example, a
+ probability distribution could be used to predict the probability
+ that a token in a document will have a given type. Formally, a
+ probability distribution can be defined as a function mapping from
+ samples to nonnegative real numbers, such that the sum of every
+ number in the function's range is 1.0. A ``ProbDist`` is often
+ used to model the probability distribution of the experiment used
+ to generate a frequency distribution.
+ """
+ SUM_TO_ONE = True
+ """True if the probabilities of the samples in this probability
+ distribution will always sum to one."""
+
+ def __init__(self):
+ if self.__class__ == ProbDistI:
+ raise NotImplementedError("Interfaces can't be instantiated")
+
+ def prob(self, sample):
+ """
+ Return the probability for a given sample. Probabilities
+ are always real numbers in the range [0, 1].
+
+ :param sample: The sample whose probability
+ should be returned.
+ :type sample: any
+ :rtype: float
+ """
+ raise NotImplementedError()
+
+ def logprob(self, sample):
+ """
+ Return the base 2 logarithm of the probability for a given sample.
+
+ :param sample: The sample whose probability
+ should be returned.
+ :type sample: any
+ :rtype: float
+ """
+ # Default definition, in terms of prob()
+ p = self.prob(sample)
+ return (math.log(p, 2) if p != 0 else _NINF)
+
+ def max(self):
+ """
+ Return the sample with the greatest probability. If two or
+ more samples have the same probability, return one of them;
+ which sample is returned is undefined.
+
+ :rtype: any
+ """
+ raise NotImplementedError()
+
+ def samples(self):
+ """
+ Return a list of all samples that have nonzero probabilities.
+ Use ``prob`` to find the probability of each sample.
+
+ :rtype: list
+ """
+ raise NotImplementedError()
+
+ # cf self.SUM_TO_ONE
+ def discount(self):
+ """
+ Return the ratio by which counts are discounted on average: c*/c
+
+ :rtype: float
+ """
+ return 0.0
+
+ # Subclasses should define more efficient implementations of this,
+ # where possible.
+ def generate(self):
+ """
+ Return a randomly selected sample from this probability distribution.
+ The probability of returning each sample ``samp`` is equal to
+ ``self.prob(samp)``.
+ """
+ p = random.random()
+ p_init = p
+ for sample in self.samples():
+ p -= self.prob(sample)
+ if p <= 0: return sample
+ # allow for some rounding error:
+ if p < .0001:
+ return sample
+ # we *should* never get here
+ if self.SUM_TO_ONE:
+ warnings.warn("Probability distribution %r sums to %r; generate()"
+ " is returning an arbitrary sample." % (self, p_init-p))
+ return random.choice(list(self.samples()))
+
+
+ at compat.python_2_unicode_compatible
+class UniformProbDist(ProbDistI):
+ """
+ A probability distribution that assigns equal probability to each
+ sample in a given set; and a zero probability to all other
+ samples.
+ """
+ def __init__(self, samples):
+ """
+ Construct a new uniform probability distribution, that assigns
+ equal probability to each sample in ``samples``.
+
+ :param samples: The samples that should be given uniform
+ probability.
+ :type samples: list
+ :raise ValueError: If ``samples`` is empty.
+ """
+ if len(samples) == 0:
+ raise ValueError('A Uniform probability distribution must '+
+ 'have at least one sample.')
+ self._sampleset = set(samples)
+ self._prob = 1.0/len(self._sampleset)
+ self._samples = list(self._sampleset)
+
+ def prob(self, sample):
+ return (self._prob if sample in self._sampleset else 0)
+
+ def max(self):
+ return self._samples[0]
+
+ def samples(self):
+ return self._samples
+
+ def __repr__(self):
+ return '<UniformProbDist with %d samples>' % len(self._sampleset)
+
+
+ at compat.python_2_unicode_compatible
+class RandomProbDist(ProbDistI):
+ """
+ Generates a random probability distribution whereby each sample
+ will be between 0 and 1 with equal probability (uniform random distribution.
+ Also called a continuous uniform distribution).
+ """
+ def __init__(self, samples):
+ if len(samples) == 0:
+ raise ValueError('A probability distribution must '+
+ 'have at least one sample.')
+ self._probs = self.unirand(samples)
+ self._samples = list(self._probs.keys())
+
+ @classmethod
+ def unirand(cls, samples):
+ """
+ The key function that creates a randomized initial distribution
+ that still sums to 1. Set as a dictionary of prob values so that
+ it can still be passed to MutableProbDist and called with identical
+ syntax to UniformProbDist
+ """
+ randrow = [random.random() for i in range(len(samples))]
+ total = sum(randrow)
+ for i, x in enumerate(randrow):
+ randrow[i] = x/total
+
+ total = sum(randrow)
+ if total != 1:
+ #this difference, if present, is so small (near NINF) that it
+ #can be subtracted from any element without risking probs not (0 1)
+ randrow[-1] -= total - 1
+
+ return dict((s, randrow[i]) for i, s in enumerate(samples))
+
+ def prob(self, sample):
+ return self._probs.get(sample, 0)
+
+ def samples(self):
+ return self._samples
+
+ def __repr__(self):
+ return '<RandomUniformProbDist with %d samples>' %len(self._probs)
+
+
+ at compat.python_2_unicode_compatible
+class DictionaryProbDist(ProbDistI):
+ """
+ A probability distribution whose probabilities are directly
+ specified by a given dictionary. The given dictionary maps
+ samples to probabilities.
+ """
+ def __init__(self, prob_dict=None, log=False, normalize=False):
+ """
+ Construct a new probability distribution from the given
+ dictionary, which maps values to probabilities (or to log
+ probabilities, if ``log`` is true). If ``normalize`` is
+ true, then the probability values are scaled by a constant
+ factor such that they sum to 1.
+
+ If called without arguments, the resulting probability
+ distribution assigns zero probability to all values.
+ """
+
+ self._prob_dict = (prob_dict.copy() if prob_dict is not None else {})
+ self._log = log
+
+ # Normalize the distribution, if requested.
+ if normalize:
+ if len(prob_dict) == 0:
+ raise ValueError('A DictionaryProbDist must have at least one sample ' +
+ 'before it can be normalized.')
+ if log:
+ value_sum = sum_logs(list(self._prob_dict.values()))
+ if value_sum <= _NINF:
+ logp = math.log(1.0/len(prob_dict), 2)
+ for x in prob_dict:
+ self._prob_dict[x] = logp
+ else:
+ for (x, p) in self._prob_dict.items():
+ self._prob_dict[x] -= value_sum
+ else:
+ value_sum = sum(self._prob_dict.values())
+ if value_sum == 0:
+ p = 1.0/len(prob_dict)
+ for x in prob_dict:
+ self._prob_dict[x] = p
+ else:
+ norm_factor = 1.0/value_sum
+ for (x, p) in self._prob_dict.items():
+ self._prob_dict[x] *= norm_factor
+
+ def prob(self, sample):
+ if self._log:
+ return (2**(self._prob_dict[sample]) if sample in self._prob_dict else 0)
+ else:
+ return self._prob_dict.get(sample, 0)
+
+ def logprob(self, sample):
+ if self._log:
+ return self._prob_dict.get(sample, _NINF)
+ else:
+ if sample not in self._prob_dict: return _NINF
+ elif self._prob_dict[sample] == 0: return _NINF
+ else: return math.log(self._prob_dict[sample], 2)
+
+ def max(self):
+ if not hasattr(self, '_max'):
+ self._max = max((p,v) for (v,p) in self._prob_dict.items())[1]
+ return self._max
+ def samples(self):
+ return self._prob_dict.keys()
+ def __repr__(self):
+ return '<ProbDist with %d samples>' % len(self._prob_dict)
+
+
+ at compat.python_2_unicode_compatible
+class MLEProbDist(ProbDistI):
+ """
+ The maximum likelihood estimate for the probability distribution
+ of the experiment used to generate a frequency distribution. The
+ "maximum likelihood estimate" approximates the probability of
+ each sample as the frequency of that sample in the frequency
+ distribution.
+ """
+ def __init__(self, freqdist, bins=None):
+ """
+ Use the maximum likelihood estimate to create a probability
+ distribution for the experiment used to generate ``freqdist``.
+
+ :type freqdist: FreqDist
+ :param freqdist: The frequency distribution that the
+ probability estimates should be based on.
+ """
+ self._freqdist = freqdist
+
+ def freqdist(self):
+ """
+ Return the frequency distribution that this probability
+ distribution is based on.
+
+ :rtype: FreqDist
+ """
+ return self._freqdist
+
+ def prob(self, sample):
+ return self._freqdist.freq(sample)
+
+ def max(self):
+ return self._freqdist.max()
+
+ def samples(self):
+ return self._freqdist.keys()
+
+ def __repr__(self):
+ """
+ :rtype: str
+ :return: A string representation of this ``ProbDist``.
+ """
+ return '<MLEProbDist based on %d samples>' % self._freqdist.N()
+
+
+ at compat.python_2_unicode_compatible
+class LidstoneProbDist(ProbDistI):
+ """
+ The Lidstone estimate for the probability distribution of the
+ experiment used to generate a frequency distribution. The
+ "Lidstone estimate" is parameterized by a real number *gamma*,
+ which typically ranges from 0 to 1. The Lidstone estimate
+ approximates the probability of a sample with count *c* from an
+ experiment with *N* outcomes and *B* bins as
+ ``c+gamma)/(N+B*gamma)``. This is equivalent to adding
+ *gamma* to the count for each bin, and taking the maximum
+ likelihood estimate of the resulting frequency distribution.
+ """
+ SUM_TO_ONE = False
+ def __init__(self, freqdist, gamma, bins=None):
+ """
+ Use the Lidstone estimate to create a probability distribution
+ for the experiment used to generate ``freqdist``.
+
+ :type freqdist: FreqDist
+ :param freqdist: The frequency distribution that the
+ probability estimates should be based on.
+ :type gamma: float
+ :param gamma: A real number used to parameterize the
+ estimate. The Lidstone estimate is equivalent to adding
+ *gamma* to the count for each bin, and taking the
+ maximum likelihood estimate of the resulting frequency
+ distribution.
+ :type bins: int
+ :param bins: The number of sample values that can be generated
+ by the experiment that is described by the probability
+ distribution. This value must be correctly set for the
+ probabilities of the sample values to sum to one. If
+ ``bins`` is not specified, it defaults to ``freqdist.B()``.
+ """
+ if (bins == 0) or (bins is None and freqdist.N() == 0):
+ name = self.__class__.__name__[:-8]
+ raise ValueError('A %s probability distribution ' % name +
+ 'must have at least one bin.')
+ if (bins is not None) and (bins < freqdist.B()):
+ name = self.__class__.__name__[:-8]
+ raise ValueError('\nThe number of bins in a %s distribution ' % name +
+ '(%d) must be greater than or equal to\n' % bins +
+ 'the number of bins in the FreqDist used ' +
+ 'to create it (%d).' % freqdist.B())
+
+ self._freqdist = freqdist
+ self._gamma = float(gamma)
+ self._N = self._freqdist.N()
+
+ if bins is None:
+ bins = freqdist.B()
+ self._bins = bins
+
+ self._divisor = self._N + bins * gamma
+ if self._divisor == 0.0:
+ # In extreme cases we force the probability to be 0,
+ # which it will be, since the count will be 0:
+ self._gamma = 0
+ self._divisor = 1
+
+ def freqdist(self):
+ """
+ Return the frequency distribution that this probability
+ distribution is based on.
+
+ :rtype: FreqDist
+ """
+ return self._freqdist
+
+ def prob(self, sample):
+ c = self._freqdist[sample]
+ return (c + self._gamma) / self._divisor
+
+ def max(self):
+ # For Lidstone distributions, probability is monotonic with
+ # frequency, so the most probable sample is the one that
+ # occurs most frequently.
+ return self._freqdist.max()
+
+ def samples(self):
+ return self._freqdist.keys()
+
+ def discount(self):
+ gb = self._gamma * self._bins
+ return gb / (self._N + gb)
+
+ def __repr__(self):
+ """
+ Return a string representation of this ``ProbDist``.
+
+ :rtype: str
+ """
+ return '<LidstoneProbDist based on %d samples>' % self._freqdist.N()
+
+
+ at compat.python_2_unicode_compatible
+class LaplaceProbDist(LidstoneProbDist):
+ """
+ The Laplace estimate for the probability distribution of the
+ experiment used to generate a frequency distribution. The
+ "Laplace estimate" approximates the probability of a sample with
+ count *c* from an experiment with *N* outcomes and *B* bins as
+ *(c+1)/(N+B)*. This is equivalent to adding one to the count for
+ each bin, and taking the maximum likelihood estimate of the
+ resulting frequency distribution.
+ """
+ def __init__(self, freqdist, bins=None):
+ """
+ Use the Laplace estimate to create a probability distribution
+ for the experiment used to generate ``freqdist``.
+
+ :type freqdist: FreqDist
+ :param freqdist: The frequency distribution that the
+ probability estimates should be based on.
+ :type bins: int
+ :param bins: The number of sample values that can be generated
+ by the experiment that is described by the probability
+ distribution. This value must be correctly set for the
+ probabilities of the sample values to sum to one. If
+ ``bins`` is not specified, it defaults to ``freqdist.B()``.
+ """
+ LidstoneProbDist.__init__(self, freqdist, 1, bins)
+
+ def __repr__(self):
+ """
+ :rtype: str
+ :return: A string representation of this ``ProbDist``.
+ """
+ return '<LaplaceProbDist based on %d samples>' % self._freqdist.N()
+
+
+ at compat.python_2_unicode_compatible
+class ELEProbDist(LidstoneProbDist):
+ """
+ The expected likelihood estimate for the probability distribution
+ of the experiment used to generate a frequency distribution. The
+ "expected likelihood estimate" approximates the probability of a
+ sample with count *c* from an experiment with *N* outcomes and
+ *B* bins as *(c+0.5)/(N+B/2)*. This is equivalent to adding 0.5
+ to the count for each bin, and taking the maximum likelihood
+ estimate of the resulting frequency distribution.
+ """
+ def __init__(self, freqdist, bins=None):
+ """
+ Use the expected likelihood estimate to create a probability
+ distribution for the experiment used to generate ``freqdist``.
+
+ :type freqdist: FreqDist
+ :param freqdist: The frequency distribution that the
+ probability estimates should be based on.
+ :type bins: int
+ :param bins: The number of sample values that can be generated
+ by the experiment that is described by the probability
+ distribution. This value must be correctly set for the
+ probabilities of the sample values to sum to one. If
+ ``bins`` is not specified, it defaults to ``freqdist.B()``.
+ """
+ LidstoneProbDist.__init__(self, freqdist, 0.5, bins)
+
+ def __repr__(self):
+ """
+ Return a string representation of this ``ProbDist``.
+
+ :rtype: str
+ """
+ return '<ELEProbDist based on %d samples>' % self._freqdist.N()
+
+
+ at compat.python_2_unicode_compatible
+class HeldoutProbDist(ProbDistI):
+ """
+ The heldout estimate for the probability distribution of the
+ experiment used to generate two frequency distributions. These
+ two frequency distributions are called the "heldout frequency
+ distribution" and the "base frequency distribution." The
+ "heldout estimate" uses uses the "heldout frequency
+ distribution" to predict the probability of each sample, given its
+ frequency in the "base frequency distribution".
+
+ In particular, the heldout estimate approximates the probability
+ for a sample that occurs *r* times in the base distribution as
+ the average frequency in the heldout distribution of all samples
+ that occur *r* times in the base distribution.
+
+ This average frequency is *Tr[r]/(Nr[r].N)*, where:
+
+ - *Tr[r]* is the total count in the heldout distribution for
+ all samples that occur *r* times in the base distribution.
+ - *Nr[r]* is the number of samples that occur *r* times in
+ the base distribution.
+ - *N* is the number of outcomes recorded by the heldout
+ frequency distribution.
+
+ In order to increase the efficiency of the ``prob`` member
+ function, *Tr[r]/(Nr[r].N)* is precomputed for each value of *r*
+ when the ``HeldoutProbDist`` is created.
+
+ :type _estimate: list(float)
+ :ivar _estimate: A list mapping from *r*, the number of
+ times that a sample occurs in the base distribution, to the
+ probability estimate for that sample. ``_estimate[r]`` is
+ calculated by finding the average frequency in the heldout
+ distribution of all samples that occur *r* times in the base
+ distribution. In particular, ``_estimate[r]`` =
+ *Tr[r]/(Nr[r].N)*.
+ :type _max_r: int
+ :ivar _max_r: The maximum number of times that any sample occurs
+ in the base distribution. ``_max_r`` is used to decide how
+ large ``_estimate`` must be.
+ """
+ SUM_TO_ONE = False
+ def __init__(self, base_fdist, heldout_fdist, bins=None):
+ """
+ Use the heldout estimate to create a probability distribution
+ for the experiment used to generate ``base_fdist`` and
+ ``heldout_fdist``.
+
+ :type base_fdist: FreqDist
+ :param base_fdist: The base frequency distribution.
+ :type heldout_fdist: FreqDist
+ :param heldout_fdist: The heldout frequency distribution.
+ :type bins: int
+ :param bins: The number of sample values that can be generated
+ by the experiment that is described by the probability
+ distribution. This value must be correctly set for the
+ probabilities of the sample values to sum to one. If
+ ``bins`` is not specified, it defaults to ``freqdist.B()``.
+ """
+
+ self._base_fdist = base_fdist
+ self._heldout_fdist = heldout_fdist
+
+ # The max number of times any sample occurs in base_fdist.
+ self._max_r = base_fdist[base_fdist.max()]
+
+ # Calculate Tr, Nr, and N.
+ Tr = self._calculate_Tr()
+ r_Nr = base_fdist.r_Nr(bins)
+ Nr = [r_Nr[r] for r in range(self._max_r+1)]
+ N = heldout_fdist.N()
+
+ # Use Tr, Nr, and N to compute the probability estimate for
+ # each value of r.
+ self._estimate = self._calculate_estimate(Tr, Nr, N)
+
+ def _calculate_Tr(self):
+ """
+ Return the list *Tr*, where *Tr[r]* is the total count in
+ ``heldout_fdist`` for all samples that occur *r*
+ times in ``base_fdist``.
+
+ :rtype: list(float)
+ """
+ Tr = [0.0] * (self._max_r+1)
+ for sample in self._heldout_fdist:
+ r = self._base_fdist[sample]
+ Tr[r] += self._heldout_fdist[sample]
+ return Tr
+
+ def _calculate_estimate(self, Tr, Nr, N):
+ """
+ Return the list *estimate*, where *estimate[r]* is the probability
+ estimate for any sample that occurs *r* times in the base frequency
+ distribution. In particular, *estimate[r]* is *Tr[r]/(N[r].N)*.
+ In the special case that *N[r]=0*, *estimate[r]* will never be used;
+ so we define *estimate[r]=None* for those cases.
+
+ :rtype: list(float)
+ :type Tr: list(float)
+ :param Tr: the list *Tr*, where *Tr[r]* is the total count in
+ the heldout distribution for all samples that occur *r*
+ times in base distribution.
+ :type Nr: list(float)
+ :param Nr: The list *Nr*, where *Nr[r]* is the number of
+ samples that occur *r* times in the base distribution.
+ :type N: int
+ :param N: The total number of outcomes recorded by the heldout
+ frequency distribution.
+ """
+ estimate = []
+ for r in range(self._max_r+1):
+ if Nr[r] == 0: estimate.append(None)
+ else: estimate.append(Tr[r]/(Nr[r]*N))
+ return estimate
+
+ def base_fdist(self):
+ """
+ Return the base frequency distribution that this probability
+ distribution is based on.
+
+ :rtype: FreqDist
+ """
+ return self._base_fdist
+
+ def heldout_fdist(self):
+ """
+ Return the heldout frequency distribution that this
+ probability distribution is based on.
+
+ :rtype: FreqDist
+ """
+ return self._heldout_fdist
+
+ def samples(self):
+ return self._base_fdist.keys()
+
+ def prob(self, sample):
+ # Use our precomputed probability estimate.
+ r = self._base_fdist[sample]
+ return self._estimate[r]
+
+ def max(self):
+ # Note: the Heldout estimation is *not* necessarily monotonic;
+ # so this implementation is currently broken. However, it
+ # should give the right answer *most* of the time. :)
+ return self._base_fdist.max()
+
+ def discount(self):
+ raise NotImplementedError()
+
+ def __repr__(self):
+ """
+ :rtype: str
+ :return: A string representation of this ``ProbDist``.
+ """
+ s = '<HeldoutProbDist: %d base samples; %d heldout samples>'
+ return s % (self._base_fdist.N(), self._heldout_fdist.N())
+
+
+ at compat.python_2_unicode_compatible
+class CrossValidationProbDist(ProbDistI):
+ """
+ The cross-validation estimate for the probability distribution of
+ the experiment used to generate a set of frequency distribution.
+ The "cross-validation estimate" for the probability of a sample
+ is found by averaging the held-out estimates for the sample in
+ each pair of frequency distributions.
+ """
+ SUM_TO_ONE = False
+ def __init__(self, freqdists, bins):
+ """
+ Use the cross-validation estimate to create a probability
+ distribution for the experiment used to generate
+ ``freqdists``.
+
+ :type freqdists: list(FreqDist)
+ :param freqdists: A list of the frequency distributions
+ generated by the experiment.
+ :type bins: int
+ :param bins: The number of sample values that can be generated
+ by the experiment that is described by the probability
+ distribution. This value must be correctly set for the
+ probabilities of the sample values to sum to one. If
+ ``bins`` is not specified, it defaults to ``freqdist.B()``.
+ """
+ self._freqdists = freqdists
+
+ # Create a heldout probability distribution for each pair of
+ # frequency distributions in freqdists.
+ self._heldout_probdists = []
+ for fdist1 in freqdists:
+ for fdist2 in freqdists:
+ if fdist1 is not fdist2:
+ probdist = HeldoutProbDist(fdist1, fdist2, bins)
+ self._heldout_probdists.append(probdist)
+
+ def freqdists(self):
+ """
+ Return the list of frequency distributions that this ``ProbDist`` is based on.
+
+ :rtype: list(FreqDist)
+ """
+ return self._freqdists
+
+ def samples(self):
+ # [xx] nb: this is not too efficient
+ return set(sum([list(fd) for fd in self._freqdists], []))
+
+ def prob(self, sample):
+ # Find the average probability estimate returned by each
+ # heldout distribution.
+ prob = 0.0
+ for heldout_probdist in self._heldout_probdists:
+ prob += heldout_probdist.prob(sample)
+ return prob/len(self._heldout_probdists)
+
+ def discount(self):
+ raise NotImplementedError()
+
+ def __repr__(self):
+ """
+ Return a string representation of this ``ProbDist``.
+
+ :rtype: str
+ """
+ return '<CrossValidationProbDist: %d-way>' % len(self._freqdists)
+
+
+ at compat.python_2_unicode_compatible
+class WittenBellProbDist(ProbDistI):
+ """
+ The Witten-Bell estimate of a probability distribution. This distribution
+ allocates uniform probability mass to as yet unseen events by using the
+ number of events that have only been seen once. The probability mass
+ reserved for unseen events is equal to *T / (N + T)*
+ where *T* is the number of observed event types and *N* is the total
+ number of observed events. This equates to the maximum likelihood estimate
+ of a new type event occurring. The remaining probability mass is discounted
+ such that all probability estimates sum to one, yielding:
+
+ - *p = T / Z (N + T)*, if count = 0
+ - *p = c / (N + T)*, otherwise
+ """
+
+ def __init__(self, freqdist, bins=None):
+ """
+ Creates a distribution of Witten-Bell probability estimates. This
+ distribution allocates uniform probability mass to as yet unseen
+ events by using the number of events that have only been seen once. The
+ probability mass reserved for unseen events is equal to *T / (N + T)*
+ where *T* is the number of observed event types and *N* is the total
+ number of observed events. This equates to the maximum likelihood
+ estimate of a new type event occurring. The remaining probability mass
+ is discounted such that all probability estimates sum to one,
+ yielding:
+
+ - *p = T / Z (N + T)*, if count = 0
+ - *p = c / (N + T)*, otherwise
+
+ The parameters *T* and *N* are taken from the ``freqdist`` parameter
+ (the ``B()`` and ``N()`` values). The normalizing factor *Z* is
+ calculated using these values along with the ``bins`` parameter.
+
+ :param freqdist: The frequency counts upon which to base the
+ estimation.
+ :type freqdist: FreqDist
+ :param bins: The number of possible event types. This must be at least
+ as large as the number of bins in the ``freqdist``. If None, then
+ it's assumed to be equal to that of the ``freqdist``
+ :type bins: int
+ """
+ assert bins is None or bins >= freqdist.B(),\
+ 'bins parameter must not be less than %d=freqdist.B()' % freqdist.B()
+ if bins is None:
+ bins = freqdist.B()
+ self._freqdist = freqdist
+ self._T = self._freqdist.B()
+ self._Z = bins - self._freqdist.B()
+ self._N = self._freqdist.N()
+ # self._P0 is P(0), precalculated for efficiency:
+ if self._N==0:
+ # if freqdist is empty, we approximate P(0) by a UniformProbDist:
+ self._P0 = 1.0 / self._Z
+ else:
+ self._P0 = self._T / float(self._Z * (self._N + self._T))
+
+ def prob(self, sample):
+ # inherit docs from ProbDistI
+ c = self._freqdist[sample]
+ return (c / float(self._N + self._T) if c != 0 else self._P0)
+
+ def max(self):
+ return self._freqdist.max()
+
+ def samples(self):
+ return self._freqdist.keys()
+
+ def freqdist(self):
+ return self._freqdist
+
+ def discount(self):
+ raise NotImplementedError()
+
+ def __repr__(self):
+ """
+ Return a string representation of this ``ProbDist``.
+
+ :rtype: str
+ """
+ return '<WittenBellProbDist based on %d samples>' % self._freqdist.N()
+
+
+##//////////////////////////////////////////////////////
+## Good-Turing Probability Distributions
+##//////////////////////////////////////////////////////
+
+# Good-Turing frequency estimation was contributed by Alan Turing and
+# his statistical assistant I.J. Good, during their collaboration in
+# the WWII. It is a statistical technique for predicting the
+# probability of occurrence of objects belonging to an unknown number
+# of species, given past observations of such objects and their
+# species. (In drawing balls from an urn, the 'objects' would be balls
+# and the 'species' would be the distinct colors of the balls (finite
+# but unknown in number).
+#
+# Good-Turing method calculates the probability mass to assign to
+# events with zero or low counts based on the number of events with
+# higher counts. It does so by using the adjusted count *c\**:
+#
+# - *c\* = (c + 1) N(c + 1) / N(c)* for c >= 1
+# - *things with frequency zero in training* = N(1) for c == 0
+#
+# where *c* is the original count, *N(i)* is the number of event types
+# observed with count *i*. We can think the count of unseen as the count
+# of frequency one (see Jurafsky & Martin 2nd Edition, p101).
+#
+# This method is problematic because the situation ``N(c+1) == 0``
+# is quite common in the original Good-Turing estimation; smoothing or
+# interpolation of *N(i)* values is essential in practice.
+#
+# Bill Gale and Geoffrey Sampson present a simple and effective approach,
+# Simple Good-Turing. As a smoothing curve they simply use a power curve:
+#
+# Nr = a*r^b (with b < -1 to give the appropriate hyperbolic
+# relationship)
+#
+# They estimate a and b by simple linear regression technique on the
+# logarithmic form of the equation:
+#
+# log Nr = a + b*log(r)
+#
+# However, they suggest that such a simple curve is probably only
+# appropriate for high values of r. For low values of r, they use the
+# measured Nr directly. (see M&S, p.213)
+#
+# Gale and Sampson propose to use r while the difference between r and
+# r* is 1.96 greater than the standard deviation, and switch to r* if
+# it is less or equal:
+#
+# |r - r*| > 1.96 * sqrt((r + 1)^2 (Nr+1 / Nr^2) (1 + Nr+1 / Nr))
+#
+# The 1.96 coefficient correspond to a 0.05 significance criterion,
+# some implementations can use a coefficient of 1.65 for a 0.1
+# significance criterion.
+#
+
+##//////////////////////////////////////////////////////
+## Simple Good-Turing Probablity Distributions
+##//////////////////////////////////////////////////////
+
+ at compat.python_2_unicode_compatible
+class SimpleGoodTuringProbDist(ProbDistI):
+ """
+ SimpleGoodTuring ProbDist approximates from frequency to frequency of
+ frequency into a linear line under log space by linear regression.
+ Details of Simple Good-Turing algorithm can be found in:
+
+ - Good Turing smoothing without tears" (Gale & Sampson 1995),
+ Journal of Quantitative Linguistics, vol. 2 pp. 217-237.
+ - "Speech and Language Processing (Jurafsky & Martin),
+ 2nd Edition, Chapter 4.5 p103 (log(Nc) = a + b*log(c))
+ - http://www.grsampson.net/RGoodTur.html
+
+ Given a set of pair (xi, yi), where the xi denotes the frequency and
+ yi denotes the frequency of frequency, we want to minimize their
+ square variation. E(x) and E(y) represent the mean of xi and yi.
+
+ - slope: b = sigma ((xi-E(x)(yi-E(y))) / sigma ((xi-E(x))(xi-E(x)))
+ - intercept: a = E(y) - b.E(x)
+ """
+ SUM_TO_ONE = False
+ def __init__(self, freqdist, bins=None):
+ """
+ :param freqdist: The frequency counts upon which to base the
+ estimation.
+ :type freqdist: FreqDist
+ :param bins: The number of possible event types. This must be
+ larger than the number of bins in the ``freqdist``. If None,
+ then it's assumed to be equal to ``freqdist``.B() + 1
+ :type bins: int
+ """
+ assert bins is None or bins > freqdist.B(),\
+ 'bins parameter must not be less than %d=freqdist.B()+1' % (freqdist.B()+1)
+ if bins is None:
+ bins = freqdist.B() + 1
+ self._freqdist = freqdist
+ self._bins = bins
+ r, nr = self._r_Nr()
+ self.find_best_fit(r, nr)
+ self._switch(r, nr)
+ self._renormalize(r, nr)
+
+ def _r_Nr_non_zero(self):
+ r_Nr = self._freqdist.r_Nr()
+ del r_Nr[0]
+ return r_Nr
+
+ def _r_Nr(self):
+ """
+ Split the frequency distribution in two list (r, Nr), where Nr(r) > 0
+ """
+ nonzero = self._r_Nr_non_zero()
+
+ if not nonzero:
+ return [], []
+ return zip(*sorted(nonzero.items()))
+
+ def find_best_fit(self, r, nr):
+ """
+ Use simple linear regression to tune parameters self._slope and
+ self._intercept in the log-log space based on count and Nr(count)
+ (Work in log space to avoid floating point underflow.)
+ """
+ # For higher sample frequencies the data points becomes horizontal
+ # along line Nr=1. To create a more evident linear model in log-log
+ # space, we average positive Nr values with the surrounding zero
+ # values. (Church and Gale, 1991)
+
+ if not r or not nr:
+ # Empty r or nr?
+ return
+
+ zr = []
+ for j in range(len(r)):
+ i = (r[j-1] if j > 0 else 0)
+ k = (2 * r[j] - i if j == len(r) - 1 else r[j+1])
+ zr_ = 2.0 * nr[j] / (k - i)
+ zr.append(zr_)
+
+ log_r = [math.log(i) for i in r]
+ log_zr = [math.log(i) for i in zr]
+
+ xy_cov = x_var = 0.0
+ x_mean = 1.0 * sum(log_r) / len(log_r)
+ y_mean = 1.0 * sum(log_zr) / len(log_zr)
+ for (x, y) in zip(log_r, log_zr):
+ xy_cov += (x - x_mean) * (y - y_mean)
+ x_var += (x - x_mean)**2
+ self._slope = (xy_cov / x_var if x_var != 0 else 0.0)
+ self._intercept = y_mean - self._slope * x_mean
+
+ def _switch(self, r, nr):
+ """
+ Calculate the r frontier where we must switch from Nr to Sr
+ when estimating E[Nr].
+ """
+ for i, r_ in enumerate(r):
+ if len(r) == i + 1 or r[i+1] != r_ + 1:
+ # We are at the end of r, or there is a gap in r
+ self._switch_at = r_
+ break
+
+ Sr = self.smoothedNr
+ smooth_r_star = (r_ + 1) * Sr(r_+1) / Sr(r_)
+ unsmooth_r_star = 1.0 * (r_ + 1) * nr[i+1] / nr[i]
+
+ std = math.sqrt(self._variance(r_, nr[i], nr[i+1]))
+ if abs(unsmooth_r_star-smooth_r_star) <= 1.96 * std:
+ self._switch_at = r_
+ break
+
+ def _variance(self, r, nr, nr_1):
+ r = float(r)
+ nr = float(nr)
+ nr_1 = float(nr_1)
+ return (r + 1.0)**2 * (nr_1 / nr**2) * (1.0 + nr_1 / nr)
+
+ def _renormalize(self, r, nr):
+ """
+ It is necessary to renormalize all the probability estimates to
+ ensure a proper probability distribution results. This can be done
+ by keeping the estimate of the probability mass for unseen items as
+ N(1)/N and renormalizing all the estimates for previously seen items
+ (as Gale and Sampson (1995) propose). (See M&S P.213, 1999)
+ """
+ prob_cov = 0.0
+ for r_, nr_ in zip(r, nr):
+ prob_cov += nr_ * self._prob_measure(r_)
+ if prob_cov:
+ self._renormal = (1 - self._prob_measure(0)) / prob_cov
+
+ def smoothedNr(self, r):
+ """
+ Return the number of samples with count r.
+
+ :param r: The amount of frequency.
+ :type r: int
+ :rtype: float
+ """
+
+ # Nr = a*r^b (with b < -1 to give the appropriate hyperbolic
+ # relationship)
+ # Estimate a and b by simple linear regression technique on
+ # the logarithmic form of the equation: log Nr = a + b*log(r)
+
+ return math.exp(self._intercept + self._slope * math.log(r))
+
+ def prob(self, sample):
+ """
+ Return the sample's probability.
+
+ :param sample: sample of the event
+ :type sample: str
+ :rtype: float
+ """
+ count = self._freqdist[sample]
+ p = self._prob_measure(count)
+ if count == 0:
+ if self._bins == self._freqdist.B():
+ p = 0.0
+ else:
+ p = p / (1.0 * self._bins - self._freqdist.B())
+ else:
+ p = p * self._renormal
+ return p
+
+ def _prob_measure(self, count):
+ if count == 0 and self._freqdist.N() == 0 :
+ return 1.0
+ elif count == 0 and self._freqdist.N() != 0:
+ return 1.0 * self._freqdist.Nr(1) / self._freqdist.N()
+
+ if self._switch_at > count:
+ Er_1 = 1.0 * self._freqdist.Nr(count+1)
+ Er = 1.0 * self._freqdist.Nr(count)
+ else:
+ Er_1 = self.smoothedNr(count+1)
+ Er = self.smoothedNr(count)
+
+ r_star = (count + 1) * Er_1 / Er
+ return r_star / self._freqdist.N()
+
+ def check(self):
+ prob_sum = 0.0
+ for i in range(0, len(self._Nr)):
+ prob_sum += self._Nr[i] * self._prob_measure(i) / self._renormal
+ print("Probability Sum:", prob_sum)
+ #assert prob_sum != 1.0, "probability sum should be one!"
+
+ def discount(self):
+ """
+ This function returns the total mass of probability transfers from the
+ seen samples to the unseen samples.
+ """
+ return 1.0 * self.smoothedNr(1) / self._freqdist.N()
+
+ def max(self):
+ return self._freqdist.max()
+
+ def samples(self):
+ return self._freqdist.keys()
+
+ def freqdist(self):
+ return self._freqdist
+
+ def __repr__(self):
+ """
+ Return a string representation of this ``ProbDist``.
+
+ :rtype: str
+ """
+ return '<SimpleGoodTuringProbDist based on %d samples>'\
+ % self._freqdist.N()
+
+
+class MutableProbDist(ProbDistI):
+ """
+ An mutable probdist where the probabilities may be easily modified. This
+ simply copies an existing probdist, storing the probability values in a
+ mutable dictionary and providing an update method.
+ """
+
+ def __init__(self, prob_dist, samples, store_logs=True):
+ """
+ Creates the mutable probdist based on the given prob_dist and using
+ the list of samples given. These values are stored as log
+ probabilities if the store_logs flag is set.
+
+ :param prob_dist: the distribution from which to garner the
+ probabilities
+ :type prob_dist: ProbDist
+ :param samples: the complete set of samples
+ :type samples: sequence of any
+ :param store_logs: whether to store the probabilities as logarithms
+ :type store_logs: bool
+ """
+ self._samples = samples
+ self._sample_dict = dict((samples[i], i) for i in range(len(samples)))
+ self._data = array.array(str("d"), [0.0]) * len(samples)
+ for i in range(len(samples)):
+ if store_logs:
+ self._data[i] = prob_dist.logprob(samples[i])
+ else:
+ self._data[i] = prob_dist.prob(samples[i])
+ self._logs = store_logs
+
+ def samples(self):
+ # inherit documentation
+ return self._samples
+
+ def prob(self, sample):
+ # inherit documentation
+ i = self._sample_dict.get(sample)
+ if i is None:
+ return 0.0
+ return (2**(self._data[i]) if self._logs else self._data[i])
+
+ def logprob(self, sample):
+ # inherit documentation
+ i = self._sample_dict.get(sample)
+ if i is None:
+ return float('-inf')
+ return (self._data[i] if self._logs else math.log(self._data[i], 2))
+
+ def update(self, sample, prob, log=True):
+ """
+ Update the probability for the given sample. This may cause the object
+ to stop being the valid probability distribution - the user must
+ ensure that they update the sample probabilities such that all samples
+ have probabilities between 0 and 1 and that all probabilities sum to
+ one.
+
+ :param sample: the sample for which to update the probability
+ :type sample: any
+ :param prob: the new probability
+ :type prob: float
+ :param log: is the probability already logged
+ :type log: bool
+ """
+ i = self._sample_dict.get(sample)
+ assert i is not None
+ if self._logs:
+ self._data[i] = (prob if log else math.log(prob, 2))
+ else:
+ self._data[i] = (2**(prob) if log else prob)
+
+##/////////////////////////////////////////////////////
+## Kneser-Ney Probability Distribution
+##//////////////////////////////////////////////////////
+
+# This method for calculating probabilities was introduced in 1995 by Reinhard
+# Kneser and Hermann Ney. It was meant to improve the accuracy of language
+# models that use backing-off to deal with sparse data. The authors propose two
+# ways of doing so: a marginal distribution constraint on the back-off
+# distribution and a leave-one-out distribution. For a start, the first one is
+# implemented as a class below.
+#
+# The idea behind a back-off n-gram model is that we have a series of
+# frequency distributions for our n-grams so that in case we have not seen a
+# given n-gram during training (and as a result have a 0 probability for it) we
+# can 'back off' (hence the name!) and try testing whether we've seen the
+# n-1-gram part of the n-gram in training.
+#
+# The novelty of Kneser and Ney's approach was that they decided to fiddle
+# around with the way this latter, backed off probability was being calculated
+# whereas their peers seemed to focus on the primary probability.
+#
+# The implementation below uses one of the techniques described in their paper
+# titled "Improved backing-off for n-gram language modeling." In the same paper
+# another technique is introduced to attempt to smooth the back-off
+# distribution as well as the primary one. There is also a much-cited
+# modification of this method proposed by Chen and Goodman.
+#
+# In order for the implementation of Kneser-Ney to be more efficient, some
+# changes have been made to the original algorithm. Namely, the calculation of
+# the normalizing function gamma has been significantly simplified and
+# combined slightly differently with beta. None of these changes affect the
+# nature of the algorithm, but instead aim to cut out unnecessary calculations
+# and take advantage of storing and retrieving information in dictionaries
+# where possible.
+
+ at compat.python_2_unicode_compatible
+class KneserNeyProbDist(ProbDistI):
+ """
+ Kneser-Ney estimate of a probability distribution. This is a version of
+ back-off that counts how likely an n-gram is provided the n-1-gram had
+ been seen in training. Extends the ProbDistI interface, requires a trigram
+ FreqDist instance to train on. Optionally, a different from default discount
+ value can be specified. The default discount is set to 0.75.
+
+ """
+ def __init__(self, freqdist, bins=None, discount=0.75):
+ """
+ :param trigrams: The trigram frequency distribution upon which to base
+ the estimation
+ :type trigrams: FreqDist
+ :param bins: Included for compatibility with nltk.tag.hmm
+ :type bins: int or float
+ :param discount: The discount applied when retrieving counts of
+ trigrams
+ :type discount: float (preferred, but can be set to int)
+ """
+
+ if not bins:
+ self._bins = freqdist.B()
+ else:
+ self._bins = bins
+ self._D = discount
+
+ # cache for probability calculation
+ self._cache = {}
+
+ # internal bigram and trigram frequency distributions
+ self._bigrams = defaultdict(int)
+ self._trigrams = freqdist
+
+ # helper dictionaries used to calculate probabilities
+ self._wordtypes_after = defaultdict(float)
+ self._trigrams_contain = defaultdict(float)
+ self._wordtypes_before = defaultdict(float)
+ for w0, w1, w2 in freqdist:
+ self._bigrams[(w0,w1)] += freqdist[(w0, w1, w2)]
+ self._wordtypes_after[(w0,w1)] += 1
+ self._trigrams_contain[w1] += 1
+ self._wordtypes_before[(w1,w2)] += 1
+
+ def prob(self, trigram):
+ # sample must be a triple
+ if len(trigram) != 3:
+ raise ValueError('Expected an iterable with 3 members.')
+ trigram = tuple(trigram)
+ w0, w1, w2 = trigram
+
+ if trigram in self._cache:
+ return self._cache[trigram]
+ else:
+ # if the sample trigram was seen during training
+ if trigram in self._trigrams:
+ prob = (self._trigrams[trigram]
+ - self.discount())/self._bigrams[(w0, w1)]
+
+ # else if the 'rougher' environment was seen during training
+ elif (w0,w1) in self._bigrams and (w1,w2) in self._wordtypes_before:
+ aftr = self._wordtypes_after[(w0, w1)]
+ bfr = self._wordtypes_before[(w1, w2)]
+
+ # the probability left over from alphas
+ leftover_prob = ((aftr * self.discount())
+ / self._bigrams[(w0, w1)])
+
+ # the beta (including normalization)
+ beta = bfr /(self._trigrams_contain[w1] - aftr)
+
+ prob = leftover_prob * beta
+
+ # else the sample was completely unseen during training
+ else:
+ prob = 0.0
+
+ self._cache[trigram] = prob
+ return prob
+
+ def discount(self):
+ """
+ Return the value by which counts are discounted. By default set to 0.75.
+
+ :rtype: float
+ """
+ return self._D
+
+ def set_discount(self, discount):
+ """
+ Set the value by which counts are discounted to the value of discount.
+
+ :param discount: the new value to discount counts by
+ :type discount: float (preferred, but int possible)
+ :rtype: None
+ """
+ self._D = discount
+
+ def samples(self):
+ return self._trigrams.keys()
+
+ def max(self):
+ return self._trigrams.max()
+
+ def __repr__(self):
+ '''
+ Return a string representation of this ProbDist
+
+ :rtype: str
+ '''
+ return '<KneserNeyProbDist based on {0} trigrams'.format(self._trigrams.N())
+
+##//////////////////////////////////////////////////////
+## Probability Distribution Operations
+##//////////////////////////////////////////////////////
+
+def log_likelihood(test_pdist, actual_pdist):
+ if (not isinstance(test_pdist, ProbDistI) or
+ not isinstance(actual_pdist, ProbDistI)):
+ raise ValueError('expected a ProbDist.')
+ # Is this right?
+ return sum(actual_pdist.prob(s) * math.log(test_pdist.prob(s), 2)
+ for s in actual_pdist)
+
+def entropy(pdist):
+ probs = (pdist.prob(s) for s in pdist.samples())
+ return -sum(p * math.log(p,2) for p in probs)
+
+##//////////////////////////////////////////////////////
+## Conditional Distributions
+##//////////////////////////////////////////////////////
+
+ at compat.python_2_unicode_compatible
+class ConditionalFreqDist(defaultdict):
+ """
+ A collection of frequency distributions for a single experiment
+ run under different conditions. Conditional frequency
+ distributions are used to record the number of times each sample
+ occurred, given the condition under which the experiment was run.
+ For example, a conditional frequency distribution could be used to
+ record the frequency of each word (type) in a document, given its
+ length. Formally, a conditional frequency distribution can be
+ defined as a function that maps from each condition to the
+ FreqDist for the experiment under that condition.
+
+ Conditional frequency distributions are typically constructed by
+ repeatedly running an experiment under a variety of conditions,
+ and incrementing the sample outcome counts for the appropriate
+ conditions. For example, the following code will produce a
+ conditional frequency distribution that encodes how often each
+ word type occurs, given the length of that word type:
+
+ >>> from nltk.probability import ConditionalFreqDist
+ >>> from nltk.tokenize import word_tokenize
+ >>> sent = "the the the dog dog some other words that we do not care about"
+ >>> cfdist = ConditionalFreqDist()
+ >>> for word in word_tokenize(sent):
+ ... condition = len(word)
+ ... cfdist[condition][word] += 1
+
+ An equivalent way to do this is with the initializer:
+
+ >>> cfdist = ConditionalFreqDist((len(word), word) for word in word_tokenize(sent))
+
+ The frequency distribution for each condition is accessed using
+ the indexing operator:
+
+ >>> cfdist[3]
+ <FreqDist with 3 samples and 6 outcomes>
+ >>> cfdist[3].freq('the')
+ 0.5
+ >>> cfdist[3]['dog']
+ 2
+
+ When the indexing operator is used to access the frequency
+ distribution for a condition that has not been accessed before,
+ ``ConditionalFreqDist`` creates a new empty FreqDist for that
+ condition.
+
+ """
+ def __init__(self, cond_samples=None):
+ """
+ Construct a new empty conditional frequency distribution. In
+ particular, the count for every sample, under every condition,
+ is zero.
+
+ :param cond_samples: The samples to initialize the conditional
+ frequency distribution with
+ :type cond_samples: Sequence of (condition, sample) tuples
+ """
+ defaultdict.__init__(self, FreqDist)
+ if cond_samples:
+ for (cond, sample) in cond_samples:
+ self[cond][sample] += 1
+
+ def __reduce__(self):
+ kv_pairs = ((cond, self[cond]) for cond in self.conditions())
+ return (self.__class__, (), None, None, kv_pairs)
+
+ def conditions(self):
+ """
+ Return a list of the conditions that have been accessed for
+ this ``ConditionalFreqDist``. Use the indexing operator to
+ access the frequency distribution for a given condition.
+ Note that the frequency distributions for some conditions
+ may contain zero sample outcomes.
+
+ :rtype: list
+ """
+ return list(self.keys())
+
+ def N(self):
+ """
+ Return the total number of sample outcomes that have been
+ recorded by this ``ConditionalFreqDist``.
+
+ :rtype: int
+ """
+ return sum(fdist.N() for fdist in compat.itervalues(self))
+
+ def plot(self, *args, **kwargs):
+ """
+ Plot the given samples from the conditional frequency distribution.
+ For a cumulative plot, specify cumulative=True.
+ (Requires Matplotlib to be installed.)
+
+ :param samples: The samples to plot
+ :type samples: list
+ :param title: The title for the graph
+ :type title: str
+ :param conditions: The conditions to plot (default is all)
+ :type conditions: list
+ """
+ try:
+ import pylab
+ except ImportError:
+ raise ValueError('The plot function requires the matplotlib package (aka pylab).'
+ 'See http://matplotlib.sourceforge.net/')
+
+ cumulative = _get_kwarg(kwargs, 'cumulative', False)
+ conditions = _get_kwarg(kwargs, 'conditions', sorted(self.conditions()))
+ title = _get_kwarg(kwargs, 'title', '')
+ samples = _get_kwarg(kwargs, 'samples',
+ sorted(set(v for c in conditions for v in self[c]))) # this computation could be wasted
+ if not "linewidth" in kwargs:
+ kwargs["linewidth"] = 2
+
+ for condition in conditions:
+ if cumulative:
+ freqs = list(self[condition]._cumulative_frequencies(samples))
+ ylabel = "Cumulative Counts"
+ legend_loc = 'lower right'
+ else:
+ freqs = [self[condition][sample] for sample in samples]
+ ylabel = "Counts"
+ legend_loc = 'upper right'
+ # percents = [f * 100 for f in freqs] only in ConditionalProbDist?
+ kwargs['label'] = "%s" % condition
+ pylab.plot(freqs, *args, **kwargs)
+
+ pylab.legend(loc=legend_loc)
+ pylab.grid(True, color="silver")
+ pylab.xticks(range(len(samples)), [compat.text_type(s) for s in samples], rotation=90)
+ if title:
+ pylab.title(title)
+ pylab.xlabel("Samples")
+ pylab.ylabel(ylabel)
+ pylab.show()
+
+ def tabulate(self, *args, **kwargs):
+ """
+ Tabulate the given samples from the conditional frequency distribution.
+
+ :param samples: The samples to plot
+ :type samples: list
+ :param title: The title for the graph
+ :type title: str
+ :param conditions: The conditions to plot (default is all)
+ :type conditions: list
+ """
+
+ cumulative = _get_kwarg(kwargs, 'cumulative', False)
+ conditions = _get_kwarg(kwargs, 'conditions', sorted(self.conditions()))
+ samples = _get_kwarg(kwargs, 'samples',
+ sorted(set(v for c in conditions for v in self[c]))) # this computation could be wasted
+
+ condition_size = max(len("%s" % c) for c in conditions)
+ print(' ' * condition_size, end=' ')
+ for s in samples:
+ print("%4s" % s, end=' ')
+ print()
+ for c in conditions:
+ print("%*s" % (condition_size, c), end=' ')
+ if cumulative:
+ freqs = list(self[c]._cumulative_frequencies(samples))
+ else:
+ freqs = [self[c][sample] for sample in samples]
+
+ for f in freqs:
+ print("%4d" % f, end=' ')
+ print()
+
+ # @total_ordering doesn't work here, since the class inherits from a builtin class
+ def __le__(self, other):
+ if not isinstance(other, ConditionalFreqDist):
+ raise_unorderable_types("<=", self, other)
+ return set(self.conditions()).issubset(other.conditions()) \
+ and all(self[c] <= other[c] for c in self.conditions())
+ def __lt__(self, other):
+ if not isinstance(other, ConditionalFreqDist):
+ raise_unorderable_types("<", self, other)
+ return self <= other and self != other
+ def __ge__(self, other):
+ if not isinstance(other, ConditionalFreqDist):
+ raise_unorderable_types(">=", self, other)
+ return other <= self
+ def __gt__(self, other):
+ if not isinstance(other, ConditionalFreqDist):
+ raise_unorderable_types(">", self, other)
+ return other < self
+
+ def __repr__(self):
+ """
+ Return a string representation of this ``ConditionalFreqDist``.
+
+ :rtype: str
+ """
+ return '<ConditionalFreqDist with %d conditions>' % len(self)
+
+
+ at compat.python_2_unicode_compatible
+class ConditionalProbDistI(dict):
+ """
+ A collection of probability distributions for a single experiment
+ run under different conditions. Conditional probability
+ distributions are used to estimate the likelihood of each sample,
+ given the condition under which the experiment was run. For
+ example, a conditional probability distribution could be used to
+ estimate the probability of each word type in a document, given
+ the length of the word type. Formally, a conditional probability
+ distribution can be defined as a function that maps from each
+ condition to the ``ProbDist`` for the experiment under that
+ condition.
+ """
+ def __init__(self):
+ raise NotImplementedError("Interfaces can't be instantiated")
+
+ def conditions(self):
+ """
+ Return a list of the conditions that are represented by
+ this ``ConditionalProbDist``. Use the indexing operator to
+ access the probability distribution for a given condition.
+
+ :rtype: list
+ """
+ return list(self.keys())
+
+ def __repr__(self):
+ """
+ Return a string representation of this ``ConditionalProbDist``.
+
+ :rtype: str
+ """
+ return '<%s with %d conditions>' % (type(self).__name__, len(self))
+
+
+class ConditionalProbDist(ConditionalProbDistI):
+ """
+ A conditional probability distribution modeling the experiments
+ that were used to generate a conditional frequency distribution.
+ A ConditionalProbDist is constructed from a
+ ``ConditionalFreqDist`` and a ``ProbDist`` factory:
+
+ - The ``ConditionalFreqDist`` specifies the frequency
+ distribution for each condition.
+ - The ``ProbDist`` factory is a function that takes a
+ condition's frequency distribution, and returns its
+ probability distribution. A ``ProbDist`` class's name (such as
+ ``MLEProbDist`` or ``HeldoutProbDist``) can be used to specify
+ that class's constructor.
+
+ The first argument to the ``ProbDist`` factory is the frequency
+ distribution that it should model; and the remaining arguments are
+ specified by the ``factory_args`` parameter to the
+ ``ConditionalProbDist`` constructor. For example, the following
+ code constructs a ``ConditionalProbDist``, where the probability
+ distribution for each condition is an ``ELEProbDist`` with 10 bins:
+
+ >>> from nltk.corpus import brown
+ >>> from nltk.probability import ConditionalFreqDist
+ >>> from nltk.probability import ConditionalProbDist, ELEProbDist
+ >>> cfdist = ConditionalFreqDist(brown.tagged_words()[:5000])
+ >>> cpdist = ConditionalProbDist(cfdist, ELEProbDist, 10)
+ >>> cpdist['passed'].max()
+ 'VBD'
+ >>> cpdist['passed'].prob('VBD')
+ 0.423...
+
+ """
+ def __init__(self, cfdist, probdist_factory,
+ *factory_args, **factory_kw_args):
+ """
+ Construct a new conditional probability distribution, based on
+ the given conditional frequency distribution and ``ProbDist``
+ factory.
+
+ :type cfdist: ConditionalFreqDist
+ :param cfdist: The ``ConditionalFreqDist`` specifying the
+ frequency distribution for each condition.
+ :type probdist_factory: class or function
+ :param probdist_factory: The function or class that maps
+ a condition's frequency distribution to its probability
+ distribution. The function is called with the frequency
+ distribution as its first argument,
+ ``factory_args`` as its remaining arguments, and
+ ``factory_kw_args`` as keyword arguments.
+ :type factory_args: (any)
+ :param factory_args: Extra arguments for ``probdist_factory``.
+ These arguments are usually used to specify extra
+ properties for the probability distributions of individual
+ conditions, such as the number of bins they contain.
+ :type factory_kw_args: (any)
+ :param factory_kw_args: Extra keyword arguments for ``probdist_factory``.
+ """
+ self._probdist_factory = probdist_factory
+ self._factory_args = factory_args
+ self._factory_kw_args = factory_kw_args
+
+ for condition in cfdist:
+ self[condition] = probdist_factory(cfdist[condition],
+ *factory_args, **factory_kw_args)
+
+ def __missing__(self, key):
+ self[key] = self._probdist_factory(FreqDist(),
+ *self._factory_args,
+ **self._factory_kw_args)
+ return self[key]
+
+class DictionaryConditionalProbDist(ConditionalProbDistI):
+ """
+ An alternative ConditionalProbDist that simply wraps a dictionary of
+ ProbDists rather than creating these from FreqDists.
+ """
+
+ def __init__(self, probdist_dict):
+ """
+ :param probdist_dict: a dictionary containing the probdists indexed
+ by the conditions
+ :type probdist_dict: dict any -> probdist
+ """
+ self.update(probdist_dict)
+
+ def __missing__(self, key):
+ self[key] = DictionaryProbDist()
+ return self[key]
+
+##//////////////////////////////////////////////////////
+## Adding in log-space.
+##//////////////////////////////////////////////////////
+
+# If the difference is bigger than this, then just take the bigger one:
+_ADD_LOGS_MAX_DIFF = math.log(1e-30, 2)
+
+def add_logs(logx, logy):
+ """
+ Given two numbers ``logx`` = *log(x)* and ``logy`` = *log(y)*, return
+ *log(x+y)*. Conceptually, this is the same as returning
+ ``log(2**(logx)+2**(logy))``, but the actual implementation
+ avoids overflow errors that could result from direct computation.
+ """
+ if (logx < logy + _ADD_LOGS_MAX_DIFF):
+ return logy
+ if (logy < logx + _ADD_LOGS_MAX_DIFF):
+ return logx
+ base = min(logx, logy)
+ return base + math.log(2**(logx-base) + 2**(logy-base), 2)
+
+def sum_logs(logs):
+ return (reduce(add_logs, logs[1:], logs[0]) if len(logs) != 0 else _NINF)
+
+##//////////////////////////////////////////////////////
+## Probabilistic Mix-in
+##//////////////////////////////////////////////////////
+
+class ProbabilisticMixIn(object):
+ """
+ A mix-in class to associate probabilities with other classes
+ (trees, rules, etc.). To use the ``ProbabilisticMixIn`` class,
+ define a new class that derives from an existing class and from
+ ProbabilisticMixIn. You will need to define a new constructor for
+ the new class, which explicitly calls the constructors of both its
+ parent classes. For example:
+
+ >>> from nltk.probability import ProbabilisticMixIn
+ >>> class A:
+ ... def __init__(self, x, y): self.data = (x,y)
+ ...
+ >>> class ProbabilisticA(A, ProbabilisticMixIn):
+ ... def __init__(self, x, y, **prob_kwarg):
+ ... A.__init__(self, x, y)
+ ... ProbabilisticMixIn.__init__(self, **prob_kwarg)
+
+ See the documentation for the ProbabilisticMixIn
+ ``constructor<__init__>`` for information about the arguments it
+ expects.
+
+ You should generally also redefine the string representation
+ methods, the comparison methods, and the hashing method.
+ """
+ def __init__(self, **kwargs):
+ """
+ Initialize this object's probability. This initializer should
+ be called by subclass constructors. ``prob`` should generally be
+ the first argument for those constructors.
+
+ :param prob: The probability associated with the object.
+ :type prob: float
+ :param logprob: The log of the probability associated with
+ the object.
+ :type logprob: float
+ """
+ if 'prob' in kwargs:
+ if 'logprob' in kwargs:
+ raise TypeError('Must specify either prob or logprob '
+ '(not both)')
+ else:
+ ProbabilisticMixIn.set_prob(self, kwargs['prob'])
+ elif 'logprob' in kwargs:
+ ProbabilisticMixIn.set_logprob(self, kwargs['logprob'])
+ else:
+ self.__prob = self.__logprob = None
+
+ def set_prob(self, prob):
+ """
+ Set the probability associated with this object to ``prob``.
+
+ :param prob: The new probability
+ :type prob: float
+ """
+ self.__prob = prob
+ self.__logprob = None
+
+ def set_logprob(self, logprob):
+ """
+ Set the log probability associated with this object to
+ ``logprob``. I.e., set the probability associated with this
+ object to ``2**(logprob)``.
+
+ :param logprob: The new log probability
+ :type logprob: float
+ """
+ self.__logprob = logprob
+ self.__prob = None
+
+ def prob(self):
+ """
+ Return the probability associated with this object.
+
+ :rtype: float
+ """
+ if self.__prob is None:
+ if self.__logprob is None: return None
+ self.__prob = 2**(self.__logprob)
+ return self.__prob
+
+ def logprob(self):
+ """
+ Return ``log(p)``, where ``p`` is the probability associated
+ with this object.
+
+ :rtype: float
+ """
+ if self.__logprob is None:
+ if self.__prob is None: return None
+ self.__logprob = math.log(self.__prob, 2)
+ return self.__logprob
+
+class ImmutableProbabilisticMixIn(ProbabilisticMixIn):
+ def set_prob(self, prob):
+ raise ValueError('%s is immutable' % self.__class__.__name__)
+ def set_logprob(self, prob):
+ raise ValueError('%s is immutable' % self.__class__.__name__)
+
+## Helper function for processing keyword arguments
+
+def _get_kwarg(kwargs, key, default):
+ if key in kwargs:
+ arg = kwargs[key]
+ del kwargs[key]
+ else:
+ arg = default
+ return arg
+
+##//////////////////////////////////////////////////////
+## Demonstration
+##//////////////////////////////////////////////////////
+
+def _create_rand_fdist(numsamples, numoutcomes):
+ """
+ Create a new frequency distribution, with random samples. The
+ samples are numbers from 1 to ``numsamples``, and are generated by
+ summing two numbers, each of which has a uniform distribution.
+ """
+ import random
+ fdist = FreqDist()
+ for x in range(numoutcomes):
+ y = (random.randint(1, (1 + numsamples) // 2) +
+ random.randint(0, numsamples // 2))
+ fdist[y] += 1
+ return fdist
+
+def _create_sum_pdist(numsamples):
+ """
+ Return the true probability distribution for the experiment
+ ``_create_rand_fdist(numsamples, x)``.
+ """
+ fdist = FreqDist()
+ for x in range(1, (1 + numsamples) // 2 + 1):
+ for y in range(0, numsamples // 2 + 1):
+ fdist[x+y] += 1
+ return MLEProbDist(fdist)
+
+def demo(numsamples=6, numoutcomes=500):
+ """
+ A demonstration of frequency distributions and probability
+ distributions. This demonstration creates three frequency
+ distributions with, and uses them to sample a random process with
+ ``numsamples`` samples. Each frequency distribution is sampled
+ ``numoutcomes`` times. These three frequency distributions are
+ then used to build six probability distributions. Finally, the
+ probability estimates of these distributions are compared to the
+ actual probability of each sample.
+
+ :type numsamples: int
+ :param numsamples: The number of samples to use in each demo
+ frequency distributions.
+ :type numoutcomes: int
+ :param numoutcomes: The total number of outcomes for each
+ demo frequency distribution. These outcomes are divided into
+ ``numsamples`` bins.
+ :rtype: None
+ """
+
+ # Randomly sample a stochastic process three times.
+ fdist1 = _create_rand_fdist(numsamples, numoutcomes)
+ fdist2 = _create_rand_fdist(numsamples, numoutcomes)
+ fdist3 = _create_rand_fdist(numsamples, numoutcomes)
+
+ # Use our samples to create probability distributions.
+ pdists = [
+ MLEProbDist(fdist1),
+ LidstoneProbDist(fdist1, 0.5, numsamples),
+ HeldoutProbDist(fdist1, fdist2, numsamples),
+ HeldoutProbDist(fdist2, fdist1, numsamples),
+ CrossValidationProbDist([fdist1, fdist2, fdist3], numsamples),
+ SimpleGoodTuringProbDist(fdist1),
+ SimpleGoodTuringProbDist(fdist1, 7),
+ _create_sum_pdist(numsamples),
+ ]
+
+ # Find the probability of each sample.
+ vals = []
+ for n in range(1,numsamples+1):
+ vals.append(tuple([n, fdist1.freq(n)] +
+ [pdist.prob(n) for pdist in pdists]))
+
+ # Print the results in a formatted table.
+ print(('%d samples (1-%d); %d outcomes were sampled for each FreqDist' %
+ (numsamples, numsamples, numoutcomes)))
+ print('='*9*(len(pdists)+2))
+ FORMATSTR = ' FreqDist '+ '%8s '*(len(pdists)-1) + '| Actual'
+ print(FORMATSTR % tuple(repr(pdist)[1:9] for pdist in pdists[:-1]))
+ print('-'*9*(len(pdists)+2))
+ FORMATSTR = '%3d %8.6f ' + '%8.6f '*(len(pdists)-1) + '| %8.6f'
+ for val in vals:
+ print(FORMATSTR % val)
+
+ # Print the totals for each column (should all be 1.0)
+ zvals = list(zip(*vals))
+ sums = [sum(val) for val in zvals[1:]]
+ print('-'*9*(len(pdists)+2))
+ FORMATSTR = 'Total ' + '%8.6f '*(len(pdists)) + '| %8.6f'
+ print(FORMATSTR % tuple(sums))
+ print('='*9*(len(pdists)+2))
+
+ # Display the distributions themselves, if they're short enough.
+ if len("%s" % fdist1) < 70:
+ print(' fdist1: %s' % fdist1)
+ print(' fdist2: %s' % fdist2)
+ print(' fdist3: %s' % fdist3)
+ print()
+
+ print('Generating:')
+ for pdist in pdists:
+ fdist = FreqDist(pdist.generate() for i in range(5000))
+ print('%20s %s' % (pdist.__class__.__name__[:20], ("%s" % fdist)[:55]))
+ print()
+
+def gt_demo():
+ from nltk import corpus
+ emma_words = corpus.gutenberg.words('austen-emma.txt')
+ fd = FreqDist(emma_words)
+ sgt = SimpleGoodTuringProbDist(fd)
+ print('%18s %8s %14s' \
+ % ("word", "freqency", "SimpleGoodTuring"))
+ fd_keys_sorted=(key for key, value in sorted(fd.items(), key=lambda item: item[1], reverse=True))
+ for key in fd_keys_sorted:
+ print('%18s %8d %14e' \
+ % (key, fd[key], sgt.prob(key)))
+
+if __name__ == '__main__':
+ demo(6, 10)
+ demo(5, 5000)
+ gt_demo()
+
+__all__ = ['ConditionalFreqDist', 'ConditionalProbDist',
+ 'ConditionalProbDistI', 'CrossValidationProbDist',
+ 'DictionaryConditionalProbDist', 'DictionaryProbDist', 'ELEProbDist',
+ 'FreqDist', 'SimpleGoodTuringProbDist', 'HeldoutProbDist',
+ 'ImmutableProbabilisticMixIn', 'LaplaceProbDist', 'LidstoneProbDist',
+ 'MLEProbDist', 'MutableProbDist', 'KneserNeyProbDist', 'ProbDistI', 'ProbabilisticMixIn',
+ 'UniformProbDist', 'WittenBellProbDist', 'add_logs',
+ 'log_likelihood', 'sum_logs', 'entropy']
diff --git a/nltk/sem/__init__.py b/nltk/sem/__init__.py
new file mode 100644
index 0000000..a5c2757
--- /dev/null
+++ b/nltk/sem/__init__.py
@@ -0,0 +1,61 @@
+# Natural Language Toolkit: Semantic Interpretation
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+NLTK Semantic Interpretation Package
+
+This package contains classes for representing semantic structure in
+formulas of first-order logic and for evaluating such formulas in
+set-theoretic models.
+
+ >>> from nltk.sem import logic
+ >>> logic._counter._value = 0
+
+The package has two main components:
+
+ - ``logic`` provides support for analyzing expressions of First
+ Order Logic (FOL).
+ - ``evaluate`` allows users to recursively determine truth in a
+ model for formulas of FOL.
+
+A model consists of a domain of discourse and a valuation function,
+which assigns values to non-logical constants. We assume that entities
+in the domain are represented as strings such as ``'b1'``, ``'g1'``,
+etc. A ``Valuation`` is initialized with a list of (symbol, value)
+pairs, where values are entities, sets of entities or sets of tuples
+of entities.
+The domain of discourse can be inferred from the valuation, and model
+is then created with domain and valuation as parameters.
+
+ >>> from nltk.sem import Valuation, Model
+ >>> v = [('adam', 'b1'), ('betty', 'g1'), ('fido', 'd1'),
+ ... ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])),
+ ... ('dog', set(['d1'])),
+ ... ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]))]
+ >>> val = Valuation(v)
+ >>> dom = val.domain
+ >>> m = Model(dom, val)
+"""
+
+from nltk.sem.util import (parse_sents, interpret_sents, evaluate_sents,
+ root_semrep, parse_valuation)
+from nltk.sem.evaluate import (Valuation, Assignment, Model, Undefined,
+ is_rel, set2rel, arity)
+from nltk.sem.logic import (boolean_ops, binding_ops, equality_preds,
+ parse_logic, Variable, Expression,
+ ApplicationExpression, LogicalExpressionException)
+from nltk.sem.skolemize import skolemize
+from nltk.sem.lfg import FStructure
+from nltk.sem.relextract import (extract_rels, rtuple, clause)
+from nltk.sem.boxer import Boxer
+from nltk.sem.drt import DrtExpression, DRS
+
+# from nltk.sem.glue import Glue
+# from nltk.sem.hole import HoleSemantics
+# from nltk.sem.cooper_storage import CooperStore
+
+# don't import chat80 as its names are too generic
diff --git a/nltk/sem/boxer.py b/nltk/sem/boxer.py
new file mode 100644
index 0000000..e13a73e
--- /dev/null
+++ b/nltk/sem/boxer.py
@@ -0,0 +1,1226 @@
+# Natural Language Toolkit: Interface to Boxer
+# <http://svn.ask.it.usyd.edu.au/trac/candc/wiki/boxer>
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+An interface to Boxer.
+
+This interface relies on the latest version of the development (subversion) version of
+C&C and Boxer.
+
+Usage:
+ Set the environment variable CANDCHOME to the bin directory of your CandC installation.
+ The models directory should be in the CandC root directory.
+ For example:
+ /path/to/candc/
+ bin/
+ candc
+ boxer
+ models/
+ boxer/
+"""
+from __future__ import print_function, unicode_literals
+
+import os
+import re
+import operator
+import subprocess
+from optparse import OptionParser
+import tempfile
+from functools import reduce
+
+from nltk.internals import Counter, find_binary
+
+from nltk.sem.logic import (ExpectedMoreTokensException, LogicalExpressionException,
+ UnexpectedTokenException, Variable)
+
+from nltk.sem.drt import (DRS, DrtApplicationExpression, DrtEqualityExpression,
+ DrtNegatedExpression, DrtOrExpression, _DrtParser,
+ DrtProposition, DrtTokens, DrtVariableExpression)
+
+from nltk.compat import python_2_unicode_compatible
+
+class Boxer(object):
+ """
+ This class is an interface to Johan Bos's program Boxer, a wide-coverage
+ semantic parser that produces Discourse Representation Structures (DRSs).
+ """
+
+ def __init__(self, boxer_drs_interpreter=None, elimeq=False, bin_dir=None, verbose=False):
+ """
+ :param boxer_drs_interpreter: A class that converts from the
+ ``AbstractBoxerDrs`` object hierarchy to a different object. The
+ default is ``NltkDrtBoxerDrsInterpreter``, which converts to the NLTK
+ DRT hierarchy.
+ :param elimeq: When set to true, Boxer removes all equalities from the
+ DRSs and discourse referents standing in the equality relation are
+ unified, but only if this can be done in a meaning-preserving manner.
+ """
+ if boxer_drs_interpreter is None:
+ boxer_drs_interpreter = NltkDrtBoxerDrsInterpreter()
+ self._boxer_drs_interpreter = boxer_drs_interpreter
+
+ self._elimeq = elimeq
+
+ self.set_bin_dir(bin_dir, verbose)
+
+ def set_bin_dir(self, bin_dir, verbose=False):
+ self._candc_bin = self._find_binary('candc', bin_dir, verbose)
+ self._candc_models_path = os.path.normpath(os.path.join(self._candc_bin[:-5], '../models'))
+ self._boxer_bin = self._find_binary('boxer', bin_dir, verbose)
+
+ def interpret(self, input, discourse_id=None, question=False, verbose=False):
+ """
+ Use Boxer to give a first order representation.
+
+ :param input: str Input sentence to parse
+ :param occur_index: bool Should predicates be occurrence indexed?
+ :param discourse_id: str An identifier to be inserted to each occurrence-indexed predicate.
+ :return: ``drt.AbstractDrs``
+ """
+ discourse_ids = ([discourse_id] if discourse_id is not None else None)
+ d, = self.interpret_multi_sents([[input]], discourse_ids, question, verbose)
+ if not d:
+ raise Exception('Unable to interpret: "%s"' % input)
+ return d
+
+ def interpret_multi(self, input, discourse_id=None, question=False, verbose=False):
+ """
+ Use Boxer to give a first order representation.
+
+ :param input: list of str Input sentences to parse as a single discourse
+ :param occur_index: bool Should predicates be occurrence indexed?
+ :param discourse_id: str An identifier to be inserted to each occurrence-indexed predicate.
+ :return: ``drt.AbstractDrs``
+ """
+ discourse_ids = ([discourse_id] if discourse_id is not None else None)
+ d, = self.interpret_multi_sents([input], discourse_ids, question, verbose)
+ if not d:
+ raise Exception('Unable to interpret: "%s"' % input)
+ return d
+
+ def interpret_sents(self, inputs, discourse_ids=None, question=False, verbose=False):
+ """
+ Use Boxer to give a first order representation.
+
+ :param inputs: list of str Input sentences to parse as individual discourses
+ :param occur_index: bool Should predicates be occurrence indexed?
+ :param discourse_ids: list of str Identifiers to be inserted to each occurrence-indexed predicate.
+ :return: list of ``drt.AbstractDrs``
+ """
+ return self.interpret_multi_sents([[input] for input in inputs], discourse_ids, question, verbose)
+
+ def interpret_multi_sents(self, inputs, discourse_ids=None, question=False, verbose=False):
+ """
+ Use Boxer to give a first order representation.
+
+ :param inputs: list of list of str Input discourses to parse
+ :param occur_index: bool Should predicates be occurrence indexed?
+ :param discourse_ids: list of str Identifiers to be inserted to each occurrence-indexed predicate.
+ :return: ``drt.AbstractDrs``
+ """
+ if discourse_ids is not None:
+ assert len(inputs) == len(discourse_ids)
+ assert reduce(operator.and_, (id is not None for id in discourse_ids))
+ use_disc_id = True
+ else:
+ discourse_ids = list(map(str, range(len(inputs))))
+ use_disc_id = False
+
+ candc_out = self._call_candc(inputs, discourse_ids, question, verbose=verbose)
+ boxer_out = self._call_boxer(candc_out, verbose=verbose)
+
+# if 'ERROR: input file contains no ccg/2 terms.' in boxer_out:
+# raise UnparseableInputException('Could not parse with candc: "%s"' % input_str)
+
+ drs_dict = self._parse_to_drs_dict(boxer_out, use_disc_id)
+ return [drs_dict.get(id, None) for id in discourse_ids]
+
+ def _call_candc(self, inputs, discourse_ids, question, verbose=False):
+ """
+ Call the ``candc`` binary with the given input.
+
+ :param inputs: list of list of str Input discourses to parse
+ :param discourse_ids: list of str Identifiers to be inserted to each occurrence-indexed predicate.
+ :param filename: str A filename for the output file
+ :return: stdout
+ """
+ args = ['--models', os.path.join(self._candc_models_path, ['boxer','questions'][question]),
+ '--candc-printer', 'boxer']
+ return self._call('\n'.join(sum((["<META>'%s'" % id] + d for d,id in zip(inputs,discourse_ids)), [])), self._candc_bin, args, verbose)
+
+ def _call_boxer(self, candc_out, verbose=False):
+ """
+ Call the ``boxer`` binary with the given input.
+
+ :param candc_out: str output from C&C parser
+ :return: stdout
+ """
+ f = None
+ try:
+ fd, temp_filename = tempfile.mkstemp(prefix='boxer-', suffix='.in', text=True)
+ f = os.fdopen(fd, 'w')
+ f.write(candc_out)
+ finally:
+ if f: f.close()
+
+ args = ['--box', 'false',
+ '--semantics', 'drs',
+ '--flat', 'false',
+ '--resolve', 'true',
+ '--elimeq', ['false','true'][self._elimeq],
+ '--format', 'prolog',
+ '--instantiate', 'true',
+ '--input', temp_filename]
+ stdout = self._call(None, self._boxer_bin, args, verbose)
+ os.remove(temp_filename)
+ return stdout
+
+ def _find_binary(self, name, bin_dir, verbose=False):
+ return find_binary(name,
+ path_to_bin=bin_dir,
+ env_vars=['CANDCHOME'],
+ url='http://svn.ask.it.usyd.edu.au/trac/candc/',
+ binary_names=[name, name + '.exe'],
+ verbose=verbose)
+
+ def _call(self, input_str, binary, args=[], verbose=False):
+ """
+ Call the binary with the given input.
+
+ :param input_str: A string whose contents are used as stdin.
+ :param binary: The location of the binary to call
+ :param args: A list of command-line arguments.
+ :return: stdout
+ """
+ if verbose:
+ print('Calling:', binary)
+ print('Args:', args)
+ print('Input:', input_str)
+ print('Command:', binary + ' ' + ' '.join(args))
+
+ # Call via a subprocess
+ if input_str is None:
+ cmd = [binary] + args
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ else:
+ cmd = 'echo "%s" | %s %s' % (input_str, binary, ' '.join(args))
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ stdout, stderr = p.communicate()
+
+ if verbose:
+ print('Return code:', p.returncode)
+ if stdout: print('stdout:\n', stdout, '\n')
+ if stderr: print('stderr:\n', stderr, '\n')
+ if p.returncode != 0:
+ raise Exception('ERROR CALLING: %s %s\nReturncode: %d\n%s' % (binary, ' '.join(args), p.returncode, stderr))
+
+ return stdout
+
+ def _parse_to_drs_dict(self, boxer_out, use_disc_id):
+ lines = boxer_out.split('\n')
+ drs_dict = {}
+ i = 0
+ while i < len(lines):
+ line = lines[i]
+ if line.startswith('id('):
+ comma_idx = line.index(',')
+ discourse_id = line[3:comma_idx]
+ if discourse_id[0] == "'" and discourse_id[-1] == "'":
+ discourse_id = discourse_id[1:-1]
+ drs_id = line[comma_idx+1:line.index(')')]
+ i += 1
+ line = lines[i]
+ assert line.startswith('sem(%s,' % drs_id)
+ assert line.endswith(').')
+
+ search_start = len('sem(%s,[' % drs_id)
+ brace_count = 1
+ drs_start = -1
+ for j,c in enumerate(line[search_start:]):
+ if(c == '['):
+ brace_count += 1
+ if(c == ']'):
+ brace_count -= 1
+ if(brace_count == 0):
+ drs_start = search_start + j + 2
+ break
+ assert drs_start > -1
+
+ drs_input = line[drs_start:-2].strip()
+ parsed = self._parse_drs(drs_input, discourse_id, use_disc_id)
+ drs_dict[discourse_id] = self._boxer_drs_interpreter.interpret(parsed)
+ i += 1
+ return drs_dict
+
+ def _parse_drs(self, drs_string, discourse_id, use_disc_id):
+ return BoxerOutputDrsParser([None,discourse_id][use_disc_id]).parse(drs_string)
+
+
+class BoxerOutputDrsParser(_DrtParser):
+ def __init__(self, discourse_id=None):
+ """
+ This class is used to parse the Prolog DRS output from Boxer into a
+ hierarchy of python objects.
+ """
+ _DrtParser.__init__(self)
+ self.discourse_id = discourse_id
+ self.sentence_id_offset = None
+ self.quote_chars = [("'", "'", "\\", False)]
+ self._label_counter = None
+
+ def parse(self, data, signature=None):
+ self._label_counter = Counter(-1)
+ return _DrtParser.parse(self, data, signature)
+
+ def get_all_symbols(self):
+ return ['(', ')', ',', '[', ']',':']
+
+ def handle(self, tok, context):
+ return self.handle_drs(tok)
+
+ def attempt_adjuncts(self, expression, context):
+ return expression
+
+ def parse_condition(self, indices):
+ """
+ Parse a DRS condition
+
+ :return: list of ``AbstractDrs``
+ """
+ tok = self.token()
+ accum = self.handle_condition(tok, indices)
+ if accum is None:
+ raise UnexpectedTokenException(tok)
+ return accum
+
+ def handle_drs(self, tok):
+ if tok == 'drs':
+ return self.parse_drs()
+ elif tok in ['merge', 'smerge']:
+ return self._handle_binary_expression(self._make_merge_expression)(None, [])
+ elif tok in ['alfa']:
+ return self._handle_alfa(self._make_merge_expression)(None, [])
+
+ def handle_condition(self, tok, indices):
+ """
+ Handle a DRS condition
+
+ :param indices: list of int
+ :return: list of ``AbstractDrs``
+ """
+ if tok == 'not':
+ return [self._handle_not()]
+
+ if tok == 'or':
+ conds = [self._handle_binary_expression(self._make_or_expression)]
+ elif tok == 'imp':
+ conds = [self._handle_binary_expression(self._make_imp_expression)]
+ elif tok == 'eq':
+ conds = [self._handle_eq()]
+ elif tok == 'prop':
+ conds = [self._handle_prop()]
+
+ elif tok == 'pred':
+ conds = [self._handle_pred()]
+ elif tok == 'named':
+ conds = [self._handle_named()]
+ elif tok == 'rel':
+ conds = [self._handle_rel()]
+ elif tok == 'timex':
+ conds = self._handle_timex()
+ elif tok == 'card':
+ conds = [self._handle_card()]
+
+ elif tok == 'whq':
+ conds = [self._handle_whq()]
+
+ else:
+ conds = []
+
+ return sum([[cond(sent_index, word_indices) for cond in conds] for sent_index, word_indices in self._sent_and_word_indices(indices)], [])
+
+ def _handle_not(self):
+ self.assertToken(self.token(), '(')
+ drs = self.parse_Expression(None)
+ self.assertToken(self.token(), ')')
+ return BoxerNot(drs)
+
+ def _handle_pred(self):
+ #pred(_G3943, dog, n, 0)
+ self.assertToken(self.token(), '(')
+ variable = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ name = self.token()
+ self.assertToken(self.token(), ',')
+ pos = self.token()
+ self.assertToken(self.token(), ',')
+ sense = int(self.token())
+ self.assertToken(self.token(), ')')
+
+ def _handle_pred_f(sent_index, word_indices):
+ return BoxerPred(self.discourse_id, sent_index, word_indices, variable, name, pos, sense)
+ return _handle_pred_f
+
+ def _handle_named(self):
+ #named(x0, john, per, 0)
+ self.assertToken(self.token(), '(')
+ variable = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ name = self.token()
+ self.assertToken(self.token(), ',')
+ type = self.token()
+ self.assertToken(self.token(), ',')
+ sense = int(self.token())
+ self.assertToken(self.token(), ')')
+ return lambda sent_index, word_indices: BoxerNamed(self.discourse_id, sent_index, word_indices, variable, name, type, sense)
+
+ def _handle_rel(self):
+ #rel(_G3993, _G3943, agent, 0)
+ self.assertToken(self.token(), '(')
+ var1 = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ var2 = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ rel = self.token()
+ self.assertToken(self.token(), ',')
+ sense = int(self.token())
+ self.assertToken(self.token(), ')')
+ return lambda sent_index, word_indices: BoxerRel(self.discourse_id, sent_index, word_indices, var1, var2, rel, sense)
+
+ def _handle_timex(self):
+ #timex(_G18322, date([]: (+), []:'XXXX', [1004]:'04', []:'XX'))
+ self.assertToken(self.token(), '(')
+ arg = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ new_conds = self._handle_time_expression(arg)
+ self.assertToken(self.token(), ')')
+ return new_conds
+
+ def _handle_time_expression(self, arg):
+ #date([]: (+), []:'XXXX', [1004]:'04', []:'XX')
+ tok = self.token()
+ self.assertToken(self.token(), '(')
+ if tok == 'date':
+ conds = self._handle_date(arg)
+ elif tok == 'time':
+ conds = self._handle_time(arg)
+ else:
+ return None
+ self.assertToken(self.token(), ')')
+ return [lambda sent_index, word_indices: BoxerPred(self.discourse_id, sent_index, word_indices, arg, tok, 'n', 0)] + \
+ [lambda sent_index, word_indices: cond for cond in conds]
+
+ def _handle_date(self, arg):
+ #[]: (+), []:'XXXX', [1004]:'04', []:'XX'
+ conds = []
+ (sent_index, word_indices), = self._sent_and_word_indices(self._parse_index_list())
+ self.assertToken(self.token(), '(')
+ pol = self.token()
+ self.assertToken(self.token(), ')')
+ conds.append(BoxerPred(self.discourse_id, sent_index, word_indices, arg, 'date_pol_%s' % (pol), 'a', 0))
+ self.assertToken(self.token(), ',')
+
+ (sent_index, word_indices), = self._sent_and_word_indices(self._parse_index_list())
+ year = self.token()
+ if year != 'XXXX':
+ year = year.replace(':', '_')
+ conds.append(BoxerPred(self.discourse_id, sent_index, word_indices, arg, 'date_year_%s' % (year), 'a', 0))
+ self.assertToken(self.token(), ',')
+
+ (sent_index, word_indices), = self._sent_and_word_indices(self._parse_index_list())
+ month = self.token()
+ if month != 'XX':
+ conds.append(BoxerPred(self.discourse_id, sent_index, word_indices, arg, 'date_month_%s' % (month), 'a', 0))
+ self.assertToken(self.token(), ',')
+
+ (sent_index, word_indices), = self._sent_and_word_indices(self._parse_index_list())
+ day = self.token()
+ if day != 'XX':
+ conds.append(BoxerPred(self.discourse_id, sent_index, word_indices, arg, 'date_day_%s' % (day), 'a', 0))
+
+ return conds
+
+ def _handle_time(self, arg):
+ #time([1018]:'18', []:'XX', []:'XX')
+ conds = []
+ self._parse_index_list()
+ hour = self.token()
+ if hour != 'XX':
+ conds.append(self._make_atom('r_hour_2',arg,hour))
+ self.assertToken(self.token(), ',')
+
+ self._parse_index_list()
+ min = self.token()
+ if min != 'XX':
+ conds.append(self._make_atom('r_min_2',arg,min))
+ self.assertToken(self.token(), ',')
+
+ self._parse_index_list()
+ sec = self.token()
+ if sec != 'XX':
+ conds.append(self._make_atom('r_sec_2',arg,sec))
+
+ return conds
+
+ def _handle_card(self):
+ #card(_G18535, 28, ge)
+ self.assertToken(self.token(), '(')
+ variable = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ value = self.token()
+ self.assertToken(self.token(), ',')
+ type = self.token()
+ self.assertToken(self.token(), ')')
+ return lambda sent_index, word_indices: BoxerCard(self.discourse_id, sent_index, word_indices, variable, value, type)
+
+ def _handle_prop(self):
+ #prop(_G15949, drs(...))
+ self.assertToken(self.token(), '(')
+ variable = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ drs = self.parse_Expression(None)
+ self.assertToken(self.token(), ')')
+ return lambda sent_index, word_indices: BoxerProp(self.discourse_id, sent_index, word_indices, variable, drs)
+
+ def _parse_index_list(self):
+ #[1001,1002]:
+ indices = []
+ self.assertToken(self.token(), '[')
+ while self.token(0) != ']':
+ indices.append(self.parse_index())
+ if self.token(0) == ',':
+ self.token() #swallow ','
+ self.token() #swallow ']'
+ self.assertToken(self.token(), ':')
+ return indices
+
+ def parse_drs(self):
+ #drs([[1001]:_G3943],
+ # [[1002]:pred(_G3943, dog, n, 0)]
+ # )
+ label = self._label_counter.get()
+ self.assertToken(self.token(), '(')
+ self.assertToken(self.token(), '[')
+ refs = set()
+ while self.token(0) != ']':
+ indices = self._parse_index_list()
+ refs.add(self.parse_variable())
+ if self.token(0) == ',':
+ self.token() #swallow ','
+ self.token() #swallow ']'
+ self.assertToken(self.token(), ',')
+ self.assertToken(self.token(), '[')
+ conds = []
+ while self.token(0) != ']':
+ indices = self._parse_index_list()
+ conds.extend(self.parse_condition(indices))
+ if self.token(0) == ',':
+ self.token() #swallow ','
+ self.token() #swallow ']'
+ self.assertToken(self.token(), ')')
+ return BoxerDrs(label, list(refs), conds)
+
+ def _handle_binary_expression(self, make_callback):
+ self.assertToken(self.token(), '(')
+ drs1 = self.parse_Expression(None)
+ self.assertToken(self.token(), ',')
+ drs2 = self.parse_Expression(None)
+ self.assertToken(self.token(), ')')
+ return lambda sent_index, word_indices: make_callback(sent_index, word_indices, drs1, drs2)
+
+ def _handle_alfa(self, make_callback):
+ self.assertToken(self.token(), '(')
+ type = self.token()
+ self.assertToken(self.token(), ',')
+ drs1 = self.parse_Expression(None)
+ self.assertToken(self.token(), ',')
+ drs2 = self.parse_Expression(None)
+ self.assertToken(self.token(), ')')
+ return lambda sent_index, word_indices: make_callback(sent_index, word_indices, drs1, drs2)
+
+ def _handle_eq(self):
+ self.assertToken(self.token(), '(')
+ var1 = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ var2 = self.parse_variable()
+ self.assertToken(self.token(), ')')
+ return lambda sent_index, word_indices: BoxerEq(self.discourse_id, sent_index, word_indices, var1, var2)
+
+
+ def _handle_whq(self):
+ self.assertToken(self.token(), '(')
+ self.assertToken(self.token(), '[')
+ ans_types = []
+ while self.token(0) != ']':
+ cat = self.token()
+ self.assertToken(self.token(), ':')
+ if cat == 'des':
+ ans_types.append(self.token())
+ elif cat == 'num':
+ ans_types.append('number')
+ typ = self.token()
+ if typ == 'cou':
+ ans_types.append('count')
+ else:
+ ans_types.append(typ)
+ else:
+ ans_types.append(self.token())
+ self.token() #swallow the ']'
+
+ self.assertToken(self.token(), ',')
+ d1 = self.parse_Expression(None)
+ self.assertToken(self.token(), ',')
+ ref = self.parse_variable()
+ self.assertToken(self.token(), ',')
+ d2 = self.parse_Expression(None)
+ self.assertToken(self.token(), ')')
+ return lambda sent_index, word_indices: BoxerWhq(self.discourse_id, sent_index, word_indices, ans_types, d1, ref, d2)
+
+ def _make_merge_expression(self, sent_index, word_indices, drs1, drs2):
+ return BoxerDrs(drs1.label, drs1.refs + drs2.refs, drs1.conds + drs2.conds)
+
+ def _make_or_expression(self, sent_index, word_indices, drs1, drs2):
+ return BoxerOr(self.discourse_id, sent_index, word_indices, drs1, drs2)
+
+ def _make_imp_expression(self, sent_index, word_indices, drs1, drs2):
+ return BoxerDrs(drs1.label, drs1.refs, drs1.conds, drs2)
+
+ def parse_variable(self):
+ var = self.token()
+ assert re.match('^[ex]\d+$', var), var
+ return int(var[1:])
+
+ def parse_index(self):
+ return int(self.token())
+
+ def _sent_and_word_indices(self, indices):
+ """
+ :return: list of (sent_index, word_indices) tuples
+ """
+ sent_indices = set((i / 1000)-1 for i in indices if i>=0)
+ if sent_indices:
+ pairs = []
+ for sent_index in sent_indices:
+ word_indices = [(i % 1000)-1 for i in indices if sent_index == (i / 1000)-1]
+ pairs.append((sent_index, word_indices))
+ return pairs
+ else:
+ word_indices = [(i % 1000)-1 for i in indices]
+ return [(None, word_indices)]
+
+
+class BoxerDrsParser(_DrtParser):
+ """
+ Reparse the str form of subclasses of ``AbstractBoxerDrs``
+ """
+ def __init__(self, discourse_id=None):
+ _DrtParser.__init__(self)
+ self.discourse_id = discourse_id
+
+ def get_all_symbols(self):
+ return [DrtTokens.OPEN, DrtTokens.CLOSE, DrtTokens.COMMA, DrtTokens.OPEN_BRACKET, DrtTokens.CLOSE_BRACKET]
+
+ def attempt_adjuncts(self, expression, context):
+ return expression
+
+ def handle(self, tok, context):
+ try:
+ if tok == 'drs':
+ self.assertNextToken(DrtTokens.OPEN)
+ label = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ refs = list(map(int, self.handle_refs()))
+ self.assertNextToken(DrtTokens.COMMA)
+ conds = self.handle_conds(None)
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerDrs(label, refs, conds)
+ elif tok == 'pred':
+ self.assertNextToken(DrtTokens.OPEN)
+ disc_id = (self.token(), self.discourse_id)[self.discourse_id is not None]
+ self.assertNextToken(DrtTokens.COMMA)
+ sent_id = self.nullableIntToken()
+ self.assertNextToken(DrtTokens.COMMA)
+ word_ids = list(map(int, self.handle_refs()))
+ self.assertNextToken(DrtTokens.COMMA)
+ variable = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ name = self.token()
+ self.assertNextToken(DrtTokens.COMMA)
+ pos = self.token()
+ self.assertNextToken(DrtTokens.COMMA)
+ sense = int(self.token())
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerPred(disc_id, sent_id, word_ids, variable, name, pos, sense)
+ elif tok == 'named':
+ self.assertNextToken(DrtTokens.OPEN)
+ disc_id = (self.token(), self.discourse_id)[self.discourse_id is not None]
+ self.assertNextToken(DrtTokens.COMMA)
+ sent_id = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ word_ids = map(int, self.handle_refs())
+ self.assertNextToken(DrtTokens.COMMA)
+ variable = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ name = self.token()
+ self.assertNextToken(DrtTokens.COMMA)
+ type = self.token()
+ self.assertNextToken(DrtTokens.COMMA)
+ sense = int(self.token())
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerNamed(disc_id, sent_id, word_ids, variable, name, type, sense)
+ elif tok == 'rel':
+ self.assertNextToken(DrtTokens.OPEN)
+ disc_id = (self.token(), self.discourse_id)[self.discourse_id is not None]
+ self.assertNextToken(DrtTokens.COMMA)
+ sent_id = self.nullableIntToken()
+ self.assertNextToken(DrtTokens.COMMA)
+ word_ids = list(map(int, self.handle_refs()))
+ self.assertNextToken(DrtTokens.COMMA)
+ var1 = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ var2 = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ rel = self.token()
+ self.assertNextToken(DrtTokens.COMMA)
+ sense = int(self.token())
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerRel(disc_id, sent_id, word_ids, var1, var2, rel, sense)
+ elif tok == 'prop':
+ self.assertNextToken(DrtTokens.OPEN)
+ disc_id = (self.token(), self.discourse_id)[self.discourse_id is not None]
+ self.assertNextToken(DrtTokens.COMMA)
+ sent_id = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ word_ids = list(map(int, self.handle_refs()))
+ self.assertNextToken(DrtTokens.COMMA)
+ variable = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ drs = self.parse_Expression(None)
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerProp(disc_id, sent_id, word_ids, variable, drs)
+ elif tok == 'not':
+ self.assertNextToken(DrtTokens.OPEN)
+ drs = self.parse_Expression(None)
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerNot(drs)
+ elif tok == 'imp':
+ self.assertNextToken(DrtTokens.OPEN)
+ drs1 = self.parse_Expression(None)
+ self.assertNextToken(DrtTokens.COMMA)
+ drs2 = self.parse_Expression(None)
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerDrs(drs1.label, drs1.refs, drs1.conds, drs2)
+ elif tok == 'or':
+ self.assertNextToken(DrtTokens.OPEN)
+ disc_id = (self.token(), self.discourse_id)[self.discourse_id is not None]
+ self.assertNextToken(DrtTokens.COMMA)
+ sent_id = self.nullableIntToken()
+ self.assertNextToken(DrtTokens.COMMA)
+ word_ids = map(int, self.handle_refs())
+ self.assertNextToken(DrtTokens.COMMA)
+ drs1 = self.parse_Expression(None)
+ self.assertNextToken(DrtTokens.COMMA)
+ drs2 = self.parse_Expression(None)
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerOr(disc_id, sent_id, word_ids, drs1, drs2)
+ elif tok == 'eq':
+ self.assertNextToken(DrtTokens.OPEN)
+ disc_id = (self.token(), self.discourse_id)[self.discourse_id is not None]
+ self.assertNextToken(DrtTokens.COMMA)
+ sent_id = self.nullableIntToken()
+ self.assertNextToken(DrtTokens.COMMA)
+ word_ids = list(map(int, self.handle_refs()))
+ self.assertNextToken(DrtTokens.COMMA)
+ var1 = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ var2 = int(self.token())
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerEq(disc_id, sent_id, word_ids, var1, var2)
+ elif tok == 'card':
+ self.assertNextToken(DrtTokens.OPEN)
+ disc_id = (self.token(), self.discourse_id)[self.discourse_id is not None]
+ self.assertNextToken(DrtTokens.COMMA)
+ sent_id = self.nullableIntToken()
+ self.assertNextToken(DrtTokens.COMMA)
+ word_ids = map(int, self.handle_refs())
+ self.assertNextToken(DrtTokens.COMMA)
+ var = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ value = self.token()
+ self.assertNextToken(DrtTokens.COMMA)
+ type = self.token()
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerCard(disc_id, sent_id, word_ids, var, value, type)
+ elif tok == 'whq':
+ self.assertNextToken(DrtTokens.OPEN)
+ disc_id = (self.token(), self.discourse_id)[self.discourse_id is not None]
+ self.assertNextToken(DrtTokens.COMMA)
+ sent_id = self.nullableIntToken()
+ self.assertNextToken(DrtTokens.COMMA)
+ word_ids = list(map(int, self.handle_refs()))
+ self.assertNextToken(DrtTokens.COMMA)
+ ans_types = self.handle_refs()
+ self.assertNextToken(DrtTokens.COMMA)
+ drs1 = self.parse_Expression(None)
+ self.assertNextToken(DrtTokens.COMMA)
+ var = int(self.token())
+ self.assertNextToken(DrtTokens.COMMA)
+ drs2 = self.parse_Expression(None)
+ self.assertNextToken(DrtTokens.CLOSE)
+ return BoxerWhq(disc_id, sent_id, word_ids, ans_types, drs1, var, drs2)
+ except Exception as e:
+ raise LogicalExpressionException(self._currentIndex, str(e))
+ assert False, repr(tok)
+
+ def nullableIntToken(self):
+ t = self.token()
+ return [None,int(t)][t != 'None']
+
+ def get_next_token_variable(self, description):
+ try:
+ return self.token()
+ except ExpectedMoreTokensException as e:
+ raise ExpectedMoreTokensException(e.index, 'Variable expected.')
+
+
+
+class AbstractBoxerDrs(object):
+ def variables(self):
+ """
+ :return: (set<variables>, set<events>, set<propositions>)
+ """
+ variables, events, propositions = self._variables()
+ return (variables - (events | propositions), events, propositions - events)
+
+ def variable_types(self):
+ vartypes = {}
+ for t,vars in zip(('z','e','p'), self.variables()):
+ for v in vars:
+ vartypes[v] = t
+ return vartypes
+
+ def _variables(self):
+ """
+ :return: (set<variables>, set<events>, set<propositions>)
+ """
+ return (set(), set(), set())
+
+ def atoms(self):
+ return set()
+
+ def clean(self):
+ return self
+
+ def _clean_name(self, name):
+ return name.replace('-','_').replace("'", "_")
+
+ def renumber_sentences(self, f):
+ return self
+
+ def __hash__(self):
+ return hash("%s" % self)
+
+
+ at python_2_unicode_compatible
+class BoxerDrs(AbstractBoxerDrs):
+ def __init__(self, label, refs, conds, consequent=None):
+ AbstractBoxerDrs.__init__(self)
+ self.label = label
+ self.refs = refs
+ self.conds = conds
+ self.consequent = consequent
+
+ def _variables(self):
+ variables = (set(), set(), set())
+ for cond in self.conds:
+ for s,v in zip(variables, cond._variables()):
+ s.update(v)
+ if self.consequent is not None:
+ for s,v in zip(variables, self.consequent._variables()):
+ s.update(v)
+ return variables
+
+ def atoms(self):
+ atoms = reduce(operator.or_, (cond.atoms() for cond in self.conds), set())
+ if self.consequent is not None:
+ atoms.update(self.consequent.atoms())
+ return atoms
+
+ def clean(self):
+ consequent = (self.consequent.clean() if self.consequent else None)
+ return BoxerDrs(self.label, self.refs, [c.clean() for c in self.conds], consequent)
+
+ def renumber_sentences(self, f):
+ consequent = (self.consequent.renumber_sentences(f) if self.consequent else None)
+ return BoxerDrs(self.label, self.refs, [c.renumber_sentences(f) for c in self.conds], consequent)
+
+ def __repr__(self):
+ s = 'drs(%s, [%s], [%s])' % (self.label,
+ ', '.join("%s" % r for r in self.refs),
+ ', '.join("%s" % c for c in self.conds))
+ if self.consequent is not None:
+ s = 'imp(%s, %s)' % (s, self.consequent)
+ return s
+
+ def __eq__(self, other):
+ return self.__class__ == other.__class__ and \
+ self.label == other.label and \
+ self.refs == other.refs and \
+ len(self.conds) == len(other.conds) and \
+ reduce(operator.and_, (c1==c2 for c1,c2 in zip(self.conds, other.conds))) and \
+ self.consequent == other.consequent
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = AbstractBoxerDrs.__hash__
+
+
+ at python_2_unicode_compatible
+class BoxerNot(AbstractBoxerDrs):
+ def __init__(self, drs):
+ AbstractBoxerDrs.__init__(self)
+ self.drs = drs
+
+ def _variables(self):
+ return self.drs._variables()
+
+ def atoms(self):
+ return self.drs.atoms()
+
+ def clean(self):
+ return BoxerNot(self.drs.clean())
+
+ def renumber_sentences(self, f):
+ return BoxerNot(self.drs.renumber_sentences(f))
+
+ def __repr__(self):
+ return 'not(%s)' % (self.drs)
+
+ def __eq__(self, other):
+ return self.__class__ == other.__class__ and self.drs == other.drs
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = AbstractBoxerDrs.__hash__
+
+ at python_2_unicode_compatible
+class BoxerIndexed(AbstractBoxerDrs):
+ def __init__(self, discourse_id, sent_index, word_indices):
+ AbstractBoxerDrs.__init__(self)
+ self.discourse_id = discourse_id
+ self.sent_index = sent_index
+ self.word_indices = word_indices
+
+ def atoms(self):
+ return set([self])
+
+ def __eq__(self, other):
+ return self.__class__ == other.__class__ and \
+ self.discourse_id == other.discourse_id and \
+ self.sent_index == other.sent_index and \
+ self.word_indices == other.word_indices and \
+ reduce(operator.and_, (s==o for s,o in zip(self, other)))
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = AbstractBoxerDrs.__hash__
+
+ def __repr__(self):
+ s = '%s(%s, %s, [%s]' % (self._pred(), self.discourse_id,
+ self.sent_index, ', '.join("%s" % wi for wi in self.word_indices))
+ for v in self:
+ s += ', %s' % v
+ return s + ')'
+
+class BoxerPred(BoxerIndexed):
+ def __init__(self, discourse_id, sent_index, word_indices, var, name, pos, sense):
+ BoxerIndexed.__init__(self, discourse_id, sent_index, word_indices)
+ self.var = var
+ self.name = name
+ self.pos = pos
+ self.sense = sense
+
+ def _variables(self):
+ return (set([self.var]), set(), set())
+
+ def change_var(self, var):
+ return BoxerPred(self.discourse_id, self.sent_index, self.word_indices, var, self.name, self.pos, self.sense)
+
+ def clean(self):
+ return BoxerPred(self.discourse_id, self.sent_index, self.word_indices, self.var, self._clean_name(self.name), self.pos, self.sense)
+
+ def renumber_sentences(self, f):
+ new_sent_index = f(self.sent_index)
+ return BoxerPred(self.discourse_id, new_sent_index, self.word_indices, self.var, self.name, self.pos, self.sense)
+
+ def __iter__(self):
+ return iter((self.var, self.name, self.pos, self.sense))
+
+ def _pred(self):
+ return 'pred'
+
+class BoxerNamed(BoxerIndexed):
+ def __init__(self, discourse_id, sent_index, word_indices, var, name, type, sense):
+ BoxerIndexed.__init__(self, discourse_id, sent_index, word_indices)
+ self.var = var
+ self.name = name
+ self.type = type
+ self.sense = sense
+
+ def _variables(self):
+ return (set([self.var]), set(), set())
+
+ def change_var(self, var):
+ return BoxerNamed(self.discourse_id, self.sent_index, self.word_indices, var, self.name, self.type, self.sense)
+
+ def clean(self):
+ return BoxerNamed(self.discourse_id, self.sent_index, self.word_indices, self.var, self._clean_name(self.name), self.type, self.sense)
+
+ def renumber_sentences(self, f):
+ return BoxerNamed(self.discourse_id, f(self.sent_index), self.word_indices, self.var, self.name, self.type, self.sense)
+
+ def __iter__(self):
+ return iter((self.var, self.name, self.type, self.sense))
+
+ def _pred(self):
+ return 'named'
+
+class BoxerRel(BoxerIndexed):
+ def __init__(self, discourse_id, sent_index, word_indices, var1, var2, rel, sense):
+ BoxerIndexed.__init__(self, discourse_id, sent_index, word_indices)
+ self.var1 = var1
+ self.var2 = var2
+ self.rel = rel
+ self.sense = sense
+
+ def _variables(self):
+ return (set([self.var1, self.var2]), set(), set())
+
+ def clean(self):
+ return BoxerRel(self.discourse_id, self.sent_index, self.word_indices, self.var1, self.var2, self._clean_name(self.rel), self.sense)
+
+ def renumber_sentences(self, f):
+ return BoxerRel(self.discourse_id, f(self.sent_index), self.word_indices, self.var1, self.var2, self.rel, self.sense)
+
+ def __iter__(self):
+ return iter((self.var1, self.var2, self.rel, self.sense))
+
+ def _pred(self):
+ return 'rel'
+
+class BoxerProp(BoxerIndexed):
+ def __init__(self, discourse_id, sent_index, word_indices, var, drs):
+ BoxerIndexed.__init__(self, discourse_id, sent_index, word_indices)
+ self.var = var
+ self.drs = drs
+
+ def _variables(self):
+ return tuple(map(operator.or_, (set(), set(), set([self.var])), self.drs._variables()))
+
+ def referenced_labels(self):
+ return set([self.drs])
+
+ def atoms(self):
+ return self.drs.atoms()
+
+ def clean(self):
+ return BoxerProp(self.discourse_id, self.sent_index, self.word_indices, self.var, self.drs.clean())
+
+ def renumber_sentences(self, f):
+ return BoxerProp(self.discourse_id, f(self.sent_index), self.word_indices, self.var, self.drs.renumber_sentences(f))
+
+ def __iter__(self):
+ return iter((self.var, self.drs))
+
+ def _pred(self):
+ return 'prop'
+
+class BoxerEq(BoxerIndexed):
+ def __init__(self, discourse_id, sent_index, word_indices, var1, var2):
+ BoxerIndexed.__init__(self, discourse_id, sent_index, word_indices)
+ self.var1 = var1
+ self.var2 = var2
+
+ def _variables(self):
+ return (set([self.var1, self.var2]), set(), set())
+
+ def atoms(self):
+ return set()
+
+ def renumber_sentences(self, f):
+ return BoxerEq(self.discourse_id, f(self.sent_index), self.word_indices, self.var1, self.var2)
+
+ def __iter__(self):
+ return iter((self.var1, self.var2))
+
+ def _pred(self):
+ return 'eq'
+
+class BoxerCard(BoxerIndexed):
+ def __init__(self, discourse_id, sent_index, word_indices, var, value, type):
+ BoxerIndexed.__init__(self, discourse_id, sent_index, word_indices)
+ self.var = var
+ self.value = value
+ self.type = type
+
+ def _variables(self):
+ return (set([self.var]), set(), set())
+
+ def renumber_sentences(self, f):
+ return BoxerCard(self.discourse_id, f(self.sent_index), self.word_indices, self.var, self.value, self.type)
+
+ def __iter__(self):
+ return iter((self.var, self.value, self.type))
+
+ def _pred(self):
+ return 'card'
+
+class BoxerOr(BoxerIndexed):
+ def __init__(self, discourse_id, sent_index, word_indices, drs1, drs2):
+ BoxerIndexed.__init__(self, discourse_id, sent_index, word_indices)
+ self.drs1 = drs1
+ self.drs2 = drs2
+
+ def _variables(self):
+ return tuple(map(operator.or_, self.drs1._variables(), self.drs2._variables()))
+
+ def atoms(self):
+ return self.drs1.atoms() | self.drs2.atoms()
+
+ def clean(self):
+ return BoxerOr(self.discourse_id, self.sent_index, self.word_indices, self.drs1.clean(), self.drs2.clean())
+
+ def renumber_sentences(self, f):
+ return BoxerOr(self.discourse_id, f(self.sent_index), self.word_indices, self.drs1, self.drs2)
+
+ def __iter__(self):
+ return iter((self.drs1, self.drs2))
+
+ def _pred(self):
+ return 'or'
+
+class BoxerWhq(BoxerIndexed):
+ def __init__(self, discourse_id, sent_index, word_indices, ans_types, drs1, variable, drs2):
+ BoxerIndexed.__init__(self, discourse_id, sent_index, word_indices)
+ self.ans_types = ans_types
+ self.drs1 = drs1
+ self.variable = variable
+ self.drs2 = drs2
+
+ def _variables(self):
+ return tuple(map(operator.or_, (set([self.variable]), set(), set()), self.drs1._variables(), self.drs2._variables()))
+
+ def atoms(self):
+ return self.drs1.atoms() | self.drs2.atoms()
+
+ def clean(self):
+ return BoxerWhq(self.discourse_id, self.sent_index, self.word_indices, self.ans_types, self.drs1.clean(), self.variable, self.drs2.clean())
+
+ def renumber_sentences(self, f):
+ return BoxerWhq(self.discourse_id, f(self.sent_index), self.word_indices, self.ans_types, self.drs1, self.variable, self.drs2)
+
+ def __iter__(self):
+ return iter(('['+','.join(self.ans_types)+']', self.drs1, self.variable, self.drs2))
+
+ def _pred(self):
+ return 'whq'
+
+
+
+class PassthroughBoxerDrsInterpreter(object):
+ def interpret(self, ex):
+ return ex
+
+
+class NltkDrtBoxerDrsInterpreter(object):
+ def __init__(self, occur_index=False):
+ self._occur_index = occur_index
+
+ def interpret(self, ex):
+ """
+ :param ex: ``AbstractBoxerDrs``
+ :return: ``AbstractDrs``
+ """
+ if isinstance(ex, BoxerDrs):
+ drs = DRS([Variable('x%d' % r) for r in ex.refs], list(map(self.interpret, ex.conds)))
+ if ex.label is not None:
+ drs.label = Variable('x%d' % ex.label)
+ if ex.consequent is not None:
+ drs.consequent = self.interpret(ex.consequent)
+ return drs
+ elif isinstance(ex, BoxerNot):
+ return DrtNegatedExpression(self.interpret(ex.drs))
+ elif isinstance(ex, BoxerPred):
+ pred = self._add_occur_indexing('%s_%s' % (ex.pos, ex.name), ex)
+ return self._make_atom(pred, 'x%d' % ex.var)
+ elif isinstance(ex, BoxerNamed):
+ pred = self._add_occur_indexing('ne_%s_%s' % (ex.type, ex.name), ex)
+ return self._make_atom(pred, 'x%d' % ex.var)
+ elif isinstance(ex, BoxerRel):
+ pred = self._add_occur_indexing('%s' % (ex.rel), ex)
+ return self._make_atom(pred, 'x%d' % ex.var1, 'x%d' % ex.var2)
+ elif isinstance(ex, BoxerProp):
+ return DrtProposition(Variable('x%d' % ex.var), self.interpret(ex.drs))
+ elif isinstance(ex, BoxerEq):
+ return DrtEqualityExpression(DrtVariableExpression(Variable('x%d' % ex.var1)),
+ DrtVariableExpression(Variable('x%d' % ex.var2)))
+ elif isinstance(ex, BoxerCard):
+ pred = self._add_occur_indexing('card_%s_%s' % (ex.type, ex.value), ex)
+ return self._make_atom(pred, 'x%d' % ex.var)
+ elif isinstance(ex, BoxerOr):
+ return DrtOrExpression(self.interpret(ex.drs1), self.interpret(ex.drs2))
+ elif isinstance(ex, BoxerWhq):
+ drs1 = self.interpret(ex.drs1)
+ drs2 = self.interpret(ex.drs2)
+ return DRS(drs1.refs + drs2.refs, drs1.conds + drs2.conds)
+ assert False, '%s: %s' % (ex.__class__.__name__, ex)
+
+ def _make_atom(self, pred, *args):
+ accum = DrtVariableExpression(Variable(pred))
+ for arg in args:
+ accum = DrtApplicationExpression(accum, DrtVariableExpression(Variable(arg)))
+ return accum
+
+ def _add_occur_indexing(self, base, ex):
+ if self._occur_index and ex.sent_index is not None:
+ if ex.discourse_id:
+ base += '_%s' % ex.discourse_id
+ base += '_s%s' % ex.sent_index
+ base += '_w%s' % sorted(ex.word_indices)[0]
+ return base
+
+
+class UnparseableInputException(Exception):
+ pass
+
+
+if __name__ == '__main__':
+ opts = OptionParser("usage: %prog TEXT [options]")
+ opts.add_option("--verbose", "-v", help="display verbose logs", action="store_true", default=False, dest="verbose")
+ opts.add_option("--fol", "-f", help="output FOL", action="store_true", default=False, dest="fol")
+ opts.add_option("--question", "-q", help="input is a question", action="store_true", default=False, dest="question")
+ opts.add_option("--occur", "-o", help="occurrence index", action="store_true", default=False, dest="occur_index")
+ (options, args) = opts.parse_args()
+
+ if len(args) != 1:
+ opts.error("incorrect number of arguments")
+
+ interpreter = NltkDrtBoxerDrsInterpreter(occur_index=options.occur_index)
+ drs = Boxer(interpreter).interpret_multi(args[0].split(r'\n'), question=options.question, verbose=options.verbose)
+ if drs is None:
+ print(None)
+ else:
+ drs = drs.simplify().eliminate_equality()
+ if options.fol:
+ print(drs.fol().normalize())
+ else:
+ drs.normalize().pprint()
diff --git a/nltk/sem/chat80.py b/nltk/sem/chat80.py
new file mode 100644
index 0000000..7da0353
--- /dev/null
+++ b/nltk/sem/chat80.py
@@ -0,0 +1,781 @@
+# Natural Language Toolkit: Chat-80 KB Reader
+# See http://www.w3.org/TR/swbp-skos-core-guide/
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>,
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+"""
+Overview
+========
+
+Chat-80 was a natural language system which allowed the user to
+interrogate a Prolog knowledge base in the domain of world
+geography. It was developed in the early '80s by Warren and Pereira; see
+``http://www.aclweb.org/anthology/J82-3002.pdf`` for a description and
+``http://www.cis.upenn.edu/~pereira/oldies.html`` for the source
+files.
+
+This module contains functions to extract data from the Chat-80
+relation files ('the world database'), and convert then into a format
+that can be incorporated in the FOL models of
+``nltk.sem.evaluate``. The code assumes that the Prolog
+input files are available in the NLTK corpora directory.
+
+The Chat-80 World Database consists of the following files::
+
+ world0.pl
+ rivers.pl
+ cities.pl
+ countries.pl
+ contain.pl
+ borders.pl
+
+This module uses a slightly modified version of ``world0.pl``, in which
+a set of Prolog rules have been omitted. The modified file is named
+``world1.pl``. Currently, the file ``rivers.pl`` is not read in, since
+it uses a list rather than a string in the second field.
+
+Reading Chat-80 Files
+=====================
+
+Chat-80 relations are like tables in a relational database. The
+relation acts as the name of the table; the first argument acts as the
+'primary key'; and subsequent arguments are further fields in the
+table. In general, the name of the table provides a label for a unary
+predicate whose extension is all the primary keys. For example,
+relations in ``cities.pl`` are of the following form::
+
+ 'city(athens,greece,1368).'
+
+Here, ``'athens'`` is the key, and will be mapped to a member of the
+unary predicate *city*.
+
+The fields in the table are mapped to binary predicates. The first
+argument of the predicate is the primary key, while the second
+argument is the data in the relevant field. Thus, in the above
+example, the third field is mapped to the binary predicate
+*population_of*, whose extension is a set of pairs such as
+``'(athens, 1368)'``.
+
+An exception to this general framework is required by the relations in
+the files ``borders.pl`` and ``contains.pl``. These contain facts of the
+following form::
+
+ 'borders(albania,greece).'
+
+ 'contains0(africa,central_africa).'
+
+We do not want to form a unary concept out the element in
+the first field of these records, and we want the label of the binary
+relation just to be ``'border'``/``'contain'`` respectively.
+
+In order to drive the extraction process, we use 'relation metadata bundles'
+which are Python dictionaries such as the following::
+
+ city = {'label': 'city',
+ 'closures': [],
+ 'schema': ['city', 'country', 'population'],
+ 'filename': 'cities.pl'}
+
+According to this, the file ``city['filename']`` contains a list of
+relational tuples (or more accurately, the corresponding strings in
+Prolog form) whose predicate symbol is ``city['label']`` and whose
+relational schema is ``city['schema']``. The notion of a ``closure`` is
+discussed in the next section.
+
+Concepts
+========
+In order to encapsulate the results of the extraction, a class of
+``Concept`` objects is introduced. A ``Concept`` object has a number of
+attributes, in particular a ``prefLabel`` and ``extension``, which make
+it easier to inspect the output of the extraction. In addition, the
+``extension`` can be further processed: in the case of the ``'border'``
+relation, we check that the relation is symmetric, and in the case
+of the ``'contain'`` relation, we carry out the transitive
+closure. The closure properties associated with a concept is
+indicated in the relation metadata, as indicated earlier.
+
+The ``extension`` of a ``Concept`` object is then incorporated into a
+``Valuation`` object.
+
+Persistence
+===========
+The functions ``val_dump`` and ``val_load`` are provided to allow a
+valuation to be stored in a persistent database and re-loaded, rather
+than having to be re-computed each time.
+
+Individuals and Lexical Items
+=============================
+As well as deriving relations from the Chat-80 data, we also create a
+set of individual constants, one for each entity in the domain. The
+individual constants are string-identical to the entities. For
+example, given a data item such as ``'zloty'``, we add to the valuation
+a pair ``('zloty', 'zloty')``. In order to parse English sentences that
+refer to these entities, we also create a lexical item such as the
+following for each individual constant::
+
+ PropN[num=sg, sem=<\P.(P zloty)>] -> 'Zloty'
+
+The set of rules is written to the file ``chat_pnames.cfg`` in the
+current directory.
+
+"""
+from __future__ import print_function, unicode_literals
+
+import re
+import shelve
+import os
+import sys
+
+import nltk.data
+from nltk.compat import string_types, python_2_unicode_compatible
+
+###########################################################################
+# Chat-80 relation metadata bundles needed to build the valuation
+###########################################################################
+
+borders = {'rel_name': 'borders',
+ 'closures': ['symmetric'],
+ 'schema': ['region', 'border'],
+ 'filename': 'borders.pl'}
+
+contains = {'rel_name': 'contains0',
+ 'closures': ['transitive'],
+ 'schema': ['region', 'contain'],
+ 'filename': 'contain.pl'}
+
+city = {'rel_name': 'city',
+ 'closures': [],
+ 'schema': ['city', 'country', 'population'],
+ 'filename': 'cities.pl'}
+
+country = {'rel_name': 'country',
+ 'closures': [],
+ 'schema': ['country', 'region', 'latitude', 'longitude',
+ 'area', 'population', 'capital', 'currency'],
+ 'filename': 'countries.pl'}
+
+circle_of_lat = {'rel_name': 'circle_of_latitude',
+ 'closures': [],
+ 'schema': ['circle_of_latitude', 'degrees'],
+ 'filename': 'world1.pl'}
+
+circle_of_long = {'rel_name': 'circle_of_longitude',
+ 'closures': [],
+ 'schema': ['circle_of_longitude', 'degrees'],
+ 'filename': 'world1.pl'}
+
+continent = {'rel_name': 'continent',
+ 'closures': [],
+ 'schema': ['continent'],
+ 'filename': 'world1.pl'}
+
+region = {'rel_name': 'in_continent',
+ 'closures': [],
+ 'schema': ['region', 'continent'],
+ 'filename': 'world1.pl'}
+
+ocean = {'rel_name': 'ocean',
+ 'closures': [],
+ 'schema': ['ocean'],
+ 'filename': 'world1.pl'}
+
+sea = {'rel_name': 'sea',
+ 'closures': [],
+ 'schema': ['sea'],
+ 'filename': 'world1.pl'}
+
+
+
+items = ['borders', 'contains', 'city', 'country', 'circle_of_lat',
+ 'circle_of_long', 'continent', 'region', 'ocean', 'sea']
+items = tuple(sorted(items))
+
+item_metadata = {
+ 'borders': borders,
+ 'contains': contains,
+ 'city': city,
+ 'country': country,
+ 'circle_of_lat': circle_of_lat,
+ 'circle_of_long': circle_of_long,
+ 'continent': continent,
+ 'region': region,
+ 'ocean': ocean,
+ 'sea': sea
+ }
+
+rels = item_metadata.values()
+
+not_unary = ['borders.pl', 'contain.pl']
+
+###########################################################################
+
+ at python_2_unicode_compatible
+class Concept(object):
+ """
+ A Concept class, loosely based on SKOS
+ (http://www.w3.org/TR/swbp-skos-core-guide/).
+ """
+ def __init__(self, prefLabel, arity, altLabels=[], closures=[], extension=set()):
+ """
+ :param prefLabel: the preferred label for the concept
+ :type prefLabel: str
+ :param arity: the arity of the concept
+ :type arity: int
+ @keyword altLabels: other (related) labels
+ :type altLabels: list
+ @keyword closures: closure properties of the extension \
+ (list items can be ``symmetric``, ``reflexive``, ``transitive``)
+ :type closures: list
+ @keyword extension: the extensional value of the concept
+ :type extension: set
+ """
+ self.prefLabel = prefLabel
+ self.arity = arity
+ self.altLabels = altLabels
+ self.closures = closures
+ #keep _extension internally as a set
+ self._extension = extension
+ #public access is via a list (for slicing)
+ self.extension = sorted(list(extension))
+
+ def __str__(self):
+ #_extension = ''
+ #for element in sorted(self.extension):
+ #if isinstance(element, tuple):
+ #element = '(%s, %s)' % (element)
+ #_extension += element + ', '
+ #_extension = _extension[:-1]
+
+ return "Label = '%s'\nArity = %s\nExtension = %s" % \
+ (self.prefLabel, self.arity, self.extension)
+
+ def __repr__(self):
+ return "Concept('%s')" % self.prefLabel
+
+ def augment(self, data):
+ """
+ Add more data to the ``Concept``'s extension set.
+
+ :param data: a new semantic value
+ :type data: string or pair of strings
+ :rtype: set
+
+ """
+ self._extension.add(data)
+ self.extension = sorted(list(self._extension))
+ return self._extension
+
+
+ def _make_graph(self, s):
+ """
+ Convert a set of pairs into an adjacency linked list encoding of a graph.
+ """
+ g = {}
+ for (x, y) in s:
+ if x in g:
+ g[x].append(y)
+ else:
+ g[x] = [y]
+ return g
+
+ def _transclose(self, g):
+ """
+ Compute the transitive closure of a graph represented as a linked list.
+ """
+ for x in g:
+ for adjacent in g[x]:
+ # check that adjacent is a key
+ if adjacent in g:
+ for y in g[adjacent]:
+ if y not in g[x]:
+ g[x].append(y)
+ return g
+
+ def _make_pairs(self, g):
+ """
+ Convert an adjacency linked list back into a set of pairs.
+ """
+ pairs = []
+ for node in g:
+ for adjacent in g[node]:
+ pairs.append((node, adjacent))
+ return set(pairs)
+
+
+ def close(self):
+ """
+ Close a binary relation in the ``Concept``'s extension set.
+
+ :return: a new extension for the ``Concept`` in which the
+ relation is closed under a given property
+ """
+ from nltk.sem import is_rel
+ assert is_rel(self._extension)
+ if 'symmetric' in self.closures:
+ pairs = []
+ for (x, y) in self._extension:
+ pairs.append((y, x))
+ sym = set(pairs)
+ self._extension = self._extension.union(sym)
+ if 'transitive' in self.closures:
+ all = self._make_graph(self._extension)
+ closed = self._transclose(all)
+ trans = self._make_pairs(closed)
+ #print sorted(trans)
+ self._extension = self._extension.union(trans)
+ self.extension = sorted(list(self._extension))
+
+
+def clause2concepts(filename, rel_name, schema, closures=[]):
+ """
+ Convert a file of Prolog clauses into a list of ``Concept`` objects.
+
+ :param filename: filename containing the relations
+ :type filename: str
+ :param rel_name: name of the relation
+ :type rel_name: str
+ :param schema: the schema used in a set of relational tuples
+ :type schema: list
+ :param closures: closure properties for the extension of the concept
+ :type closures: list
+ :return: a list of ``Concept`` objects
+ :rtype: list
+ """
+ concepts = []
+ # position of the subject of a binary relation
+ subj = 0
+ # label of the 'primary key'
+ pkey = schema[0]
+ # fields other than the primary key
+ fields = schema[1:]
+
+ # convert a file into a list of lists
+ records = _str2records(filename, rel_name)
+
+ # add a unary concept corresponding to the set of entities
+ # in the primary key position
+ # relations in 'not_unary' are more like ordinary binary relations
+ if not filename in not_unary:
+ concepts.append(unary_concept(pkey, subj, records))
+
+ # add a binary concept for each non-key field
+ for field in fields:
+ obj = schema.index(field)
+ concepts.append(binary_concept(field, closures, subj, obj, records))
+
+ return concepts
+
+def cities2table(filename, rel_name, dbname, verbose=False, setup=False):
+ """
+ Convert a file of Prolog clauses into a database table.
+
+ This is not generic, since it doesn't allow arbitrary
+ schemas to be set as a parameter.
+
+ Intended usage::
+
+ cities2table('cities.pl', 'city', 'city.db', verbose=True, setup=True)
+
+ :param filename: filename containing the relations
+ :type filename: str
+ :param rel_name: name of the relation
+ :type rel_name: str
+ :param dbname: filename of persistent store
+ :type schema: str
+ """
+ import sqlite3
+ records = _str2records(filename, rel_name)
+ connection = sqlite3.connect(dbname)
+ cur = connection.cursor()
+ if setup:
+ cur.execute('''CREATE TABLE city_table
+ (City text, Country text, Population int)''')
+
+ table_name = "city_table"
+ for t in records:
+ cur.execute('insert into %s values (?,?,?)' % table_name, t)
+ if verbose:
+ print("inserting values into %s: " % table_name, t)
+ connection.commit()
+ if verbose:
+ print("Committing update to %s" % dbname)
+ cur.close()
+
+def sql_query(dbname, query):
+ """
+ Execute an SQL query over a database.
+ :param dbname: filename of persistent store
+ :type schema: str
+ :param query: SQL query
+ :type rel_name: str
+ """
+ import sqlite3
+ try:
+ path = nltk.data.find(dbname)
+ connection = sqlite3.connect(str(path))
+ cur = connection.cursor()
+ return cur.execute(query)
+ except (ValueError, sqlite3.OperationalError):
+ import warnings
+ warnings.warn("Make sure the database file %s is installed and uncompressed." % dbname)
+ raise
+
+def _str2records(filename, rel):
+ """
+ Read a file into memory and convert each relation clause into a list.
+ """
+ recs = []
+ contents = nltk.data.load("corpora/chat80/%s" % filename, format="text")
+ for line in contents.splitlines():
+ if line.startswith(rel):
+ line = re.sub(rel+r'\(', '', line)
+ line = re.sub(r'\)\.$', '', line)
+ record = line.split(',')
+ recs.append(record)
+ return recs
+
+def unary_concept(label, subj, records):
+ """
+ Make a unary concept out of the primary key in a record.
+
+ A record is a list of entities in some relation, such as
+ ``['france', 'paris']``, where ``'france'`` is acting as the primary
+ key.
+
+ :param label: the preferred label for the concept
+ :type label: string
+ :param subj: position in the record of the subject of the predicate
+ :type subj: int
+ :param records: a list of records
+ :type records: list of lists
+ :return: ``Concept`` of arity 1
+ :rtype: Concept
+ """
+ c = Concept(label, arity=1, extension=set())
+ for record in records:
+ c.augment(record[subj])
+ return c
+
+def binary_concept(label, closures, subj, obj, records):
+ """
+ Make a binary concept out of the primary key and another field in a record.
+
+ A record is a list of entities in some relation, such as
+ ``['france', 'paris']``, where ``'france'`` is acting as the primary
+ key, and ``'paris'`` stands in the ``'capital_of'`` relation to
+ ``'france'``.
+
+ More generally, given a record such as ``['a', 'b', 'c']``, where
+ label is bound to ``'B'``, and ``obj`` bound to 1, the derived
+ binary concept will have label ``'B_of'``, and its extension will
+ be a set of pairs such as ``('a', 'b')``.
+
+
+ :param label: the base part of the preferred label for the concept
+ :type label: str
+ :param closures: closure properties for the extension of the concept
+ :type closures: list
+ :param subj: position in the record of the subject of the predicate
+ :type subj: int
+ :param obj: position in the record of the object of the predicate
+ :type obj: int
+ :param records: a list of records
+ :type records: list of lists
+ :return: ``Concept`` of arity 2
+ :rtype: Concept
+ """
+ if not label == 'border' and not label == 'contain':
+ label = label + '_of'
+ c = Concept(label, arity=2, closures=closures, extension=set())
+ for record in records:
+ c.augment((record[subj], record[obj]))
+ # close the concept's extension according to the properties in closures
+ c.close()
+ return c
+
+
+def process_bundle(rels):
+ """
+ Given a list of relation metadata bundles, make a corresponding
+ dictionary of concepts, indexed by the relation name.
+
+ :param rels: bundle of metadata needed for constructing a concept
+ :type rels: list of dict
+ :return: a dictionary of concepts, indexed by the relation name.
+ :rtype: dict
+ """
+ concepts = {}
+ for rel in rels:
+ rel_name = rel['rel_name']
+ closures = rel['closures']
+ schema = rel['schema']
+ filename = rel['filename']
+
+ concept_list = clause2concepts(filename, rel_name, schema, closures)
+ for c in concept_list:
+ label = c.prefLabel
+ if (label in concepts):
+ for data in c.extension:
+ concepts[label].augment(data)
+ concepts[label].close()
+ else:
+ concepts[label] = c
+ return concepts
+
+
+def make_valuation(concepts, read=False, lexicon=False):
+ """
+ Convert a list of ``Concept`` objects into a list of (label, extension) pairs;
+ optionally create a ``Valuation`` object.
+
+ :param concepts: concepts
+ :type concepts: list(Concept)
+ :param read: if ``True``, ``(symbol, set)`` pairs are read into a ``Valuation``
+ :type read: bool
+ :rtype: list or Valuation
+ """
+ vals = []
+
+ for c in concepts:
+ vals.append((c.prefLabel, c.extension))
+ if lexicon: read = True
+ if read:
+ from nltk.sem import Valuation
+ val = Valuation({})
+ val.update(vals)
+ # add labels for individuals
+ val = label_indivs(val, lexicon=lexicon)
+ return val
+ else: return vals
+
+
+def val_dump(rels, db):
+ """
+ Make a ``Valuation`` from a list of relation metadata bundles and dump to
+ persistent database.
+
+ :param rels: bundle of metadata needed for constructing a concept
+ :type rels: list of dict
+ :param db: name of file to which data is written.
+ The suffix '.db' will be automatically appended.
+ :type db: string
+ """
+ concepts = process_bundle(rels).values()
+ valuation = make_valuation(concepts, read=True)
+ db_out = shelve.open(db, 'n')
+
+ db_out.update(valuation)
+
+ db_out.close()
+
+
+def val_load(db):
+ """
+ Load a ``Valuation`` from a persistent database.
+
+ :param db: name of file from which data is read.
+ The suffix '.db' should be omitted from the name.
+ :type db: string
+ """
+ dbname = db+".db"
+
+ if not os.access(dbname, os.R_OK):
+ sys.exit("Cannot read file: %s" % dbname)
+ else:
+ db_in = shelve.open(db)
+ from nltk.sem import Valuation
+ val = Valuation(db_in)
+# val.read(db_in.items())
+ return val
+
+
+#def alpha(str):
+ #"""
+ #Utility to filter out non-alphabetic constants.
+
+ #:param str: candidate constant
+ #:type str: string
+ #:rtype: bool
+ #"""
+ #try:
+ #int(str)
+ #return False
+ #except ValueError:
+ ## some unknown values in records are labeled '?'
+ #if not str == '?':
+ #return True
+
+
+def label_indivs(valuation, lexicon=False):
+ """
+ Assign individual constants to the individuals in the domain of a ``Valuation``.
+
+ Given a valuation with an entry of the form ``{'rel': {'a': True}}``,
+ add a new entry ``{'a': 'a'}``.
+
+ :type valuation: Valuation
+ :rtype: Valuation
+ """
+ # collect all the individuals into a domain
+ domain = valuation.domain
+ # convert the domain into a sorted list of alphabetic terms
+ # use the same string as a label
+ pairs = [(e, e) for e in domain]
+ if lexicon:
+ lex = make_lex(domain)
+ with open("chat_pnames.cfg", 'w') as outfile:
+ outfile.writelines(lex)
+ # read the pairs into the valuation
+ valuation.update(pairs)
+ return valuation
+
+def make_lex(symbols):
+ """
+ Create lexical CFG rules for each individual symbol.
+
+ Given a valuation with an entry of the form ``{'zloty': 'zloty'}``,
+ create a lexical rule for the proper name 'Zloty'.
+
+ :param symbols: a list of individual constants in the semantic representation
+ :type symbols: sequence
+ :rtype: list
+ """
+ lex = []
+ header = """
+##################################################################
+# Lexical rules automatically generated by running 'chat80.py -x'.
+##################################################################
+
+"""
+ lex.append(header)
+ template = "PropN[num=sg, sem=<\P.(P %s)>] -> '%s'\n"
+
+ for s in symbols:
+ parts = s.split('_')
+ caps = [p.capitalize() for p in parts]
+ pname = '_'.join(caps)
+ rule = template % (s, pname)
+ lex.append(rule)
+ return lex
+
+
+###########################################################################
+# Interface function to emulate other corpus readers
+###########################################################################
+
+def concepts(items = items):
+ """
+ Build a list of concepts corresponding to the relation names in ``items``.
+
+ :param items: names of the Chat-80 relations to extract
+ :type items: list of strings
+ :return: the ``Concept`` objects which are extracted from the relations
+ :rtype: list
+ """
+ if isinstance(items, string_types): items = (items,)
+
+ rels = [item_metadata[r] for r in items]
+
+ concept_map = process_bundle(rels)
+ return concept_map.values()
+
+
+
+
+###########################################################################
+
+
+def main():
+ import sys
+ from optparse import OptionParser
+ description = \
+ """
+Extract data from the Chat-80 Prolog files and convert them into a
+Valuation object for use in the NLTK semantics package.
+ """
+
+ opts = OptionParser(description=description)
+ opts.set_defaults(verbose=True, lex=False, vocab=False)
+ opts.add_option("-s", "--store", dest="outdb",
+ help="store a valuation in DB", metavar="DB")
+ opts.add_option("-l", "--load", dest="indb",
+ help="load a stored valuation from DB", metavar="DB")
+ opts.add_option("-c", "--concepts", action="store_true",
+ help="print concepts instead of a valuation")
+ opts.add_option("-r", "--relation", dest="label",
+ help="print concept with label REL (check possible labels with '-v' option)", metavar="REL")
+ opts.add_option("-q", "--quiet", action="store_false", dest="verbose",
+ help="don't print out progress info")
+ opts.add_option("-x", "--lex", action="store_true", dest="lex",
+ help="write a file of lexical entries for country names, then exit")
+ opts.add_option("-v", "--vocab", action="store_true", dest="vocab",
+ help="print out the vocabulary of concept labels and their arity, then exit")
+
+ (options, args) = opts.parse_args()
+ if options.outdb and options.indb:
+ opts.error("Options --store and --load are mutually exclusive")
+
+
+ if options.outdb:
+ # write the valuation to a persistent database
+ if options.verbose:
+ outdb = options.outdb+".db"
+ print("Dumping a valuation to %s" % outdb)
+ val_dump(rels, options.outdb)
+ sys.exit(0)
+ else:
+ # try to read in a valuation from a database
+ if options.indb is not None:
+ dbname = options.indb+".db"
+ if not os.access(dbname, os.R_OK):
+ sys.exit("Cannot read file: %s" % dbname)
+ else:
+ valuation = val_load(options.indb)
+ # we need to create the valuation from scratch
+ else:
+ # build some concepts
+ concept_map = process_bundle(rels)
+ concepts = concept_map.values()
+ # just print out the vocabulary
+ if options.vocab:
+ items = sorted([(c.arity, c.prefLabel) for c in concepts])
+ for (arity, label) in items:
+ print(label, arity)
+ sys.exit(0)
+ # show all the concepts
+ if options.concepts:
+ for c in concepts:
+ print(c)
+ print()
+ if options.label:
+ print(concept_map[options.label])
+ sys.exit(0)
+ else:
+ # turn the concepts into a Valuation
+ if options.lex:
+ if options.verbose:
+ print("Writing out lexical rules")
+ make_valuation(concepts, lexicon=True)
+ else:
+ valuation = make_valuation(concepts, read=True)
+ print(valuation)
+
+
+def sql_demo():
+ """
+ Print out every row from the 'city.db' database.
+ """
+ print()
+ print("Using SQL to extract rows from 'city.db' RDB.")
+ for row in sql_query('corpora/city_database/city.db', "SELECT * FROM city_table"):
+ print(row)
+
+
+if __name__ == '__main__':
+ main()
+ sql_demo()
+
+
diff --git a/nltk/sem/cooper_storage.py b/nltk/sem/cooper_storage.py
new file mode 100644
index 0000000..f4f9090
--- /dev/null
+++ b/nltk/sem/cooper_storage.py
@@ -0,0 +1,118 @@
+# Natural Language Toolkit: Cooper storage for Quantifier Ambiguity
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function
+
+from nltk.sem.logic import LambdaExpression, ApplicationExpression, Variable
+from nltk.parse import load_parser
+from nltk.parse.featurechart import InstantiateVarsChart
+
+class CooperStore(object):
+ """
+ A container for handling quantifier ambiguity via Cooper storage.
+ """
+ def __init__(self, featstruct):
+ """
+ :param featstruct: The value of the ``sem`` node in a tree from
+ ``parse_with_bindops()``
+ :type featstruct: FeatStruct (with features ``core`` and ``store``)
+
+ """
+ self.featstruct = featstruct
+ self.readings = []
+ try:
+ self.core = featstruct['CORE']
+ self.store = featstruct['STORE']
+ except KeyError:
+ print("%s is not a Cooper storage structure" % featstruct)
+
+ def _permute(self, lst):
+ """
+ :return: An iterator over the permutations of the input list
+ :type lst: list
+ :rtype: iter
+ """
+ remove = lambda lst0, index: lst0[:index] + lst0[index+1:]
+ if lst:
+ for index, x in enumerate(lst):
+ for y in self._permute(remove(lst, index)):
+ yield (x,)+y
+ else: yield ()
+
+ def s_retrieve(self, trace=False):
+ """
+ Carry out S-Retrieval of binding operators in store. If hack=True,
+ serialize the bindop and core as strings and reparse. Ugh.
+
+ Each permutation of the store (i.e. list of binding operators) is
+ taken to be a possible scoping of quantifiers. We iterate through the
+ binding operators in each permutation, and successively apply them to
+ the current term, starting with the core semantic representation,
+ working from the inside out.
+
+ Binding operators are of the form::
+
+ bo(\P.all x.(man(x) -> P(x)),z1)
+ """
+ for perm, store_perm in enumerate(self._permute(self.store)):
+ if trace:
+ print("Permutation %s" % (perm+1))
+ term = self.core
+ for bindop in store_perm:
+ # we just want the arguments that are wrapped by the 'bo' predicate
+ quant, varex = tuple(bindop.args)
+ # use var to make an abstraction over the current term and then
+ # apply the quantifier to it
+ term = ApplicationExpression(quant, LambdaExpression(varex.variable, term))
+ if trace:
+ print(" ", term)
+ term = term.simplify()
+ self.readings.append(term)
+
+
+def parse_with_bindops(sentence, grammar=None, trace=0):
+ """
+ Use a grammar with Binding Operators to parse a sentence.
+ """
+ if not grammar:
+ grammar = 'grammars/book_grammars/storage.fcfg'
+ parser = load_parser(grammar, trace=trace, chart_class=InstantiateVarsChart)
+ # Parse the sentence.
+ tokens = sentence.split()
+ return list(parser.parse(tokens))
+
+
+def demo():
+ from nltk.sem import cooper_storage as cs
+ sentence = "every girl chases a dog"
+ #sentence = "a man gives a bone to every dog"
+ print()
+ print("Analyis of sentence '%s'" % sentence)
+ print("=" * 50)
+ trees = cs.parse_with_bindops(sentence, trace=0)
+ for tree in trees:
+ semrep = cs.CooperStore(tree.label()['SEM'])
+ print()
+ print("Binding operators:")
+ print("-" * 15)
+ for s in semrep.store:
+ print(s)
+ print()
+ print("Core:")
+ print("-" * 15)
+ print(semrep.core)
+ print()
+ print("S-Retrieval:")
+ print("-" * 15)
+ semrep.s_retrieve(trace=True)
+ print("Readings:")
+ print("-" * 15)
+
+ for i, reading in enumerate(semrep.readings):
+ print("%s: %s" % (i+1, reading))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/sem/drt.py b/nltk/sem/drt.py
new file mode 100644
index 0000000..9be2ced
--- /dev/null
+++ b/nltk/sem/drt.py
@@ -0,0 +1,1254 @@
+# Natural Language Toolkit: Discourse Representation Theory (DRT)
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+import operator
+from functools import reduce
+
+from nltk.compat import string_types, python_2_unicode_compatible
+from nltk.sem.logic import (APP, AbstractVariableExpression, AllExpression,
+ AndExpression, ApplicationExpression, BinaryExpression,
+ BooleanExpression, ConstantExpression, EqualityExpression,
+ EventVariableExpression, ExistsExpression, Expression,
+ FunctionVariableExpression, ImpExpression,
+ IndividualVariableExpression, LambdaExpression, Tokens,
+ _LogicParser, NegatedExpression, OrExpression, Variable,
+ is_eventvar, is_funcvar, is_indvar, unique_variable)
+
+# Import Tkinter-based modules if they are available
+try:
+ # imports are fixed for Python 2.x by nltk.compat
+ from tkinter import Canvas
+ from tkinter import Tk
+ from tkinter.font import Font
+ from nltk.util import in_idle
+
+except ImportError:
+ # No need to print a warning here, nltk.draw has already printed one.
+ pass
+
+class DrtTokens(Tokens):
+ DRS = 'DRS'
+ DRS_CONC = '+'
+ PRONOUN = 'PRO'
+ OPEN_BRACKET = '['
+ CLOSE_BRACKET = ']'
+ COLON = ':'
+
+ PUNCT = [DRS_CONC, OPEN_BRACKET, CLOSE_BRACKET, COLON]
+
+ SYMBOLS = Tokens.SYMBOLS + PUNCT
+
+ TOKENS = Tokens.TOKENS + [DRS] + PUNCT
+
+
+class _DrtParser(_LogicParser):
+ """A lambda calculus expression parser."""
+ def __init__(self):
+ _LogicParser.__init__(self)
+
+ self.operator_precedence = dict(
+ [(x,1) for x in DrtTokens.LAMBDA_LIST] + \
+ [(x,2) for x in DrtTokens.NOT_LIST] + \
+ [(APP,3)] + \
+ [(x,4) for x in DrtTokens.EQ_LIST+Tokens.NEQ_LIST] + \
+ [(DrtTokens.COLON,5)] + \
+ [(DrtTokens.DRS_CONC,6)] + \
+ [(x,7) for x in DrtTokens.OR_LIST] + \
+ [(x,8) for x in DrtTokens.IMP_LIST] + \
+ [(None,9)])
+
+ def get_all_symbols(self):
+ """This method exists to be overridden"""
+ return DrtTokens.SYMBOLS
+
+ def isvariable(self, tok):
+ return tok not in DrtTokens.TOKENS
+
+ def handle(self, tok, context):
+ """This method is intended to be overridden for logics that
+ use different operators or expressions"""
+ if tok in DrtTokens.NOT_LIST:
+ return self.handle_negation(tok, context)
+
+ elif tok in DrtTokens.LAMBDA_LIST:
+ return self.handle_lambda(tok, context)
+
+ elif tok == DrtTokens.OPEN:
+ if self.inRange(0) and self.token(0) == DrtTokens.OPEN_BRACKET:
+ return self.handle_DRS(tok, context)
+ else:
+ return self.handle_open(tok, context)
+
+ elif tok.upper() == DrtTokens.DRS:
+ self.assertNextToken(DrtTokens.OPEN)
+ return self.handle_DRS(tok, context)
+
+ elif self.isvariable(tok):
+ if self.inRange(0) and self.token(0) == DrtTokens.COLON:
+ return self.handle_prop(tok, context)
+ else:
+ return self.handle_variable(tok, context)
+
+ def make_NegatedExpression(self, expression):
+ return DrtNegatedExpression(expression)
+
+ def handle_DRS(self, tok, context):
+ # a DRS
+ refs = self.handle_refs()
+ if self.inRange(0) and self.token(0) == DrtTokens.COMMA: #if there is a comma (it's optional)
+ self.token() # swallow the comma
+ conds = self.handle_conds(context)
+ self.assertNextToken(DrtTokens.CLOSE)
+ return DRS(refs, conds, None)
+
+ def handle_refs(self):
+ self.assertNextToken(DrtTokens.OPEN_BRACKET)
+ refs = []
+ while self.inRange(0) and self.token(0) != DrtTokens.CLOSE_BRACKET:
+ # Support expressions like: DRS([x y],C) == DRS([x,y],C)
+ if refs and self.token(0) == DrtTokens.COMMA:
+ self.token() # swallow the comma
+ refs.append(self.get_next_token_variable('quantified'))
+ self.assertNextToken(DrtTokens.CLOSE_BRACKET)
+ return refs
+
+ def handle_conds(self, context):
+ self.assertNextToken(DrtTokens.OPEN_BRACKET)
+ conds = []
+ while self.inRange(0) and self.token(0) != DrtTokens.CLOSE_BRACKET:
+ # Support expressions like: DRS([x y],C) == DRS([x, y],C)
+ if conds and self.token(0) == DrtTokens.COMMA:
+ self.token() # swallow the comma
+ conds.append(self.parse_Expression(context))
+ self.assertNextToken(DrtTokens.CLOSE_BRACKET)
+ return conds
+
+ def handle_prop(self, tok, context):
+ variable = self.make_VariableExpression(tok)
+ self.assertNextToken(':')
+ drs = self.parse_Expression(DrtTokens.COLON)
+ return DrtProposition(variable, drs)
+
+ def make_EqualityExpression(self, first, second):
+ """This method serves as a hook for other logic parsers that
+ have different equality expression classes"""
+ return DrtEqualityExpression(first, second)
+
+ def get_BooleanExpression_factory(self, tok):
+ """This method serves as a hook for other logic parsers that
+ have different boolean operators"""
+ if tok == DrtTokens.DRS_CONC:
+ return lambda first, second: DrtConcatenation(first, second, None)
+ elif tok in DrtTokens.OR_LIST:
+ return DrtOrExpression
+ elif tok in DrtTokens.IMP_LIST:
+ def make_imp_expression(first, second):
+ if isinstance(first, DRS):
+ return DRS(first.refs, first.conds, second)
+ if isinstance(first, DrtConcatenation):
+ return DrtConcatenation(first.first, first.second, second)
+ raise Exception('Antecedent of implication must be a DRS')
+ return make_imp_expression
+ else:
+ return None
+
+ def make_BooleanExpression(self, factory, first, second):
+ return factory(first, second)
+
+ def make_ApplicationExpression(self, function, argument):
+ return DrtApplicationExpression(function, argument)
+
+ def make_VariableExpression(self, name):
+ return DrtVariableExpression(Variable(name))
+
+ def make_LambdaExpression(self, variables, term):
+ return DrtLambdaExpression(variables, term)
+
+
+class DrtExpression(object):
+ """
+ This is the base abstract DRT Expression from which every DRT
+ Expression extends.
+ """
+
+ _drt_parser = _DrtParser()
+
+ @classmethod
+ def fromstring(cls, s):
+ return cls._drt_parser.parse(s)
+
+ def applyto(self, other):
+ return DrtApplicationExpression(self, other)
+
+ def __neg__(self):
+ return DrtNegatedExpression(self)
+
+ def __and__(self, other):
+ raise NotImplementedError()
+
+ def __or__(self, other):
+ assert isinstance(other, DrtExpression)
+ return DrtOrExpression(self, other)
+
+ def __gt__(self, other):
+ assert isinstance(other, DrtExpression)
+ if isinstance(self, DRS):
+ return DRS(self.refs, self.conds, other)
+ if isinstance(self, DrtConcatenation):
+ return DrtConcatenation(self.first, self.second, other)
+ raise Exception('Antecedent of implication must be a DRS')
+
+ def equiv(self, other, prover=None):
+ """
+ Check for logical equivalence.
+ Pass the expression (self <-> other) to the theorem prover.
+ If the prover says it is valid, then the self and other are equal.
+
+ :param other: an ``DrtExpression`` to check equality against
+ :param prover: a ``nltk.inference.api.Prover``
+ """
+ assert isinstance(other, DrtExpression)
+
+ f1 = self.simplify().fol();
+ f2 = other.simplify().fol();
+ return f1.equiv(f2, prover)
+
+ @property
+ def type(self):
+ raise AttributeError("'%s' object has no attribute 'type'" %
+ self.__class__.__name__)
+
+ def typecheck(self, signature=None):
+ raise NotImplementedError()
+
+ def __add__(self, other):
+ return DrtConcatenation(self, other, None)
+
+ def get_refs(self, recursive=False):
+ """
+ Return the set of discourse referents in this DRS.
+ :param recursive: bool Also find discourse referents in subterms?
+ :return: list of ``Variable`` objects
+ """
+ raise NotImplementedError()
+
+ def is_pronoun_function(self):
+ """ Is self of the form "PRO(x)"? """
+ return isinstance(self, DrtApplicationExpression) and \
+ isinstance(self.function, DrtAbstractVariableExpression) and \
+ self.function.variable.name == DrtTokens.PRONOUN and \
+ isinstance(self.argument, DrtIndividualVariableExpression)
+
+ def make_EqualityExpression(self, first, second):
+ return DrtEqualityExpression(first, second)
+
+ def make_VariableExpression(self, variable):
+ return DrtVariableExpression(variable)
+
+ def resolve_anaphora(self):
+ return resolve_anaphora(self)
+
+ def eliminate_equality(self):
+ return self.visit_structured(lambda e: e.eliminate_equality(),
+ self.__class__)
+
+ def pprint(self):
+ """
+ Draw the DRS
+ """
+ print(self.pretty())
+
+ def pretty(self):
+ """
+ Draw the DRS
+ :return: the pretty print string
+ """
+ return '\n'.join(self._pretty())
+
+ def draw(self):
+ DrsDrawer(self).draw()
+
+
+ at python_2_unicode_compatible
+class DRS(DrtExpression, Expression):
+ """A Discourse Representation Structure."""
+ def __init__(self, refs, conds, consequent=None):
+ """
+ :param refs: list of ``DrtIndividualVariableExpression`` for the
+ discourse referents
+ :param conds: list of ``Expression`` for the conditions
+ """
+ self.refs = refs
+ self.conds = conds
+ self.consequent = consequent
+
+ def replace(self, variable, expression, replace_bound=False, alpha_convert=True):
+ """Replace all instances of variable v with expression E in self,
+ where v is free in self."""
+ if variable in self.refs:
+ #if a bound variable is the thing being replaced
+ if not replace_bound:
+ return self
+ else:
+ i = self.refs.index(variable)
+ if self.consequent:
+ consequent = self.consequent.replace(variable, expression, True, alpha_convert)
+ else:
+ consequent = None
+ return DRS(self.refs[:i]+[expression.variable]+self.refs[i+1:],
+ [cond.replace(variable, expression, True, alpha_convert)
+ for cond in self.conds],
+ consequent)
+ else:
+ if alpha_convert:
+ # any bound variable that appears in the expression must
+ # be alpha converted to avoid a conflict
+ for ref in (set(self.refs) & expression.free()):
+ newvar = unique_variable(ref)
+ newvarex = DrtVariableExpression(newvar)
+ i = self.refs.index(ref)
+ if self.consequent:
+ consequent = self.consequent.replace(ref, newvarex, True, alpha_convert)
+ else:
+ consequent = None
+ self = DRS(self.refs[:i]+[newvar]+self.refs[i+1:],
+ [cond.replace(ref, newvarex, True, alpha_convert)
+ for cond in self.conds],
+ consequent)
+
+ #replace in the conditions
+ if self.consequent:
+ consequent = self.consequent.replace(variable, expression, replace_bound, alpha_convert)
+ else:
+ consequent = None
+ return DRS(self.refs,
+ [cond.replace(variable, expression, replace_bound, alpha_convert)
+ for cond in self.conds],
+ consequent)
+
+ def free(self):
+ """:see: Expression.free()"""
+ conds_free = reduce(operator.or_, [c.free() for c in self.conds], set())
+ if self.consequent:
+ conds_free.update(self.consequent.free())
+ return conds_free - set(self.refs)
+
+ def get_refs(self, recursive=False):
+ """:see: AbstractExpression.get_refs()"""
+ if recursive:
+ conds_refs = self.refs + sum((c.get_refs(True) for c in self.conds), [])
+ if self.consequent:
+ conds_refs.extend(self.consequent.get_refs(True))
+ return conds_refs
+ else:
+ return self.refs
+
+ def visit(self, function, combinator):
+ """:see: Expression.visit()"""
+ parts = list(map(function, self.conds))
+ if self.consequent:
+ parts.append(function(self.consequent))
+ return combinator(parts)
+
+ def visit_structured(self, function, combinator):
+ """:see: Expression.visit_structured()"""
+ consequent = (function(self.consequent) if self.consequent else None)
+ return combinator(self.refs, list(map(function, self.conds)), consequent)
+
+ def eliminate_equality(self):
+ drs = self
+ i = 0
+ while i < len(drs.conds):
+ cond = drs.conds[i]
+ if isinstance(cond, EqualityExpression) and \
+ isinstance(cond.first, AbstractVariableExpression) and \
+ isinstance(cond.second, AbstractVariableExpression):
+ drs = DRS(list(set(drs.refs)-set([cond.second.variable])),
+ drs.conds[:i]+drs.conds[i+1:],
+ drs.consequent)
+ if cond.second.variable != cond.first.variable:
+ drs = drs.replace(cond.second.variable, cond.first, False, False)
+ i = 0
+ i -= 1
+ i += 1
+
+ conds = []
+ for cond in drs.conds:
+ new_cond = cond.eliminate_equality()
+ new_cond_simp = new_cond.simplify()
+ if not isinstance(new_cond_simp, DRS) or \
+ new_cond_simp.refs or new_cond_simp.conds or \
+ new_cond_simp.consequent:
+ conds.append(new_cond)
+
+ consequent = (drs.consequent.eliminate_equality() if drs.consequent else None)
+ return DRS(drs.refs, conds, consequent)
+
+ def fol(self):
+ if self.consequent:
+ accum = None
+ if self.conds:
+ accum = reduce(AndExpression, [c.fol() for c in self.conds])
+
+ if accum:
+ accum = ImpExpression(accum, self.consequent.fol())
+ else:
+ accum = self.consequent.fol()
+
+ for ref in self.refs[::-1]:
+ accum = AllExpression(ref, accum)
+
+ return accum
+
+ else:
+ if not self.conds:
+ raise Exception("Cannot convert DRS with no conditions to FOL.")
+ accum = reduce(AndExpression, [c.fol() for c in self.conds])
+ for ref in map(Variable, self._order_ref_strings(self.refs)[::-1]):
+ accum = ExistsExpression(ref, accum)
+ return accum
+
+ def _pretty(self):
+ refs_line = ' '.join(self._order_ref_strings(self.refs))
+
+ cond_lines = [cond for cond_line in [filter(lambda s: s.strip(), cond._pretty())
+ for cond in self.conds]
+ for cond in cond_line]
+ length = max([len(refs_line)] + list(map(len, cond_lines)))
+ drs = ([' _' + '_' * length + '_ ',
+ '| ' + refs_line.ljust(length) + ' |',
+ '|-' + '-' * length + '-|'] +
+ ['| ' + line.ljust(length) + ' |' for line in cond_lines] +
+ ['|_' + '_' * length + '_|'])
+ if self.consequent:
+ return DrtBinaryExpression._assemble_pretty(drs, DrtTokens.IMP,
+ self.consequent._pretty())
+ return drs
+
+ def _order_ref_strings(self, refs):
+ strings = ["%s" % ref for ref in refs]
+ ind_vars = []
+ func_vars = []
+ event_vars = []
+ other_vars = []
+ for s in strings:
+ if is_indvar(s):
+ ind_vars.append(s)
+ elif is_funcvar(s):
+ func_vars.append(s)
+ elif is_eventvar(s):
+ event_vars.append(s)
+ else:
+ other_vars.append(s)
+ return sorted(other_vars) + \
+ sorted(event_vars, key=lambda v: int([v[2:],-1][len(v[2:]) == 0])) + \
+ sorted(func_vars, key=lambda v: (v[0], int([v[1:],-1][len(v[1:])==0]))) + \
+ sorted(ind_vars, key=lambda v: (v[0], int([v[1:],-1][len(v[1:])==0])))
+
+ def __eq__(self, other):
+ r"""Defines equality modulo alphabetic variance.
+ If we are comparing \x.M and \y.N, then check equality of M and N[x/y]."""
+ if isinstance(other, DRS):
+ if len(self.refs) == len(other.refs):
+ converted_other = other
+ for (r1, r2) in zip(self.refs, converted_other.refs):
+ varex = self.make_VariableExpression(r1)
+ converted_other = converted_other.replace(r2, varex, True)
+ if self.consequent == converted_other.consequent and \
+ len(self.conds) == len(converted_other.conds):
+ for c1, c2 in zip(self.conds, converted_other.conds):
+ if not (c1 == c2):
+ return False
+ return True
+ return False
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Expression.__hash__
+
+ def __str__(self):
+ drs = '([%s],[%s])' % (','.join(self._order_ref_strings(self.refs)),
+ ', '.join("%s" % cond for cond in self.conds)) # map(str, self.conds)))
+ if self.consequent:
+ return DrtTokens.OPEN + drs + ' ' + DrtTokens.IMP + ' ' + \
+ "%s" % self.consequent + DrtTokens.CLOSE
+ return drs
+
+
+def DrtVariableExpression(variable):
+ """
+ This is a factory method that instantiates and returns a subtype of
+ ``DrtAbstractVariableExpression`` appropriate for the given variable.
+ """
+ if is_indvar(variable.name):
+ return DrtIndividualVariableExpression(variable)
+ elif is_funcvar(variable.name):
+ return DrtFunctionVariableExpression(variable)
+ elif is_eventvar(variable.name):
+ return DrtEventVariableExpression(variable)
+ else:
+ return DrtConstantExpression(variable)
+
+
+class DrtAbstractVariableExpression(DrtExpression, AbstractVariableExpression):
+ def fol(self):
+ return self
+
+ def get_refs(self, recursive=False):
+ """:see: AbstractExpression.get_refs()"""
+ return []
+
+ def _pretty(self):
+ s = "%s" % self
+ blank = ' '*len(s)
+ return [blank, blank, s, blank]
+
+ def eliminate_equality(self):
+ return self
+
+class DrtIndividualVariableExpression(DrtAbstractVariableExpression, IndividualVariableExpression):
+ pass
+
+class DrtFunctionVariableExpression(DrtAbstractVariableExpression, FunctionVariableExpression):
+ pass
+
+class DrtEventVariableExpression(DrtIndividualVariableExpression, EventVariableExpression):
+ pass
+
+class DrtConstantExpression(DrtAbstractVariableExpression, ConstantExpression):
+ pass
+
+
+ at python_2_unicode_compatible
+class DrtProposition(DrtExpression, Expression):
+ def __init__(self, variable, drs):
+ self.variable = variable
+ self.drs = drs
+
+ def replace(self, variable, expression, replace_bound=False, alpha_convert=True):
+ if self.variable == variable:
+ assert isinstance(expression, DrtAbstractVariableExpression), "Can only replace a proposition label with a variable"
+ return DrtProposition(expression.variable, self.drs.replace(variable, expression, replace_bound, alpha_convert))
+ else:
+ return DrtProposition(self.variable, self.drs.replace(variable, expression, replace_bound, alpha_convert))
+
+ def eliminate_equality(self):
+ return DrtProposition(self.variable, self.drs.eliminate_equality())
+
+ def get_refs(self, recursive=False):
+ return (self.drs.get_refs(True) if recursive else [])
+
+ def __eq__(self, other):
+ return self.__class__ == other.__class__ and \
+ self.variable == other.variable and \
+ self.drs == other.drs
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Expression.__hash__
+
+ def fol(self):
+ return self.drs.fol()
+
+ def _pretty(self):
+ drs_s = self.drs._pretty()
+ blank = ' ' * len("%s" % self.variable)
+ return ([blank + ' ' + line for line in drs_s[:1]] +
+ ["%s" % self.variable + ':' + line for line in drs_s[1:2]] +
+ [blank + ' ' + line for line in drs_s[2:]])
+
+ def visit(self, function, combinator):
+ """:see: Expression.visit()"""
+ return combinator([function(self.drs)])
+
+ def visit_structured(self, function, combinator):
+ """:see: Expression.visit_structured()"""
+ return combinator(self.variable, function(self.drs))
+
+ def __str__(self):
+ return 'prop(%s, %s)' % (self.variable, self.drs)
+
+
+class DrtNegatedExpression(DrtExpression, NegatedExpression):
+ def fol(self):
+ return NegatedExpression(self.term.fol())
+
+ def get_refs(self, recursive=False):
+ """:see: AbstractExpression.get_refs()"""
+ return self.term.get_refs(recursive)
+
+ def _pretty(self):
+ term_lines = self.term._pretty()
+ return ([' ' + line for line in term_lines[:2]] +
+ ['__ ' + line for line in term_lines[2:3]] +
+ [' | ' + line for line in term_lines[3:4]] +
+ [' ' + line for line in term_lines[4:]])
+
+class DrtLambdaExpression(DrtExpression, LambdaExpression):
+ def alpha_convert(self, newvar):
+ """Rename all occurrences of the variable introduced by this variable
+ binder in the expression to ``newvar``.
+ :param newvar: ``Variable``, for the new variable
+ """
+ return self.__class__(newvar, self.term.replace(self.variable,
+ DrtVariableExpression(newvar), True))
+
+ def fol(self):
+ return LambdaExpression(self.variable, self.term.fol())
+
+ def _pretty(self):
+ variables = [self.variable]
+ term = self.term
+ while term.__class__ == self.__class__:
+ variables.append(term.variable)
+ term = term.term
+ var_string = ' '.join("%s" % v for v in variables) + DrtTokens.DOT
+ term_lines = term._pretty()
+ blank = ' ' * len(var_string)
+ return ([' ' + blank + line for line in term_lines[:1]] +
+ [' \ ' + blank + line for line in term_lines[1:2]] +
+ [' /\ ' + var_string + line for line in term_lines[2:3]] +
+ [' ' + blank + line for line in term_lines[3:]])
+
+class DrtBinaryExpression(DrtExpression, BinaryExpression):
+ def get_refs(self, recursive=False):
+ """:see: AbstractExpression.get_refs()"""
+ return self.first.get_refs(True) + self.second.get_refs(True) if recursive else []
+
+ def _pretty(self):
+ return DrtBinaryExpression._assemble_pretty(self._pretty_subex(self.first), self.getOp(), self._pretty_subex(self.second))
+
+ @staticmethod
+ def _assemble_pretty(first_lines, op, second_lines):
+ max_lines = max(len(first_lines), len(second_lines))
+ first_lines = _pad_vertically(first_lines, max_lines)
+ second_lines = _pad_vertically(second_lines, max_lines)
+ blank = ' ' * len(op)
+ first_second_lines = list(zip(first_lines, second_lines))
+ return ([' ' + first_line + ' ' + blank + ' ' + second_line + ' ' for first_line, second_line in first_second_lines[:2]] +
+ ['(' + first_line + ' ' + op + ' ' + second_line + ')' for first_line, second_line in first_second_lines[2:3]] +
+ [' ' + first_line + ' ' + blank + ' ' + second_line + ' ' for first_line, second_line in first_second_lines[3:]])
+
+ def _pretty_subex(self, subex):
+ return subex._pretty()
+
+class DrtBooleanExpression(DrtBinaryExpression, BooleanExpression):
+ pass
+
+class DrtOrExpression(DrtBooleanExpression, OrExpression):
+ def fol(self):
+ return OrExpression(self.first.fol(), self.second.fol())
+
+ def _pretty_subex(self, subex):
+ if isinstance(subex, DrtOrExpression):
+ return [line[1:-1] for line in subex._pretty()]
+ return DrtBooleanExpression._pretty_subex(self, subex)
+
+class DrtEqualityExpression(DrtBinaryExpression, EqualityExpression):
+ def fol(self):
+ return EqualityExpression(self.first.fol(), self.second.fol())
+
+ at python_2_unicode_compatible
+class DrtConcatenation(DrtBooleanExpression):
+ """DRS of the form '(DRS + DRS)'"""
+ def __init__(self, first, second, consequent=None):
+ DrtBooleanExpression.__init__(self, first, second)
+ self.consequent = consequent
+
+ def replace(self, variable, expression, replace_bound=False, alpha_convert=True):
+ """Replace all instances of variable v with expression E in self,
+ where v is free in self."""
+ first = self.first
+ second = self.second
+ consequent = self.consequent
+
+ # If variable is bound
+ if variable in self.get_refs():
+ if replace_bound:
+ first = first.replace(variable, expression, replace_bound, alpha_convert)
+ second = second.replace(variable, expression, replace_bound, alpha_convert)
+ if consequent:
+ consequent = consequent.replace(variable, expression, replace_bound, alpha_convert)
+ else:
+ if alpha_convert:
+ # alpha convert every ref that is free in 'expression'
+ for ref in (set(self.get_refs(True)) & expression.free()):
+ v = DrtVariableExpression(unique_variable(ref))
+ first = first.replace(ref, v, True, alpha_convert)
+ second = second.replace(ref, v, True, alpha_convert)
+ if consequent:
+ consequent = consequent.replace(ref, v, True, alpha_convert)
+
+ first = first.replace(variable, expression, replace_bound, alpha_convert)
+ second = second.replace(variable, expression, replace_bound, alpha_convert)
+ if consequent:
+ consequent = consequent.replace(variable, expression, replace_bound, alpha_convert)
+
+ return self.__class__(first, second, consequent)
+
+ def eliminate_equality(self):
+ #TODO: at some point. for now, simplify.
+ drs = self.simplify()
+ assert not isinstance(drs, DrtConcatenation)
+ return drs.eliminate_equality()
+
+ def simplify(self):
+ first = self.first.simplify()
+ second = self.second.simplify()
+ consequent = (self.consequent.simplify() if self.consequent else None)
+
+ if isinstance(first, DRS) and isinstance(second, DRS):
+ # For any ref that is in both 'first' and 'second'
+ for ref in (set(first.get_refs(True)) & set(second.get_refs(True))):
+ # alpha convert the ref in 'second' to prevent collision
+ newvar = DrtVariableExpression(unique_variable(ref))
+ second = second.replace(ref, newvar, True)
+
+ return DRS(first.refs + second.refs, first.conds + second.conds, consequent)
+ else:
+ return self.__class__(first, second, consequent)
+
+ def get_refs(self, recursive=False):
+ """:see: AbstractExpression.get_refs()"""
+ refs = self.first.get_refs(recursive) + self.second.get_refs(recursive)
+ if self.consequent and recursive:
+ refs.extend(self.consequent.get_refs(True))
+ return refs
+
+ def getOp(self):
+ return DrtTokens.DRS_CONC
+
+ def __eq__(self, other):
+ r"""Defines equality modulo alphabetic variance.
+ If we are comparing \x.M and \y.N, then check equality of M and N[x/y]."""
+ if isinstance(other, DrtConcatenation):
+ self_refs = self.get_refs()
+ other_refs = other.get_refs()
+ if len(self_refs) == len(other_refs):
+ converted_other = other
+ for (r1,r2) in zip(self_refs, other_refs):
+ varex = self.make_VariableExpression(r1)
+ converted_other = converted_other.replace(r2, varex, True)
+ return self.first == converted_other.first and \
+ self.second == converted_other.second and \
+ self.consequent == converted_other.consequent
+ return False
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = DrtBooleanExpression.__hash__
+
+ def fol(self):
+ e = AndExpression(self.first.fol(), self.second.fol())
+ if self.consequent:
+ e = ImpExpression(e, self.consequent.fol())
+ return e
+
+ def _pretty(self):
+ drs = DrtBinaryExpression._assemble_pretty(self._pretty_subex(self.first),
+ self.getOp(),
+ self._pretty_subex(self.second))
+ if self.consequent:
+ drs = DrtBinaryExpression._assemble_pretty(drs, DrtTokens.IMP,
+ self._pretty(self.consequent))
+ return drs
+
+ def _pretty_subex(self, subex):
+ if isinstance(subex, DrtConcatenation):
+ return [line[1:-1] for line in subex._pretty()]
+ return DrtBooleanExpression._pretty_subex(self, subex)
+
+
+ def visit(self, function, combinator):
+ """:see: Expression.visit()"""
+ if self.consequent:
+ return combinator([function(self.first), function(self.second), function(self.consequent)])
+ else:
+ return combinator([function(self.first), function(self.second)])
+
+ def __str__(self):
+ first = self._str_subex(self.first)
+ second = self._str_subex(self.second)
+ drs = Tokens.OPEN + first + ' ' + self.getOp() \
+ + ' ' + second + Tokens.CLOSE
+ if self.consequent:
+ return DrtTokens.OPEN + drs + ' ' + DrtTokens.IMP + ' ' + \
+ "%s" % self.consequent + DrtTokens.CLOSE
+ return drs
+
+ def _str_subex(self, subex):
+ s = "%s" % subex
+ if isinstance(subex, DrtConcatenation) and subex.consequent is None:
+ return s[1:-1]
+ return s
+
+
+class DrtApplicationExpression(DrtExpression, ApplicationExpression):
+ def fol(self):
+ return ApplicationExpression(self.function.fol(), self.argument.fol())
+
+ def get_refs(self, recursive=False):
+ """:see: AbstractExpression.get_refs()"""
+ return (self.function.get_refs(True) + self.argument.get_refs(True)
+ if recursive else [])
+
+ def _pretty(self):
+ function, args = self.uncurry()
+ function_lines = function._pretty()
+ args_lines = [arg._pretty() for arg in args]
+ max_lines = max(map(len, [function_lines] + args_lines))
+ function_lines = _pad_vertically(function_lines, max_lines)
+ args_lines = [_pad_vertically(arg_lines, max_lines) for arg_lines in args_lines]
+ func_args_lines = list(zip(function_lines, list(zip(*args_lines))))
+ return ([func_line + ' ' + ' '.join(args_line) + ' ' for func_line, args_line in func_args_lines[:2]] +
+ [func_line + '(' + ','.join(args_line) + ')' for func_line, args_line in func_args_lines[2:3]] +
+ [func_line + ' ' + ' '.join(args_line) + ' ' for func_line, args_line in func_args_lines[3:]])
+
+
+def _pad_vertically(lines, max_lines):
+ pad_line = [' ' * len(lines[0])]
+ return lines + pad_line * (max_lines - len(lines))
+
+
+ at python_2_unicode_compatible
+class PossibleAntecedents(list, DrtExpression, Expression):
+ def free(self):
+ """Set of free variables."""
+ return set(self)
+
+ def replace(self, variable, expression, replace_bound=False, alpha_convert=True):
+ """Replace all instances of variable v with expression E in self,
+ where v is free in self."""
+ result = PossibleAntecedents()
+ for item in self:
+ if item == variable:
+ self.append(expression)
+ else:
+ self.append(item)
+ return result
+
+ def _pretty(self):
+ s = "%s" % self
+ blank = ' ' * len(s)
+ return [blank, blank, s]
+
+ def __str__(self):
+ return '[' + ','.join("%s" % it for it in self) + ']'
+
+
+class AnaphoraResolutionException(Exception):
+ pass
+
+
+def resolve_anaphora(expression, trail=[]):
+ if isinstance(expression, ApplicationExpression):
+ if expression.is_pronoun_function():
+ possible_antecedents = PossibleAntecedents()
+ for ancestor in trail:
+ for ref in ancestor.get_refs():
+ refex = expression.make_VariableExpression(ref)
+
+ #==========================================================
+ # Don't allow resolution to itself or other types
+ #==========================================================
+ if refex.__class__ == expression.argument.__class__ and \
+ not (refex == expression.argument):
+ possible_antecedents.append(refex)
+
+ if len(possible_antecedents) == 1:
+ resolution = possible_antecedents[0]
+ else:
+ resolution = possible_antecedents
+ return expression.make_EqualityExpression(expression.argument, resolution)
+ else:
+ r_function = resolve_anaphora(expression.function, trail + [expression])
+ r_argument = resolve_anaphora(expression.argument, trail + [expression])
+ return expression.__class__(r_function, r_argument)
+
+ elif isinstance(expression, DRS):
+ r_conds = []
+ for cond in expression.conds:
+ r_cond = resolve_anaphora(cond, trail + [expression])
+
+ # if the condition is of the form '(x = [])' then raise exception
+ if isinstance(r_cond, EqualityExpression):
+ if isinstance(r_cond.first, PossibleAntecedents):
+ #Reverse the order so that the variable is on the left
+ temp = r_cond.first
+ r_cond.first = r_cond.second
+ r_cond.second = temp
+ if isinstance(r_cond.second, PossibleAntecedents):
+ if not r_cond.second:
+ raise AnaphoraResolutionException("Variable '%s' does not "
+ "resolve to anything." % r_cond.first)
+
+ r_conds.append(r_cond)
+ if expression.consequent:
+ consequent = resolve_anaphora(expression.consequent, trail + [expression])
+ else:
+ consequent = None
+ return expression.__class__(expression.refs, r_conds, consequent)
+
+ elif isinstance(expression, AbstractVariableExpression):
+ return expression
+
+ elif isinstance(expression, NegatedExpression):
+ return expression.__class__(resolve_anaphora(expression.term, trail + [expression]))
+
+ elif isinstance(expression, DrtConcatenation):
+ if expression.consequent:
+ consequent = resolve_anaphora(expression.consequent, trail + [expression])
+ else:
+ consequent = None
+ return expression.__class__(resolve_anaphora(expression.first, trail + [expression]),
+ resolve_anaphora(expression.second, trail + [expression]),
+ consequent)
+
+ elif isinstance(expression, BinaryExpression):
+ return expression.__class__(resolve_anaphora(expression.first, trail + [expression]),
+ resolve_anaphora(expression.second, trail + [expression]))
+
+ elif isinstance(expression, LambdaExpression):
+ return expression.__class__(expression.variable, resolve_anaphora(expression.term, trail + [expression]))
+
+
+class DrsDrawer(object):
+ BUFFER = 3 #Space between elements
+ TOPSPACE = 10 #Space above whole DRS
+ OUTERSPACE = 6 #Space to the left, right, and bottom of the whle DRS
+
+ def __init__(self, drs, size_canvas=True, canvas=None):
+ """
+ :param drs: ``DrtExpression``, The DRS to be drawn
+ :param size_canvas: bool, True if the canvas size should be the exact size of the DRS
+ :param canvas: ``Canvas`` The canvas on which to draw the DRS. If none is given, create a new canvas.
+ """
+ master = None
+ if not canvas:
+ master = Tk()
+ master.title("DRT")
+
+ font = Font(family='helvetica', size=12)
+
+ if size_canvas:
+ canvas = Canvas(master, width=0, height=0)
+ canvas.font = font
+ self.canvas = canvas
+ (right, bottom) = self._visit(drs, self.OUTERSPACE, self.TOPSPACE)
+
+ width = max(right+self.OUTERSPACE, 100)
+ height = bottom+self.OUTERSPACE
+ canvas = Canvas(master, width=width, height=height)#, bg='white')
+ else:
+ canvas = Canvas(master, width=300, height=300)
+
+ canvas.pack()
+ canvas.font = font
+
+ self.canvas = canvas
+ self.drs = drs
+ self.master = master
+
+ def _get_text_height(self):
+ """Get the height of a line of text"""
+ return self.canvas.font.metrics("linespace")
+
+ def draw(self, x=OUTERSPACE, y=TOPSPACE):
+ """Draw the DRS"""
+ self._handle(self.drs, self._draw_command, x, y)
+
+ if self.master and not in_idle():
+ self.master.mainloop()
+ else:
+ return self._visit(self.drs, x, y)
+
+ def _visit(self, expression, x, y):
+ """
+ Return the bottom-rightmost point without actually drawing the item
+
+ :param expression: the item to visit
+ :param x: the top of the current drawing area
+ :param y: the left side of the current drawing area
+ :return: the bottom-rightmost point
+ """
+ return self._handle(expression, self._visit_command, x, y)
+
+ def _draw_command(self, item, x, y):
+ """
+ Draw the given item at the given location
+
+ :param item: the item to draw
+ :param x: the top of the current drawing area
+ :param y: the left side of the current drawing area
+ :return: the bottom-rightmost point
+ """
+ if isinstance(item, string_types):
+ self.canvas.create_text(x, y, anchor='nw', font=self.canvas.font, text=item)
+ elif isinstance(item, tuple):
+ # item is the lower-right of a box
+ (right, bottom) = item
+ self.canvas.create_rectangle(x, y, right, bottom)
+ horiz_line_y = y + self._get_text_height() + (self.BUFFER * 2) #the line separating refs from conds
+ self.canvas.create_line(x, horiz_line_y, right, horiz_line_y)
+
+ return self._visit_command(item, x, y)
+
+ def _visit_command(self, item, x, y):
+ """
+ Return the bottom-rightmost point without actually drawing the item
+
+ :param item: the item to visit
+ :param x: the top of the current drawing area
+ :param y: the left side of the current drawing area
+ :return: the bottom-rightmost point
+ """
+ if isinstance(item, string_types):
+ return (x + self.canvas.font.measure(item), y + self._get_text_height())
+ elif isinstance(item, tuple):
+ return item
+
+ def _handle(self, expression, command, x=0, y=0):
+ """
+ :param expression: the expression to handle
+ :param command: the function to apply, either _draw_command or _visit_command
+ :param x: the top of the current drawing area
+ :param y: the left side of the current drawing area
+ :return: the bottom-rightmost point
+ """
+ if command == self._visit_command:
+ #if we don't need to draw the item, then we can use the cached values
+ try:
+ #attempt to retrieve cached values
+ right = expression._drawing_width + x
+ bottom = expression._drawing_height + y
+ return (right, bottom)
+ except AttributeError:
+ #the values have not been cached yet, so compute them
+ pass
+
+ if isinstance(expression, DrtAbstractVariableExpression):
+ factory = self._handle_VariableExpression
+ elif isinstance(expression, DRS):
+ factory = self._handle_DRS
+ elif isinstance(expression, DrtNegatedExpression):
+ factory = self._handle_NegatedExpression
+ elif isinstance(expression, DrtLambdaExpression):
+ factory = self._handle_LambdaExpression
+ elif isinstance(expression, BinaryExpression):
+ factory = self._handle_BinaryExpression
+ elif isinstance(expression, DrtApplicationExpression):
+ factory = self._handle_ApplicationExpression
+ elif isinstance(expression, PossibleAntecedents):
+ factory = self._handle_VariableExpression
+ elif isinstance(expression, DrtProposition):
+ factory = self._handle_DrtProposition
+ else:
+ raise Exception(expression.__class__.__name__)
+
+ (right, bottom) = factory(expression, command, x, y)
+
+ #cache the values
+ expression._drawing_width = right - x
+ expression._drawing_height = bottom - y
+
+ return (right, bottom)
+
+ def _handle_VariableExpression(self, expression, command, x, y):
+ return command("%s" % expression, x, y)
+
+ def _handle_NegatedExpression(self, expression, command, x, y):
+ # Find the width of the negation symbol
+ right = self._visit_command(DrtTokens.NOT, x, y)[0]
+
+ # Handle term
+ (right, bottom) = self._handle(expression.term, command, right, y)
+
+ # Handle variables now that we know the y-coordinate
+ command(DrtTokens.NOT, x, self._get_centered_top(y, bottom - y, self._get_text_height()))
+
+ return (right, bottom)
+
+ def _handle_DRS(self, expression, command, x, y):
+ left = x + self.BUFFER #indent the left side
+ bottom = y + self.BUFFER #indent the top
+
+ # Handle Discourse Referents
+ if expression.refs:
+ refs = ' '.join("%s"%r for r in expression.refs)
+ else:
+ refs = ' '
+ (max_right, bottom) = command(refs, left, bottom)
+ bottom += (self.BUFFER * 2)
+
+ # Handle Conditions
+ if expression.conds:
+ for cond in expression.conds:
+ (right, bottom) = self._handle(cond, command, left, bottom)
+ max_right = max(max_right, right)
+ bottom += self.BUFFER
+ else:
+ bottom += self._get_text_height() + self.BUFFER
+
+ # Handle Box
+ max_right += self.BUFFER
+ return command((max_right, bottom), x, y)
+
+ def _handle_ApplicationExpression(self, expression, command, x, y):
+ function, args = expression.uncurry()
+ if not isinstance(function, DrtAbstractVariableExpression):
+ #It's not a predicate expression ("P(x,y)"), so leave arguments curried
+ function = expression.function
+ args = [expression.argument]
+
+ # Get the max bottom of any element on the line
+ function_bottom = self._visit(function, x, y)[1]
+ max_bottom = max([function_bottom] + [self._visit(arg, x, y)[1] for arg in args])
+
+ line_height = max_bottom - y
+
+ # Handle 'function'
+ function_drawing_top = self._get_centered_top(y, line_height, function._drawing_height)
+ right = self._handle(function, command, x, function_drawing_top)[0]
+
+ # Handle open paren
+ centred_string_top = self._get_centered_top(y, line_height, self._get_text_height())
+ right = command(DrtTokens.OPEN, right, centred_string_top)[0]
+
+ # Handle each arg
+ for (i,arg) in enumerate(args):
+ arg_drawing_top = self._get_centered_top(y, line_height, arg._drawing_height)
+ right = self._handle(arg, command, right, arg_drawing_top)[0]
+
+ if i+1 < len(args):
+ #since it's not the last arg, add a comma
+ right = command(DrtTokens.COMMA + ' ', right, centred_string_top)[0]
+
+ # Handle close paren
+ right = command(DrtTokens.CLOSE, right, centred_string_top)[0]
+
+ return (right, max_bottom)
+
+ def _handle_LambdaExpression(self, expression, command, x, y):
+ # Find the width of the lambda symbol and abstracted variables
+ variables = DrtTokens.LAMBDA + "%s" % expression.variable + DrtTokens.DOT
+ right = self._visit_command(variables, x, y)[0]
+
+ # Handle term
+ (right, bottom) = self._handle(expression.term, command, right, y)
+
+ # Handle variables now that we know the y-coordinate
+ command(variables, x, self._get_centered_top(y, bottom - y, self._get_text_height()))
+
+ return (right, bottom)
+
+ def _handle_BinaryExpression(self, expression, command, x, y):
+ # Get the full height of the line, based on the operands
+ first_height = self._visit(expression.first, 0, 0)[1]
+ second_height = self._visit(expression.second, 0, 0)[1]
+ line_height = max(first_height, second_height)
+
+ # Handle open paren
+ centred_string_top = self._get_centered_top(y, line_height, self._get_text_height())
+ right = command(DrtTokens.OPEN, x, centred_string_top)[0]
+
+ # Handle the first operand
+ first_height = expression.first._drawing_height
+ (right, first_bottom) = self._handle(expression.first, command, right, self._get_centered_top(y, line_height, first_height))
+
+ # Handle the operator
+ right = command(' %s ' % expression.getOp(), right, centred_string_top)[0]
+
+ # Handle the second operand
+ second_height = expression.second._drawing_height
+ (right, second_bottom) = self._handle(expression.second, command, right, self._get_centered_top(y, line_height, second_height))
+
+ # Handle close paren
+ right = command(DrtTokens.CLOSE, right, centred_string_top)[0]
+
+ return (right, max(first_bottom, second_bottom))
+
+ def _handle_DrtProposition(self, expression, command, x, y):
+ # Find the width of the negation symbol
+ right = command(expression.variable, x, y)[0]
+
+ # Handle term
+ (right, bottom) = self._handle(expression.term, command, right, y)
+
+ return (right, bottom)
+
+ def _get_centered_top(self, top, full_height, item_height):
+ """Get the y-coordinate of the point that a figure should start at if
+ its height is 'item_height' and it needs to be centered in an area that
+ starts at 'top' and is 'full_height' tall."""
+ return top + (full_height - item_height) / 2
+
+
+def demo():
+ print('='*20 + 'TEST PARSE' + '='*20)
+ dexpr = DrtExpression.fromstring
+ print(dexpr(r'([x,y],[sees(x,y)])'))
+ print(dexpr(r'([x],[man(x), walks(x)])'))
+ print(dexpr(r'\x.\y.([],[sees(x,y)])'))
+ print(dexpr(r'\x.([],[walks(x)])(john)'))
+ print(dexpr(r'(([x],[walks(x)]) + ([y],[runs(y)]))'))
+ print(dexpr(r'(([],[walks(x)]) -> ([],[runs(x)]))'))
+ print(dexpr(r'([x],[PRO(x), sees(John,x)])'))
+ print(dexpr(r'([x],[man(x), -([],[walks(x)])])'))
+ print(dexpr(r'([],[(([x],[man(x)]) -> ([],[walks(x)]))])'))
+
+ print('='*20 + 'Test fol()' + '='*20)
+ print(dexpr(r'([x,y],[sees(x,y)])').fol())
+
+ print('='*20 + 'Test alpha conversion and lambda expression equality' + '='*20)
+ e1 = dexpr(r'\x.([],[P(x)])')
+ print(e1)
+ e2 = e1.alpha_convert(Variable('z'))
+ print(e2)
+ print(e1 == e2)
+
+ print('='*20 + 'Test resolve_anaphora()' + '='*20)
+ print(resolve_anaphora(dexpr(r'([x,y,z],[dog(x), cat(y), walks(z), PRO(z)])')))
+ print(resolve_anaphora(dexpr(r'([],[(([x],[dog(x)]) -> ([y],[walks(y), PRO(y)]))])')))
+ print(resolve_anaphora(dexpr(r'(([x,y],[]) + ([],[PRO(x)]))')))
+
+ print('='*20 + 'Test pprint()' + '='*20)
+ dexpr(r"([],[])").pprint()
+ dexpr(r"([],[([x],[big(x), dog(x)]) -> ([],[bark(x)]) -([x],[walk(x)])])").pprint()
+ dexpr(r"([x,y],[x=y]) + ([z],[dog(z), walk(z)])").pprint()
+ dexpr(r"([],[([x],[]) | ([y],[]) | ([z],[dog(z), walk(z)])])").pprint()
+ dexpr(r"\P.\Q.(([x],[]) + P(x) + Q(x))(\x.([],[dog(x)]))").pprint()
+
+
+def test_draw():
+ expressions = [
+ r'x',
+ r'([],[])',
+ r'([x],[])',
+ r'([x],[man(x)])',
+
+ r'([x,y],[sees(x,y)])',
+ r'([x],[man(x), walks(x)])',
+ r'\x.([],[man(x), walks(x)])',
+ r'\x y.([],[sees(x,y)])',
+ r'([],[(([],[walks(x)]) + ([],[runs(x)]))])',
+
+ r'([x],[man(x), -([],[walks(x)])])',
+ r'([],[(([x],[man(x)]) -> ([],[walks(x)]))])'
+ ]
+
+ for e in expressions:
+ d = DrtExpression.fromstring(e)
+ d.draw()
+
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/sem/drt_glue_demo.py b/nltk/sem/drt_glue_demo.py
new file mode 100644
index 0000000..0f7f828
--- /dev/null
+++ b/nltk/sem/drt_glue_demo.py
@@ -0,0 +1,483 @@
+# Natural Language Toolkit: GUI Demo for Glue Semantics with Discourse
+# Representation Theory (DRT) as meaning language
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk import compat # this fixes tkinter imports for Python 2.x
+
+from tkinter.font import Font
+
+from tkinter import (Button, Frame, IntVar, Label,
+ Listbox, Menu, Scrollbar, Tk)
+
+from nltk.draw.util import CanvasFrame, ShowText
+from nltk.util import in_idle
+from nltk.tag import RegexpTagger
+from nltk.parse import MaltParser
+from nltk.sem.logic import Variable
+from nltk.sem.drt import DrsDrawer, DrtVariableExpression
+from nltk.sem.glue import DrtGlue
+
+class DrtGlueDemo(object):
+ def __init__(self, examples):
+ # Set up the main window.
+ self._top = Tk()
+ self._top.title('DRT Glue Demo')
+
+ # Set up key bindings.
+ self._init_bindings()
+
+ # Initialize the fonts.self._error = None
+ self._init_fonts(self._top)
+
+ self._examples = examples
+ self._readingCache = [None for example in examples]
+
+ # The user can hide the grammar.
+ self._show_grammar = IntVar(self._top)
+ self._show_grammar.set(1)
+
+ # Set the data to None
+ self._curExample = -1
+ self._readings = []
+ self._drs = None
+ self._drsWidget = None
+ self._error = None
+
+ self._init_glue()
+
+ # Create the basic frames.
+ self._init_menubar(self._top)
+ self._init_buttons(self._top)
+ self._init_exampleListbox(self._top)
+ self._init_readingListbox(self._top)
+ self._init_canvas(self._top)
+
+ # Resize callback
+ self._canvas.bind('<Configure>', self._configure)
+
+ #########################################
+ ## Initialization Helpers
+ #########################################
+
+ def _init_glue(self):
+ tagger = RegexpTagger(
+ [('^(David|Mary|John)$', 'NNP'),
+ ('^(walks|sees|eats|chases|believes|gives|sleeps|chases|persuades|tries|seems|leaves)$', 'VB'),
+ ('^(go|order|vanish|find|approach)$', 'VB'),
+ ('^(a)$', 'ex_quant'),
+ ('^(every)$', 'univ_quant'),
+ ('^(sandwich|man|dog|pizza|unicorn|cat|senator)$', 'NN'),
+ ('^(big|gray|former)$', 'JJ'),
+ ('^(him|himself)$', 'PRP')
+ ])
+
+ depparser = MaltParser(tagger=tagger)
+ self._glue = DrtGlue(depparser=depparser, remove_duplicates=False)
+
+ def _init_fonts(self, root):
+ # See: <http://www.astro.washington.edu/owen/ROTKFolklore.html>
+ self._sysfont = Font(font=Button()["font"])
+ root.option_add("*Font", self._sysfont)
+
+ # TWhat's our font size (default=same as sysfont)
+ self._size = IntVar(root)
+ self._size.set(self._sysfont.cget('size'))
+
+ self._boldfont = Font(family='helvetica', weight='bold',
+ size=self._size.get())
+ self._font = Font(family='helvetica',
+ size=self._size.get())
+ if self._size.get() < 0: big = self._size.get()-2
+ else: big = self._size.get()+2
+ self._bigfont = Font(family='helvetica', weight='bold',
+ size=big)
+
+ def _init_exampleListbox(self, parent):
+ self._exampleFrame = listframe = Frame(parent)
+ self._exampleFrame.pack(fill='both', side='left', padx=2)
+ self._exampleList_label = Label(self._exampleFrame, font=self._boldfont,
+ text='Examples')
+ self._exampleList_label.pack()
+ self._exampleList = Listbox(self._exampleFrame, selectmode='single',
+ relief='groove', background='white',
+ foreground='#909090', font=self._font,
+ selectforeground='#004040',
+ selectbackground='#c0f0c0')
+
+ self._exampleList.pack(side='right', fill='both', expand=1)
+
+ for example in self._examples:
+ self._exampleList.insert('end', (' %s' % example))
+ self._exampleList.config(height=min(len(self._examples), 25), width=40)
+
+ # Add a scrollbar if there are more than 25 examples.
+ if len(self._examples) > 25:
+ listscroll = Scrollbar(self._exampleFrame,
+ orient='vertical')
+ self._exampleList.config(yscrollcommand = listscroll.set)
+ listscroll.config(command=self._exampleList.yview)
+ listscroll.pack(side='left', fill='y')
+
+ # If they select a example, apply it.
+ self._exampleList.bind('<<ListboxSelect>>', self._exampleList_select)
+
+ def _init_readingListbox(self, parent):
+ self._readingFrame = listframe = Frame(parent)
+ self._readingFrame.pack(fill='both', side='left', padx=2)
+ self._readingList_label = Label(self._readingFrame, font=self._boldfont,
+ text='Readings')
+ self._readingList_label.pack()
+ self._readingList = Listbox(self._readingFrame, selectmode='single',
+ relief='groove', background='white',
+ foreground='#909090', font=self._font,
+ selectforeground='#004040',
+ selectbackground='#c0f0c0')
+
+ self._readingList.pack(side='right', fill='both', expand=1)
+
+ # Add a scrollbar if there are more than 25 examples.
+ listscroll = Scrollbar(self._readingFrame,
+ orient='vertical')
+ self._readingList.config(yscrollcommand = listscroll.set)
+ listscroll.config(command=self._readingList.yview)
+ listscroll.pack(side='right', fill='y')
+
+ self._populate_readingListbox()
+
+ def _populate_readingListbox(self):
+ # Populate the listbox with integers
+ self._readingList.delete(0, 'end')
+ for i in range(len(self._readings)):
+ self._readingList.insert('end', (' %s' % (i+1)))
+ self._readingList.config(height=min(len(self._readings), 25), width=5)
+
+ # If they select a example, apply it.
+ self._readingList.bind('<<ListboxSelect>>', self._readingList_select)
+
+ def _init_bindings(self):
+ # Key bindings are a good thing.
+ self._top.bind('<Control-q>', self.destroy)
+ self._top.bind('<Control-x>', self.destroy)
+ self._top.bind('<Escape>', self.destroy)
+ self._top.bind('n', self.next)
+ self._top.bind('<space>', self.next)
+ self._top.bind('p', self.prev)
+ self._top.bind('<BackSpace>', self.prev)
+
+ def _init_buttons(self, parent):
+ # Set up the frames.
+ self._buttonframe = buttonframe = Frame(parent)
+ buttonframe.pack(fill='none', side='bottom', padx=3, pady=2)
+ Button(buttonframe, text='Prev',
+ background='#90c0d0', foreground='black',
+ command=self.prev,).pack(side='left')
+ Button(buttonframe, text='Next',
+ background='#90c0d0', foreground='black',
+ command=self.next,).pack(side='left')
+
+ def _configure(self, event):
+ self._autostep = 0
+ (x1, y1, x2, y2) = self._cframe.scrollregion()
+ y2 = event.height - 6
+ self._canvas['scrollregion'] = '%d %d %d %d' % (x1,y1,x2,y2)
+ self._redraw()
+
+ def _init_canvas(self, parent):
+ self._cframe = CanvasFrame(parent, background='white',
+ #width=525, height=250,
+ closeenough=10,
+ border=2, relief='sunken')
+ self._cframe.pack(expand=1, fill='both', side='top', pady=2)
+ canvas = self._canvas = self._cframe.canvas()
+
+ # Initially, there's no tree or text
+ self._tree = None
+ self._textwidgets = []
+ self._textline = None
+
+ def _init_menubar(self, parent):
+ menubar = Menu(parent)
+
+ filemenu = Menu(menubar, tearoff=0)
+ filemenu.add_command(label='Exit', underline=1,
+ command=self.destroy, accelerator='q')
+ menubar.add_cascade(label='File', underline=0, menu=filemenu)
+
+ actionmenu = Menu(menubar, tearoff=0)
+ actionmenu.add_command(label='Next', underline=0,
+ command=self.next, accelerator='n, Space')
+ actionmenu.add_command(label='Previous', underline=0,
+ command=self.prev, accelerator='p, Backspace')
+ menubar.add_cascade(label='Action', underline=0, menu=actionmenu)
+
+ optionmenu = Menu(menubar, tearoff=0)
+ optionmenu.add_checkbutton(label='Remove Duplicates', underline=0,
+ variable=self._glue.remove_duplicates,
+ command=self._toggle_remove_duplicates,
+ accelerator='r')
+ menubar.add_cascade(label='Options', underline=0, menu=optionmenu)
+
+ viewmenu = Menu(menubar, tearoff=0)
+ viewmenu.add_radiobutton(label='Tiny', variable=self._size,
+ underline=0, value=10, command=self.resize)
+ viewmenu.add_radiobutton(label='Small', variable=self._size,
+ underline=0, value=12, command=self.resize)
+ viewmenu.add_radiobutton(label='Medium', variable=self._size,
+ underline=0, value=14, command=self.resize)
+ viewmenu.add_radiobutton(label='Large', variable=self._size,
+ underline=0, value=18, command=self.resize)
+ viewmenu.add_radiobutton(label='Huge', variable=self._size,
+ underline=0, value=24, command=self.resize)
+ menubar.add_cascade(label='View', underline=0, menu=viewmenu)
+
+ helpmenu = Menu(menubar, tearoff=0)
+ helpmenu.add_command(label='About', underline=0,
+ command=self.about)
+ menubar.add_cascade(label='Help', underline=0, menu=helpmenu)
+
+ parent.config(menu=menubar)
+
+ #########################################
+ ## Main draw procedure
+ #########################################
+
+ def _redraw(self):
+ canvas = self._canvas
+
+ # Delete the old DRS, widgets, etc.
+ if self._drsWidget is not None:
+ self._drsWidget.clear()
+
+ if self._drs:
+ self._drsWidget = DrsWidget( self._canvas, self._drs )
+ self._drsWidget.draw()
+
+ if self._error:
+ self._drsWidget = DrsWidget( self._canvas, self._error )
+ self._drsWidget.draw()
+
+ #########################################
+ ## Button Callbacks
+ #########################################
+
+ def destroy(self, *e):
+ self._autostep = 0
+ if self._top is None: return
+ self._top.destroy()
+ self._top = None
+
+ def prev(self, *e):
+ selection = self._readingList.curselection()
+ readingListSize = self._readingList.size()
+
+ # there are readings
+ if readingListSize > 0:
+ # if one reading is currently selected
+ if len(selection) == 1:
+ index = int(selection[0])
+
+ # if it's on (or before) the first item
+ if index <= 0:
+ self._select_previous_example()
+ else:
+ self._readingList_store_selection(index-1)
+
+ else:
+ #select its first reading
+ self._readingList_store_selection(readingListSize-1)
+
+ else:
+ self._select_previous_example()
+
+
+ def _select_previous_example(self):
+ #if the current example is not the first example
+ if self._curExample > 0:
+ self._exampleList_store_selection(self._curExample-1)
+ else:
+ #go to the last example
+ self._exampleList_store_selection(len(self._examples)-1)
+
+ def next(self, *e):
+ selection = self._readingList.curselection()
+ readingListSize = self._readingList.size()
+
+ # if there are readings
+ if readingListSize > 0:
+ # if one reading is currently selected
+ if len(selection) == 1:
+ index = int(selection[0])
+
+ # if it's on (or past) the last item
+ if index >= (readingListSize-1):
+ self._select_next_example()
+ else:
+ self._readingList_store_selection(index+1)
+
+ else:
+ #select its first reading
+ self._readingList_store_selection(0)
+
+ else:
+ self._select_next_example()
+
+ def _select_next_example(self):
+ #if the current example is not the last example
+ if self._curExample < len(self._examples)-1:
+ self._exampleList_store_selection(self._curExample+1)
+ else:
+ #go to the first example
+ self._exampleList_store_selection(0)
+
+
+ def about(self, *e):
+ ABOUT = ("NLTK Discourse Representation Theory (DRT) Glue Semantics Demo\n"+
+ "Written by Daniel H. Garrette")
+ TITLE = 'About: NLTK DRT Glue Demo'
+ try:
+ from tkMessageBox import Message
+ Message(message=ABOUT, title=TITLE).show()
+ except:
+ ShowText(self._top, TITLE, ABOUT)
+
+ def postscript(self, *e):
+ self._autostep = 0
+ self._cframe.print_to_file()
+
+ def mainloop(self, *args, **kwargs):
+ """
+ Enter the Tkinter mainloop. This function must be called if
+ this demo is created from a non-interactive program (e.g.
+ from a secript); otherwise, the demo will close as soon as
+ the script completes.
+ """
+ if in_idle(): return
+ self._top.mainloop(*args, **kwargs)
+
+ def resize(self, size=None):
+ if size is not None: self._size.set(size)
+ size = self._size.get()
+ self._font.configure(size=-(abs(size)))
+ self._boldfont.configure(size=-(abs(size)))
+ self._sysfont.configure(size=-(abs(size)))
+ self._bigfont.configure(size=-(abs(size+2)))
+ self._redraw()
+
+ def _toggle_remove_duplicates(self):
+ self._glue.remove_duplicates = not self._glue.remove_duplicates
+
+ self._exampleList.selection_clear(0, 'end')
+ self._readings = []
+ self._populate_readingListbox()
+ self._readingCache = [None for ex in self._examples]
+ self._curExample = -1
+ self._error = None
+
+ self._drs = None
+ self._redraw()
+
+
+ def _exampleList_select(self, event):
+ selection = self._exampleList.curselection()
+ if len(selection) != 1: return
+ self._exampleList_store_selection(int(selection[0]))
+
+ def _exampleList_store_selection(self, index):
+ self._curExample = index
+ example = self._examples[index]
+
+ self._exampleList.selection_clear(0, 'end')
+ if example:
+ cache = self._readingCache[index]
+ if cache:
+ if isinstance(cache, list):
+ self._readings = cache
+ self._error = None
+ else:
+ self._readings = []
+ self._error = cache
+ else:
+ try:
+ self._readings = self._glue.parse_to_meaning(example)
+ self._error = None
+ self._readingCache[index] = self._readings
+ except Exception as e:
+ self._readings = []
+ self._error = DrtVariableExpression(Variable('Error: ' + str(e)))
+ self._readingCache[index] = self._error
+
+ #add a star to the end of the example
+ self._exampleList.delete(index)
+ self._exampleList.insert(index, (' %s *' % example))
+ self._exampleList.config(height=min(len(self._examples), 25), width=40)
+
+ self._populate_readingListbox()
+
+ self._exampleList.selection_set(index)
+
+ self._drs = None
+ self._redraw()
+
+
+ def _readingList_select(self, event):
+ selection = self._readingList.curselection()
+ if len(selection) != 1: return
+ self._readingList_store_selection(int(selection[0]))
+
+ def _readingList_store_selection(self, index):
+ reading = self._readings[index]
+
+ self._readingList.selection_clear(0, 'end')
+ if reading:
+ self._readingList.selection_set(index)
+
+ self._drs = reading.simplify().normalize().resolve_anaphora()
+
+ self._redraw()
+
+
+class DrsWidget(object):
+ def __init__(self, canvas, drs, **attribs):
+ self._drs = drs
+ self._canvas = canvas
+ canvas.font = Font(font=canvas.itemcget(canvas.create_text(0, 0, text=''), 'font'))
+ canvas._BUFFER = 3
+ self.bbox = (0, 0, 0, 0)
+
+ def draw(self):
+ (right, bottom) = DrsDrawer(self._drs, canvas=self._canvas).draw()
+ self.bbox = (0, 0, right+1, bottom+1)
+
+ def clear(self):
+ self._canvas.create_rectangle(self.bbox, fill="white", width="0" )
+
+def demo():
+ examples = ['John walks',
+ 'David sees Mary',
+ 'David eats a sandwich',
+ 'every man chases a dog',
+# 'every man believes a dog yawns',
+# 'John gives David a sandwich',
+ 'John chases himself',
+# 'John persuades David to order a pizza',
+# 'John tries to go',
+# 'John tries to find a unicorn',
+# 'John seems to vanish',
+# 'a unicorn seems to approach',
+# 'every big cat leaves',
+# 'every gray cat leaves',
+# 'every big gray cat leaves',
+# 'a former senator leaves',
+# 'John likes a cat',
+# 'John likes every cat',
+# 'he walks',
+# 'John walks and he leaves'
+ ]
+ DrtGlueDemo(examples).mainloop()
+
+if __name__ == '__main__': demo()
diff --git a/nltk/sem/evaluate.py b/nltk/sem/evaluate.py
new file mode 100644
index 0000000..966f432
--- /dev/null
+++ b/nltk/sem/evaluate.py
@@ -0,0 +1,712 @@
+# Natural Language Toolkit: Models for first-order languages with lambda
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>,
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+#TODO:
+ #- fix tracing
+ #- fix iterator-based approach to existentials
+
+"""
+This module provides data structures for representing first-order
+models.
+"""
+from __future__ import print_function, unicode_literals
+
+from pprint import pformat
+import inspect
+import textwrap
+
+from nltk.decorators import decorator # this used in code that is commented out
+from nltk.compat import string_types, python_2_unicode_compatible
+
+from nltk.sem.logic import (AbstractVariableExpression, AllExpression, Expression,
+ AndExpression, ApplicationExpression, EqualityExpression,
+ ExistsExpression, IffExpression, ImpExpression,
+ IndividualVariableExpression, LambdaExpression,
+ NegatedExpression, OrExpression,
+ Variable, is_indvar)
+
+
+class Error(Exception): pass
+
+class Undefined(Error): pass
+
+def trace(f, *args, **kw):
+ argspec = inspect.getargspec(f)
+ d = dict(zip(argspec[0], args))
+ if d.pop('trace', None):
+ print()
+ for item in d.items():
+ print("%s => %s" % item)
+ return f(*args, **kw)
+
+def is_rel(s):
+ """
+ Check whether a set represents a relation (of any arity).
+
+ :param s: a set containing tuples of str elements
+ :type s: set
+ :rtype: bool
+ """
+ # we have the empty relation, i.e. set()
+ if len(s) == 0:
+ return True
+ # all the elements are tuples of the same length
+ elif all(isinstance(el, tuple) for el in s) and len(max(s))==len(min(s)):
+ return True
+ else:
+ raise ValueError("Set %r contains sequences of different lengths" % s)
+
+def set2rel(s):
+ """
+ Convert a set containing individuals (strings or numbers) into a set of
+ unary tuples. Any tuples of strings already in the set are passed through
+ unchanged.
+
+ For example:
+ - set(['a', 'b']) => set([('a',), ('b',)])
+ - set([3, 27]) => set([('3',), ('27',)])
+
+ :type s: set
+ :rtype: set of tuple of str
+ """
+ new = set()
+ for elem in s:
+ if isinstance(elem, string_types):
+ new.add((elem,))
+ elif isinstance(elem, int):
+ new.add((str(elem,)))
+ else:
+ new.add(elem)
+ return new
+
+def arity(rel):
+ """
+ Check the arity of a relation.
+ :type rel: set of tuples
+ :rtype: int of tuple of str
+ """
+ if len(rel) == 0:
+ return 0
+ return len(list(rel)[0])
+
+
+ at python_2_unicode_compatible
+class Valuation(dict):
+ """
+ A dictionary which represents a model-theoretic Valuation of non-logical constants.
+ Keys are strings representing the constants to be interpreted, and values correspond
+ to individuals (represented as strings) and n-ary relations (represented as sets of tuples
+ of strings).
+
+ An instance of ``Valuation`` will raise a KeyError exception (i.e.,
+ just behave like a standard dictionary) if indexed with an expression that
+ is not in its list of symbols.
+ """
+ def __init__(self, xs):
+ """
+ :param xs: a list of (symbol, value) pairs.
+ """
+ super(Valuation, self).__init__()
+ for (sym, val) in xs:
+ if isinstance(val, string_types) or isinstance(val, bool):
+ self[sym] = val
+ elif isinstance(val, set):
+ self[sym] = set2rel(val)
+ else:
+ msg = textwrap.fill("Error in initializing Valuation. "
+ "Unrecognized value for symbol '%s':\n%s" % (sym, val), width=66)
+
+ raise ValueError(msg)
+
+ def __getitem__(self, key):
+ if key in self:
+ return dict.__getitem__(self, key)
+ else:
+ raise Undefined("Unknown expression: '%s'" % key)
+
+ def __str__(self):
+ return pformat(self)
+
+ @property
+ def domain(self):
+ """Set-theoretic domain of the value-space of a Valuation."""
+ dom = []
+ for val in self.values():
+ if isinstance(val, string_types):
+ dom.append(val)
+ elif not isinstance(val, bool):
+ dom.extend([elem for tuple_ in val for elem in tuple_ if elem is not None])
+ return set(dom)
+
+ @property
+ def symbols(self):
+ """The non-logical constants which the Valuation recognizes."""
+ return sorted(self.keys())
+
+
+ at python_2_unicode_compatible
+class Assignment(dict):
+ """
+ A dictionary which represents an assignment of values to variables.
+
+ An assigment can only assign values from its domain.
+
+ If an unknown expression *a* is passed to a model *M*\ 's
+ interpretation function *i*, *i* will first check whether *M*\ 's
+ valuation assigns an interpretation to *a* as a constant, and if
+ this fails, *i* will delegate the interpretation of *a* to
+ *g*. *g* only assigns values to individual variables (i.e.,
+ members of the class ``IndividualVariableExpression`` in the ``logic``
+ module. If a variable is not assigned a value by *g*, it will raise
+ an ``Undefined`` exception.
+
+ A variable *Assignment* is a mapping from individual variables to
+ entities in the domain. Individual variables are usually indicated
+ with the letters ``'x'``, ``'y'``, ``'w'`` and ``'z'``, optionally
+ followed by an integer (e.g., ``'x0'``, ``'y332'``). Assignments are
+ created using the ``Assignment`` constructor, which also takes the
+ domain as a parameter.
+
+ >>> from nltk.sem.evaluate import Assignment
+ >>> dom = set(['u1', 'u2', 'u3', 'u4'])
+ >>> g3 = Assignment(dom, [('x', 'u1'), ('y', 'u2')])
+ >>> g3 == {'x': 'u1', 'y': 'u2'}
+ True
+
+ There is also a ``print`` format for assignments which uses a notation
+ closer to that in logic textbooks:
+
+ >>> print(g3)
+ g[u1/x][u2/y]
+
+ It is also possible to update an assignment using the ``add`` method:
+
+ >>> dom = set(['u1', 'u2', 'u3', 'u4'])
+ >>> g4 = Assignment(dom)
+ >>> g4.add('x', 'u1')
+ {'x': 'u1'}
+
+ With no arguments, ``purge()`` is equivalent to ``clear()`` on a dictionary:
+
+ >>> g4.purge()
+ >>> g4
+ {}
+
+ :param domain: the domain of discourse
+ :type domain: set
+ :param assign: a list of (varname, value) associations
+ :type assign: list
+ """
+
+ def __init__(self, domain, assign=None):
+ super(Assignment, self).__init__()
+ self.domain = domain
+ if assign:
+ for (var, val) in assign:
+ assert val in self.domain,\
+ "'%s' is not in the domain: %s" % (val, self.domain)
+ assert is_indvar(var),\
+ "Wrong format for an Individual Variable: '%s'" % var
+ self[var] = val
+ self.variant = None
+ self._addvariant()
+
+ def __getitem__(self, key):
+ if key in self:
+ return dict.__getitem__(self, key)
+ else:
+ raise Undefined("Not recognized as a variable: '%s'" % key)
+
+ def copy(self):
+ new = Assignment(self.domain)
+ new.update(self)
+ return new
+
+ def purge(self, var=None):
+ """
+ Remove one or all keys (i.e. logic variables) from an
+ assignment, and update ``self.variant``.
+
+ :param var: a Variable acting as a key for the assignment.
+ """
+ if var:
+ del self[var]
+ else:
+ self.clear()
+ self._addvariant()
+ return None
+
+ def __str__(self):
+ """
+ Pretty printing for assignments. {'x', 'u'} appears as 'g[u/x]'
+ """
+ gstring = "g"
+ # Deterministic output for unit testing.
+ variant = sorted(self.variant)
+ for (val, var) in variant:
+ gstring += "[%s/%s]" % (val, var)
+ return gstring
+
+ def _addvariant(self):
+ """
+ Create a more pretty-printable version of the assignment.
+ """
+ list_ = []
+ for item in self.items():
+ pair = (item[1], item[0])
+ list_.append(pair)
+ self.variant = list_
+ return None
+
+ def add(self, var, val):
+ """
+ Add a new variable-value pair to the assignment, and update
+ ``self.variant``.
+
+ """
+ assert val in self.domain,\
+ "%s is not in the domain %s" % (val, self.domain)
+ assert is_indvar(var),\
+ "Wrong format for an Individual Variable: '%s'" % var
+ self[var] = val
+ self._addvariant()
+ return self
+
+
+ at python_2_unicode_compatible
+class Model(object):
+ """
+ A first order model is a domain *D* of discourse and a valuation *V*.
+
+ A domain *D* is a set, and a valuation *V* is a map that associates
+ expressions with values in the model.
+ The domain of *V* should be a subset of *D*.
+
+ Construct a new ``Model``.
+
+ :type domain: set
+ :param domain: A set of entities representing the domain of discourse of the model.
+ :type valuation: Valuation
+ :param valuation: the valuation of the model.
+ :param prop: If this is set, then we are building a propositional\
+ model and don't require the domain of *V* to be subset of *D*.
+ """
+
+ def __init__(self, domain, valuation):
+ assert isinstance(domain, set)
+ self.domain = domain
+ self.valuation = valuation
+ if not domain.issuperset(valuation.domain):
+ raise Error("The valuation domain, %s, must be a subset of the model's domain, %s"\
+ % (valuation.domain, domain))
+
+ def __repr__(self):
+ return "(%r, %r)" % (self.domain, self.valuation)
+
+ def __str__(self):
+ return "Domain = %s,\nValuation = \n%s" % (self.domain, self.valuation)
+
+ def evaluate(self, expr, g, trace=None):
+ """
+ Read input expressions, and provide a handler for ``satisfy``
+ that blocks further propagation of the ``Undefined`` error.
+ :param expr: An ``Expression`` of ``logic``.
+ :type g: Assignment
+ :param g: an assignment to individual variables.
+ :rtype: bool or 'Undefined'
+ """
+ try:
+ parsed = Expression.fromstring(expr)
+ value = self.satisfy(parsed, g, trace=trace)
+ if trace:
+ print()
+ print("'%s' evaluates to %s under M, %s" % (expr, value, g))
+ return value
+ except Undefined:
+ if trace:
+ print()
+ print("'%s' is undefined under M, %s" % (expr, g))
+ return 'Undefined'
+
+
+ def satisfy(self, parsed, g, trace=None):
+ """
+ Recursive interpretation function for a formula of first-order logic.
+
+ Raises an ``Undefined`` error when ``parsed`` is an atomic string
+ but is not a symbol or an individual variable.
+
+ :return: Returns a truth value or ``Undefined`` if ``parsed`` is\
+ complex, and calls the interpretation function ``i`` if ``parsed``\
+ is atomic.
+
+ :param parsed: An expression of ``logic``.
+ :type g: Assignment
+ :param g: an assignment to individual variables.
+ """
+
+ if isinstance(parsed, ApplicationExpression):
+ function, arguments = parsed.uncurry()
+ if isinstance(function, AbstractVariableExpression):
+ #It's a predicate expression ("P(x,y)"), so used uncurried arguments
+ funval = self.satisfy(function, g)
+ argvals = tuple(self.satisfy(arg, g) for arg in arguments)
+ return argvals in funval
+ else:
+ #It must be a lambda expression, so use curried form
+ funval = self.satisfy(parsed.function, g)
+ argval = self.satisfy(parsed.argument, g)
+ return funval[argval]
+ elif isinstance(parsed, NegatedExpression):
+ return not self.satisfy(parsed.term, g)
+ elif isinstance(parsed, AndExpression):
+ return self.satisfy(parsed.first, g) and \
+ self.satisfy(parsed.second, g)
+ elif isinstance(parsed, OrExpression):
+ return self.satisfy(parsed.first, g) or \
+ self.satisfy(parsed.second, g)
+ elif isinstance(parsed, ImpExpression):
+ return (not self.satisfy(parsed.first, g)) or \
+ self.satisfy(parsed.second, g)
+ elif isinstance(parsed, IffExpression):
+ return self.satisfy(parsed.first, g) == \
+ self.satisfy(parsed.second, g)
+ elif isinstance(parsed, EqualityExpression):
+ return self.satisfy(parsed.first, g) == \
+ self.satisfy(parsed.second, g)
+ elif isinstance(parsed, AllExpression):
+ new_g = g.copy()
+ for u in self.domain:
+ new_g.add(parsed.variable.name, u)
+ if not self.satisfy(parsed.term, new_g):
+ return False
+ return True
+ elif isinstance(parsed, ExistsExpression):
+ new_g = g.copy()
+ for u in self.domain:
+ new_g.add(parsed.variable.name, u)
+ if self.satisfy(parsed.term, new_g):
+ return True
+ return False
+ elif isinstance(parsed, LambdaExpression):
+ cf = {}
+ var = parsed.variable.name
+ for u in self.domain:
+ val = self.satisfy(parsed.term, g.add(var, u))
+ # NB the dict would be a lot smaller if we do this:
+ # if val: cf[u] = val
+ # But then need to deal with cases where f(a) should yield
+ # a function rather than just False.
+ cf[u] = val
+ return cf
+ else:
+ return self.i(parsed, g, trace)
+
+ #@decorator(trace_eval)
+ def i(self, parsed, g, trace=False):
+ """
+ An interpretation function.
+
+ Assuming that ``parsed`` is atomic:
+
+ - if ``parsed`` is a non-logical constant, calls the valuation *V*
+ - else if ``parsed`` is an individual variable, calls assignment *g*
+ - else returns ``Undefined``.
+
+ :param parsed: an ``Expression`` of ``logic``.
+ :type g: Assignment
+ :param g: an assignment to individual variables.
+ :return: a semantic value
+ """
+ # If parsed is a propositional letter 'p', 'q', etc, it could be in valuation.symbols
+ # and also be an IndividualVariableExpression. We want to catch this first case.
+ # So there is a procedural consequence to the ordering of clauses here:
+ if parsed.variable.name in self.valuation.symbols:
+ return self.valuation[parsed.variable.name]
+ elif isinstance(parsed, IndividualVariableExpression):
+ return g[parsed.variable.name]
+
+ else:
+ raise Undefined("Can't find a value for %s" % parsed)
+
+ def satisfiers(self, parsed, varex, g, trace=None, nesting=0):
+ """
+ Generate the entities from the model's domain that satisfy an open formula.
+
+ :param parsed: an open formula
+ :type parsed: Expression
+ :param varex: the relevant free individual variable in ``parsed``.
+ :type varex: VariableExpression or str
+ :param g: a variable assignment
+ :type g: Assignment
+ :return: a set of the entities that satisfy ``parsed``.
+ """
+
+ spacer = ' '
+ indent = spacer + (spacer * nesting)
+ candidates = []
+
+ if isinstance(varex, string_types):
+ var = Variable(varex)
+ else:
+ var = varex
+
+ if var in parsed.free():
+ if trace:
+ print()
+ print((spacer * nesting) + "Open formula is '%s' with assignment %s" % (parsed, g))
+ for u in self.domain:
+ new_g = g.copy()
+ new_g.add(var.name, u)
+ if trace and trace > 1:
+ lowtrace = trace-1
+ else:
+ lowtrace = 0
+ value = self.satisfy(parsed, new_g, lowtrace)
+
+ if trace:
+ print(indent + "(trying assignment %s)" % new_g)
+
+ # parsed == False under g[u/var]?
+ if value == False:
+ if trace:
+ print(indent + "value of '%s' under %s is False" % (parsed, new_g))
+
+ # so g[u/var] is a satisfying assignment
+ else:
+ candidates.append(u)
+ if trace:
+ print(indent + "value of '%s' under %s is %s" % (parsed, new_g, value))
+
+ result = set(c for c in candidates)
+ # var isn't free in parsed
+ else:
+ raise Undefined("%s is not free in %s" % (var.name, parsed))
+
+ return result
+
+
+
+
+
+#//////////////////////////////////////////////////////////////////////
+# Demo..
+#//////////////////////////////////////////////////////////////////////
+# number of spacer chars
+mult = 30
+
+# Demo 1: Propositional Logic
+#################
+def propdemo(trace=None):
+ """Example of a propositional model."""
+
+ global val1, dom1, m1, g1
+ val1 = Valuation([('P', True), ('Q', True), ('R', False)])
+ dom1 = set([])
+ m1 = Model(dom1, val1)
+ g1 = Assignment(dom1)
+
+ print()
+ print('*' * mult)
+ print("Propositional Formulas Demo")
+ print('*' * mult)
+ print('(Propositional constants treated as nullary predicates)')
+ print()
+ print("Model m1:\n", m1)
+ print('*' * mult)
+ sentences = [
+ '(P & Q)',
+ '(P & R)',
+ '- P',
+ '- R',
+ '- - P',
+ '- (P & R)',
+ '(P | R)',
+ '(R | P)',
+ '(R | R)',
+ '(- P | R)',
+ '(P | - P)',
+ '(P -> Q)',
+ '(P -> R)',
+ '(R -> P)',
+ '(P <-> P)',
+ '(R <-> R)',
+ '(P <-> R)',
+ ]
+
+ for sent in sentences:
+ if trace:
+ print()
+ m1.evaluate(sent, g1, trace)
+ else:
+ print("The value of '%s' is: %s" % (sent, m1.evaluate(sent, g1)))
+
+# Demo 2: FOL Model
+#############
+
+def folmodel(quiet=False, trace=None):
+ """Example of a first-order model."""
+
+ global val2, v2, dom2, m2, g2
+
+ v2 = [('adam', 'b1'), ('betty', 'g1'), ('fido', 'd1'),\
+ ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])), ('dog', set(['d1'])),
+ ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]))]
+ val2 = Valuation(v2)
+ dom2 = val2.domain
+ m2 = Model(dom2, val2)
+ g2 = Assignment(dom2, [('x', 'b1'), ('y', 'g2')])
+
+ if not quiet:
+ print()
+ print('*' * mult)
+ print("Models Demo")
+ print("*" * mult)
+ print("Model m2:\n", "-" * 14,"\n", m2)
+ print("Variable assignment = ", g2)
+
+ exprs = ['adam', 'boy', 'love', 'walks', 'x', 'y', 'z']
+ parsed_exprs = [Expression.fromstring(e) for e in exprs]
+
+ print()
+ for parsed in parsed_exprs:
+ try:
+ print("The interpretation of '%s' in m2 is %s" % (parsed, m2.i(parsed, g2)))
+ except Undefined:
+ print("The interpretation of '%s' in m2 is Undefined" % parsed)
+
+
+ applications = [('boy', ('adam')), ('walks', ('adam',)), ('love', ('adam', 'y')), ('love', ('y', 'adam'))]
+
+ for (fun, args) in applications:
+ try:
+ funval = m2.i(Expression.fromstring(fun), g2)
+ argsval = tuple(m2.i(Expression.fromstring(arg), g2) for arg in args)
+ print("%s(%s) evaluates to %s" % (fun, args, argsval in funval))
+ except Undefined:
+ print("%s(%s) evaluates to Undefined" % (fun, args))
+
+# Demo 3: FOL
+#########
+
+def foldemo(trace=None):
+ """
+ Interpretation of closed expressions in a first-order model.
+ """
+ folmodel(quiet=True)
+
+ print()
+ print('*' * mult)
+ print("FOL Formulas Demo")
+ print('*' * mult)
+
+ formulas = [
+ 'love (adam, betty)',
+ '(adam = mia)',
+ '\\x. (boy(x) | girl(x))',
+ '\\x. boy(x)(adam)',
+ '\\x y. love(x, y)',
+ '\\x y. love(x, y)(adam)(betty)',
+ '\\x y. love(x, y)(adam, betty)',
+ '\\x y. (boy(x) & love(x, y))',
+ '\\x. exists y. (boy(x) & love(x, y))',
+ 'exists z1. boy(z1)',
+ 'exists x. (boy(x) & -(x = adam))',
+ 'exists x. (boy(x) & all y. love(y, x))',
+ 'all x. (boy(x) | girl(x))',
+ 'all x. (girl(x) -> exists y. boy(y) & love(x, y))', #Every girl loves exists boy.
+ 'exists x. (boy(x) & all y. (girl(y) -> love(y, x)))', #There is exists boy that every girl loves.
+ 'exists x. (boy(x) & all y. (girl(y) -> love(x, y)))', #exists boy loves every girl.
+ 'all x. (dog(x) -> - girl(x))',
+ 'exists x. exists y. (love(x, y) & love(x, y))'
+ ]
+
+
+ for fmla in formulas:
+ g2.purge()
+ if trace:
+ m2.evaluate(fmla, g2, trace)
+ else:
+ print("The value of '%s' is: %s" % (fmla, m2.evaluate(fmla, g2)))
+
+
+# Demo 3: Satisfaction
+#############
+
+def satdemo(trace=None):
+ """Satisfiers of an open formula in a first order model."""
+
+ print()
+ print('*' * mult)
+ print("Satisfiers Demo")
+ print('*' * mult)
+
+ folmodel(quiet=True)
+
+ formulas = [
+ 'boy(x)',
+ '(x = x)',
+ '(boy(x) | girl(x))',
+ '(boy(x) & girl(x))',
+ 'love(adam, x)',
+ 'love(x, adam)',
+ '-(x = adam)',
+ 'exists z22. love(x, z22)',
+ 'exists y. love(y, x)',
+ 'all y. (girl(y) -> love(x, y))',
+ 'all y. (girl(y) -> love(y, x))',
+ 'all y. (girl(y) -> (boy(x) & love(y, x)))',
+ '(boy(x) & all y. (girl(y) -> love(x, y)))',
+ '(boy(x) & all y. (girl(y) -> love(y, x)))',
+ '(boy(x) & exists y. (girl(y) & love(y, x)))',
+ '(girl(x) -> dog(x))',
+ 'all y. (dog(y) -> (x = y))',
+ 'exists y. love(y, x)',
+ 'exists y. (love(adam, y) & love(y, x))'
+ ]
+
+ if trace:
+ print(m2)
+
+ for fmla in formulas:
+ print(fmla)
+ Expression.fromstring(fmla)
+
+ parsed = [Expression.fromstring(fmla) for fmla in formulas]
+
+ for p in parsed:
+ g2.purge()
+ print("The satisfiers of '%s' are: %s" % (p, m2.satisfiers(p, 'x', g2, trace)))
+
+
+def demo(num=0, trace=None):
+ """
+ Run exists demos.
+
+ - num = 1: propositional logic demo
+ - num = 2: first order model demo (only if trace is set)
+ - num = 3: first order sentences demo
+ - num = 4: satisfaction of open formulas demo
+ - any other value: run all the demos
+
+ :param trace: trace = 1, or trace = 2 for more verbose tracing
+ """
+ demos = {
+ 1: propdemo,
+ 2: folmodel,
+ 3: foldemo,
+ 4: satdemo}
+
+ try:
+ demos[num](trace=trace)
+ except KeyError:
+ for num in demos:
+ demos[num](trace=trace)
+
+
+if __name__ == "__main__":
+ demo(2, trace=0)
diff --git a/nltk/sem/glue.py b/nltk/sem/glue.py
new file mode 100644
index 0000000..8a5eedd
--- /dev/null
+++ b/nltk/sem/glue.py
@@ -0,0 +1,661 @@
+# Natural Language Toolkit: Glue Semantics
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, division, unicode_literals
+
+import os
+
+import nltk
+from nltk.internals import Counter
+from nltk.compat import string_types
+from nltk.corpus import brown
+from nltk.tag import UnigramTagger, BigramTagger, TrigramTagger, RegexpTagger
+from nltk.sem.logic import (Expression, Variable, VariableExpression,
+ LambdaExpression, AbstractVariableExpression)
+from nltk.compat import python_2_unicode_compatible
+from nltk.sem import drt
+from nltk.sem import linearlogic
+
+SPEC_SEMTYPES = {'a' : 'ex_quant',
+ 'an' : 'ex_quant',
+ 'every' : 'univ_quant',
+ 'the' : 'def_art',
+ 'no' : 'no_quant',
+ 'default' : 'ex_quant'}
+
+OPTIONAL_RELATIONSHIPS = ['nmod', 'vmod', 'punct']
+
+ at python_2_unicode_compatible
+class GlueFormula(object):
+ def __init__(self, meaning, glue, indices=None):
+ if not indices:
+ indices = set()
+
+ if isinstance(meaning, string_types):
+ self.meaning = Expression.fromstring(meaning)
+ elif isinstance(meaning, Expression):
+ self.meaning = meaning
+ else:
+ raise RuntimeError('Meaning term neither string or expression: %s, %s' % (meaning, meaning.__class__))
+
+ if isinstance(glue, string_types):
+ self.glue = linearlogic.LinearLogicParser().parse(glue)
+ elif isinstance(glue, linearlogic.Expression):
+ self.glue = glue
+ else:
+ raise RuntimeError('Glue term neither string or expression: %s, %s' % (glue, glue.__class__))
+
+ self.indices = indices
+
+ def applyto(self, arg):
+ """ self = (\\x.(walk x), (subj -o f))
+ arg = (john , subj)
+ returns ((walk john), f)
+ """
+ if self.indices & arg.indices: # if the sets are NOT disjoint
+ raise linearlogic.LinearLogicApplicationException("'%s' applied to '%s'. Indices are not disjoint." % (self, arg))
+ else: # if the sets ARE disjoint
+ return_indices = (self.indices | arg.indices)
+
+ try:
+ return_glue = linearlogic.ApplicationExpression(self.glue, arg.glue, arg.indices)
+ except linearlogic.LinearLogicApplicationException:
+ raise linearlogic.LinearLogicApplicationException("'%s' applied to '%s'" % (self.simplify(), arg.simplify()))
+
+ arg_meaning_abstracted = arg.meaning
+ if return_indices:
+ for dep in self.glue.simplify().antecedent.dependencies[::-1]: # if self.glue is (A -o B), dep is in A.dependencies
+ arg_meaning_abstracted = self.make_LambdaExpression(Variable('v%s' % dep),
+ arg_meaning_abstracted)
+ return_meaning = self.meaning.applyto(arg_meaning_abstracted)
+
+ return self.__class__(return_meaning, return_glue, return_indices)
+
+ def make_VariableExpression(self, name):
+ return VariableExpression(name)
+
+ def make_LambdaExpression(self, variable, term):
+ return LambdaExpression(variable, term)
+
+ def lambda_abstract(self, other):
+ assert isinstance(other, GlueFormula)
+ assert isinstance(other.meaning, AbstractVariableExpression)
+ return self.__class__(self.make_LambdaExpression(other.meaning.variable,
+ self.meaning),
+ linearlogic.ImpExpression(other.glue, self.glue))
+
+ def compile(self, counter=None):
+ """From Iddo Lev's PhD Dissertation p108-109"""
+ if not counter:
+ counter = Counter()
+ (compiled_glue, new_forms) = self.glue.simplify().compile_pos(counter, self.__class__)
+ return new_forms + [self.__class__(self.meaning, compiled_glue, set([counter.get()]))]
+
+ def simplify(self):
+ return self.__class__(self.meaning.simplify(), self.glue.simplify(), self.indices)
+
+ def __eq__(self, other):
+ return self.__class__ == other.__class__ and self.meaning == other.meaning and self.glue == other.glue
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __str__(self):
+ assert isinstance(self.indices, set)
+ accum = '%s : %s' % (self.meaning, self.glue)
+ if self.indices:
+ accum += ' : {' + ', '.join(str(index) for index in self.indices) + '}'
+ return accum
+
+ def __repr__(self):
+ return "%s" % self
+
+ at python_2_unicode_compatible
+class GlueDict(dict):
+ def __init__(self, filename, encoding=None):
+ self.filename = filename
+ self.file_encoding = encoding
+ self.read_file()
+
+ def read_file(self, empty_first=True):
+ if empty_first:
+ self.clear()
+
+ try:
+ contents = nltk.data.load(self.filename, format='text', encoding=self.file_encoding)
+ # TODO: the above can't handle zip files, but this should anyway be fixed in nltk.data.load()
+ except LookupError as e:
+ try:
+ contents = nltk.data.load('file:' + self.filename, format='text', encoding=self.file_encoding)
+ except LookupError:
+ raise e
+ lines = contents.splitlines()
+
+ for line in lines: # example: 'n : (\\x.(<word> x), (v-or))'
+ # lambdacalc -^ linear logic -^
+ line = line.strip() # remove trailing newline
+ if not len(line): continue # skip empty lines
+ if line[0] == '#': continue # skip commented out lines
+
+ parts = line.split(' : ', 2) # ['verb', '(\\x.(<word> x), ( subj -o f ))', '[subj]']
+
+ glue_formulas = []
+ paren_count = 0
+ tuple_start = 0
+ tuple_comma = 0
+
+ relationships = None
+
+ if len(parts) > 1:
+ for (i, c) in enumerate(parts[1]):
+ if c == '(':
+ if paren_count == 0: # if it's the first '(' of a tuple
+ tuple_start = i+1 # then save the index
+ paren_count += 1
+ elif c == ')':
+ paren_count -= 1
+ if paren_count == 0: # if it's the last ')' of a tuple
+ meaning_term = parts[1][tuple_start:tuple_comma] # '\\x.(<word> x)'
+ glue_term = parts[1][tuple_comma+1:i] # '(v-r)'
+ glue_formulas.append([meaning_term, glue_term]) # add the GlueFormula to the list
+ elif c == ',':
+ if paren_count == 1: # if it's a comma separating the parts of the tuple
+ tuple_comma = i # then save the index
+ elif c == '#': # skip comments at the ends of lines
+ if paren_count != 0: # if the line hasn't parsed correctly so far
+ raise RuntimeError('Formula syntax is incorrect for entry ' + line)
+ break # break to the next line
+
+ if len(parts) > 2: #if there is a relationship entry at the end
+ rel_start = parts[2].index('[')+1
+ rel_end = parts[2].index(']')
+ if rel_start == rel_end:
+ relationships = frozenset()
+ else:
+ relationships = frozenset(r.strip() for r in parts[2][rel_start:rel_end].split(','))
+
+ try:
+ start_inheritance = parts[0].index('(')
+ end_inheritance = parts[0].index(')')
+ sem = parts[0][:start_inheritance].strip()
+ supertype = parts[0][start_inheritance+1:end_inheritance]
+ except:
+ sem = parts[0].strip()
+ supertype = None
+
+ if sem not in self:
+ self[sem] = {}
+
+ if relationships is None: #if not specified for a specific relationship set
+ #add all relationship entries for parents
+ if supertype:
+ for rels in self[supertype]:
+ if rels not in self[sem]:
+ self[sem][rels] = []
+ glue = self[supertype][rels]
+ self[sem][rels].extend(glue)
+ self[sem][rels].extend(glue_formulas) # add the glue formulas to every rel entry
+ else:
+ if None not in self[sem]:
+ self[sem][None] = []
+ self[sem][None].extend(glue_formulas) # add the glue formulas to every rel entry
+ else:
+ if relationships not in self[sem]:
+ self[sem][relationships] = []
+ if supertype:
+ self[sem][relationships].extend(self[supertype][relationships])
+ self[sem][relationships].extend(glue_formulas) # add the glue entry to the dictionary
+
+
+ def __str__(self):
+ accum = ''
+ for pos in self:
+ str_pos = "%s" % pos
+ for relset in self[pos]:
+ i = 1
+ for gf in self[pos][relset]:
+ if i==1:
+ accum += str_pos + ': '
+ else:
+ accum += ' '*(len(str_pos)+2)
+ accum += "%s" % gf
+ if relset and i==len(self[pos][relset]):
+ accum += ' : %s' % relset
+ accum += '\n'
+ i += 1
+ return accum
+
+ def to_glueformula_list(self, depgraph, node=None, counter=None, verbose=False):
+ if node is None:
+ top = depgraph.nodelist[0]
+ root = depgraph.nodelist[top['deps'][0]]
+ return self.to_glueformula_list(depgraph, root, Counter(), verbose)
+
+ glueformulas = self.lookup(node, depgraph, counter)
+ for dep_idx in node['deps']:
+ dep = depgraph.nodelist[dep_idx]
+ glueformulas.extend(self.to_glueformula_list(depgraph, dep, counter, verbose))
+ return glueformulas
+
+ def lookup(self, node, depgraph, counter):
+ semtype_names = self.get_semtypes(node)
+
+ semtype = None
+ for name in semtype_names:
+ if name in self:
+ semtype = self[name]
+ break
+ if semtype is None:
+# raise KeyError, "There is no GlueDict entry for sem type '%s' (for '%s')" % (sem, word)
+ return []
+
+ self.add_missing_dependencies(node, depgraph)
+
+ lookup = self._lookup_semtype_option(semtype, node, depgraph)
+
+ if not len(lookup):
+ raise KeyError("There is no GlueDict entry for sem type of '%s'"\
+ " with tag '%s', and rel '%s'" %\
+ (node['word'], node['tag'], node['rel']))
+
+ return self.get_glueformulas_from_semtype_entry(lookup, node['word'], node, depgraph, counter)
+
+ def add_missing_dependencies(self, node, depgraph):
+ rel = node['rel'].lower()
+
+ if rel == 'main':
+ headnode = depgraph.nodelist[node['head']]
+ subj = self.lookup_unique('subj', headnode, depgraph)
+ node['deps'].append(subj['address'])
+
+ def _lookup_semtype_option(self, semtype, node, depgraph):
+ relationships = frozenset(depgraph.nodelist[dep]['rel'].lower()
+ for dep in node['deps']
+ if depgraph.nodelist[dep]['rel'].lower()
+ not in OPTIONAL_RELATIONSHIPS)
+
+ try:
+ lookup = semtype[relationships]
+ except KeyError:
+ # An exact match is not found, so find the best match where
+ # 'best' is defined as the glue entry whose relationship set has the
+ # most relations of any possible relationship set that is a subset
+ # of the actual depgraph
+ best_match = frozenset()
+ for relset_option in set(semtype)-set([None]):
+ if len(relset_option) > len(best_match) and \
+ relset_option < relationships:
+ best_match = relset_option
+ if not best_match:
+ if None in semtype:
+ best_match = None
+ else:
+ return None
+ lookup = semtype[best_match]
+
+ return lookup
+
+ def get_semtypes(self, node):
+ """
+ Based on the node, return a list of plausible semtypes in order of
+ plausibility.
+ """
+ rel = node['rel'].lower()
+ word = node['word'].lower()
+
+ if rel == 'spec':
+ if word in SPEC_SEMTYPES:
+ return [SPEC_SEMTYPES[word]]
+ else:
+ return [SPEC_SEMTYPES['default']]
+ elif rel in ['nmod', 'vmod']:
+ return [node['tag'], rel]
+ else:
+ return [node['tag']]
+
+ def get_glueformulas_from_semtype_entry(self, lookup, word, node, depgraph, counter):
+ glueformulas = []
+
+ glueFormulaFactory = self.get_GlueFormula_factory()
+ for meaning, glue in lookup:
+ gf = glueFormulaFactory(self.get_meaning_formula(meaning, word), glue)
+ if not len(glueformulas):
+ gf.word = word
+ else:
+ gf.word = '%s%s' % (word, len(glueformulas)+1)
+
+ gf.glue = self.initialize_labels(gf.glue, node, depgraph, counter.get())
+
+ glueformulas.append(gf)
+ return glueformulas
+
+ def get_meaning_formula(self, generic, word):
+ """
+ :param generic: A meaning formula string containing the
+ parameter "<word>"
+ :param word: The actual word to be replace "<word>"
+ """
+ word = word.replace('.', '')
+ return generic.replace('<word>', word)
+
+ def initialize_labels(self, expr, node, depgraph, unique_index):
+ if isinstance(expr, linearlogic.AtomicExpression):
+ name = self.find_label_name(expr.name.lower(), node, depgraph, unique_index)
+ if name[0].isupper():
+ return linearlogic.VariableExpression(name)
+ else:
+ return linearlogic.ConstantExpression(name)
+ else:
+ return linearlogic.ImpExpression(
+ self.initialize_labels(expr.antecedent, node, depgraph, unique_index),
+ self.initialize_labels(expr.consequent, node, depgraph, unique_index))
+
+ def find_label_name(self, name, node, depgraph, unique_index):
+ try:
+ dot = name.index('.')
+
+ before_dot = name[:dot]
+ after_dot = name[dot+1:]
+ if before_dot == 'super':
+ return self.find_label_name(after_dot, depgraph.nodelist[node['head']], depgraph, unique_index)
+ else:
+ return self.find_label_name(after_dot, self.lookup_unique(before_dot, node, depgraph), depgraph, unique_index)
+ except ValueError:
+ lbl = self.get_label(node)
+ if name=='f': return lbl
+ elif name=='v': return '%sv' % lbl
+ elif name=='r': return '%sr' % lbl
+ elif name=='super': return self.get_label(depgraph.nodelist[node['head']])
+ elif name=='var': return '%s%s' % (lbl.upper(), unique_index)
+ elif name=='a': return self.get_label(self.lookup_unique('conja', node, depgraph))
+ elif name=='b': return self.get_label(self.lookup_unique('conjb', node, depgraph))
+ else: return self.get_label(self.lookup_unique(name, node, depgraph))
+
+ def get_label(self, node):
+ """
+ Pick an alphabetic character as identifier for an entity in the model.
+
+ :param value: where to index into the list of characters
+ :type value: int
+ """
+ value = node['address']
+
+ letter = ['f','g','h','i','j','k','l','m','n','o','p','q','r','s',
+ 't','u','v','w','x','y','z','a','b','c','d','e'][value-1]
+ num = int(value) // 26
+ if num > 0:
+ return letter + str(num)
+ else:
+ return letter
+
+ def lookup_unique(self, rel, node, depgraph):
+ """
+ Lookup 'key'. There should be exactly one item in the associated relation.
+ """
+ deps = [depgraph.nodelist[dep] for dep in node['deps']
+ if depgraph.nodelist[dep]['rel'].lower() == rel.lower()]
+
+ if len(deps) == 0:
+ raise KeyError("'%s' doesn't contain a feature '%s'" % (node['word'], rel))
+ elif len(deps) > 1:
+ raise KeyError("'%s' should only have one feature '%s'" % (node['word'], rel))
+ else:
+ return deps[0]
+
+ def get_GlueFormula_factory(self):
+ return GlueFormula
+
+class Glue(object):
+ def __init__(self, semtype_file=None, remove_duplicates=False,
+ depparser=None, verbose=False):
+ self.verbose = verbose
+ self.remove_duplicates = remove_duplicates
+ self.depparser = depparser
+
+ from nltk import Prover9
+ self.prover = Prover9()
+
+ if semtype_file:
+ self.semtype_file = semtype_file
+ else:
+ self.semtype_file = os.path.join('grammars', 'sample_grammars','glue.semtype')
+
+ def train_depparser(self, depgraphs=None):
+ if depgraphs:
+ self.depparser.train(depgraphs)
+ else:
+ self.depparser.train_from_file(nltk.data.find(
+ os.path.join('grammars', 'sample_grammars',
+ 'glue_train.conll')))
+
+ def parse_to_meaning(self, sentence):
+ readings = []
+ for agenda in self.parse_to_compiled(sentence):
+ readings.extend(self.get_readings(agenda))
+ return readings
+
+ def get_readings(self, agenda):
+ readings = []
+ agenda_length = len(agenda)
+ atomics = dict()
+ nonatomics = dict()
+ while agenda: # is not empty
+ cur = agenda.pop()
+ glue_simp = cur.glue.simplify()
+ if isinstance(glue_simp, linearlogic.ImpExpression): # if cur.glue is non-atomic
+ for key in atomics:
+ try:
+ if isinstance(cur.glue, linearlogic.ApplicationExpression):
+ bindings = cur.glue.bindings
+ else:
+ bindings = linearlogic.BindingDict()
+ glue_simp.antecedent.unify(key, bindings)
+ for atomic in atomics[key]:
+ if not (cur.indices & atomic.indices): # if the sets of indices are disjoint
+ try:
+ agenda.append(cur.applyto(atomic))
+ except linearlogic.LinearLogicApplicationException:
+ pass
+ except linearlogic.UnificationException:
+ pass
+ try:
+ nonatomics[glue_simp.antecedent].append(cur)
+ except KeyError:
+ nonatomics[glue_simp.antecedent] = [cur]
+
+ else: # else cur.glue is atomic
+ for key in nonatomics:
+ for nonatomic in nonatomics[key]:
+ try:
+ if isinstance(nonatomic.glue, linearlogic.ApplicationExpression):
+ bindings = nonatomic.glue.bindings
+ else:
+ bindings = linearlogic.BindingDict()
+ glue_simp.unify(key, bindings)
+ if not (cur.indices & nonatomic.indices): # if the sets of indices are disjoint
+ try:
+ agenda.append(nonatomic.applyto(cur))
+ except linearlogic.LinearLogicApplicationException:
+ pass
+ except linearlogic.UnificationException:
+ pass
+ try:
+ atomics[glue_simp].append(cur)
+ except KeyError:
+ atomics[glue_simp] = [cur]
+
+ for entry in atomics:
+ for gf in atomics[entry]:
+ if len(gf.indices) == agenda_length:
+ self._add_to_reading_list(gf, readings)
+ for entry in nonatomics:
+ for gf in nonatomics[entry]:
+ if len(gf.indices) == agenda_length:
+ self._add_to_reading_list(gf, readings)
+ return readings
+
+ def _add_to_reading_list(self, glueformula, reading_list):
+ add_reading = True
+ if self.remove_duplicates:
+ for reading in reading_list:
+ try:
+ if reading.equiv(glueformula.meaning, self.prover):
+ add_reading = False
+ break
+ except Exception as e:
+ #if there is an exception, the syntax of the formula
+ #may not be understandable by the prover, so don't
+ #throw out the reading.
+ print('Error when checking logical equality of statements', e)
+ pass
+ if add_reading:
+ reading_list.append(glueformula.meaning)
+
+ def parse_to_compiled(self, sentence):
+ gfls = [self.depgraph_to_glue(dg) for dg in self.dep_parse(sentence)]
+ return [self.gfl_to_compiled(gfl) for gfl in gfls]
+
+ def dep_parse(self, sentence):
+ #Lazy-initialize the depparser
+ if self.depparser is None:
+ from nltk.parse import MaltParser
+ self.depparser = MaltParser(tagger=self.get_pos_tagger())
+ if not self.depparser._trained:
+ self.train_depparser()
+ return [self.depparser.parse(sentence, verbose=self.verbose)]
+
+ def depgraph_to_glue(self, depgraph):
+ return self.get_glue_dict().to_glueformula_list(depgraph)
+
+ def get_glue_dict(self):
+ return GlueDict(self.semtype_file)
+
+ def gfl_to_compiled(self, gfl):
+ index_counter = Counter()
+ return_list = []
+ for gf in gfl:
+ return_list.extend(gf.compile(index_counter))
+
+ if self.verbose:
+ print('Compiled Glue Premises:')
+ for cgf in return_list:
+ print(cgf)
+
+ return return_list
+
+ def get_pos_tagger(self):
+ regexp_tagger = RegexpTagger(
+ [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
+ (r'(The|the|A|a|An|an)$', 'AT'), # articles
+ (r'.*able$', 'JJ'), # adjectives
+ (r'.*ness$', 'NN'), # nouns formed from adjectives
+ (r'.*ly$', 'RB'), # adverbs
+ (r'.*s$', 'NNS'), # plural nouns
+ (r'.*ing$', 'VBG'), # gerunds
+ (r'.*ed$', 'VBD'), # past tense verbs
+ (r'.*', 'NN') # nouns (default)
+ ])
+ brown_train = brown.tagged_sents(categories='news')
+ unigram_tagger = UnigramTagger(brown_train, backoff=regexp_tagger)
+ bigram_tagger = BigramTagger(brown_train, backoff=unigram_tagger)
+ trigram_tagger = TrigramTagger(brown_train, backoff=bigram_tagger)
+
+ #Override particular words
+ main_tagger = RegexpTagger(
+ [(r'(A|a|An|an)$', 'ex_quant'),
+ (r'(Every|every|All|all)$', 'univ_quant')
+ ], backoff=trigram_tagger)
+
+ return main_tagger
+
+
+class DrtGlueFormula(GlueFormula):
+ def __init__(self, meaning, glue, indices=None):
+ if not indices:
+ indices = set()
+
+ if isinstance(meaning, string_types):
+ self.meaning = drt.DrtExpression.fromstring(meaning)
+ elif isinstance(meaning, drt.AbstractDrs):
+ self.meaning = meaning
+ else:
+ raise RuntimeError('Meaning term neither string or expression: %s, %s' % (meaning, meaning.__class__))
+
+ if isinstance(glue, string_types):
+ self.glue = linearlogic.LinearLogicParser().parse(glue)
+ elif isinstance(glue, linearlogic.Expression):
+ self.glue = glue
+ else:
+ raise RuntimeError('Glue term neither string or expression: %s, %s' % (glue, glue.__class__))
+
+ self.indices = indices
+
+ def make_VariableExpression(self, name):
+ return drt.DrtVariableExpression(name)
+
+ def make_LambdaExpression(self, variable, term):
+ return drt.DrtLambdaExpression(variable, term)
+
+class DrtGlueDict(GlueDict):
+ def get_GlueFormula_factory(self):
+ return DrtGlueFormula
+
+class DrtGlue(Glue):
+ def __init__(self, semtype_file=None, remove_duplicates=False,
+ depparser=None, verbose=False):
+ if not semtype_file:
+ semtype_file = os.path.join('grammars', 'sample_grammars','drt_glue.semtype')
+ Glue.__init__(self, semtype_file, remove_duplicates, depparser, verbose)
+
+ def get_glue_dict(self):
+ return DrtGlueDict(self.semtype_file)
+
+
+def demo(show_example=-1):
+ from nltk.parse import MaltParser
+ examples = ['David sees Mary',
+ 'David eats a sandwich',
+ 'every man chases a dog',
+ 'every man believes a dog sleeps',
+ 'John gives David a sandwich',
+ 'John chases himself']
+# 'John persuades David to order a pizza',
+# 'John tries to go',
+# 'John tries to find a unicorn',
+# 'John seems to vanish',
+# 'a unicorn seems to approach',
+# 'every big cat leaves',
+# 'every gray cat leaves',
+# 'every big gray cat leaves',
+# 'a former senator leaves',
+
+ print('============== DEMO ==============')
+
+ tagger = RegexpTagger(
+ [('^(David|Mary|John)$', 'NNP'),
+ ('^(sees|eats|chases|believes|gives|sleeps|chases|persuades|tries|seems|leaves)$', 'VB'),
+ ('^(go|order|vanish|find|approach)$', 'VB'),
+ ('^(a)$', 'ex_quant'),
+ ('^(every)$', 'univ_quant'),
+ ('^(sandwich|man|dog|pizza|unicorn|cat|senator)$', 'NN'),
+ ('^(big|gray|former)$', 'JJ'),
+ ('^(him|himself)$', 'PRP')
+ ])
+
+ depparser = MaltParser(tagger=tagger)
+ glue = Glue(depparser=depparser, verbose=False)
+
+ for (i, sentence) in enumerate(examples):
+ if i==show_example or show_example==-1:
+ print('[[[Example %s]]] %s' % (i, sentence))
+ for reading in glue.parse_to_meaning(sentence.split()):
+ print(reading.simplify())
+ print('')
+
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/sem/hole.py b/nltk/sem/hole.py
new file mode 100644
index 0000000..daf337c
--- /dev/null
+++ b/nltk/sem/hole.py
@@ -0,0 +1,374 @@
+# Natural Language Toolkit: Logic
+#
+# Author: Peter Wang
+# Updated by: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+
+"""
+An implementation of the Hole Semantics model, following Blackburn and Bos,
+Representation and Inference for Natural Language (CSLI, 2005).
+
+The semantic representations are built by the grammar hole.fcfg.
+This module contains driver code to read in sentences and parse them
+according to a hole semantics grammar.
+
+After parsing, the semantic representation is in the form of an underspecified
+representation that is not easy to read. We use a "plugging" algorithm to
+convert that representation into first-order logic formulas.
+"""
+from __future__ import print_function, unicode_literals
+
+from functools import reduce
+
+from nltk import compat
+from nltk.parse import load_parser
+from nltk.draw.tree import draw_trees
+from nltk.sem.skolemize import skolemize
+from nltk.sem.logic import (AllExpression, AndExpression, ApplicationExpression,
+ ExistsExpression, IffExpression, ImpExpression,
+ LambdaExpression, NegatedExpression, OrExpression)
+
+
+# Note that in this code there may be multiple types of trees being referred to:
+#
+# 1. parse trees
+# 2. the underspecified representation
+# 3. first-order logic formula trees
+# 4. the search space when plugging (search tree)
+#
+
+class Constants(object):
+ ALL = 'ALL'
+ EXISTS = 'EXISTS'
+ NOT = 'NOT'
+ AND = 'AND'
+ OR = 'OR'
+ IMP = 'IMP'
+ IFF = 'IFF'
+ PRED = 'PRED'
+ LEQ = 'LEQ'
+ HOLE = 'HOLE'
+ LABEL = 'LABEL'
+
+ MAP = {ALL: lambda v,e: AllExpression(v.variable, e),
+ EXISTS: lambda v,e: ExistsExpression(v.variable, e),
+ NOT: NegatedExpression,
+ AND: AndExpression,
+ OR: OrExpression,
+ IMP: ImpExpression,
+ IFF: IffExpression,
+ PRED: ApplicationExpression}
+
+class HoleSemantics(object):
+ """
+ This class holds the broken-down components of a hole semantics, i.e. it
+ extracts the holes, labels, logic formula fragments and constraints out of
+ a big conjunction of such as produced by the hole semantics grammar. It
+ then provides some operations on the semantics dealing with holes, labels
+ and finding legal ways to plug holes with labels.
+ """
+ def __init__(self, usr):
+ """
+ Constructor. `usr' is a ``sem.Expression`` representing an
+ Underspecified Representation Structure (USR). A USR has the following
+ special predicates:
+ ALL(l,v,n),
+ EXISTS(l,v,n),
+ AND(l,n,n),
+ OR(l,n,n),
+ IMP(l,n,n),
+ IFF(l,n,n),
+ PRED(l,v,n,v[,v]*) where the brackets and star indicate zero or more repetitions,
+ LEQ(n,n),
+ HOLE(n),
+ LABEL(n)
+ where l is the label of the node described by the predicate, n is either
+ a label or a hole, and v is a variable.
+ """
+ self.holes = set()
+ self.labels = set()
+ self.fragments = {} # mapping of label -> formula fragment
+ self.constraints = set() # set of Constraints
+ self._break_down(usr)
+ self.top_most_labels = self._find_top_most_labels()
+ self.top_hole = self._find_top_hole()
+
+ def is_node(self, x):
+ """
+ Return true if x is a node (label or hole) in this semantic
+ representation.
+ """
+ return x in (self.labels | self.holes)
+
+ def _break_down(self, usr):
+ """
+ Extract holes, labels, formula fragments and constraints from the hole
+ semantics underspecified representation (USR).
+ """
+ if isinstance(usr, AndExpression):
+ self._break_down(usr.first)
+ self._break_down(usr.second)
+ elif isinstance(usr, ApplicationExpression):
+ func, args = usr.uncurry()
+ if func.variable.name == Constants.LEQ:
+ self.constraints.add(Constraint(args[0], args[1]))
+ elif func.variable.name == Constants.HOLE:
+ self.holes.add(args[0])
+ elif func.variable.name == Constants.LABEL:
+ self.labels.add(args[0])
+ else:
+ label = args[0]
+ assert label not in self.fragments
+ self.fragments[label] = (func, args[1:])
+ else:
+ raise ValueError(usr.label())
+
+ def _find_top_nodes(self, node_list):
+ top_nodes = node_list.copy()
+ for f in compat.itervalues(self.fragments):
+ #the label is the first argument of the predicate
+ args = f[1]
+ for arg in args:
+ if arg in node_list:
+ top_nodes.discard(arg)
+ return top_nodes
+
+ def _find_top_most_labels(self):
+ """
+ Return the set of labels which are not referenced directly as part of
+ another formula fragment. These will be the top-most labels for the
+ subtree that they are part of.
+ """
+ return self._find_top_nodes(self.labels)
+
+ def _find_top_hole(self):
+ """
+ Return the hole that will be the top of the formula tree.
+ """
+ top_holes = self._find_top_nodes(self.holes)
+ assert len(top_holes) == 1 # it must be unique
+ return top_holes.pop()
+
+ def pluggings(self):
+ """
+ Calculate and return all the legal pluggings (mappings of labels to
+ holes) of this semantics given the constraints.
+ """
+ record = []
+ self._plug_nodes([(self.top_hole, [])], self.top_most_labels, {}, record)
+ return record
+
+ def _plug_nodes(self, queue, potential_labels, plug_acc, record):
+ """
+ Plug the nodes in `queue' with the labels in `potential_labels'.
+
+ Each element of `queue' is a tuple of the node to plug and the list of
+ ancestor holes from the root of the graph to that node.
+
+ `potential_labels' is a set of the labels which are still available for
+ plugging.
+
+ `plug_acc' is the incomplete mapping of holes to labels made on the
+ current branch of the search tree so far.
+
+ `record' is a list of all the complete pluggings that we have found in
+ total so far. It is the only parameter that is destructively updated.
+ """
+ if queue != []:
+ (node, ancestors) = queue[0]
+ if node in self.holes:
+ # The node is a hole, try to plug it.
+ self._plug_hole(node, ancestors, queue[1:], potential_labels, plug_acc, record)
+ else:
+ assert node in self.labels
+ # The node is a label. Replace it in the queue by the holes and
+ # labels in the formula fragment named by that label.
+ args = self.fragments[node][1]
+ head = [(a, ancestors) for a in args if self.is_node(a)]
+ self._plug_nodes(head + queue[1:], potential_labels, plug_acc, record)
+ else:
+ raise Exception('queue empty')
+
+ def _plug_hole(self, hole, ancestors0, queue, potential_labels0,
+ plug_acc0, record):
+ """
+ Try all possible ways of plugging a single hole.
+ See _plug_nodes for the meanings of the parameters.
+ """
+ # Add the current hole we're trying to plug into the list of ancestors.
+ assert hole not in ancestors0
+ ancestors = [hole] + ancestors0
+
+ # Try each potential label in this hole in turn.
+ for l in potential_labels0:
+ # Is the label valid in this hole?
+ if self._violates_constraints(l, ancestors):
+ continue
+
+ plug_acc = plug_acc0.copy()
+ plug_acc[hole] = l
+ potential_labels = potential_labels0.copy()
+ potential_labels.remove(l)
+
+ if len(potential_labels) == 0:
+ # No more potential labels. That must mean all the holes have
+ # been filled so we have found a legal plugging so remember it.
+ #
+ # Note that the queue might not be empty because there might
+ # be labels on there that point to formula fragments with
+ # no holes in them. _sanity_check_plugging will make sure
+ # all holes are filled.
+ self._sanity_check_plugging(plug_acc, self.top_hole, [])
+ record.append(plug_acc)
+ else:
+ # Recursively try to fill in the rest of the holes in the
+ # queue. The label we just plugged into the hole could have
+ # holes of its own so at the end of the queue. Putting it on
+ # the end of the queue gives us a breadth-first search, so that
+ # all the holes at level i of the formula tree are filled
+ # before filling level i+1.
+ # A depth-first search would work as well since the trees must
+ # be finite but the bookkeeping would be harder.
+ self._plug_nodes(queue + [(l, ancestors)], potential_labels, plug_acc, record)
+
+ def _violates_constraints(self, label, ancestors):
+ """
+ Return True if the `label' cannot be placed underneath the holes given
+ by the set `ancestors' because it would violate the constraints imposed
+ on it.
+ """
+ for c in self.constraints:
+ if c.lhs == label:
+ if c.rhs not in ancestors:
+ return True
+ return False
+
+ def _sanity_check_plugging(self, plugging, node, ancestors):
+ """
+ Make sure that a given plugging is legal. We recursively go through
+ each node and make sure that no constraints are violated.
+ We also check that all holes have been filled.
+ """
+ if node in self.holes:
+ ancestors = [node] + ancestors
+ label = plugging[node]
+ else:
+ label = node
+ assert label in self.labels
+ for c in self.constraints:
+ if c.lhs == label:
+ assert c.rhs in ancestors
+ args = self.fragments[label][1]
+ for arg in args:
+ if self.is_node(arg):
+ self._sanity_check_plugging(plugging, arg, [label] + ancestors)
+
+ def formula_tree(self, plugging):
+ """
+ Return the first-order logic formula tree for this underspecified
+ representation using the plugging given.
+ """
+ return self._formula_tree(plugging, self.top_hole)
+
+ def _formula_tree(self, plugging, node):
+ if node in plugging:
+ return self._formula_tree(plugging, plugging[node])
+ elif node in self.fragments:
+ pred,args = self.fragments[node]
+ children = [self._formula_tree(plugging, arg) for arg in args]
+ return reduce(Constants.MAP[pred.variable.name], children)
+ else:
+ return node
+
+
+ at compat.python_2_unicode_compatible
+class Constraint(object):
+ """
+ This class represents a constraint of the form (L =< N),
+ where L is a label and N is a node (a label or a hole).
+ """
+ def __init__(self, lhs, rhs):
+ self.lhs = lhs
+ self.rhs = rhs
+ def __eq__(self, other):
+ if self.__class__ == other.__class__:
+ return self.lhs == other.lhs and self.rhs == other.rhs
+ else:
+ return False
+ def __ne__(self, other):
+ return not (self == other)
+ def __hash__(self):
+ return hash(repr(self))
+ def __repr__(self):
+ return '(%s < %s)' % (self.lhs, self.rhs)
+
+
+def hole_readings(sentence, grammar_filename=None, verbose=False):
+ if not grammar_filename:
+ grammar_filename = 'grammars/sample_grammars/hole.fcfg'
+
+ if verbose: print('Reading grammar file', grammar_filename)
+
+ parser = load_parser(grammar_filename)
+
+ # Parse the sentence.
+ tokens = sentence.split()
+ trees = list(parser.parse(tokens))
+ if verbose: print('Got %d different parses' % len(trees))
+
+ all_readings = []
+ for tree in trees:
+ # Get the semantic feature from the top of the parse tree.
+ sem = tree.label()['SEM'].simplify()
+
+ # Print the raw semantic representation.
+ if verbose: print('Raw: ', sem)
+
+ # Skolemize away all quantifiers. All variables become unique.
+ while isinstance(sem, LambdaExpression):
+ sem = sem.term
+ skolemized = skolemize(sem)
+
+ if verbose: print('Skolemized:', skolemized)
+
+ # Break the hole semantics representation down into its components
+ # i.e. holes, labels, formula fragments and constraints.
+ hole_sem = HoleSemantics(skolemized)
+
+ # Maybe show the details of the semantic representation.
+ if verbose:
+ print('Holes: ', hole_sem.holes)
+ print('Labels: ', hole_sem.labels)
+ print('Constraints: ', hole_sem.constraints)
+ print('Top hole: ', hole_sem.top_hole)
+ print('Top labels: ', hole_sem.top_most_labels)
+ print('Fragments:')
+ for (l,f) in hole_sem.fragments.items():
+ print('\t%s: %s' % (l, f))
+
+ # Find all the possible ways to plug the formulas together.
+ pluggings = hole_sem.pluggings()
+
+ # Build FOL formula trees using the pluggings.
+ readings = list(map(hole_sem.formula_tree, pluggings))
+
+ # Print out the formulas in a textual format.
+ if verbose:
+ for i,r in enumerate(readings):
+ print()
+ print('%d. %s' % (i, r))
+ print()
+
+ all_readings.extend(readings)
+
+ return all_readings
+
+
+if __name__ == '__main__':
+ for r in hole_readings('a dog barks'): print(r)
+ print()
+ for r in hole_readings('every girl chases a dog'): print(r)
+
diff --git a/nltk/sem/lfg.py b/nltk/sem/lfg.py
new file mode 100644
index 0000000..1e1d82e
--- /dev/null
+++ b/nltk/sem/lfg.py
@@ -0,0 +1,199 @@
+# Natural Language Toolkit: Lexical Functional Grammar
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, division, unicode_literals
+
+from nltk.internals import Counter
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class FStructure(dict):
+ def safeappend(self, key, item):
+ """
+ Append 'item' to the list at 'key'. If no list exists for 'key', then
+ construct one.
+ """
+ if key not in self:
+ self[key] = []
+ self[key].append(item)
+
+ def __setitem__(self, key, value):
+ dict.__setitem__(self, key.lower(), value)
+
+ def __getitem__(self, key):
+ return dict.__getitem__(self, key.lower())
+
+ def __contains__(self, key):
+ return dict.__contains__(self, key.lower())
+
+ def to_glueformula_list(self, glue_dict):
+ depgraph = self.to_depgraph()
+ return glue_dict.to_glueformula_list(depgraph)
+
+ def to_depgraph(self, rel=None):
+ from nltk.parse.dependencygraph import DependencyGraph
+ depgraph = DependencyGraph()
+ nodelist = depgraph.nodelist
+
+ self._to_depgraph(nodelist, 0, 'ROOT')
+
+ #Add all the dependencies for all the nodes
+ for node_addr, node in enumerate(nodelist):
+ for n2 in nodelist[1:]:
+ if n2['head'] == node_addr:
+ node['deps'].append(n2['address'])
+
+ depgraph.root = nodelist[1]
+
+ return depgraph
+
+ def _to_depgraph(self, nodelist, head, rel):
+ index = len(nodelist)
+
+ nodelist.append({'address': index,
+ 'word': self.pred[0],
+ 'tag': self.pred[1],
+ 'head': head,
+ 'rel': rel,
+ 'deps': []})
+
+ for feature in self:
+ for item in self[feature]:
+ if isinstance(item, FStructure):
+ item._to_depgraph(nodelist, index, feature)
+ elif isinstance(item, tuple):
+ nodelist.append({'address': len(nodelist),
+ 'word': item[0],
+ 'tag': item[1],
+ 'head': index,
+ 'rel': feature,
+ 'deps': []})
+ elif isinstance(item, list):
+ for n in item:
+ n._to_depgraph(nodelist, index, feature)
+ else: # ERROR
+ raise Exception('feature %s is not an FStruct, a list, or a tuple' % feature)
+
+ @staticmethod
+ def read_depgraph(depgraph):
+ return FStructure._read_depgraph(depgraph.root, depgraph)
+
+ @staticmethod
+ def _read_depgraph(node, depgraph, label_counter=None, parent=None):
+ if not label_counter:
+ label_counter = Counter()
+
+ if node['rel'].lower() in ['spec', 'punct']:
+ # the value of a 'spec' entry is a word, not an FStructure
+ return (node['word'], node['tag'])
+
+ else:
+ fstruct = FStructure()
+ fstruct.pred = None
+ fstruct.label = FStructure._make_label(label_counter.get())
+
+ fstruct.parent = parent
+
+ word, tag = node['word'], node['tag']
+ if tag[:2] == 'VB':
+ if tag[2:3] == 'D':
+ fstruct.safeappend('tense', ('PAST', 'tense'))
+ fstruct.pred = (word, tag[:2])
+
+ if not fstruct.pred:
+ fstruct.pred = (word, tag)
+
+ children = [depgraph.nodelist[idx] for idx in node['deps']]
+ for child in children:
+ fstruct.safeappend(child['rel'], FStructure._read_depgraph(child, depgraph, label_counter, fstruct))
+
+ return fstruct
+
+ @staticmethod
+ def _make_label(value):
+ """
+ Pick an alphabetic character as identifier for an entity in the model.
+
+ :param value: where to index into the list of characters
+ :type value: int
+ """
+ letter = ['f','g','h','i','j','k','l','m','n','o','p','q','r','s',
+ 't','u','v','w','x','y','z','a','b','c','d','e'][value-1]
+ num = int(value) // 26
+ if num > 0:
+ return letter + str(num)
+ else:
+ return letter
+
+ def __repr__(self):
+ return self.__unicode__().replace('\n', '')
+
+ def __str__(self):
+ return self.pprint()
+
+ def pprint(self, indent=3):
+ try:
+ accum = '%s:[' % self.label
+ except NameError:
+ accum = '['
+ try:
+ accum += 'pred \'%s\'' % (self.pred[0])
+ except NameError:
+ pass
+
+ for feature in sorted(self):
+ for item in self[feature]:
+ if isinstance(item, FStructure):
+ next_indent = indent+len(feature)+3+len(self.label)
+ accum += '\n%s%s %s' % (' '*(indent), feature, item.pprint(next_indent))
+ elif isinstance(item, tuple):
+ accum += '\n%s%s \'%s\'' % (' '*(indent), feature, item[0])
+ elif isinstance(item, list):
+ accum += '\n%s%s {%s}' % (' '*(indent), feature, ('\n%s' % (' '*(indent+len(feature)+2))).join(item))
+ else: # ERROR
+ raise Exception('feature %s is not an FStruct, a list, or a tuple' % feature)
+ return accum+']'
+
+
+
+def demo_read_depgraph():
+ from nltk.parse.dependencygraph import DependencyGraph
+ dg1 = DependencyGraph("""\
+Esso NNP 2 SUB
+said VBD 0 ROOT
+the DT 5 NMOD
+Whiting NNP 5 NMOD
+field NN 6 SUB
+started VBD 2 VMOD
+production NN 6 OBJ
+Tuesday NNP 6 VMOD
+""")
+ dg2 = DependencyGraph("""\
+John NNP 2 SUB
+sees VBP 0 ROOT
+Mary NNP 2 OBJ
+""")
+ dg3 = DependencyGraph("""\
+a DT 2 SPEC
+man NN 3 SUBJ
+walks VB 0 ROOT
+""")
+ dg4 = DependencyGraph("""\
+every DT 2 SPEC
+girl NN 3 SUBJ
+chases VB 0 ROOT
+a DT 5 SPEC
+dog NN 3 OBJ
+""")
+
+ depgraphs = [dg1,dg2,dg3,dg4]
+ for dg in depgraphs:
+ print(FStructure.read_depgraph(dg))
+
+if __name__ == '__main__':
+ demo_read_depgraph()
+
diff --git a/nltk/sem/linearlogic.py b/nltk/sem/linearlogic.py
new file mode 100644
index 0000000..c7a6af0
--- /dev/null
+++ b/nltk/sem/linearlogic.py
@@ -0,0 +1,449 @@
+# Natural Language Toolkit: Linear Logic
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function, unicode_literals
+
+from nltk.internals import Counter
+from nltk.compat import string_types, python_2_unicode_compatible
+from nltk.sem.logic import _LogicParser, APP
+
+_counter = Counter()
+
+class Tokens(object):
+ #Punctuation
+ OPEN = '('
+ CLOSE = ')'
+
+ #Operations
+ IMP = '-o'
+
+ PUNCT = [OPEN, CLOSE]
+ TOKENS = PUNCT + [IMP]
+
+class _LinearLogicParser(_LogicParser):
+ """A linear logic expression parser."""
+ def __init__(self):
+ _LogicParser.__init__(self)
+
+ self.operator_precedence = {APP: 1, Tokens.IMP: 2, None: 3}
+ self.right_associated_operations += [Tokens.IMP]
+
+ def get_all_symbols(self):
+ return Tokens.TOKENS
+
+ def handle(self, tok, context):
+ if tok not in Tokens.TOKENS:
+ return self.handle_variable(tok, context)
+ elif tok == Tokens.OPEN:
+ return self.handle_open(tok, context)
+
+ def get_BooleanExpression_factory(self, tok):
+ if tok == Tokens.IMP:
+ return ImpExpression
+ else:
+ return None
+
+ def make_BooleanExpression(self, factory, first, second):
+ return factory(first, second)
+
+ def attempt_ApplicationExpression(self, expression, context):
+ """Attempt to make an application expression. If the next tokens
+ are an argument in parens, then the argument expression is a
+ function being applied to the arguments. Otherwise, return the
+ argument expression."""
+ if self.has_priority(APP, context):
+ if self.inRange(0) and self.token(0) == Tokens.OPEN:
+ self.token() #swallow then open paren
+ argument = self.parse_Expression(APP)
+ self.assertNextToken(Tokens.CLOSE)
+ expression = ApplicationExpression(expression, argument, None)
+ return expression
+
+ def make_VariableExpression(self, name):
+ if name[0].isupper():
+ return VariableExpression(name)
+ else:
+ return ConstantExpression(name)
+
+
+ at python_2_unicode_compatible
+class Expression(object):
+
+ _linear_logic_parser = _LinearLogicParser()
+
+ @classmethod
+ def fromstring(cls, s):
+ return cls._linear_logic_parser.parse(s)
+
+ def applyto(self, other, other_indices=None):
+ return ApplicationExpression(self, other, other_indices)
+
+ def __call__(self, other):
+ return self.applyto(other)
+
+ def __repr__(self):
+ return '<%s %s>' % (self.__class__.__name__, self)
+
+
+ at python_2_unicode_compatible
+class AtomicExpression(Expression):
+ def __init__(self, name, dependencies=None):
+ """
+ :param name: str for the constant name
+ :param dependencies: list of int for the indices on which this atom is dependent
+ """
+ assert isinstance(name, string_types)
+ self.name = name
+
+ if not dependencies:
+ dependencies = []
+ self.dependencies = dependencies
+
+ def simplify(self, bindings=None):
+ """
+ If 'self' is bound by 'bindings', return the atomic to which it is bound.
+ Otherwise, return self.
+
+ :param bindings: ``BindingDict`` A dictionary of bindings used to simplify
+ :return: ``AtomicExpression``
+ """
+ if bindings and self in bindings:
+ return bindings[self]
+ else:
+ return self
+
+ def compile_pos(self, index_counter, glueFormulaFactory):
+ """
+ From Iddo Lev's PhD Dissertation p108-109
+
+ :param index_counter: ``Counter`` for unique indices
+ :param glueFormulaFactory: ``GlueFormula`` for creating new glue formulas
+ :return: (``Expression``,set) for the compiled linear logic and any newly created glue formulas
+ """
+ self.dependencies = []
+ return (self, [])
+
+ def compile_neg(self, index_counter, glueFormulaFactory):
+ """
+ From Iddo Lev's PhD Dissertation p108-109
+
+ :param index_counter: ``Counter`` for unique indices
+ :param glueFormulaFactory: ``GlueFormula`` for creating new glue formulas
+ :return: (``Expression``,set) for the compiled linear logic and any newly created glue formulas
+ """
+ self.dependencies = []
+ return (self, [])
+
+ def initialize_labels(self, fstruct):
+ self.name = fstruct.initialize_label(self.name.lower())
+
+ def __eq__(self, other):
+ return self.__class__ == other.__class__ and self.name == other.name
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __str__(self):
+ accum = self.name
+ if self.dependencies:
+ accum += "%s" % self.dependencies
+ return accum
+
+ def __hash__(self):
+ return hash(self.name)
+
+class ConstantExpression(AtomicExpression):
+ def unify(self, other, bindings):
+ """
+ If 'other' is a constant, then it must be equal to 'self'. If 'other' is a variable,
+ then it must not be bound to anything other than 'self'.
+
+ :param other: ``Expression``
+ :param bindings: ``BindingDict`` A dictionary of all current bindings
+ :return: ``BindingDict`` A new combined dictionary of of 'bindings' and any new binding
+ :raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
+ """
+ assert isinstance(other, Expression)
+ if isinstance(other, VariableExpression):
+ try:
+ return bindings + BindingDict([(other, self)])
+ except VariableBindingException:
+ pass
+ elif self == other:
+ return bindings
+ raise UnificationException(self, other, bindings)
+
+class VariableExpression(AtomicExpression):
+ def unify(self, other, bindings):
+ """
+ 'self' must not be bound to anything other than 'other'.
+
+ :param other: ``Expression``
+ :param bindings: ``BindingDict`` A dictionary of all current bindings
+ :return: ``BindingDict`` A new combined dictionary of of 'bindings' and the new binding
+ :raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
+ """
+ assert isinstance(other, Expression)
+ try:
+ if self == other:
+ return bindings
+ else:
+ return bindings + BindingDict([(self, other)])
+ except VariableBindingException:
+ raise UnificationException(self, other, bindings)
+
+ at python_2_unicode_compatible
+class ImpExpression(Expression):
+ def __init__(self, antecedent, consequent):
+ """
+ :param antecedent: ``Expression`` for the antecedent
+ :param consequent: ``Expression`` for the consequent
+ """
+ assert isinstance(antecedent, Expression)
+ assert isinstance(consequent, Expression)
+ self.antecedent = antecedent
+ self.consequent = consequent
+
+ def simplify(self, bindings=None):
+ return self.__class__(self.antecedent.simplify(bindings), self.consequent.simplify(bindings))
+
+ def unify(self, other, bindings):
+ """
+ Both the antecedent and consequent of 'self' and 'other' must unify.
+
+ :param other: ``ImpExpression``
+ :param bindings: ``BindingDict`` A dictionary of all current bindings
+ :return: ``BindingDict`` A new combined dictionary of of 'bindings' and any new bindings
+ :raise UnificationException: If 'self' and 'other' cannot be unified in the context of 'bindings'
+ """
+ assert isinstance(other, ImpExpression)
+ try:
+ return bindings + self.antecedent.unify(other.antecedent, bindings) + self.consequent.unify(other.consequent, bindings)
+ except VariableBindingException:
+ raise UnificationException(self, other, bindings)
+
+ def compile_pos(self, index_counter, glueFormulaFactory):
+ """
+ From Iddo Lev's PhD Dissertation p108-109
+
+ :param index_counter: ``Counter`` for unique indices
+ :param glueFormulaFactory: ``GlueFormula`` for creating new glue formulas
+ :return: (``Expression``,set) for the compiled linear logic and any newly created glue formulas
+ """
+ (a, a_new) = self.antecedent.compile_neg(index_counter, glueFormulaFactory)
+ (c, c_new) = self.consequent.compile_pos(index_counter, glueFormulaFactory)
+ return (ImpExpression(a,c), a_new + c_new)
+
+ def compile_neg(self, index_counter, glueFormulaFactory):
+ """
+ From Iddo Lev's PhD Dissertation p108-109
+
+ :param index_counter: ``Counter`` for unique indices
+ :param glueFormulaFactory: ``GlueFormula`` for creating new glue formulas
+ :return: (``Expression``,list of ``GlueFormula``) for the compiled linear logic and any newly created glue formulas
+ """
+ (a, a_new) = self.antecedent.compile_pos(index_counter, glueFormulaFactory)
+ (c, c_new) = self.consequent.compile_neg(index_counter, glueFormulaFactory)
+ fresh_index = index_counter.get()
+ c.dependencies.append(fresh_index)
+ new_v = glueFormulaFactory('v%s' % fresh_index, a, set([fresh_index]))
+ return (c, a_new + c_new + [new_v])
+
+ def initialize_labels(self, fstruct):
+ self.antecedent.initialize_labels(fstruct)
+ self.consequent.initialize_labels(fstruct)
+
+ def __eq__(self, other):
+ return self.__class__ == other.__class__ and \
+ self.antecedent == other.antecedent and self.consequent == other.consequent
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __str__(self):
+ return "%s%s %s %s%s" % (
+ Tokens.OPEN, self.antecedent, Tokens.IMP, self.consequent, Tokens.CLOSE)
+
+ def __hash__(self):
+ return hash('%s%s%s' % (hash(self.antecedent), Tokens.IMP, hash(self.consequent)))
+
+ at python_2_unicode_compatible
+class ApplicationExpression(Expression):
+ def __init__(self, function, argument, argument_indices=None):
+ """
+ :param function: ``Expression`` for the function
+ :param argument: ``Expression`` for the argument
+ :param argument_indices: set for the indices of the glue formula from which the argument came
+ :raise LinearLogicApplicationException: If 'function' cannot be applied to 'argument' given 'argument_indices'.
+ """
+ function_simp = function.simplify()
+ argument_simp = argument.simplify()
+
+ assert isinstance(function_simp, ImpExpression)
+ assert isinstance(argument_simp, Expression)
+
+ bindings = BindingDict()
+
+ try:
+ if isinstance(function, ApplicationExpression):
+ bindings += function.bindings
+ if isinstance(argument, ApplicationExpression):
+ bindings += argument.bindings
+ bindings += function_simp.antecedent.unify(argument_simp, bindings)
+ except UnificationException as e:
+ raise LinearLogicApplicationException('Cannot apply %s to %s. %s' % (function_simp, argument_simp, e))
+
+ # If you are running it on complied premises, more conditions apply
+ if argument_indices:
+ # A.dependencies of (A -o (B -o C)) must be a proper subset of argument_indices
+ if not set(function_simp.antecedent.dependencies) < argument_indices:
+ raise LinearLogicApplicationException('Dependencies unfulfilled when attempting to apply Linear Logic formula %s to %s' % (function_simp, argument_simp))
+ if set(function_simp.antecedent.dependencies) == argument_indices:
+ raise LinearLogicApplicationException('Dependencies not a proper subset of indices when attempting to apply Linear Logic formula %s to %s' % (function_simp, argument_simp))
+
+ self.function = function
+ self.argument = argument
+ self.bindings = bindings
+
+ def simplify(self, bindings=None):
+ """
+ Since function is an implication, return its consequent. There should be
+ no need to check that the application is valid since the checking is done
+ by the constructor.
+
+ :param bindings: ``BindingDict`` A dictionary of bindings used to simplify
+ :return: ``Expression``
+ """
+ if not bindings:
+ bindings = self.bindings
+
+ return self.function.simplify(bindings).consequent
+
+ def __eq__(self, other):
+ return self.__class__ == other.__class__ and \
+ self.function == other.function and self.argument == other.argument
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __str__(self):
+ return "%s" % self.function + Tokens.OPEN + "%s" % self.argument + Tokens.CLOSE
+
+ def __hash__(self):
+ return hash('%s%s%s' % (hash(self.antecedent), Tokens.OPEN, hash(self.consequent)))
+
+ at python_2_unicode_compatible
+class BindingDict(object):
+ def __init__(self, bindings=None):
+ """
+ :param bindings:
+ list [(``VariableExpression``, ``AtomicExpression``)] to initialize the dictionary
+ dict {``VariableExpression``: ``AtomicExpression``} to initialize the dictionary
+ """
+ self.d = {}
+
+ if isinstance(bindings, dict):
+ bindings = bindings.items()
+
+ if bindings:
+ for (v, b) in bindings:
+ self[v] = b
+
+ def __setitem__(self, variable, binding):
+ """
+ A binding is consistent with the dict if its variable is not already bound, OR if its
+ variable is already bound to its argument.
+
+ :param variable: ``VariableExpression`` The variable bind
+ :param binding: ``Expression`` The expression to which 'variable' should be bound
+ :raise VariableBindingException: If the variable cannot be bound in this dictionary
+ """
+ assert isinstance(variable, VariableExpression)
+ assert isinstance(binding, Expression)
+
+ assert variable != binding
+
+ existing = self.d.get(variable, None)
+
+ if not existing or binding == existing:
+ self.d[variable] = binding
+ else:
+ raise VariableBindingException('Variable %s already bound to another value' % (variable))
+
+ def __getitem__(self, variable):
+ """
+ Return the expression to which 'variable' is bound
+ """
+ assert isinstance(variable, VariableExpression)
+
+ intermediate = self.d[variable]
+ while intermediate:
+ try:
+ intermediate = self.d[intermediate]
+ except KeyError:
+ return intermediate
+
+ def __contains__(self, item):
+ return item in self.d
+
+ def __add__(self, other):
+ """
+ :param other: ``BindingDict`` The dict with which to combine self
+ :return: ``BindingDict`` A new dict containing all the elements of both parameters
+ :raise VariableBindingException: If the parameter dictionaries are not consistent with each other
+ """
+ try:
+ combined = BindingDict()
+ for v in self.d:
+ combined[v] = self.d[v]
+ for v in other.d:
+ combined[v] = other.d[v]
+ return combined
+ except VariableBindingException:
+ raise VariableBindingException('Attempting to add two contradicting'\
+ ' VariableBindingsLists: %s, %s' % (self, other))
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __eq__(self, other):
+ if not isinstance(other, BindingDict):
+ raise TypeError
+ return self.d == other.d
+
+ def __str__(self):
+ return '{' + ', '.join('%s: %s' % (v, self.d[v]) for v in self.d) + '}'
+
+ def __repr__(self):
+ return 'BindingDict: %s' % self
+
+class VariableBindingException(Exception):
+ pass
+
+class UnificationException(Exception):
+ def __init__(self, a, b, bindings):
+ Exception.__init__(self, 'Cannot unify %s with %s given %s' % (a, b, bindings))
+
+class LinearLogicApplicationException(Exception):
+ pass
+
+
+def demo():
+ lexpr = Expression.fromstring
+
+ print(lexpr(r'f'))
+ print(lexpr(r'(g -o f)'))
+ print(lexpr(r'((g -o G) -o G)'))
+ print(lexpr(r'g -o h -o f'))
+ print(lexpr(r'(g -o f)(g)').simplify())
+ print(lexpr(r'(H -o f)(g)').simplify())
+ print(lexpr(r'((g -o G) -o G)((g -o f))').simplify())
+ print(lexpr(r'(H -o H)((g -o f))').simplify())
+
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/sem/logic.py b/nltk/sem/logic.py
new file mode 100644
index 0000000..9534b59
--- /dev/null
+++ b/nltk/sem/logic.py
@@ -0,0 +1,1918 @@
+# Natural Language Toolkit: Logic
+#
+# Author: Dan Garrette <dhgarrette at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+
+"""
+A version of first order predicate logic, built on
+top of the typed lambda calculus.
+"""
+from __future__ import print_function, unicode_literals
+
+import re
+import operator
+from collections import defaultdict
+from functools import reduce
+
+from nltk.internals import Counter
+from nltk.compat import (total_ordering, string_types,
+ python_2_unicode_compatible)
+
+APP = 'APP'
+
+_counter = Counter()
+
+class Tokens(object):
+ LAMBDA = '\\'; LAMBDA_LIST = ['\\']
+
+ #Quantifiers
+ EXISTS = 'exists'; EXISTS_LIST = ['some', 'exists', 'exist']
+ ALL = 'all'; ALL_LIST = ['all', 'forall']
+
+ #Punctuation
+ DOT = '.'
+ OPEN = '('
+ CLOSE = ')'
+ COMMA = ','
+
+ #Operations
+ NOT = '-'; NOT_LIST = ['not', '-', '!']
+ AND = '&'; AND_LIST = ['and', '&', '^']
+ OR = '|'; OR_LIST = ['or', '|']
+ IMP = '->'; IMP_LIST = ['implies', '->', '=>']
+ IFF = '<->'; IFF_LIST = ['iff', '<->', '<=>']
+ EQ = '='; EQ_LIST = ['=', '==']
+ NEQ = '!='; NEQ_LIST = ['!=']
+
+ #Collections of tokens
+ BINOPS = AND_LIST + OR_LIST + IMP_LIST + IFF_LIST
+ QUANTS = EXISTS_LIST + ALL_LIST
+ PUNCT = [DOT, OPEN, CLOSE, COMMA]
+
+ TOKENS = BINOPS + EQ_LIST + NEQ_LIST + QUANTS + LAMBDA_LIST + PUNCT + NOT_LIST
+
+ #Special
+ SYMBOLS = [x for x in TOKENS if re.match(r'^[-\\.(),!&^|>=<]*$', x)]
+
+
+def boolean_ops():
+ """
+ Boolean operators
+ """
+ names = ["negation", "conjunction", "disjunction", "implication", "equivalence"]
+ for pair in zip(names, [Tokens.NOT, Tokens.AND, Tokens.OR, Tokens.IMP, Tokens.IFF]):
+ print("%-15s\t%s" % pair)
+
+def equality_preds():
+ """
+ Equality predicates
+ """
+ names = ["equality", "inequality"]
+ for pair in zip(names, [Tokens.EQ, Tokens.NEQ]):
+ print("%-15s\t%s" % pair)
+
+def binding_ops():
+ """
+ Binding operators
+ """
+ names = ["existential", "universal", "lambda"]
+ for pair in zip(names, [Tokens.EXISTS, Tokens.ALL, Tokens.LAMBDA]):
+ print("%-15s\t%s" % pair)
+
+
+ at python_2_unicode_compatible
+class _LogicParser(object):
+ """A lambda calculus expression parser."""
+
+ def __init__(self, type_check=False):
+ """
+ :param type_check: bool should type checking be performed?
+ to their types.
+ """
+ assert isinstance(type_check, bool)
+
+ self._currentIndex = 0
+ self._buffer = []
+ self.type_check = type_check
+
+ """A list of tuples of quote characters. The 4-tuple is comprised
+ of the start character, the end character, the escape character, and
+ a boolean indicating whether the quotes should be included in the
+ result. Quotes are used to signify that a token should be treated as
+ atomic, ignoring any special characters within the token. The escape
+ character allows the quote end character to be used within the quote.
+ If True, the boolean indicates that the final token should contain the
+ quote and escape characters.
+ This method exists to be overridden"""
+ self.quote_chars = []
+
+ self.operator_precedence = dict(
+ [(x,1) for x in Tokens.LAMBDA_LIST] + \
+ [(x,2) for x in Tokens.NOT_LIST] + \
+ [(APP,3)] + \
+ [(x,4) for x in Tokens.EQ_LIST+Tokens.NEQ_LIST] + \
+ [(x,5) for x in Tokens.QUANTS] + \
+ [(x,6) for x in Tokens.AND_LIST] + \
+ [(x,7) for x in Tokens.OR_LIST] + \
+ [(x,8) for x in Tokens.IMP_LIST] + \
+ [(x,9) for x in Tokens.IFF_LIST] + \
+ [(None,10)])
+ self.right_associated_operations = [APP]
+
+ def parse(self, data, signature=None):
+ """
+ Parse the expression.
+
+ :param data: str for the input to be parsed
+ :param signature: ``dict<str, str>`` that maps variable names to type
+ strings
+ :returns: a parsed Expression
+ """
+ data = data.rstrip()
+
+ self._currentIndex = 0
+ self._buffer, mapping = self.process(data)
+
+ try:
+ result = self.parse_Expression(None)
+ if self.inRange(0):
+ raise UnexpectedTokenException(self._currentIndex+1, self.token(0))
+ except LogicalExpressionException as e:
+ msg = '%s\n%s\n%s^' % (e, data, ' '*mapping[e.index-1])
+ raise LogicalExpressionException(None, msg)
+
+ if self.type_check:
+ result.typecheck(signature)
+
+ return result
+
+ def process(self, data):
+ """Split the data into tokens"""
+ out = []
+ mapping = {}
+ tokenTrie = StringTrie(self.get_all_symbols())
+ token = ''
+ data_idx = 0
+ token_start_idx = data_idx
+ while data_idx < len(data):
+ cur_data_idx = data_idx
+ quoted_token, data_idx = self.process_quoted_token(data_idx, data)
+ if quoted_token:
+ if not token:
+ token_start_idx = cur_data_idx
+ token += quoted_token
+ continue
+
+ st = tokenTrie
+ c = data[data_idx]
+ symbol = ''
+ while c in st:
+ symbol += c
+ st = st[c]
+ if len(data)-data_idx > len(symbol):
+ c = data[data_idx+len(symbol)]
+ else:
+ break
+ if StringTrie.LEAF in st:
+ #token is a complete symbol
+ if token:
+ mapping[len(out)] = token_start_idx
+ out.append(token)
+ token = ''
+ mapping[len(out)] = data_idx
+ out.append(symbol)
+ data_idx += len(symbol)
+ else:
+ if data[data_idx] in ' \t\n': #any whitespace
+ if token:
+ mapping[len(out)] = token_start_idx
+ out.append(token)
+ token = ''
+ else:
+ if not token:
+ token_start_idx = data_idx
+ token += data[data_idx]
+ data_idx += 1
+ if token:
+ mapping[len(out)] = token_start_idx
+ out.append(token)
+ mapping[len(out)] = len(data)
+ mapping[len(out)+1] = len(data)+1
+ return out, mapping
+
+ def process_quoted_token(self, data_idx, data):
+ token = ''
+ c = data[data_idx]
+ i = data_idx
+ for start, end, escape, incl_quotes in self.quote_chars:
+ if c == start:
+ if incl_quotes:
+ token += c
+ i += 1
+ while data[i] != end:
+ if data[i] == escape:
+ if incl_quotes:
+ token += data[i]
+ i += 1
+ if len(data) == i: #if there are no more chars
+ raise LogicalExpressionException(None, "End of input reached. "
+ "Escape character [%s] found at end."
+ % escape)
+ token += data[i]
+ else:
+ token += data[i]
+ i += 1
+ if len(data) == i:
+ raise LogicalExpressionException(None, "End of input reached. "
+ "Expected: [%s]" % end)
+ if incl_quotes:
+ token += data[i]
+ i += 1
+ if not token:
+ raise LogicalExpressionException(None, 'Empty quoted token found')
+ break
+ return token, i
+
+ def get_all_symbols(self):
+ """This method exists to be overridden"""
+ return Tokens.SYMBOLS
+
+ def inRange(self, location):
+ """Return TRUE if the given location is within the buffer"""
+ return self._currentIndex+location < len(self._buffer)
+
+ def token(self, location=None):
+ """Get the next waiting token. If a location is given, then
+ return the token at currentIndex+location without advancing
+ currentIndex; setting it gives lookahead/lookback capability."""
+ try:
+ if location is None:
+ tok = self._buffer[self._currentIndex]
+ self._currentIndex += 1
+ else:
+ tok = self._buffer[self._currentIndex+location]
+ return tok
+ except IndexError:
+ raise ExpectedMoreTokensException(self._currentIndex+1)
+
+ def isvariable(self, tok):
+ return tok not in Tokens.TOKENS
+
+ def parse_Expression(self, context):
+ """Parse the next complete expression from the stream and return it."""
+ try:
+ tok = self.token()
+ except ExpectedMoreTokensException:
+ raise ExpectedMoreTokensException(self._currentIndex+1, message='Expression expected.')
+
+ accum = self.handle(tok, context)
+
+ if not accum:
+ raise UnexpectedTokenException(self._currentIndex, tok, message='Expression expected.')
+
+ return self.attempt_adjuncts(accum, context)
+
+ def handle(self, tok, context):
+ """This method is intended to be overridden for logics that
+ use different operators or expressions"""
+ if self.isvariable(tok):
+ return self.handle_variable(tok, context)
+
+ elif tok in Tokens.NOT_LIST:
+ return self.handle_negation(tok, context)
+
+ elif tok in Tokens.LAMBDA_LIST:
+ return self.handle_lambda(tok, context)
+
+ elif tok in Tokens.QUANTS:
+ return self.handle_quant(tok, context)
+
+ elif tok == Tokens.OPEN:
+ return self.handle_open(tok, context)
+
+ def attempt_adjuncts(self, expression, context):
+ cur_idx = None
+ while cur_idx != self._currentIndex: #while adjuncts are added
+ cur_idx = self._currentIndex
+ expression = self.attempt_EqualityExpression(expression, context)
+ expression = self.attempt_ApplicationExpression(expression, context)
+ expression = self.attempt_BooleanExpression(expression, context)
+ return expression
+
+ def handle_negation(self, tok, context):
+ return self.make_NegatedExpression(self.parse_Expression(Tokens.NOT))
+
+ def make_NegatedExpression(self, expression):
+ return NegatedExpression(expression)
+
+ def handle_variable(self, tok, context):
+ #It's either: 1) a predicate expression: sees(x,y)
+ # 2) an application expression: P(x)
+ # 3) a solo variable: john OR x
+ accum = self.make_VariableExpression(tok)
+ if self.inRange(0) and self.token(0) == Tokens.OPEN:
+ #The predicate has arguments
+ if not isinstance(accum, FunctionVariableExpression) and \
+ not isinstance(accum, ConstantExpression):
+ raise LogicalExpressionException(self._currentIndex,
+ "'%s' is an illegal predicate name. "
+ "Individual variables may not be used as "
+ "predicates." % tok)
+ self.token() #swallow the Open Paren
+
+ #curry the arguments
+ accum = self.make_ApplicationExpression(accum, self.parse_Expression(APP))
+ while self.inRange(0) and self.token(0) == Tokens.COMMA:
+ self.token() #swallow the comma
+ accum = self.make_ApplicationExpression(accum, self.parse_Expression(APP))
+ self.assertNextToken(Tokens.CLOSE)
+ return accum
+
+ def get_next_token_variable(self, description):
+ try:
+ tok = self.token()
+ except ExpectedMoreTokensException as e:
+ raise ExpectedMoreTokensException(e.index, 'Variable expected.')
+ if isinstance(self.make_VariableExpression(tok), ConstantExpression):
+ raise LogicalExpressionException(self._currentIndex,
+ "'%s' is an illegal variable name. "
+ "Constants may not be %s." % (tok, description))
+ return Variable(tok)
+
+ def handle_lambda(self, tok, context):
+ # Expression is a lambda expression
+ if not self.inRange(0):
+ raise ExpectedMoreTokensException(self._currentIndex+2,
+ message="Variable and Expression expected following lambda operator.")
+ vars = [self.get_next_token_variable('abstracted')]
+ while True:
+ if not self.inRange(0) or (self.token(0) == Tokens.DOT and not self.inRange(1)):
+ raise ExpectedMoreTokensException(self._currentIndex+2, message="Expression expected.")
+ if not self.isvariable(self.token(0)):
+ break
+ # Support expressions like: \x y.M == \x.\y.M
+ vars.append(self.get_next_token_variable('abstracted'))
+ if self.inRange(0) and self.token(0) == Tokens.DOT:
+ self.token() #swallow the dot
+
+ accum = self.parse_Expression(tok)
+ while vars:
+ accum = self.make_LambdaExpression(vars.pop(), accum)
+ return accum
+
+ def handle_quant(self, tok, context):
+ # Expression is a quantified expression: some x.M
+ factory = self.get_QuantifiedExpression_factory(tok)
+
+ if not self.inRange(0):
+ raise ExpectedMoreTokensException(self._currentIndex+2,
+ message="Variable and Expression expected following quantifier '%s'." % tok)
+ vars = [self.get_next_token_variable('quantified')]
+ while True:
+ if not self.inRange(0) or (self.token(0) == Tokens.DOT and not self.inRange(1)):
+ raise ExpectedMoreTokensException(self._currentIndex+2, message="Expression expected.")
+ if not self.isvariable(self.token(0)):
+ break
+ # Support expressions like: some x y.M == some x.some y.M
+ vars.append(self.get_next_token_variable('quantified'))
+ if self.inRange(0) and self.token(0) == Tokens.DOT:
+ self.token() #swallow the dot
+
+ accum = self.parse_Expression(tok)
+ while vars:
+ accum = self.make_QuanifiedExpression(factory, vars.pop(), accum)
+ return accum
+
+ def get_QuantifiedExpression_factory(self, tok):
+ """This method serves as a hook for other logic parsers that
+ have different quantifiers"""
+ if tok in Tokens.EXISTS_LIST:
+ return ExistsExpression
+ elif tok in Tokens.ALL_LIST:
+ return AllExpression
+ else:
+ self.assertToken(tok, Tokens.QUANTS)
+
+ def make_QuanifiedExpression(self, factory, variable, term):
+ return factory(variable, term)
+
+ def handle_open(self, tok, context):
+ #Expression is in parens
+ accum = self.parse_Expression(None)
+ self.assertNextToken(Tokens.CLOSE)
+ return accum
+
+ def attempt_EqualityExpression(self, expression, context):
+ """Attempt to make an equality expression. If the next token is an
+ equality operator, then an EqualityExpression will be returned.
+ Otherwise, the parameter will be returned."""
+ if self.inRange(0):
+ tok = self.token(0)
+ if tok in Tokens.EQ_LIST + Tokens.NEQ_LIST and self.has_priority(tok, context):
+ self.token() #swallow the "=" or "!="
+ expression = self.make_EqualityExpression(expression, self.parse_Expression(tok))
+ if tok in Tokens.NEQ_LIST:
+ expression = self.make_NegatedExpression(expression)
+ return expression
+
+ def make_EqualityExpression(self, first, second):
+ """This method serves as a hook for other logic parsers that
+ have different equality expression classes"""
+ return EqualityExpression(first, second)
+
+ def attempt_BooleanExpression(self, expression, context):
+ """Attempt to make a boolean expression. If the next token is a boolean
+ operator, then a BooleanExpression will be returned. Otherwise, the
+ parameter will be returned."""
+ while self.inRange(0):
+ tok = self.token(0)
+ factory = self.get_BooleanExpression_factory(tok)
+ if factory and self.has_priority(tok, context):
+ self.token() #swallow the operator
+ expression = self.make_BooleanExpression(factory, expression,
+ self.parse_Expression(tok))
+ else:
+ break
+ return expression
+
+ def get_BooleanExpression_factory(self, tok):
+ """This method serves as a hook for other logic parsers that
+ have different boolean operators"""
+ if tok in Tokens.AND_LIST:
+ return AndExpression
+ elif tok in Tokens.OR_LIST:
+ return OrExpression
+ elif tok in Tokens.IMP_LIST:
+ return ImpExpression
+ elif tok in Tokens.IFF_LIST:
+ return IffExpression
+ else:
+ return None
+
+ def make_BooleanExpression(self, factory, first, second):
+ return factory(first, second)
+
+ def attempt_ApplicationExpression(self, expression, context):
+ """Attempt to make an application expression. The next tokens are
+ a list of arguments in parens, then the argument expression is a
+ function being applied to the arguments. Otherwise, return the
+ argument expression."""
+ if self.has_priority(APP, context):
+ if self.inRange(0) and self.token(0) == Tokens.OPEN:
+ if not isinstance(expression, LambdaExpression) and \
+ not isinstance(expression, ApplicationExpression) and \
+ not isinstance(expression, FunctionVariableExpression) and \
+ not isinstance(expression, ConstantExpression):
+ raise LogicalExpressionException(self._currentIndex,
+ ("The function '%s" % expression) +
+ "' is not a Lambda Expression, an "
+ "Application Expression, or a "
+ "functional predicate, so it may "
+ "not take arguments.")
+ self.token() #swallow then open paren
+ #curry the arguments
+ accum = self.make_ApplicationExpression(expression, self.parse_Expression(APP))
+ while self.inRange(0) and self.token(0) == Tokens.COMMA:
+ self.token() #swallow the comma
+ accum = self.make_ApplicationExpression(accum, self.parse_Expression(APP))
+ self.assertNextToken(Tokens.CLOSE)
+ return accum
+ return expression
+
+ def make_ApplicationExpression(self, function, argument):
+ return ApplicationExpression(function, argument)
+
+ def make_VariableExpression(self, name):
+ return VariableExpression(Variable(name))
+
+ def make_LambdaExpression(self, variable, term):
+ return LambdaExpression(variable, term)
+
+ def has_priority(self, operation, context):
+ return self.operator_precedence[operation] < self.operator_precedence[context] or \
+ (operation in self.right_associated_operations and \
+ self.operator_precedence[operation] == self.operator_precedence[context])
+
+ def assertNextToken(self, expected):
+ try:
+ tok = self.token()
+ except ExpectedMoreTokensException as e:
+ raise ExpectedMoreTokensException(e.index, message="Expected token '%s'." % expected)
+
+ if isinstance(expected, list):
+ if tok not in expected:
+ raise UnexpectedTokenException(self._currentIndex, tok, expected)
+ else:
+ if tok != expected:
+ raise UnexpectedTokenException(self._currentIndex, tok, expected)
+
+ def assertToken(self, tok, expected):
+ if isinstance(expected, list):
+ if tok not in expected:
+ raise UnexpectedTokenException(self._currentIndex, tok, expected)
+ else:
+ if tok != expected:
+ raise UnexpectedTokenException(self._currentIndex, tok, expected)
+
+ def __repr__(self):
+ if self.inRange(0):
+ msg = 'Next token: ' + self.token(0)
+ else:
+ msg = 'No more tokens'
+ return '<' + self.__class__.__name__ + ': ' + msg + '>'
+
+
+def parse_logic(s, logic_parser=None, encoding=None):
+ """
+ Convert a file of First Order Formulas into a list of {Expression}s.
+
+ :param s: the contents of the file
+ :type s: str
+ :param logic_parser: The parser to be used to parse the logical expression
+ :type logic_parser: _LogicParser
+ :param encoding: the encoding of the input string, if it is binary
+ :type encoding: str
+ :return: a list of parsed formulas.
+ :rtype: list(Expression)
+ """
+ if encoding is not None:
+ s = s.decode(encoding)
+ if logic_parser is None:
+ logic_parser = _LogicParser()
+
+ statements = []
+ for linenum, line in enumerate(s.splitlines()):
+ line = line.strip()
+ if line.startswith('#') or line=='': continue
+ try:
+ statements.append(logic_parser.parse(line))
+ except LogicalExpressionException:
+ raise ValueError('Unable to parse line %s: %s' % (linenum, line))
+ return statements
+
+
+ at total_ordering
+ at python_2_unicode_compatible
+class Variable(object):
+ def __init__(self, name):
+ """
+ :param name: the name of the variable
+ """
+ assert isinstance(name, string_types), "%s is not a string" % name
+ self.name = name
+
+ def __eq__(self, other):
+ return isinstance(other, Variable) and self.name == other.name
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, Variable):
+ raise TypeError
+ return self.name < other.name
+
+ def substitute_bindings(self, bindings):
+ return bindings.get(self, self)
+
+ def __hash__(self):
+ return hash(self.name)
+
+ def __str__(self):
+ return self.name
+
+ def __repr__(self):
+ return "Variable('%s')" % self.name
+
+
+def unique_variable(pattern=None, ignore=None):
+ """
+ Return a new, unique variable.
+
+ :param pattern: ``Variable`` that is being replaced. The new variable must
+ be the same type.
+ :param term: a set of ``Variable`` objects that should not be returned from
+ this function.
+ :rtype: Variable
+ """
+ if pattern is not None:
+ if is_indvar(pattern.name):
+ prefix = 'z'
+ elif is_funcvar(pattern.name):
+ prefix = 'F'
+ elif is_eventvar(pattern.name):
+ prefix = 'e0'
+ else:
+ assert False, "Cannot generate a unique constant"
+ else:
+ prefix = 'z'
+
+ v = Variable("%s%s" % (prefix, _counter.get()))
+ while ignore is not None and v in ignore:
+ v = Variable("%s%s" % (prefix, _counter.get()))
+ return v
+
+def skolem_function(univ_scope=None):
+ """
+ Return a skolem function over the variables in univ_scope
+ param univ_scope
+ """
+ skolem = VariableExpression(Variable('F%s' % _counter.get()))
+ if univ_scope:
+ for v in list(univ_scope):
+ skolem = skolem(VariableExpression(v))
+ return skolem
+
+
+ at python_2_unicode_compatible
+class Type(object):
+ def __repr__(self):
+ return "%s" % self
+
+ def __hash__(self):
+ return hash("%s" % self)
+
+ at python_2_unicode_compatible
+class ComplexType(Type):
+ def __init__(self, first, second):
+ assert(isinstance(first, Type)), "%s is not a Type" % first
+ assert(isinstance(second, Type)), "%s is not a Type" % second
+ self.first = first
+ self.second = second
+
+ def __eq__(self, other):
+ return isinstance(other, ComplexType) and \
+ self.first == other.first and \
+ self.second == other.second
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Type.__hash__
+
+ def matches(self, other):
+ if isinstance(other, ComplexType):
+ return self.first.matches(other.first) and \
+ self.second.matches(other.second)
+ else:
+ return self == ANY_TYPE
+
+ def resolve(self, other):
+ if other == ANY_TYPE:
+ return self
+ elif isinstance(other, ComplexType):
+ f = self.first.resolve(other.first)
+ s = self.second.resolve(other.second)
+ if f and s:
+ return ComplexType(f,s)
+ else:
+ return None
+ elif self == ANY_TYPE:
+ return other
+ else:
+ return None
+
+ def __str__(self):
+ if self == ANY_TYPE:
+ return "%s" % ANY_TYPE
+ else:
+ return '<%s,%s>' % (self.first, self.second)
+
+ def str(self):
+ if self == ANY_TYPE:
+ return ANY_TYPE.str()
+ else:
+ return '(%s -> %s)' % (self.first.str(), self.second.str())
+
+class BasicType(Type):
+ def __eq__(self, other):
+ return isinstance(other, BasicType) and ("%s" % self) == ("%s" % other)
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Type.__hash__
+
+ def matches(self, other):
+ return other == ANY_TYPE or self == other
+
+ def resolve(self, other):
+ if self.matches(other):
+ return self
+ else:
+ return None
+
+ at python_2_unicode_compatible
+class EntityType(BasicType):
+ def __str__(self):
+ return 'e'
+
+ def str(self):
+ return 'IND'
+
+ at python_2_unicode_compatible
+class TruthValueType(BasicType):
+ def __str__(self):
+ return 't'
+
+ def str(self):
+ return 'BOOL'
+
+ at python_2_unicode_compatible
+class EventType(BasicType):
+ def __str__(self):
+ return 'v'
+
+ def str(self):
+ return 'EVENT'
+
+ at python_2_unicode_compatible
+class AnyType(BasicType, ComplexType):
+ def __init__(self):
+ pass
+
+ @property
+ def first(self): return self
+
+ @property
+ def second(self): return self
+
+ def __eq__(self, other):
+ return isinstance(other, AnyType) or other.__eq__(self)
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Type.__hash__
+
+ def matches(self, other):
+ return True
+
+ def resolve(self, other):
+ return other
+
+ def __str__(self):
+ return '?'
+
+ def str(self):
+ return 'ANY'
+
+
+TRUTH_TYPE = TruthValueType()
+ENTITY_TYPE = EntityType()
+EVENT_TYPE = EventType()
+ANY_TYPE = AnyType()
+
+
+def parse_type(type_string):
+ assert isinstance(type_string, string_types)
+ type_string = type_string.replace(' ', '') #remove spaces
+
+ if type_string[0] == '<':
+ assert type_string[-1] == '>'
+ paren_count = 0
+ for i,char in enumerate(type_string):
+ if char == '<':
+ paren_count += 1
+ elif char == '>':
+ paren_count -= 1
+ assert paren_count > 0
+ elif char == ',':
+ if paren_count == 1:
+ break
+ return ComplexType(parse_type(type_string[1 :i ]),
+ parse_type(type_string[i+1:-1]))
+ elif type_string[0] == "%s" % ENTITY_TYPE:
+ return ENTITY_TYPE
+ elif type_string[0] == "%s" % TRUTH_TYPE:
+ return TRUTH_TYPE
+ elif type_string[0] == "%s" % ANY_TYPE:
+ return ANY_TYPE
+ else:
+ raise LogicalExpressionException("Unexpected character: '%s'." % type_string[0])
+
+
+class TypeException(Exception):
+ def __init__(self, msg):
+ Exception.__init__(self, msg)
+
+class InconsistentTypeHierarchyException(TypeException):
+ def __init__(self, variable, expression=None):
+ if expression:
+ msg = "The variable '%s' was found in multiple places with different"\
+ " types in '%s'." % (variable, expression)
+ else:
+ msg = "The variable '%s' was found in multiple places with different"\
+ " types." % (variable)
+ Exception.__init__(self, msg)
+
+class TypeResolutionException(TypeException):
+ def __init__(self, expression, other_type):
+ Exception.__init__(self, "The type of '%s', '%s', cannot be "
+ "resolved with type '%s'" % \
+ (expression, expression.type, other_type))
+
+class IllegalTypeException(TypeException):
+ def __init__(self, expression, other_type, allowed_type):
+ Exception.__init__(self, "Cannot set type of %s '%s' to '%s'; "
+ "must match type '%s'." %
+ (expression.__class__.__name__, expression,
+ other_type, allowed_type))
+
+
+def typecheck(expressions, signature=None):
+ """
+ Ensure correct typing across a collection of ``Expression`` objects.
+ :param expressions: a collection of expressions
+ :param signature: dict that maps variable names to types (or string
+ representations of types)
+ """
+ #typecheck and create master signature
+ for expression in expressions:
+ signature = expression.typecheck(signature)
+ #apply master signature to all expressions
+ for expression in expressions[:-1]:
+ expression.typecheck(signature)
+ return signature
+
+
+class SubstituteBindingsI(object):
+ """
+ An interface for classes that can perform substitutions for
+ variables.
+ """
+ def substitute_bindings(self, bindings):
+ """
+ :return: The object that is obtained by replacing
+ each variable bound by ``bindings`` with its values.
+ Aliases are already resolved. (maybe?)
+ :rtype: (any)
+ """
+ raise NotImplementedError()
+
+ def variables(self):
+ """
+ :return: A list of all variables in this object.
+ """
+ raise NotImplementedError()
+
+
+ at python_2_unicode_compatible
+class Expression(SubstituteBindingsI):
+ """This is the base abstract object for all logical expressions"""
+
+ _logic_parser = _LogicParser()
+
+ @classmethod
+ def fromstring(cls, s):
+ return cls._logic_parser.parse(s)
+
+ def __call__(self, other, *additional):
+ accum = self.applyto(other)
+ for a in additional:
+ accum = accum(a)
+ return accum
+
+ def applyto(self, other):
+ assert isinstance(other, Expression), "%s is not an Expression" % other
+ return ApplicationExpression(self, other)
+
+ def __neg__(self):
+ return NegatedExpression(self)
+
+ def negate(self):
+ """If this is a negated expression, remove the negation.
+ Otherwise add a negation."""
+ return -self
+
+ def __and__(self, other):
+ if not isinstance(other, Expression):
+ raise TypeError("%s is not an Expression" % other)
+ return AndExpression(self, other)
+
+ def __or__(self, other):
+ if not isinstance(other, Expression):
+ raise TypeError("%s is not an Expression" % other)
+ return OrExpression(self, other)
+
+ def __gt__(self, other):
+ if not isinstance(other, Expression):
+ raise TypeError("%s is not an Expression" % other)
+ return ImpExpression(self, other)
+
+ def __lt__(self, other):
+ if not isinstance(other, Expression):
+ raise TypeError("%s is not an Expression" % other)
+ return IffExpression(self, other)
+
+ def __eq__(self, other):
+ raise NotImplementedError()
+
+ def __ne__(self, other):
+ return not self == other
+
+ def equiv(self, other, prover=None):
+ """
+ Check for logical equivalence.
+ Pass the expression (self <-> other) to the theorem prover.
+ If the prover says it is valid, then the self and other are equal.
+
+ :param other: an ``Expression`` to check equality against
+ :param prover: a ``nltk.inference.api.Prover``
+ """
+ assert isinstance(other, Expression), "%s is not an Expression" % other
+
+ if prover is None:
+ from nltk.inference import Prover9
+ prover = Prover9()
+ bicond = IffExpression(self.simplify(), other.simplify())
+ return prover.prove(bicond)
+
+ def __hash__(self):
+ return hash(repr(self))
+
+ def substitute_bindings(self, bindings):
+ expr = self
+ for var in expr.variables():
+ if var in bindings:
+ val = bindings[var]
+ if isinstance(val, Variable):
+ val = self.make_VariableExpression(val)
+ elif not isinstance(val, Expression):
+ raise ValueError('Can not substitute a non-expression '
+ 'value into an expression: %r' % (val,))
+ # Substitute bindings in the target value.
+ val = val.substitute_bindings(bindings)
+ # Replace var w/ the target value.
+ expr = expr.replace(var, val)
+ return expr.simplify()
+
+ def typecheck(self, signature=None):
+ """
+ Infer and check types. Raise exceptions if necessary.
+
+ :param signature: dict that maps variable names to types (or string
+ representations of types)
+ :return: the signature, plus any additional type mappings
+ """
+ sig = defaultdict(list)
+ if signature:
+ for key in signature:
+ val = signature[key]
+ varEx = VariableExpression(Variable(key))
+ if isinstance(val, Type):
+ varEx.type = val
+ else:
+ varEx.type = parse_type(val)
+ sig[key].append(varEx)
+
+ self._set_type(signature=sig)
+
+ return dict((key, sig[key][0].type) for key in sig)
+
+ def findtype(self, variable):
+ """
+ Find the type of the given variable as it is used in this expression.
+ For example, finding the type of "P" in "P(x) & Q(x,y)" yields "<e,t>"
+
+ :param variable: Variable
+ """
+ raise NotImplementedError()
+
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """
+ Set the type of this expression to be the given type. Raise type
+ exceptions where applicable.
+
+ :param other_type: Type
+ :param signature: dict(str -> list(AbstractVariableExpression))
+ """
+ raise NotImplementedError()
+
+ def replace(self, variable, expression, replace_bound=False, alpha_convert=True):
+ """
+ Replace every instance of 'variable' with 'expression'
+ :param variable: ``Variable`` The variable to replace
+ :param expression: ``Expression`` The expression with which to replace it
+ :param replace_bound: bool Should bound variables be replaced?
+ :param alpha_convert: bool Alpha convert automatically to avoid name clashes?
+ """
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ assert isinstance(expression, Expression), "%s is not an Expression" % expression
+
+ return self.visit_structured(lambda e: e.replace(variable, expression,
+ replace_bound, alpha_convert),
+ self.__class__)
+
+ def normalize(self, newvars=None):
+ """Rename auto-generated unique variables"""
+ def get_indiv_vars(e):
+ if isinstance(e, IndividualVariableExpression):
+ return set([e])
+ elif isinstance(e, AbstractVariableExpression):
+ return set()
+ else:
+ return e.visit(get_indiv_vars,
+ lambda parts: reduce(operator.or_, parts, set()))
+
+ result = self
+ for i,e in enumerate(sorted(get_indiv_vars(self), key=lambda e: e.variable)):
+ if isinstance(e,EventVariableExpression):
+ newVar = e.__class__(Variable('e0%s' % (i+1)))
+ elif isinstance(e,IndividualVariableExpression):
+ newVar = e.__class__(Variable('z%s' % (i+1)))
+ else:
+ newVar = e
+ result = result.replace(e.variable, newVar, True)
+ return result
+
+ def visit(self, function, combinator):
+ """
+ Recursively visit subexpressions. Apply 'function' to each
+ subexpression and pass the result of each function application
+ to the 'combinator' for aggregation:
+
+ return combinator(map(function, self.subexpressions))
+
+ Bound variables are neither applied upon by the function nor given to
+ the combinator.
+ :param function: ``Function<Expression,T>`` to call on each subexpression
+ :param combinator: ``Function<list<T>,R>`` to combine the results of the
+ function calls
+ :return: result of combination ``R``
+ """
+ raise NotImplementedError()
+
+ def visit_structured(self, function, combinator):
+ """
+ Recursively visit subexpressions. Apply 'function' to each
+ subexpression and pass the result of each function application
+ to the 'combinator' for aggregation. The combinator must have
+ the same signature as the constructor. The function is not
+ applied to bound variables, but they are passed to the
+ combinator.
+ :param function: ``Function`` to call on each subexpression
+ :param combinator: ``Function`` with the same signature as the
+ constructor, to combine the results of the function calls
+ :return: result of combination
+ """
+ return self.visit(function, lambda parts: combinator(*parts))
+
+ def __repr__(self):
+ return '<%s %s>' % (self.__class__.__name__, self)
+
+ def __str__(self):
+ return self.str()
+
+ def variables(self):
+ """
+ Return a set of all the variables for binding substitution.
+ The variables returned include all free (non-bound) individual
+ variables and any variable starting with '?' or '@'.
+ :return: set of ``Variable`` objects
+ """
+ return self.free() | set(p for p in self.predicates()|self.constants()
+ if re.match('^[?@]', p.name))
+
+ def free(self):
+ """
+ Return a set of all the free (non-bound) variables. This includes
+ both individual and predicate variables, but not constants.
+ :return: set of ``Variable`` objects
+ """
+ return self.visit(lambda e: e.free(),
+ lambda parts: reduce(operator.or_, parts, set()))
+
+ def constants(self):
+ """
+ Return a set of individual constants (non-predicates).
+ :return: set of ``Variable`` objects
+ """
+ return self.visit(lambda e: e.constants(),
+ lambda parts: reduce(operator.or_, parts, set()))
+
+ def predicates(self):
+ """
+ Return a set of predicates (constants, not variables).
+ :return: set of ``Variable`` objects
+ """
+ return self.visit(lambda e: e.predicates(),
+ lambda parts: reduce(operator.or_, parts, set()))
+
+ def simplify(self):
+ """
+ :return: beta-converted version of this expression
+ """
+ return self.visit_structured(lambda e: e.simplify(), self.__class__)
+
+ def make_VariableExpression(self, variable):
+ return VariableExpression(variable)
+
+
+ at python_2_unicode_compatible
+class ApplicationExpression(Expression):
+ r"""
+ This class is used to represent two related types of logical expressions.
+
+ The first is a Predicate Expression, such as "P(x,y)". A predicate
+ expression is comprised of a ``FunctionVariableExpression`` or
+ ``ConstantExpression`` as the predicate and a list of Expressions as the
+ arguments.
+
+ The second is a an application of one expression to another, such as
+ "(\x.dog(x))(fido)".
+
+ The reason Predicate Expressions are treated as Application Expressions is
+ that the Variable Expression predicate of the expression may be replaced
+ with another Expression, such as a LambdaExpression, which would mean that
+ the Predicate should be thought of as being applied to the arguments.
+
+ The logical expression parser will always curry arguments in a application expression.
+ So, "\x y.see(x,y)(john,mary)" will be represented internally as
+ "((\x y.(see(x))(y))(john))(mary)". This simplifies the internals since
+ there will always be exactly one argument in an application.
+
+ The str() method will usually print the curried forms of application
+ expressions. The one exception is when the the application expression is
+ really a predicate expression (ie, underlying function is an
+ ``AbstractVariableExpression``). This means that the example from above
+ will be returned as "(\x y.see(x,y)(john))(mary)".
+ """
+ def __init__(self, function, argument):
+ """
+ :param function: ``Expression``, for the function expression
+ :param argument: ``Expression``, for the argument
+ """
+ assert isinstance(function, Expression), "%s is not an Expression" % function
+ assert isinstance(argument, Expression), "%s is not an Expression" % argument
+ self.function = function
+ self.argument = argument
+
+ def simplify(self):
+ function = self.function.simplify()
+ argument = self.argument.simplify()
+ if isinstance(function, LambdaExpression):
+ return function.term.replace(function.variable, argument).simplify()
+ else:
+ return self.__class__(function, argument)
+
+ @property
+ def type(self):
+ if isinstance(self.function.type, ComplexType):
+ return self.function.type.second
+ else:
+ return ANY_TYPE
+
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ self.argument._set_type(ANY_TYPE, signature)
+ try:
+ self.function._set_type(ComplexType(self.argument.type, other_type), signature)
+ except TypeResolutionException:
+ raise TypeException(
+ "The function '%s' is of type '%s' and cannot be applied "
+ "to '%s' of type '%s'. Its argument must match type '%s'."
+ % (self.function, self.function.type, self.argument,
+ self.argument.type, self.function.type.first))
+
+ def findtype(self, variable):
+ """:see Expression.findtype()"""
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ if self.is_atom():
+ function, args = self.uncurry()
+ else:
+ #It's not a predicate expression ("P(x,y)"), so leave args curried
+ function = self.function
+ args = [self.argument]
+
+ found = [arg.findtype(variable) for arg in [function]+args]
+
+ unique = []
+ for f in found:
+ if f != ANY_TYPE:
+ if unique:
+ for u in unique:
+ if f.matches(u):
+ break
+ else:
+ unique.append(f)
+
+ if len(unique) == 1:
+ return list(unique)[0]
+ else:
+ return ANY_TYPE
+
+ def constants(self):
+ """:see: Expression.constants()"""
+ if isinstance(self.function, AbstractVariableExpression):
+ function_constants = set()
+ else:
+ function_constants = self.function.constants()
+ return function_constants | self.argument.constants()
+
+ def predicates(self):
+ """:see: Expression.predicates()"""
+ if isinstance(self.function, ConstantExpression):
+ function_preds = set([self.function.variable])
+ else:
+ function_preds = self.function.predicates()
+ return function_preds | self.argument.predicates()
+
+ def visit(self, function, combinator):
+ """:see: Expression.visit()"""
+ return combinator([function(self.function), function(self.argument)])
+
+ def __eq__(self, other):
+ return isinstance(other, ApplicationExpression) and \
+ self.function == other.function and \
+ self.argument == other.argument
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Expression.__hash__
+
+ def __str__(self):
+ # uncurry the arguments and find the base function
+ if self.is_atom():
+ function, args = self.uncurry()
+ arg_str = ','.join("%s" % arg for arg in args)
+ else:
+ #Leave arguments curried
+ function = self.function
+ arg_str = "%s" % self.argument
+
+ function_str = "%s" % function
+ parenthesize_function = False
+ if isinstance(function, LambdaExpression):
+ if isinstance(function.term, ApplicationExpression):
+ if not isinstance(function.term.function,
+ AbstractVariableExpression):
+ parenthesize_function = True
+ elif not isinstance(function.term, BooleanExpression):
+ parenthesize_function = True
+ elif isinstance(function, ApplicationExpression):
+ parenthesize_function = True
+
+ if parenthesize_function:
+ function_str = Tokens.OPEN + function_str + Tokens.CLOSE
+
+ return function_str + Tokens.OPEN + arg_str + Tokens.CLOSE
+
+ def uncurry(self):
+ """
+ Uncurry this application expression
+
+ return: A tuple (base-function, arg-list)
+ """
+ function = self.function
+ args = [self.argument]
+ while isinstance(function, ApplicationExpression):
+ #(\x.\y.sees(x,y)(john))(mary)
+ args.insert(0, function.argument)
+ function = function.function
+ return (function, args)
+
+ @property
+ def pred(self):
+ """
+ Return uncurried base-function.
+ If this is an atom, then the result will be a variable expression.
+ Otherwise, it will be a lambda expression.
+ """
+ return self.uncurry()[0]
+
+ @property
+ def args(self):
+ """
+ Return uncurried arg-list
+ """
+ return self.uncurry()[1]
+
+ def is_atom(self):
+ """
+ Is this expression an atom (as opposed to a lambda expression applied
+ to a term)?
+ """
+ return isinstance(self.pred, AbstractVariableExpression)
+
+
+ at total_ordering
+ at python_2_unicode_compatible
+class AbstractVariableExpression(Expression):
+ """This class represents a variable to be used as a predicate or entity"""
+ def __init__(self, variable):
+ """
+ :param variable: ``Variable``, for the variable
+ """
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ self.variable = variable
+
+ def simplify(self):
+ return self
+
+ def replace(self, variable, expression, replace_bound=False, alpha_convert=True):
+ """:see: Expression.replace()"""
+ assert isinstance(variable, Variable), "%s is not an Variable" % variable
+ assert isinstance(expression, Expression), "%s is not an Expression" % expression
+ if self.variable == variable:
+ return expression
+ else:
+ return self
+
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ resolution = other_type
+ for varEx in signature[self.variable.name]:
+ resolution = varEx.type.resolve(resolution)
+ if not resolution:
+ raise InconsistentTypeHierarchyException(self)
+
+ signature[self.variable.name].append(self)
+ for varEx in signature[self.variable.name]:
+ varEx.type = resolution
+
+ def findtype(self, variable):
+ """:see Expression.findtype()"""
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ if self.variable == variable:
+ return self.type
+ else:
+ return ANY_TYPE
+
+ def predicates(self):
+ """:see: Expression.predicates()"""
+ return set()
+
+ def __eq__(self, other):
+ """Allow equality between instances of ``AbstractVariableExpression``
+ subtypes."""
+ return isinstance(other, AbstractVariableExpression) and \
+ self.variable == other.variable
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if not isinstance(other, AbstractVariableExpression):
+ raise TypeError
+ return self.variable < other.variable
+
+ __hash__ = Expression.__hash__
+
+ def __str__(self):
+ return "%s" % self.variable
+
+class IndividualVariableExpression(AbstractVariableExpression):
+ """This class represents variables that take the form of a single lowercase
+ character (other than 'e') followed by zero or more digits."""
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ if not other_type.matches(ENTITY_TYPE):
+ raise IllegalTypeException(self, other_type, ENTITY_TYPE)
+
+ signature[self.variable.name].append(self)
+
+ def _get_type(self): return ENTITY_TYPE
+ type = property(_get_type, _set_type)
+
+ def free(self):
+ """:see: Expression.free()"""
+ return set([self.variable])
+
+ def constants(self):
+ """:see: Expression.constants()"""
+ return set()
+
+class FunctionVariableExpression(AbstractVariableExpression):
+ """This class represents variables that take the form of a single uppercase
+ character followed by zero or more digits."""
+ type = ANY_TYPE
+
+ def free(self):
+ """:see: Expression.free()"""
+ return set([self.variable])
+
+ def constants(self):
+ """:see: Expression.constants()"""
+ return set()
+
+class EventVariableExpression(IndividualVariableExpression):
+ """This class represents variables that take the form of a single lowercase
+ 'e' character followed by zero or more digits."""
+ type = EVENT_TYPE
+
+class ConstantExpression(AbstractVariableExpression):
+ """This class represents variables that do not take the form of a single
+ character followed by zero or more digits."""
+ type = ENTITY_TYPE
+
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ if other_type == ANY_TYPE:
+ #entity type by default, for individuals
+ resolution = ENTITY_TYPE
+ else:
+ resolution = other_type
+ if self.type != ENTITY_TYPE:
+ resolution = resolution.resolve(self.type)
+
+ for varEx in signature[self.variable.name]:
+ resolution = varEx.type.resolve(resolution)
+ if not resolution:
+ raise InconsistentTypeHierarchyException(self)
+
+ signature[self.variable.name].append(self)
+ for varEx in signature[self.variable.name]:
+ varEx.type = resolution
+
+ def free(self):
+ """:see: Expression.free()"""
+ return set()
+
+ def constants(self):
+ """:see: Expression.constants()"""
+ return set([self.variable])
+
+
+def VariableExpression(variable):
+ """
+ This is a factory method that instantiates and returns a subtype of
+ ``AbstractVariableExpression`` appropriate for the given variable.
+ """
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ if is_indvar(variable.name):
+ return IndividualVariableExpression(variable)
+ elif is_funcvar(variable.name):
+ return FunctionVariableExpression(variable)
+ elif is_eventvar(variable.name):
+ return EventVariableExpression(variable)
+ else:
+ return ConstantExpression(variable)
+
+
+class VariableBinderExpression(Expression):
+ """This an abstract class for any Expression that binds a variable in an
+ Expression. This includes LambdaExpressions and Quantified Expressions"""
+ def __init__(self, variable, term):
+ """
+ :param variable: ``Variable``, for the variable
+ :param term: ``Expression``, for the term
+ """
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ assert isinstance(term, Expression), "%s is not an Expression" % term
+ self.variable = variable
+ self.term = term
+
+ def replace(self, variable, expression, replace_bound=False, alpha_convert=True):
+ """:see: Expression.replace()"""
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ assert isinstance(expression, Expression), "%s is not an Expression" % expression
+ #if the bound variable is the thing being replaced
+ if self.variable == variable:
+ if replace_bound:
+ assert isinstance(expression, AbstractVariableExpression),\
+ "%s is not a AbstractVariableExpression" % expression
+ return self.__class__(expression.variable,
+ self.term.replace(variable, expression, True, alpha_convert))
+ else:
+ return self
+ else:
+ # if the bound variable appears in the expression, then it must
+ # be alpha converted to avoid a conflict
+ if alpha_convert and self.variable in expression.free():
+ self = self.alpha_convert(unique_variable(pattern=self.variable))
+
+ #replace in the term
+ return self.__class__(self.variable,
+ self.term.replace(variable, expression, replace_bound, alpha_convert))
+
+ def alpha_convert(self, newvar):
+ """Rename all occurrences of the variable introduced by this variable
+ binder in the expression to ``newvar``.
+ :param newvar: ``Variable``, for the new variable
+ """
+ assert isinstance(newvar, Variable), "%s is not a Variable" % newvar
+ return self.__class__(newvar,
+ self.term.replace(self.variable,
+ VariableExpression(newvar),
+ True))
+
+ def free(self):
+ """:see: Expression.free()"""
+ return self.term.free() - set([self.variable])
+
+ def findtype(self, variable):
+ """:see Expression.findtype()"""
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ if variable == self.variable:
+ return ANY_TYPE
+ else:
+ return self.term.findtype(variable)
+
+ def visit(self, function, combinator):
+ """:see: Expression.visit()"""
+ return combinator([function(self.term)])
+
+ def visit_structured(self, function, combinator):
+ """:see: Expression.visit_structured()"""
+ return combinator(self.variable, function(self.term))
+
+ def __eq__(self, other):
+ r"""Defines equality modulo alphabetic variance. If we are comparing
+ \x.M and \y.N, then check equality of M and N[x/y]."""
+ if isinstance(self, other.__class__) or \
+ isinstance(other, self.__class__):
+ if self.variable == other.variable:
+ return self.term == other.term
+ else:
+ # Comparing \x.M and \y.N. Relabel y in N with x and continue.
+ varex = VariableExpression(self.variable)
+ return self.term == other.term.replace(other.variable, varex)
+ else:
+ return False
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Expression.__hash__
+
+
+ at python_2_unicode_compatible
+class LambdaExpression(VariableBinderExpression):
+ @property
+ def type(self):
+ return ComplexType(self.term.findtype(self.variable),
+ self.term.type)
+
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ self.term._set_type(other_type.second, signature)
+ if not self.type.resolve(other_type):
+ raise TypeResolutionException(self, other_type)
+
+ def __str__(self):
+ variables = [self.variable]
+ term = self.term
+ while term.__class__ == self.__class__:
+ variables.append(term.variable)
+ term = term.term
+ return Tokens.LAMBDA + ' '.join("%s" % v for v in variables) + \
+ Tokens.DOT + "%s" % term
+
+
+ at python_2_unicode_compatible
+class QuantifiedExpression(VariableBinderExpression):
+ @property
+ def type(self): return TRUTH_TYPE
+
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ if not other_type.matches(TRUTH_TYPE):
+ raise IllegalTypeException(self, other_type, TRUTH_TYPE)
+ self.term._set_type(TRUTH_TYPE, signature)
+
+ def __str__(self):
+ variables = [self.variable]
+ term = self.term
+ while term.__class__ == self.__class__:
+ variables.append(term.variable)
+ term = term.term
+ return self.getQuantifier() + ' ' + ' '.join("%s" % v for v in variables) + \
+ Tokens.DOT + "%s" % term
+
+class ExistsExpression(QuantifiedExpression):
+ def getQuantifier(self):
+ return Tokens.EXISTS
+
+class AllExpression(QuantifiedExpression):
+ def getQuantifier(self):
+ return Tokens.ALL
+
+
+ at python_2_unicode_compatible
+class NegatedExpression(Expression):
+ def __init__(self, term):
+ assert isinstance(term, Expression), "%s is not an Expression" % term
+ self.term = term
+
+ @property
+ def type(self): return TRUTH_TYPE
+
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ if not other_type.matches(TRUTH_TYPE):
+ raise IllegalTypeException(self, other_type, TRUTH_TYPE)
+ self.term._set_type(TRUTH_TYPE, signature)
+
+ def findtype(self, variable):
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ return self.term.findtype(variable)
+
+ def visit(self, function, combinator):
+ """:see: Expression.visit()"""
+ return combinator([function(self.term)])
+
+ def negate(self):
+ """:see: Expression.negate()"""
+ return self.term
+
+ def __eq__(self, other):
+ return isinstance(other, NegatedExpression) and self.term == other.term
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Expression.__hash__
+
+ def __str__(self):
+ return Tokens.NOT + "%s" % self.term
+
+
+ at python_2_unicode_compatible
+class BinaryExpression(Expression):
+ def __init__(self, first, second):
+ assert isinstance(first, Expression), "%s is not an Expression" % first
+ assert isinstance(second, Expression), "%s is not an Expression" % second
+ self.first = first
+ self.second = second
+
+ @property
+ def type(self): return TRUTH_TYPE
+
+ def findtype(self, variable):
+ """:see Expression.findtype()"""
+ assert isinstance(variable, Variable), "%s is not a Variable" % variable
+ f = self.first.findtype(variable)
+ s = self.second.findtype(variable)
+ if f == s or s == ANY_TYPE:
+ return f
+ elif f == ANY_TYPE:
+ return s
+ else:
+ return ANY_TYPE
+
+ def visit(self, function, combinator):
+ """:see: Expression.visit()"""
+ return combinator([function(self.first), function(self.second)])
+
+ def __eq__(self, other):
+ return (isinstance(self, other.__class__) or \
+ isinstance(other, self.__class__)) and \
+ self.first == other.first and self.second == other.second
+
+ def __ne__(self, other):
+ return not self == other
+
+ __hash__ = Expression.__hash__
+
+ def __str__(self):
+ first = self._str_subex(self.first)
+ second = self._str_subex(self.second)
+ return Tokens.OPEN + first + ' ' + self.getOp() \
+ + ' ' + second + Tokens.CLOSE
+
+ def _str_subex(self, subex):
+ return "%s" % subex
+
+
+class BooleanExpression(BinaryExpression):
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ if not other_type.matches(TRUTH_TYPE):
+ raise IllegalTypeException(self, other_type, TRUTH_TYPE)
+ self.first._set_type(TRUTH_TYPE, signature)
+ self.second._set_type(TRUTH_TYPE, signature)
+
+class AndExpression(BooleanExpression):
+ """This class represents conjunctions"""
+ def getOp(self):
+ return Tokens.AND
+
+ def _str_subex(self, subex):
+ s = "%s" % subex
+ if isinstance(subex, AndExpression):
+ return s[1:-1]
+ return s
+
+class OrExpression(BooleanExpression):
+ """This class represents disjunctions"""
+ def getOp(self):
+ return Tokens.OR
+
+ def _str_subex(self, subex):
+ s = "%s" % subex
+ if isinstance(subex, OrExpression):
+ return s[1:-1]
+ return s
+
+class ImpExpression(BooleanExpression):
+ """This class represents implications"""
+ def getOp(self):
+ return Tokens.IMP
+
+class IffExpression(BooleanExpression):
+ """This class represents biconditionals"""
+ def getOp(self):
+ return Tokens.IFF
+
+
+class EqualityExpression(BinaryExpression):
+ """This class represents equality expressions like "(x = y)"."""
+ def _set_type(self, other_type=ANY_TYPE, signature=None):
+ """:see Expression._set_type()"""
+ assert isinstance(other_type, Type)
+
+ if signature is None:
+ signature = defaultdict(list)
+
+ if not other_type.matches(TRUTH_TYPE):
+ raise IllegalTypeException(self, other_type, TRUTH_TYPE)
+ self.first._set_type(ENTITY_TYPE, signature)
+ self.second._set_type(ENTITY_TYPE, signature)
+
+ def getOp(self):
+ return Tokens.EQ
+
+
+### Utilities
+
+class StringTrie(defaultdict):
+ LEAF = "<leaf>"
+
+ def __init__(self, strings=None):
+ defaultdict.__init__(self, StringTrie)
+ if strings:
+ for string in strings:
+ self.insert(string)
+
+ def insert(self, string):
+ if len(string):
+ self[string[0]].insert(string[1:])
+ else:
+ #mark the string is complete
+ self[StringTrie.LEAF] = None
+
+
+class LogicalExpressionException(Exception):
+ def __init__(self, index, message):
+ self.index = index
+ Exception.__init__(self, message)
+
+class UnexpectedTokenException(LogicalExpressionException):
+ def __init__(self, index, unexpected=None, expected=None, message=None):
+ if unexpected and expected:
+ msg = "Unexpected token: '%s'. " \
+ "Expected token '%s'." % (unexpected, expected)
+ elif unexpected:
+ msg = "Unexpected token: '%s'." % unexpected
+ if message:
+ msg += ' '+message
+ else:
+ msg = "Expected token '%s'." % expected
+ LogicalExpressionException.__init__(self, index, msg)
+
+class ExpectedMoreTokensException(LogicalExpressionException):
+ def __init__(self, index, message=None):
+ if not message:
+ message = 'More tokens expected.'
+ LogicalExpressionException.__init__(self, index, 'End of input found. ' + message)
+
+
+def is_indvar(expr):
+ """
+ An individual variable must be a single lowercase character other than 'e',
+ followed by zero or more digits.
+
+ :param expr: str
+ :return: bool True if expr is of the correct form
+ """
+ assert isinstance(expr, string_types), "%s is not a string" % expr
+ return re.match(r'^[a-df-z]\d*$', expr) is not None
+
+def is_funcvar(expr):
+ """
+ A function variable must be a single uppercase character followed by
+ zero or more digits.
+
+ :param expr: str
+ :return: bool True if expr is of the correct form
+ """
+ assert isinstance(expr, string_types), "%s is not a string" % expr
+ return re.match(r'^[A-Z]\d*$', expr) is not None
+
+def is_eventvar(expr):
+ """
+ An event variable must be a single lowercase 'e' character followed by
+ zero or more digits.
+
+ :param expr: str
+ :return: bool True if expr is of the correct form
+ """
+ assert isinstance(expr, string_types), "%s is not a string" % expr
+ return re.match(r'^e\d*$', expr) is not None
+
+
+def demo():
+ lexpr = Expression.fromstring
+ print('='*20 + 'Test parser' + '='*20)
+ print(lexpr(r'john'))
+ print(lexpr(r'man(x)'))
+ print(lexpr(r'-man(x)'))
+ print(lexpr(r'(man(x) & tall(x) & walks(x))'))
+ print(lexpr(r'exists x.(man(x) & tall(x) & walks(x))'))
+ print(lexpr(r'\x.man(x)'))
+ print(lexpr(r'\x.man(x)(john)'))
+ print(lexpr(r'\x y.sees(x,y)'))
+ print(lexpr(r'\x y.sees(x,y)(a,b)'))
+ print(lexpr(r'(\x.exists y.walks(x,y))(x)'))
+ print(lexpr(r'exists x.x = y'))
+ print(lexpr(r'exists x.(x = y)'))
+ print(lexpr('P(x) & x=y & P(y)'))
+ print(lexpr(r'\P Q.exists x.(P(x) & Q(x))'))
+ print(lexpr(r'man(x) <-> tall(x)'))
+
+ print('='*20 + 'Test simplify' + '='*20)
+ print(lexpr(r'\x.\y.sees(x,y)(john)(mary)').simplify())
+ print(lexpr(r'\x.\y.sees(x,y)(john, mary)').simplify())
+ print(lexpr(r'all x.(man(x) & (\x.exists y.walks(x,y))(x))').simplify())
+ print(lexpr(r'(\P.\Q.exists x.(P(x) & Q(x)))(\x.dog(x))(\x.bark(x))').simplify())
+
+ print('='*20 + 'Test alpha conversion and binder expression equality' + '='*20)
+ e1 = p('exists x.P(x)')
+ print(e1)
+ e2 = e1.alpha_convert(Variable('z'))
+ print(e2)
+ print(e1 == e2)
+
+def demo_errors():
+ print('='*20 + 'Test parser errors' + '='*20)
+ demoException('(P(x) & Q(x)')
+ demoException('((P(x) &) & Q(x))')
+ demoException('P(x) -> ')
+ demoException('P(x')
+ demoException('P(x,')
+ demoException('P(x,)')
+ demoException('exists')
+ demoException('exists x.')
+ demoException('\\')
+ demoException('\\ x y.')
+ demoException('P(x)Q(x)')
+ demoException('(P(x)Q(x)')
+ demoException('exists x -> y')
+
+def demoException(s):
+ try:
+ Expression.fromstring(s)
+ except LogicalExpressionException as e:
+ print("%s: %s" % (e.__class__.__name__, e))
+
+def printtype(ex):
+ print("%s : %s" % (ex.str(), ex.type))
+
+if __name__ == '__main__':
+ demo()
+ demo_errors()
diff --git a/nltk/sem/relextract.py b/nltk/sem/relextract.py
new file mode 100644
index 0000000..1a9ca44
--- /dev/null
+++ b/nltk/sem/relextract.py
@@ -0,0 +1,483 @@
+# Natural Language Toolkit: Relation Extraction
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Code for extracting relational triples from the ieer and conll2002 corpora.
+
+Relations are stored internally as dictionaries ('reldicts').
+
+The two serialization outputs are "rtuple" and "clause".
+
+- An rtuple is a tuple of the form ``(subj, filler, obj)``,
+ where ``subj`` and ``obj`` are pairs of Named Entity mentions, and ``filler`` is the string of words
+ occurring between ``sub`` and ``obj`` (with no intervening NEs). Strings are printed via ``repr()`` to
+ circumvent locale variations in rendering utf-8 encoded strings.
+- A clause is an atom of the form ``relsym(subjsym, objsym)``,
+ where the relation, subject and object have been canonicalized to single strings.
+"""
+from __future__ import print_function
+
+# todo: get a more general solution to canonicalized symbols for clauses -- maybe use xmlcharrefs?
+
+from collections import defaultdict
+import re
+from nltk.compat import htmlentitydefs
+
+# Dictionary that associates corpora with NE classes
+NE_CLASSES = {
+ 'ieer': ['LOCATION', 'ORGANIZATION', 'PERSON', 'DURATION',
+ 'DATE', 'CARDINAL', 'PERCENT', 'MONEY', 'MEASURE'],
+ 'conll2002': ['LOC', 'PER', 'ORG'],
+ 'ace': ['LOCATION', 'ORGANIZATION', 'PERSON', 'DURATION',
+ 'DATE', 'CARDINAL', 'PERCENT', 'MONEY', 'MEASURE', 'FACILITY', 'GPE'],
+ }
+
+# Allow abbreviated class labels
+short2long = dict(LOC = 'LOCATION', ORG = 'ORGANIZATION', PER = 'PERSON')
+long2short = dict(LOCATION ='LOC', ORGANIZATION = 'ORG', PERSON = 'PER')
+
+
+def _expand(type):
+ """
+ Expand an NE class name.
+ :type type: str
+ :rtype: str
+ """
+ try:
+ return short2long[type]
+ except KeyError:
+ return type
+
+def class_abbrev(type):
+ """
+ Abbreviate an NE class name.
+ :type type: str
+ :rtype: str
+ """
+ try:
+ return long2short[type]
+ except KeyError:
+ return type
+
+
+def _join(lst, sep=' ', untag=False):
+ """
+ Join a list into a string, turning tags tuples into tag strings or just words.
+ :param untag: if ``True``, omit the tag from tagged input strings.
+ :type lst: list
+ :rtype: str
+ """
+ try:
+ return sep.join(lst)
+ except TypeError:
+ if untag:
+ return sep.join(tup[0] for tup in lst)
+ from nltk.tag import tuple2str
+ return sep.join(tuple2str(tup) for tup in lst)
+
+def descape_entity(m, defs=htmlentitydefs.entitydefs):
+ """
+ Translate one entity to its ISO Latin value.
+ Inspired by example from effbot.org
+
+
+ """
+ #s = 'mcglashan_&_sarrail'
+ #l = ['mcglashan', '&', 'sarrail']
+ #pattern = re.compile("&(\w+?);")
+ #new = list2sym(l)
+ #s = pattern.sub(descape_entity, s)
+ #print s, new
+ try:
+ return defs[m.group(1)]
+
+ except KeyError:
+ return m.group(0) # use as is
+
+def list2sym(lst):
+ """
+ Convert a list of strings into a canonical symbol.
+ :type lst: list
+ :return: a Unicode string without whitespace
+ :rtype: unicode
+ """
+ sym = _join(lst, '_', untag=True)
+ sym = sym.lower()
+ ENT = re.compile("&(\w+?);")
+ sym = ENT.sub(descape_entity, sym)
+ sym = sym.replace('.', '')
+ return sym
+
+def tree2semi_rel(tree):
+ """
+ Group a chunk structure into a list of 'semi-relations' of the form (list(str), ``Tree``).
+
+ In order to facilitate the construction of (``Tree``, string, ``Tree``) triples, this
+ identifies pairs whose first member is a list (possibly empty) of terminal
+ strings, and whose second member is a ``Tree`` of the form (NE_label, terminals).
+
+ :param tree: a chunk tree
+ :return: a list of pairs (list(str), ``Tree``)
+ :rtype: list of tuple
+ """
+
+ from nltk.tree import Tree
+
+ semi_rels = []
+ semi_rel = [[], None]
+
+ for dtr in tree:
+ if not isinstance(dtr, Tree):
+ semi_rel[0].append(dtr)
+ else:
+ # dtr is a Tree
+ semi_rel[1] = dtr
+ semi_rels.append(semi_rel)
+ semi_rel = [[], None]
+ return semi_rels
+
+
+def semi_rel2reldict(pairs, window=5, trace=False):
+ """
+ Converts the pairs generated by ``tree2semi_rel`` into a 'reldict': a dictionary which
+ stores information about the subject and object NEs plus the filler between them.
+ Additionally, a left and right context of length =< window are captured (within
+ a given input sentence).
+
+ :param pairs: a pair of list(str) and ``Tree``, as generated by
+ :param window: a threshold for the number of items to include in the left and right context
+ :type window: int
+ :return: 'relation' dictionaries whose keys are 'lcon', 'subjclass', 'subjtext', 'subjsym', 'filler', objclass', objtext', 'objsym' and 'rcon'
+ :rtype: list(defaultdict)
+ """
+ result = []
+ while len(pairs) > 2:
+ reldict = defaultdict(str)
+ reldict['lcon'] = _join(pairs[0][0][-window:])
+ reldict['subjclass'] = pairs[0][1].label()
+ reldict['subjtext'] = _join(pairs[0][1].leaves())
+ reldict['subjsym'] = list2sym(pairs[0][1].leaves())
+ reldict['filler'] = _join(pairs[1][0])
+ reldict['untagged_filler'] = _join(pairs[1][0], untag=True)
+ reldict['objclass'] = pairs[1][1].label()
+ reldict['objtext'] = _join(pairs[1][1].leaves())
+ reldict['objsym'] = list2sym(pairs[1][1].leaves())
+ reldict['rcon'] = _join(pairs[2][0][:window])
+ if trace:
+ print("(%s(%s, %s)" % (reldict['untagged_filler'], reldict['subjclass'], reldict['objclass']))
+ result.append(reldict)
+ pairs = pairs[1:]
+ return result
+
+def extract_rels(subjclass, objclass, doc, corpus='ace', pattern=None, window=10):
+ """
+ Filter the output of ``semi_rel2reldict`` according to specified NE classes and a filler pattern.
+
+ The parameters ``subjclass`` and ``objclass`` can be used to restrict the
+ Named Entities to particular types (any of 'LOCATION', 'ORGANIZATION',
+ 'PERSON', 'DURATION', 'DATE', 'CARDINAL', 'PERCENT', 'MONEY', 'MEASURE').
+
+ :param subjclass: the class of the subject Named Entity.
+ :type subjclass: str
+ :param objclass: the class of the object Named Entity.
+ :type objclass: str
+ :param doc: input document
+ :type doc: ieer document or a list of chunk trees
+ :param corpus: name of the corpus to take as input; possible values are
+ 'ieer' and 'conll2002'
+ :type corpus: str
+ :param pattern: a regular expression for filtering the fillers of
+ retrieved triples.
+ :type pattern: SRE_Pattern
+ :param window: filters out fillers which exceed this threshold
+ :type window: int
+ :return: see ``mk_reldicts``
+ :rtype: list(defaultdict)
+ """
+
+ if subjclass and subjclass not in NE_CLASSES[corpus]:
+ if _expand(subjclass) in NE_CLASSES[corpus]:
+ subjclass = _expand(subjclass)
+ else:
+ raise ValueError("your value for the subject type has not been recognized: %s" % subjclass)
+ if objclass and objclass not in NE_CLASSES[corpus]:
+ if _expand(objclass) in NE_CLASSES[corpus]:
+ objclass = _expand(objclass)
+ else:
+ raise ValueError("your value for the object type has not been recognized: %s" % objclass)
+
+ if corpus == 'ace' or corpus == 'conll2002':
+ pairs = tree2semi_rel(doc)
+ elif corpus == 'ieer':
+ pairs = tree2semi_rel(doc.text) + tree2semi_rel(doc.headline)
+ else:
+ raise ValueError("corpus type not recognized")
+
+ reldicts = semi_rel2reldict(pairs)
+
+ relfilter = lambda x: (x['subjclass'] == subjclass and
+ len(x['filler'].split()) <= window and
+ pattern.match(x['filler']) and
+ x['objclass'] == objclass)
+
+ return list(filter(relfilter, reldicts))
+
+
+def rtuple(reldict, lcon=False, rcon=False):
+ """
+ Pretty print the reldict as an rtuple.
+ :param reldict: a relation dictionary
+ :type reldict: defaultdict
+ """
+ items = [class_abbrev(reldict['subjclass']), reldict['subjtext'], reldict['filler'], class_abbrev(reldict['objclass']), reldict['objtext']]
+ format = '[%s: %r] %r [%s: %r]'
+ if lcon:
+ items = [reldict['lcon']] + items
+ format = '...%r)' + format
+ if rcon:
+ items.append(reldict['rcon'])
+ format = format + '(%r...'
+ printargs = tuple(items)
+ return format % printargs
+
+def clause(reldict, relsym):
+ """
+ Print the relation in clausal form.
+ :param reldict: a relation dictionary
+ :type reldict: defaultdict
+ :param relsym: a label for the relation
+ :type relsym: str
+ """
+ items = (relsym, reldict['subjsym'], reldict['objsym'])
+ return "%s(%r, %r)" % items
+
+
+#######################################################
+# Demos of relation extraction with regular expressions
+#######################################################
+
+############################################
+# Example of in(ORG, LOC)
+############################################
+def in_demo(trace=0, sql=True):
+ """
+ Select pairs of organizations and locations whose mentions occur with an
+ intervening occurrence of the preposition "in".
+
+ If the sql parameter is set to True, then the entity pairs are loaded into
+ an in-memory database, and subsequently pulled out using an SQL "SELECT"
+ query.
+ """
+ from nltk.corpus import ieer
+ if sql:
+ try:
+ import sqlite3
+ connection = sqlite3.connect(":memory:")
+ connection.text_factory = sqlite3.OptimizedUnicode
+ cur = connection.cursor()
+ cur.execute("""create table Locations
+ (OrgName text, LocationName text, DocID text)""")
+ except ImportError:
+ import warnings
+ warnings.warn("Cannot import sqlite; sql flag will be ignored.")
+
+
+ IN = re.compile(r'.*\bin\b(?!\b.+ing)')
+
+ print()
+ print("IEER: in(ORG, LOC) -- just the clauses:")
+ print("=" * 45)
+
+ for file in ieer.fileids():
+ for doc in ieer.parsed_docs(file):
+ if trace:
+ print(doc.docno)
+ print("=" * 15)
+ for rel in extract_rels('ORG', 'LOC', doc, corpus='ieer', pattern=IN):
+ print(clause(rel, relsym='IN'))
+ if sql:
+ try:
+ rtuple = (rel['subjtext'], rel['objtext'], doc.docno)
+ cur.execute("""insert into Locations
+ values (?, ?, ?)""", rtuple)
+ connection.commit()
+ except NameError:
+ pass
+
+ if sql:
+ try:
+ cur.execute("""select OrgName from Locations
+ where LocationName = 'Atlanta'""")
+ print()
+ print("Extract data from SQL table: ORGs in Atlanta")
+ print("-" * 15)
+ for row in cur:
+ print(row)
+ except NameError:
+ pass
+
+
+############################################
+# Example of has_role(PER, LOC)
+############################################
+
+def roles_demo(trace=0):
+ from nltk.corpus import ieer
+ roles = """
+ (.*( # assorted roles
+ analyst|
+ chair(wo)?man|
+ commissioner|
+ counsel|
+ director|
+ economist|
+ editor|
+ executive|
+ foreman|
+ governor|
+ head|
+ lawyer|
+ leader|
+ librarian).*)|
+ manager|
+ partner|
+ president|
+ producer|
+ professor|
+ researcher|
+ spokes(wo)?man|
+ writer|
+ ,\sof\sthe?\s* # "X, of (the) Y"
+ """
+ ROLES = re.compile(roles, re.VERBOSE)
+
+ print()
+ print("IEER: has_role(PER, ORG) -- raw rtuples:")
+ print("=" * 45)
+
+ for file in ieer.fileids():
+ for doc in ieer.parsed_docs(file):
+ lcon = rcon = False
+ if trace:
+ print(doc.docno)
+ print("=" * 15)
+ lcon = rcon = True
+ for rel in extract_rels('PER', 'ORG', doc, corpus='ieer', pattern=ROLES):
+ print(rtuple(rel, lcon=lcon, rcon=rcon))
+
+
+##############################################
+### Show what's in the IEER Headlines
+##############################################
+
+
+def ieer_headlines():
+
+ from nltk.corpus import ieer
+ from nltk.tree import Tree
+
+ print("IEER: First 20 Headlines")
+ print("=" * 45)
+
+ trees = [(doc.docno, doc.headline) for file in ieer.fileids() for doc in ieer.parsed_docs(file)]
+ for tree in trees[:20]:
+ print()
+ print("%s:\n%s" % tree)
+
+
+
+#############################################
+## Dutch CONLL2002: take_on_role(PER, ORG
+#############################################
+
+def conllned(trace=1):
+ """
+ Find the copula+'van' relation ('of') in the Dutch tagged training corpus
+ from CoNLL 2002.
+ """
+
+ from nltk.corpus import conll2002
+
+ vnv = """
+ (
+ is/V| # 3rd sing present and
+ was/V| # past forms of the verb zijn ('be')
+ werd/V| # and also present
+ wordt/V # past of worden ('become)
+ )
+ .* # followed by anything
+ van/Prep # followed by van ('of')
+ """
+ VAN = re.compile(vnv, re.VERBOSE)
+
+ print()
+ print("Dutch CoNLL2002: van(PER, ORG) -- raw rtuples with context:")
+ print("=" * 45)
+
+
+ for doc in conll2002.chunked_sents('ned.train'):
+ lcon = rcon = False
+ if trace:
+ lcon = rcon = True
+ for rel in extract_rels('PER', 'ORG', doc, corpus='conll2002', pattern=VAN, window=10):
+ print(rtuple(rel, lcon=True, rcon=True))
+
+#############################################
+## Spanish CONLL2002: (PER, ORG)
+#############################################
+
+def conllesp():
+ from nltk.corpus import conll2002
+
+ de = """
+ .*
+ (
+ de/SP|
+ del/SP
+ )
+ """
+ DE = re.compile(de, re.VERBOSE)
+
+ print()
+ print("Spanish CoNLL2002: de(ORG, LOC) -- just the first 10 clauses:")
+ print("=" * 45)
+ rels = [rel for doc in conll2002.chunked_sents('esp.train')
+ for rel in extract_rels('ORG', 'LOC', doc, corpus='conll2002', pattern = DE)]
+ for r in rels[:10]: print(clause(r, relsym='DE'))
+ print()
+
+
+def ne_chunked():
+ print()
+ print("1500 Sentences from Penn Treebank, as processed by NLTK NE Chunker")
+ print("=" * 45)
+ ROLE = re.compile(r'.*(chairman|president|trader|scientist|economist|analyst|partner).*')
+ rels = []
+ for i, sent in enumerate(nltk.corpus.treebank.tagged_sents()[:1500]):
+ sent = nltk.ne_chunk(sent)
+ rels = extract_rels('PER', 'ORG', sent, corpus='ace', pattern=ROLE, window=7)
+ for rel in rels:
+ print('{0:<5}{1}'.format(i, rtuple(rel)))
+
+
+if __name__ == '__main__':
+ import nltk
+ from nltk.sem import relextract
+ in_demo(trace=0)
+ roles_demo(trace=0)
+ conllned()
+ conllesp()
+ ieer_headlines()
+ ne_chunked()
+
+
+
+
+
+
+
+
diff --git a/nltk/sem/skolemize.py b/nltk/sem/skolemize.py
new file mode 100644
index 0000000..120fd9b
--- /dev/null
+++ b/nltk/sem/skolemize.py
@@ -0,0 +1,101 @@
+# Natural Language Toolkit: Semantic Interpretation
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk.sem.logic import (AllExpression, AndExpression, ApplicationExpression,
+ EqualityExpression, ExistsExpression, IffExpression,
+ ImpExpression, NegatedExpression, OrExpression,
+ VariableExpression, skolem_function, unique_variable)
+
+def skolemize(expression, univ_scope=None, used_variables=None):
+ """
+ Skolemize the expression and convert to conjunctive normal form (CNF)
+ """
+ if univ_scope is None:
+ univ_scope = set()
+ if used_variables is None:
+ used_variables = set()
+
+ if isinstance(expression, AllExpression):
+ term = skolemize(expression.term, univ_scope|set([expression.variable]), used_variables|set([expression.variable]))
+ return term.replace(expression.variable, VariableExpression(unique_variable(ignore=used_variables)))
+ elif isinstance(expression, AndExpression):
+ return skolemize(expression.first, univ_scope, used_variables) &\
+ skolemize(expression.second, univ_scope, used_variables)
+ elif isinstance(expression, OrExpression):
+ return to_cnf(skolemize(expression.first, univ_scope, used_variables),
+ skolemize(expression.second, univ_scope, used_variables))
+ elif isinstance(expression, ImpExpression):
+ return to_cnf(skolemize(-expression.first, univ_scope, used_variables),
+ skolemize(expression.second, univ_scope, used_variables))
+ elif isinstance(expression, IffExpression):
+ return to_cnf(skolemize(-expression.first, univ_scope, used_variables),
+ skolemize(expression.second, univ_scope, used_variables)) &\
+ to_cnf(skolemize(expression.first, univ_scope, used_variables),
+ skolemize(-expression.second, univ_scope, used_variables))
+ elif isinstance(expression, EqualityExpression):
+ return expression
+ elif isinstance(expression, NegatedExpression):
+ negated = expression.term
+ if isinstance(negated, AllExpression):
+ term = skolemize(-negated.term, univ_scope, used_variables|set([negated.variable]))
+ if univ_scope:
+ return term.replace(negated.variable, skolem_function(univ_scope))
+ else:
+ skolem_constant = VariableExpression(unique_variable(ignore=used_variables))
+ return term.replace(negated.variable, skolem_constant)
+ elif isinstance(negated, AndExpression):
+ return to_cnf(skolemize(-negated.first, univ_scope, used_variables),
+ skolemize(-negated.second, univ_scope, used_variables))
+ elif isinstance(negated, OrExpression):
+ return skolemize(-negated.first, univ_scope, used_variables) &\
+ skolemize(-negated.second, univ_scope, used_variables)
+ elif isinstance(negated, ImpExpression):
+ return skolemize(negated.first, univ_scope, used_variables) &\
+ skolemize(-negated.second, univ_scope, used_variables)
+ elif isinstance(negated, IffExpression):
+ return to_cnf(skolemize(-negated.first, univ_scope, used_variables),
+ skolemize(-negated.second, univ_scope, used_variables)) &\
+ to_cnf(skolemize(negated.first, univ_scope, used_variables),
+ skolemize(negated.second, univ_scope, used_variables))
+ elif isinstance(negated, EqualityExpression):
+ return expression
+ elif isinstance(negated, NegatedExpression):
+ return skolemize(negated.term, univ_scope, used_variables)
+ elif isinstance(negated, ExistsExpression):
+ term = skolemize(-negated.term, univ_scope|set([negated.variable]), used_variables|set([negated.variable]))
+ return term.replace(negated.variable, VariableExpression(unique_variable(ignore=used_variables)))
+ elif isinstance(negated, ApplicationExpression):
+ return expression
+ else:
+ raise Exception('\'%s\' cannot be skolemized' % expression)
+ elif isinstance(expression, ExistsExpression):
+ term = skolemize(expression.term, univ_scope, used_variables|set([expression.variable]))
+ if univ_scope:
+ return term.replace(expression.variable, skolem_function(univ_scope))
+ else:
+ skolem_constant = VariableExpression(unique_variable(ignore=used_variables))
+ return term.replace(expression.variable, skolem_constant)
+ elif isinstance(expression, ApplicationExpression):
+ return expression
+ else:
+ raise Exception('\'%s\' cannot be skolemized' % expression)
+
+def to_cnf(first, second):
+ """
+ Convert this split disjunction to conjunctive normal form (CNF)
+ """
+ if isinstance(first, AndExpression):
+ r_first = to_cnf(first.first, second)
+ r_second = to_cnf(first.second, second)
+ return r_first & r_second
+ elif isinstance(second, AndExpression):
+ r_first = to_cnf(first, second.first)
+ r_second = to_cnf(first, second.second)
+ return r_first & r_second
+ else:
+ return first | second
diff --git a/nltk/sem/util.py b/nltk/sem/util.py
new file mode 100644
index 0000000..62bf06f
--- /dev/null
+++ b/nltk/sem/util.py
@@ -0,0 +1,317 @@
+# Natural Language Toolkit: Semantic Interpretation
+#
+# Author: Ewan Klein <ewan at inf.ed.ac.uk>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Utility functions for batch-processing sentences: parsing and
+extraction of the semantic representation of the root node of the the
+syntax tree, followed by evaluation of the semantic representation in
+a first-order model.
+"""
+from __future__ import print_function, unicode_literals
+
+import re
+import codecs
+from nltk.sem import evaluate
+
+
+##############################################################
+## Utility functions for connecting parse output to semantics
+##############################################################
+
+def parse_sents(inputs, grammar, trace=0):
+ """
+ Convert input sentences into syntactic trees.
+
+ :param inputs: sentences to be parsed
+ :type inputs: list of str
+ :param grammar: ``FeatureGrammar`` or name of feature-based grammar
+ :rtype: dict
+ :return: a mapping from input sentences to a list of ``Tree``s
+ """
+
+ # put imports here to avoid circult dependencies
+ from nltk.grammar import FeatureGrammar
+ from nltk.parse import FeatureChartParser, load_parser
+
+ if isinstance(grammar, FeatureGrammar):
+ cp = FeatureChartParser(grammar)
+ else:
+ cp = load_parser(grammar, trace=trace)
+ parses = []
+ for sent in inputs:
+ tokens = sent.split() # use a tokenizer?
+ syntrees = list(cp.parse(tokens))
+ parses.append(syntrees)
+ return parses
+
+def root_semrep(syntree, semkey='SEM'):
+ """
+ Find the semantic representation at the root of a tree.
+
+ :param syntree: a parse ``Tree``
+ :param semkey: the feature label to use for the root semantics in the tree
+ :return: the semantic representation at the root of a ``Tree``
+ :rtype: sem.Expression
+ """
+ from nltk.grammar import FeatStructNonterminal
+
+ node = syntree.label()
+ assert isinstance(node, FeatStructNonterminal)
+ try:
+ return node[semkey]
+ except KeyError:
+ print(node, end=' ')
+ print("has no specification for the feature %s" % semkey)
+ raise
+
+def interpret_sents(inputs, grammar, semkey='SEM', trace=0):
+ """
+ Add the semantic representation to each syntactic parse tree
+ of each input sentence.
+
+ :param inputs: a list of sentences
+ :param grammar: ``FeatureGrammar`` or name of feature-based grammar
+ :return: a mapping from sentences to lists of pairs (parse-tree, semantic-representations)
+ :rtype: dict
+ """
+ return [[(syn, root_semrep(syn, semkey)) for syn in syntrees]
+ for syntrees in parse_sents(inputs, grammar, trace=trace)]
+
+def evaluate_sents(inputs, grammar, model, assignment, trace=0):
+ """
+ Add the truth-in-a-model value to each semantic representation
+ for each syntactic parse of each input sentences.
+
+ :param inputs: a list of sentences
+ :param grammar: ``FeatureGrammar`` or name of feature-based grammar
+ :return: a mapping from sentences to lists of triples (parse-tree, semantic-representations, evaluation-in-model)
+ :rtype: dict
+ """
+ return [[(syn, sem, model.evaluate("%s" % sem, assignment, trace=trace))
+ for (syn, sem) in interpretations]
+ for interpretations in interpret_sents(inputs, grammar)]
+
+
+##########################################
+# REs used by the parse_valuation function
+##########################################
+_VAL_SPLIT_RE = re.compile(r'\s*=+>\s*')
+_ELEMENT_SPLIT_RE = re.compile(r'\s*,\s*')
+_TUPLES_RE = re.compile(r"""\s*
+ (\([^)]+\)) # tuple-expression
+ \s*""", re.VERBOSE)
+
+def parse_valuation_line(s, encoding=None):
+ """
+ Parse a line in a valuation file.
+
+ Lines are expected to be of the form::
+
+ noosa => n
+ girl => {g1, g2}
+ chase => {(b1, g1), (b2, g1), (g1, d1), (g2, d2)}
+
+ :param s: input line
+ :type s: str
+ :param encoding: the encoding of the input string, if it is binary
+ :type encoding: str
+ :return: a pair (symbol, value)
+ :rtype: tuple
+ """
+ if encoding is not None:
+ s = s.decode(encoding)
+ pieces = _VAL_SPLIT_RE.split(s)
+ symbol = pieces[0]
+ value = pieces[1]
+ # check whether the value is meant to be a set
+ if value.startswith('{'):
+ value = value[1:-1]
+ tuple_strings = _TUPLES_RE.findall(value)
+ # are the set elements tuples?
+ if tuple_strings:
+ set_elements = []
+ for ts in tuple_strings:
+ ts = ts[1:-1]
+ element = tuple(_ELEMENT_SPLIT_RE.split(ts))
+ set_elements.append(element)
+ else:
+ set_elements = _ELEMENT_SPLIT_RE.split(value)
+ value = set(set_elements)
+ return symbol, value
+
+def parse_valuation(s, encoding=None):
+ """
+ Convert a valuation file into a valuation.
+
+ :param s: the contents of a valuation file
+ :type s: str
+ :param encoding: the encoding of the input string, if it is binary
+ :type encoding: str
+ :return: a ``nltk.sem`` valuation
+ :rtype: Valuation
+ """
+ if encoding is not None:
+ s = s.decode(encoding)
+ statements = []
+ for linenum, line in enumerate(s.splitlines()):
+ line = line.strip()
+ if line.startswith('#') or line=='': continue
+ try: statements.append(parse_valuation_line(line))
+ except ValueError:
+ raise ValueError('Unable to parse line %s: %s' % (linenum, line))
+ val = evaluate.Valuation(statements)
+ return val
+
+
+def demo_model0():
+ global m0, g0
+ #Initialize a valuation of non-logical constants."""
+ v = [('john', 'b1'),
+ ('mary', 'g1'),
+ ('suzie', 'g2'),
+ ('fido', 'd1'),
+ ('tess', 'd2'),
+ ('noosa', 'n'),
+ ('girl', set(['g1', 'g2'])),
+ ('boy', set(['b1', 'b2'])),
+ ('dog', set(['d1', 'd2'])),
+ ('bark', set(['d1', 'd2'])),
+ ('walk', set(['b1', 'g2', 'd1'])),
+ ('chase', set([('b1', 'g1'), ('b2', 'g1'), ('g1', 'd1'), ('g2', 'd2')])),
+ ('see', set([('b1', 'g1'), ('b2', 'd2'), ('g1', 'b1'),('d2', 'b1'), ('g2', 'n')])),
+ ('in', set([('b1', 'n'), ('b2', 'n'), ('d2', 'n')])),
+ ('with', set([('b1', 'g1'), ('g1', 'b1'), ('d1', 'b1'), ('b1', 'd1')]))
+ ]
+ #Read in the data from ``v``
+ val = evaluate.Valuation(v)
+ #Bind ``dom`` to the ``domain`` property of ``val``
+ dom = val.domain
+ #Initialize a model with parameters ``dom`` and ``val``.
+ m0 = evaluate.Model(dom, val)
+ #Initialize a variable assignment with parameter ``dom``
+ g0 = evaluate.Assignment(dom)
+
+
+def read_sents(filename, encoding='utf8'):
+ with codecs.open(filename, 'r', encoding) as fp:
+ sents = [l.rstrip() for l in fp]
+
+ # get rid of blank lines
+ sents = [l for l in sents if len(l) > 0]
+ sents = [l for l in sents if not l[0] == '#']
+ return sents
+
+def demo_legacy_grammar():
+ """
+ Check that interpret_sents() is compatible with legacy grammars that use
+ a lowercase 'sem' feature.
+
+ Define 'test.fcfg' to be the following
+
+ """
+ from nltk.grammar import parse_fcfg
+
+ g = parse_fcfg("""
+ % start S
+ S[sem=<hello>] -> 'hello'
+ """)
+ print("Reading grammar: %s" % g)
+ print("*" * 20)
+ for reading in interpret_sents(['hello'], g, semkey='sem'):
+ syn, sem = reading[0]
+ print()
+ print("output: ", sem)
+
+def demo():
+ import sys
+ from optparse import OptionParser
+ description = \
+ """
+ Parse and evaluate some sentences.
+ """
+
+ opts = OptionParser(description=description)
+
+ opts.set_defaults(evaluate=True, beta=True, syntrace=0,
+ semtrace=0, demo='default', grammar='', sentences='')
+
+ opts.add_option("-d", "--demo", dest="demo",
+ help="choose demo D; omit this for the default demo, or specify 'chat80'", metavar="D")
+ opts.add_option("-g", "--gram", dest="grammar",
+ help="read in grammar G", metavar="G")
+ opts.add_option("-m", "--model", dest="model",
+ help="import model M (omit '.py' suffix)", metavar="M")
+ opts.add_option("-s", "--sentences", dest="sentences",
+ help="read in a file of test sentences S", metavar="S")
+ opts.add_option("-e", "--no-eval", action="store_false", dest="evaluate",
+ help="just do a syntactic analysis")
+ opts.add_option("-b", "--no-beta-reduction", action="store_false",
+ dest="beta", help="don't carry out beta-reduction")
+ opts.add_option("-t", "--syntrace", action="count", dest="syntrace",
+ help="set syntactic tracing on; requires '-e' option")
+ opts.add_option("-T", "--semtrace", action="count", dest="semtrace",
+ help="set semantic tracing on")
+
+ (options, args) = opts.parse_args()
+
+ SPACER = '-' * 30
+
+ demo_model0()
+
+ sents = [
+ 'Fido sees a boy with Mary',
+ 'John sees Mary',
+ 'every girl chases a dog',
+ 'every boy chases a girl',
+ 'John walks with a girl in Noosa',
+ 'who walks']
+
+ gramfile = 'grammars/sample_grammars/sem2.fcfg'
+
+ if options.sentences:
+ sentsfile = options.sentences
+ if options.grammar:
+ gramfile = options.grammar
+ if options.model:
+ exec("import %s as model" % options.model)
+
+ if sents is None:
+ sents = read_sents(sentsfile)
+
+ # Set model and assignment
+ model = m0
+ g = g0
+
+ if options.evaluate:
+ evaluations = \
+ evaluate_sents(sents, gramfile, model, g, trace=options.semtrace)
+ else:
+ semreps = \
+ interpret_sents(sents, gramfile, trace=options.syntrace)
+
+ for i, sent in enumerate(sents):
+ n = 1
+ print('\nSentence: %s' % sent)
+ print(SPACER)
+ if options.evaluate:
+
+ for (syntree, semrep, value) in evaluations[i]:
+ if isinstance(value, dict):
+ value = set(value.keys())
+ print('%d: %s' % (n, semrep))
+ print(value)
+ n += 1
+ else:
+
+ for (syntree, semrep) in semreps[i]:
+ print('%d: %s' % (n, semrep))
+ n += 1
+
+if __name__ == "__main__":
+ #demo()
+ demo_legacy_grammar()
diff --git a/nltk/stem/__init__.py b/nltk/stem/__init__.py
new file mode 100644
index 0000000..294b018
--- /dev/null
+++ b/nltk/stem/__init__.py
@@ -0,0 +1,36 @@
+# Natural Language Toolkit: Stemmers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+NLTK Stemmers
+
+Interfaces used to remove morphological affixes from words, leaving
+only the word stem. Stemming algorithms aim to remove those affixes
+required for eg. grammatical role, tense, derivational morphology
+leaving only the stem of the word. This is a difficult problem due to
+irregular words (eg. common verbs in English), complicated
+morphological rules, and part-of-speech and sense ambiguities
+(eg. ``ceil-`` is not the stem of ``ceiling``).
+
+StemmerI defines a standard interface for stemmers.
+"""
+
+from nltk.stem.api import StemmerI
+from nltk.stem.regexp import RegexpStemmer
+from nltk.stem.lancaster import LancasterStemmer
+from nltk.stem.isri import ISRIStemmer
+from nltk.stem.porter import PorterStemmer
+from nltk.stem.snowball import SnowballStemmer
+from nltk.stem.wordnet import WordNetLemmatizer
+from nltk.stem.rslp import RSLPStemmer
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/stem/api.py b/nltk/stem/api.py
new file mode 100644
index 0000000..c603242
--- /dev/null
+++ b/nltk/stem/api.py
@@ -0,0 +1,28 @@
+# Natural Language Toolkit: Stemmer Interface
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+class StemmerI(object):
+ """
+ A processing interface for removing morphological affixes from
+ words. This process is known as stemming.
+
+ """
+ def stem(self, token):
+ """
+ Strip affixes from the token and return the stem.
+
+ :param token: The token that should be stemmed.
+ :type token: str
+ """
+ raise NotImplementedError()
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/stem/isri.py b/nltk/stem/isri.py
new file mode 100644
index 0000000..8ce4ee6
--- /dev/null
+++ b/nltk/stem/isri.py
@@ -0,0 +1,348 @@
+# -*- coding: utf-8 -*-
+#
+# Natural Language Toolkit: The ISRI Arabic Stemmer
+#
+# Copyright (C) 2001-2014 NLTK Proejct
+# Algorithm: Kazem Taghva, Rania Elkhoury, and Jeffrey Coombs (2005)
+# Author: Hosam Algasaier <hosam_hme at yahoo.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+ISRI Arabic Stemmer
+
+The algorithm for this stemmer is described in:
+
+Taghva, K., Elkoury, R., and Coombs, J. 2005. Arabic Stemming without a root dictionary.
+Information Science Research Institute. University of Nevada, Las Vegas, USA.
+
+The Information Science Research Institute’s (ISRI) Arabic stemmer shares many features
+with the Khoja stemmer. However, the main difference is that ISRI stemmer does not use root
+dictionary. Also, if a root is not found, ISRI stemmer returned normalized form, rather than
+returning the original unmodified word.
+
+Additional adjustments were made to improve the algorithm:
+
+1- Adding 60 stop words.
+2- Adding the pattern (تفاعيل) to ISRI pattern set.
+3- The step 2 in the original algorithm was normalizing all hamza. This step is discarded because it
+increases the word ambiguities and changes the original root.
+
+"""
+from __future__ import unicode_literals
+import re
+
+from nltk.stem.api import StemmerI
+
+
+class ISRIStemmer(StemmerI):
+ '''
+ ISRI Arabic stemmer based on algorithm: Arabic Stemming without a root dictionary.
+ Information Science Research Institute. University of Nevada, Las Vegas, USA.
+
+ A few minor modifications have been made to ISRI basic algorithm.
+ See the source code of this module for more information.
+
+ isri.stem(token) returns Arabic root for the given token.
+
+ The ISRI Stemmer requires that all tokens have Unicode string types.
+ If you use Python IDLE on Arabic Windows you have to decode text first
+ using Arabic '1256' coding.
+ '''
+
+ def __init__(self):
+ # length three prefixes
+ self.p3 = ['\u0643\u0627\u0644', '\u0628\u0627\u0644',
+ '\u0648\u0644\u0644', '\u0648\u0627\u0644']
+
+ # length two prefixes
+ self.p2 = ['\u0627\u0644', '\u0644\u0644']
+
+ # length one prefixes
+ self.p1 = ['\u0644', '\u0628', '\u0641', '\u0633', '\u0648',
+ '\u064a', '\u062a', '\u0646', '\u0627']
+
+ # length three suffixes
+ self.s3 = ['\u062a\u0645\u0644', '\u0647\u0645\u0644',
+ '\u062a\u0627\u0646', '\u062a\u064a\u0646',
+ '\u0643\u0645\u0644']
+
+ # length two suffixes
+ self.s2 = ['\u0648\u0646', '\u0627\u062a', '\u0627\u0646',
+ '\u064a\u0646', '\u062a\u0646', '\u0643\u0645',
+ '\u0647\u0646', '\u0646\u0627', '\u064a\u0627',
+ '\u0647\u0627', '\u062a\u0645', '\u0643\u0646',
+ '\u0646\u064a', '\u0648\u0627', '\u0645\u0627',
+ '\u0647\u0645']
+
+ # length one suffixes
+ self.s1 = ['\u0629', '\u0647', '\u064a', '\u0643', '\u062a',
+ '\u0627', '\u0646']
+
+ # groups of length four patterns
+ self.pr4 = {0: ['\u0645'], 1: ['\u0627'],
+ 2: ['\u0627', '\u0648', '\u064A'], 3: ['\u0629']}
+
+ # Groups of length five patterns and length three roots
+ self.pr53 = {0: ['\u0627', '\u062a'],
+ 1: ['\u0627', '\u064a', '\u0648'],
+ 2: ['\u0627', '\u062a', '\u0645'],
+ 3: ['\u0645', '\u064a', '\u062a'],
+ 4: ['\u0645', '\u062a'],
+ 5: ['\u0627', '\u0648'],
+ 6: ['\u0627', '\u0645']}
+
+ self.re_short_vowels = re.compile(r'[\u064B-\u0652]')
+ self.re_hamza = re.compile(r'[\u0621\u0624\u0626]')
+ self.re_initial_hamza = re.compile(r'^[\u0622\u0623\u0625]')
+
+ self.stop_words = ['\u064a\u0643\u0648\u0646',
+ '\u0648\u0644\u064a\u0633',
+ '\u0648\u0643\u0627\u0646',
+ '\u0643\u0630\u0644\u0643',
+ '\u0627\u0644\u062a\u064a',
+ '\u0648\u0628\u064a\u0646',
+ '\u0639\u0644\u064a\u0647\u0627',
+ '\u0645\u0633\u0627\u0621',
+ '\u0627\u0644\u0630\u064a',
+ '\u0648\u0643\u0627\u0646\u062a',
+ '\u0648\u0644\u0643\u0646',
+ '\u0648\u0627\u0644\u062a\u064a',
+ '\u062a\u0643\u0648\u0646',
+ '\u0627\u0644\u064a\u0648\u0645',
+ '\u0627\u0644\u0644\u0630\u064a\u0646',
+ '\u0639\u0644\u064a\u0647',
+ '\u0643\u0627\u0646\u062a',
+ '\u0644\u0630\u0644\u0643',
+ '\u0623\u0645\u0627\u0645',
+ '\u0647\u0646\u0627\u0643',
+ '\u0645\u0646\u0647\u0627',
+ '\u0645\u0627\u0632\u0627\u0644',
+ '\u0644\u0627\u0632\u0627\u0644',
+ '\u0644\u0627\u064a\u0632\u0627\u0644',
+ '\u0645\u0627\u064a\u0632\u0627\u0644',
+ '\u0627\u0635\u0628\u062d',
+ '\u0623\u0635\u0628\u062d',
+ '\u0623\u0645\u0633\u0649',
+ '\u0627\u0645\u0633\u0649',
+ '\u0623\u0636\u062d\u0649',
+ '\u0627\u0636\u062d\u0649',
+ '\u0645\u0627\u0628\u0631\u062d',
+ '\u0645\u0627\u0641\u062a\u0626',
+ '\u0645\u0627\u0627\u0646\u0641\u0643',
+ '\u0644\u0627\u0633\u064a\u0645\u0627',
+ '\u0648\u0644\u0627\u064a\u0632\u0627\u0644',
+ '\u0627\u0644\u062d\u0627\u0644\u064a',
+ '\u0627\u0644\u064a\u0647\u0627',
+ '\u0627\u0644\u0630\u064a\u0646',
+ '\u0641\u0627\u0646\u0647',
+ '\u0648\u0627\u0644\u0630\u064a',
+ '\u0648\u0647\u0630\u0627',
+ '\u0644\u0647\u0630\u0627',
+ '\u0641\u0643\u0627\u0646',
+ '\u0633\u062a\u0643\u0648\u0646',
+ '\u0627\u0644\u064a\u0647',
+ '\u064a\u0645\u0643\u0646',
+ '\u0628\u0647\u0630\u0627',
+ '\u0627\u0644\u0630\u0649']
+
+ def stem(self, token):
+ """
+ Stemming a word token using the ISRI stemmer.
+ """
+ token = self.norm(token, 1) # remove diacritics which representing Arabic short vowels
+ if token in self.stop_words:
+ return token # exclude stop words from being processed
+ token = self.pre32(token) # remove length three and length two prefixes in this order
+ token = self.suf32(token) # remove length three and length two suffixes in this order
+ token = self.waw(token) # remove connective ‘و’ if it precedes a word beginning with ‘و’
+ token = self.norm(token, 2) # normalize initial hamza to bare alif
+ # if 4 <= word length <= 7, then stem; otherwise, no stemming
+ if len(token) == 4: # length 4 word
+ token = self.pro_w4(token)
+ elif len(token) == 5: # length 5 word
+ token = self.pro_w53(token)
+ token = self.end_w5(token)
+ elif len(token) == 6: # length 6 word
+ token = self.pro_w6(token)
+ token = self.end_w6(token)
+ elif len(token) == 7: # length 7 word
+ token = self.suf1(token)
+ if len(token) == 7:
+ token = self.pre1(token)
+ if len(token) == 6:
+ token = self.pro_w6(token)
+ token = self.end_w6(token)
+ return token
+
+ def norm(self, word, num=3):
+ """
+ normalization:
+ num=1 normalize diacritics
+ num=2 normalize initial hamza
+ num=3 both 1&2
+ """
+ if num == 1:
+ word = self.re_short_vowels.sub('', word)
+ elif num == 2:
+ word = self.re_initial_hamza.sub('\u0627', word)
+ elif num == 3:
+ word = self.re_short_vowels.sub('', word)
+ word = self.re_initial_hamza.sub('\u0627', word)
+ return word
+
+ def pre32(self, word):
+ """remove length three and length two prefixes in this order"""
+ if len(word) >= 6:
+ for pre3 in self.p3:
+ if word.startswith(pre3):
+ return word[3:]
+ if len(word) >= 5:
+ for pre2 in self.p2:
+ if word.startswith(pre2):
+ return word[2:]
+ return word
+
+ def suf32(self, word):
+ """remove length three and length two suffixes in this order"""
+ if len(word) >= 6:
+ for suf3 in self.s3:
+ if word.endswith(suf3):
+ return word[:-3]
+ if len(word) >= 5:
+ for suf2 in self.s2:
+ if word.endswith(suf2):
+ return word[:-2]
+ return word
+
+ def waw(self, word):
+ """remove connective ‘و’ if it precedes a word beginning with ‘و’ """
+ if len(word) >= 4 and word[:2] == '\u0648\u0648':
+ word = word[1:]
+ return word
+
+ def pro_w4(self, word):
+ """process length four patterns and extract length three roots"""
+ if word[0] in self.pr4[0]: # مفعل
+ word = word[1:]
+ elif word[1] in self.pr4[1]: # فاعل
+ word = word[:1] + word[2:]
+ elif word[2] in self.pr4[2]: # فعال - فعول - فعيل
+ word = word[:2] + word[3]
+ elif word[3] in self.pr4[3]: # فعلة
+ word = word[:-1]
+ else:
+ word = self.suf1(word) # do - normalize short sufix
+ if len(word) == 4:
+ word = self.pre1(word) # do - normalize short prefix
+ return word
+
+ def pro_w53(self, word):
+ """process length five patterns and extract length three roots"""
+ if word[2] in self.pr53[0] and word[0] == '\u0627': # افتعل - افاعل
+ word = word[1] + word[3:]
+ elif word[3] in self.pr53[1] and word[0] == '\u0645': # مفعول - مفعال - مفعيل
+ word = word[1:3] + word[4]
+ elif word[0] in self.pr53[2] and word[4] == '\u0629': # مفعلة - تفعلة - افعلة
+ word = word[1:4]
+ elif word[0] in self.pr53[3] and word[2] == '\u062a': # مفتعل - يفتعل - تفتعل
+ word = word[1] + word[3:]
+ elif word[0] in self.pr53[4] and word[2] == '\u0627': # مفاعل - تفاعل
+ word = word[1] + word[3:]
+ elif word[2] in self.pr53[5] and word[4] == '\u0629': # فعولة - فعالة
+ word = word[:2] + word[3]
+ elif word[0] in self.pr53[6] and word[1] == '\u0646': # انفعل - منفعل
+ word = word[2:]
+ elif word[3] == '\u0627' and word[0] == '\u0627': # افعال
+ word = word[1:3] + word[4]
+ elif word[4] == '\u0646' and word[3] == '\u0627': # فعلان
+ word = word[:3]
+ elif word[3] == '\u064a' and word[0] == '\u062a': # تفعيل
+ word = word[1:3] + word[4]
+ elif word[3] == '\u0648' and word[1] == '\u0627': # فاعول
+ word = word[0] + word[2] + word[4]
+ elif word[2] == '\u0627' and word[1] == '\u0648': # فواعل
+ word = word[0] + word[3:]
+ elif word[3] == '\u0626' and word[2] == '\u0627': # فعائل
+ word = word[:2] + word[4]
+ elif word[4] == '\u0629' and word[1] == '\u0627': # فاعلة
+ word = word[0] + word[2:4]
+ elif word[4] == '\u064a' and word[2] == '\u0627': # فعالي
+ word = word[:2] + word[3]
+ else:
+ word = self.suf1(word) # do - normalize short sufix
+ if len(word) == 5:
+ word = self.pre1(word) # do - normalize short prefix
+ return word
+
+ def pro_w54(self, word):
+ """process length five patterns and extract length four roots"""
+ if word[0] in self.pr53[2]: # تفعلل - افعلل - مفعلل
+ word = word[1:]
+ elif word[4] == '\u0629': # فعللة
+ word = word[:4]
+ elif word[2] == '\u0627': # فعالل
+ word = word[:2] + word[3:]
+ return word
+
+ def end_w5(self, word):
+ """ending step (word of length five)"""
+ if len(word) == 4:
+ word = self.pro_w4(word)
+ elif len(word) == 5:
+ word = self.pro_w54(word)
+ return word
+
+ def pro_w6(self, word):
+ """process length six patterns and extract length three roots"""
+ if word.startswith('\u0627\u0633\u062a') or word.startswith('\u0645\u0633\u062a'): # مستفعل - استفعل
+ word = word[3:]
+ elif word[0] == '\u0645' and word[3] == '\u0627' and word[5] == '\u0629': # مفعالة
+ word = word[1:3] + word[4]
+ elif word[0] == '\u0627' and word[2] == '\u062a' and word[4] == '\u0627': # افتعال
+ word = word[1] + word[3] + word[5]
+ elif word[0] == '\u0627' and word[3] == '\u0648' and word[2] == word[4]: # افعوعل
+ word = word[1] + word[4:]
+ elif word[0] == '\u062a' and word[2] == '\u0627' and word[4] == '\u064a': # تفاعيل new pattern
+ word = word[1] + word[3] + word[5]
+ else:
+ word = self.suf1(word) # do - normalize short sufix
+ if len(word) == 6:
+ word = self.pre1(word) # do - normalize short prefix
+ return word
+
+ def pro_w64(self, word):
+ """process length six patterns and extract length four roots"""
+ if word[0] == '\u0627' and word[4] == '\u0627': # افعلال
+ word = word[1:4] + word[5]
+ elif word.startswith('\u0645\u062a'): # متفعلل
+ word = word[2:]
+ return word
+
+ def end_w6(self, word):
+ """ending step (word of length six)"""
+ if len(word) == 5:
+ word = self.pro_w53(word)
+ word = self.end_w5(word)
+ elif len(word) == 6:
+ word = self.pro_w64(word)
+ return word
+
+ def suf1(self, word):
+ """normalize short sufix"""
+ for sf1 in self.s1:
+ if word.endswith(sf1):
+ return word[:-1]
+ return word
+
+ def pre1(self, word):
+ """normalize short prefix"""
+ for sp1 in self.p1:
+ if word.startswith(sp1):
+ return word[1:]
+ return word
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/stem/lancaster.py b/nltk/stem/lancaster.py
new file mode 100644
index 0000000..bd169bd
--- /dev/null
+++ b/nltk/stem/lancaster.py
@@ -0,0 +1,313 @@
+# Natural Language Toolkit: Stemmers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Tomcavage <stomcava at law.upenn.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A word stemmer based on the Lancaster stemming algorithm.
+Paice, Chris D. "Another Stemmer." ACM SIGIR Forum 24.3 (1990): 56-61.
+"""
+from __future__ import unicode_literals
+import re
+
+from nltk.stem.api import StemmerI
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class LancasterStemmer(StemmerI):
+ """
+ Lancaster Stemmer
+
+ >>> from nltk.stem.lancaster import LancasterStemmer
+ >>> st = LancasterStemmer()
+ >>> st.stem('maximum') # Remove "-um" when word is intact
+ 'maxim'
+ >>> st.stem('presumably') # Don't remove "-um" when word is not intact
+ 'presum'
+ >>> st.stem('multiply') # No action taken if word ends with "-ply"
+ 'multiply'
+ >>> st.stem('provision') # Replace "-sion" with "-j" to trigger "j" set of rules
+ 'provid'
+ >>> st.stem('owed') # Word starting with vowel must contain at least 2 letters
+ 'ow'
+ >>> st.stem('ear') # ditto
+ 'ear'
+ >>> st.stem('saying') # Words starting with consonant must contain at least 3
+ 'say'
+ >>> st.stem('crying') # letters and one of those letters must be a vowel
+ 'cry'
+ >>> st.stem('string') # ditto
+ 'string'
+ >>> st.stem('meant') # ditto
+ 'meant'
+ >>> st.stem('cement') # ditto
+ 'cem'
+ """
+
+ # The rule list is static since it doesn't change between instances
+ rule_tuple = (
+ "ai*2.", # -ia > - if intact
+ "a*1.", # -a > - if intact
+ "bb1.", # -bb > -b
+ "city3s.", # -ytic > -ys
+ "ci2>", # -ic > -
+ "cn1t>", # -nc > -nt
+ "dd1.", # -dd > -d
+ "dei3y>", # -ied > -y
+ "deec2ss.", # -ceed >", -cess
+ "dee1.", # -eed > -ee
+ "de2>", # -ed > -
+ "dooh4>", # -hood > -
+ "e1>", # -e > -
+ "feil1v.", # -lief > -liev
+ "fi2>", # -if > -
+ "gni3>", # -ing > -
+ "gai3y.", # -iag > -y
+ "ga2>", # -ag > -
+ "gg1.", # -gg > -g
+ "ht*2.", # -th > - if intact
+ "hsiug5ct.", # -guish > -ct
+ "hsi3>", # -ish > -
+ "i*1.", # -i > - if intact
+ "i1y>", # -i > -y
+ "ji1d.", # -ij > -id -- see nois4j> & vis3j>
+ "juf1s.", # -fuj > -fus
+ "ju1d.", # -uj > -ud
+ "jo1d.", # -oj > -od
+ "jeh1r.", # -hej > -her
+ "jrev1t.", # -verj > -vert
+ "jsim2t.", # -misj > -mit
+ "jn1d.", # -nj > -nd
+ "j1s.", # -j > -s
+ "lbaifi6.", # -ifiabl > -
+ "lbai4y.", # -iabl > -y
+ "lba3>", # -abl > -
+ "lbi3.", # -ibl > -
+ "lib2l>", # -bil > -bl
+ "lc1.", # -cl > c
+ "lufi4y.", # -iful > -y
+ "luf3>", # -ful > -
+ "lu2.", # -ul > -
+ "lai3>", # -ial > -
+ "lau3>", # -ual > -
+ "la2>", # -al > -
+ "ll1.", # -ll > -l
+ "mui3.", # -ium > -
+ "mu*2.", # -um > - if intact
+ "msi3>", # -ism > -
+ "mm1.", # -mm > -m
+ "nois4j>", # -sion > -j
+ "noix4ct.", # -xion > -ct
+ "noi3>", # -ion > -
+ "nai3>", # -ian > -
+ "na2>", # -an > -
+ "nee0.", # protect -een
+ "ne2>", # -en > -
+ "nn1.", # -nn > -n
+ "pihs4>", # -ship > -
+ "pp1.", # -pp > -p
+ "re2>", # -er > -
+ "rae0.", # protect -ear
+ "ra2.", # -ar > -
+ "ro2>", # -or > -
+ "ru2>", # -ur > -
+ "rr1.", # -rr > -r
+ "rt1>", # -tr > -t
+ "rei3y>", # -ier > -y
+ "sei3y>", # -ies > -y
+ "sis2.", # -sis > -s
+ "si2>", # -is > -
+ "ssen4>", # -ness > -
+ "ss0.", # protect -ss
+ "suo3>", # -ous > -
+ "su*2.", # -us > - if intact
+ "s*1>", # -s > - if intact
+ "s0.", # -s > -s
+ "tacilp4y.", # -plicat > -ply
+ "ta2>", # -at > -
+ "tnem4>", # -ment > -
+ "tne3>", # -ent > -
+ "tna3>", # -ant > -
+ "tpir2b.", # -ript > -rib
+ "tpro2b.", # -orpt > -orb
+ "tcud1.", # -duct > -duc
+ "tpmus2.", # -sumpt > -sum
+ "tpec2iv.", # -cept > -ceiv
+ "tulo2v.", # -olut > -olv
+ "tsis0.", # protect -sist
+ "tsi3>", # -ist > -
+ "tt1.", # -tt > -t
+ "uqi3.", # -iqu > -
+ "ugo1.", # -ogu > -og
+ "vis3j>", # -siv > -j
+ "vie0.", # protect -eiv
+ "vi2>", # -iv > -
+ "ylb1>", # -bly > -bl
+ "yli3y>", # -ily > -y
+ "ylp0.", # protect -ply
+ "yl2>", # -ly > -
+ "ygo1.", # -ogy > -og
+ "yhp1.", # -phy > -ph
+ "ymo1.", # -omy > -om
+ "ypo1.", # -opy > -op
+ "yti3>", # -ity > -
+ "yte3>", # -ety > -
+ "ytl2.", # -lty > -l
+ "yrtsi5.", # -istry > -
+ "yra3>", # -ary > -
+ "yro3>", # -ory > -
+ "yfi3.", # -ify > -
+ "ycn2t>", # -ncy > -nt
+ "yca3>", # -acy > -
+ "zi2>", # -iz > -
+ "zy1s." # -yz > -ys
+ )
+
+
+ def __init__(self):
+ """Create an instance of the Lancaster stemmer.
+ """
+ # Setup an empty rule dictionary - this will be filled in later
+ self.rule_dictionary = {}
+
+ def parseRules(self, rule_tuple):
+ """Validate the set of rules used in this stemmer.
+ """
+ valid_rule = re.compile("^[a-z]+\*?\d[a-z]*[>\.]?$")
+ # Empty any old rules from the rule set before adding new ones
+ self.rule_dictionary = {}
+
+ for rule in rule_tuple:
+ if not valid_rule.match(rule):
+ raise ValueError("The rule %s is invalid" % rule)
+ first_letter = rule[0:1]
+ if first_letter in self.rule_dictionary:
+ self.rule_dictionary[first_letter].append(rule)
+ else:
+ self.rule_dictionary[first_letter] = [rule]
+
+ def stem(self, word):
+ """Stem a word using the Lancaster stemmer.
+ """
+ # Lower-case the word, since all the rules are lower-cased
+ word = word.lower()
+
+ # Save a copy of the original word
+ intact_word = word
+
+ # If the user hasn't supplied any rules, setup the default rules
+ if len(self.rule_dictionary) == 0:
+ self.parseRules(LancasterStemmer.rule_tuple)
+
+ return self.__doStemming(word, intact_word)
+
+ def __doStemming(self, word, intact_word):
+ """Perform the actual word stemming
+ """
+
+ valid_rule = re.compile("^([a-z]+)(\*?)(\d)([a-z]*)([>\.]?)$")
+
+ proceed = True
+
+ while proceed:
+
+ # Find the position of the last letter of the word to be stemmed
+ last_letter_position = self.__getLastLetter(word)
+
+ # Only stem the word if it has a last letter and a rule matching that last letter
+ if last_letter_position < 0 or word[last_letter_position] not in self.rule_dictionary:
+ proceed = False
+
+ else:
+ rule_was_applied = False
+
+ # Go through each rule that matches the word's final letter
+ for rule in self.rule_dictionary[word[last_letter_position]]:
+ rule_match = valid_rule.match(rule)
+ if rule_match:
+ (ending_string,
+ intact_flag,
+ remove_total,
+ append_string,
+ cont_flag) = rule_match.groups()
+
+ # Convert the number of chars to remove when stemming
+ # from a string to an integer
+ remove_total = int(remove_total)
+
+ # Proceed if word's ending matches rule's word ending
+ if word.endswith(ending_string[::-1]):
+ if intact_flag:
+ if (word == intact_word and
+ self.__isAcceptable(word, remove_total)):
+ word = self.__applyRule(word,
+ remove_total,
+ append_string)
+ rule_was_applied = True
+ if cont_flag == '.':
+ proceed = False
+ break
+ elif self.__isAcceptable(word, remove_total):
+ word = self.__applyRule(word,
+ remove_total,
+ append_string)
+ rule_was_applied = True
+ if cont_flag == '.':
+ proceed = False
+ break
+ # If no rules apply, the word doesn't need any more stemming
+ if rule_was_applied == False:
+ proceed = False
+ return word
+
+ def __getLastLetter(self, word):
+ """Get the zero-based index of the last alphabetic character in this string
+ """
+ last_letter = -1
+ for position in range(len(word)):
+ if word[position].isalpha():
+ last_letter = position
+ else:
+ break
+ return last_letter
+
+ def __isAcceptable(self, word, remove_total):
+ """Determine if the word is acceptable for stemming.
+ """
+ word_is_acceptable = False
+ # If the word starts with a vowel, it must be at least 2
+ # characters long to be stemmed
+ if word[0] in "aeiouy":
+ if (len(word) - remove_total >= 2):
+ word_is_acceptable = True
+ # If the word starts with a consonant, it must be at least 3
+ # characters long (including one vowel) to be stemmed
+ elif (len(word) - remove_total >= 3):
+ if word[1] in "aeiouy":
+ word_is_acceptable = True
+ elif word[2] in "aeiouy":
+ word_is_acceptable = True
+ return word_is_acceptable
+
+
+ def __applyRule(self, word, remove_total, append_string):
+ """Apply the stemming rule to the word
+ """
+ # Remove letters from the end of the word
+ new_word_length = len(word) - remove_total
+ word = word[0:new_word_length]
+
+ # And add new letters to the end of the truncated word
+ if append_string:
+ word += append_string
+ return word
+
+ def __repr__(self):
+ return '<LancasterStemmer>'
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/stem/porter.py b/nltk/stem/porter.py
new file mode 100644
index 0000000..467a4d4
--- /dev/null
+++ b/nltk/stem/porter.py
@@ -0,0 +1,697 @@
+# Copyright (c) 2002 Vivake Gupta (vivakeATomniscia.org). All rights reserved.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+# This software is maintained by Vivake (vivakeATomniscia.org) and is available at:
+# http://www.omniscia.org/~vivake/python/PorterStemmer.py
+#
+# Additional modifications were made to incorporate this module into
+# NLTK. All such modifications are marked with "--NLTK--". The NLTK
+# version of this module is maintained by NLTK developers,
+# and is available via http://nltk.org/
+#
+# GNU Linking Exception:
+# Using this module statically or dynamically with other modules is
+# making a combined work based on this module. Thus, the terms and
+# conditions of the GNU General Public License cover the whole combination.
+# As a special exception, the copyright holders of this module give
+# you permission to combine this module with independent modules to
+# produce an executable program, regardless of the license terms of these
+# independent modules, and to copy and distribute the resulting
+# program under terms of your choice, provided that you also meet,
+# for each linked independent module, the terms and conditions of
+# the license of that module. An independent module is a module which
+# is not derived from or based on this module. If you modify this module,
+# you may extend this exception to your version of the module, but you
+# are not obliged to do so. If you do not wish to do so, delete this
+# exception statement from your version.
+
+"""
+Porter Stemmer
+
+This is the Porter stemming algorithm, ported to Python from the
+version coded up in ANSI C by the author. It follows the algorithm
+presented in
+
+Porter, M. "An algorithm for suffix stripping." Program 14.3 (1980): 130-137.
+
+only differing from it at the points marked --DEPARTURE-- and --NEW--
+below.
+
+For a more faithful version of the Porter algorithm, see
+
+ http://www.tartarus.org/~martin/PorterStemmer/
+
+Later additions:
+
+ June 2000
+
+ The 'l' of the 'logi' -> 'log' rule is put with the stem, so that
+ short stems like 'geo' 'theo' etc work like 'archaeo' 'philo' etc.
+
+ This follows a suggestion of Barry Wilkins, research student at
+ Birmingham.
+
+
+ February 2000
+
+ the cvc test for not dropping final -e now looks after vc at the
+ beginning of a word, so are, eve, ice, ore, use keep final -e. In this
+ test c is any consonant, including w, x and y. This extension was
+ suggested by Chris Emerson.
+
+ -fully -> -ful treated like -fulness -> -ful, and
+ -tionally -> -tion treated like -tional -> -tion
+
+ both in Step 2. These were suggested by Hiranmay Ghosh, of New Delhi.
+
+ Invariants proceed, succeed, exceed. Also suggested by Hiranmay Ghosh.
+
+Additional modifications were made to incorperate this module into
+nltk. All such modifications are marked with \"--NLTK--\". The nltk
+version of this module is maintained by the NLTK developers, and is
+available from <http://nltk.sourceforge.net>
+"""
+from __future__ import print_function, unicode_literals
+
+## --NLTK--
+## Declare this module's documentation format.
+__docformat__ = 'plaintext'
+
+import re
+
+from nltk.stem.api import StemmerI
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class PorterStemmer(StemmerI):
+
+ ## --NLTK--
+ ## Add a module docstring
+ """
+ A word stemmer based on the Porter stemming algorithm.
+
+ Porter, M. \"An algorithm for suffix stripping.\"
+ Program 14.3 (1980): 130-137.
+
+ A few minor modifications have been made to Porter's basic
+ algorithm. See the source code of this module for more
+ information.
+
+ The Porter Stemmer requires that all tokens have string types.
+ """
+
+ # The main part of the stemming algorithm starts here.
+ # Note that only lower case sequences are stemmed. Forcing to lower case
+ # should be done before stem(...) is called.
+
+ def __init__(self):
+
+ ## --NEW--
+ ## This is a table of irregular forms. It is quite short, but still
+ ## reflects the errors actually drawn to Martin Porter's attention over
+ ## a 20 year period!
+ ##
+ ## Extend it as necessary.
+ ##
+ ## The form of the table is:
+ ## {
+ ## "p1" : ["s11","s12","s13", ... ],
+ ## "p2" : ["s21","s22","s23", ... ],
+ ## ...
+ ## "pn" : ["sn1","sn2","sn3", ... ]
+ ## }
+ ##
+ ## String sij is mapped to paradigm form pi, and the main stemming
+ ## process is then bypassed.
+
+ irregular_forms = {
+ "sky" : ["sky", "skies"],
+ "die" : ["dying"],
+ "lie" : ["lying"],
+ "tie" : ["tying"],
+ "news" : ["news"],
+ "inning" : ["innings", "inning"],
+ "outing" : ["outings", "outing"],
+ "canning" : ["cannings", "canning"],
+ "howe" : ["howe"],
+
+ # --NEW--
+ "proceed" : ["proceed"],
+ "exceed" : ["exceed"],
+ "succeed" : ["succeed"], # Hiranmay Ghosh
+ }
+
+ self.pool = {}
+ for key in irregular_forms:
+ for val in irregular_forms[key]:
+ self.pool[val] = key
+
+ self.vowels = frozenset(['a', 'e', 'i', 'o', 'u'])
+
+ def _cons(self, word, i):
+ """cons(i) is TRUE <=> b[i] is a consonant."""
+ if word[i] in self.vowels:
+ return False
+ if word[i] == 'y':
+ if i == 0:
+ return True
+ else:
+ return (not self._cons(word, i - 1))
+ return True
+
+ def _m(self, word, j):
+ """m() measures the number of consonant sequences between k0 and j.
+ if c is a consonant sequence and v a vowel sequence, and <..>
+ indicates arbitrary presence,
+
+ <c><v> gives 0
+ <c>vc<v> gives 1
+ <c>vcvc<v> gives 2
+ <c>vcvcvc<v> gives 3
+ ....
+ """
+ n = 0
+ i = 0
+ while True:
+ if i > j:
+ return n
+ if not self._cons(word, i):
+ break
+ i = i + 1
+ i = i + 1
+
+ while True:
+ while True:
+ if i > j:
+ return n
+ if self._cons(word, i):
+ break
+ i = i + 1
+ i = i + 1
+ n = n + 1
+
+ while True:
+ if i > j:
+ return n
+ if not self._cons(word, i):
+ break
+ i = i + 1
+ i = i + 1
+
+ def _vowelinstem(self, stem):
+ """vowelinstem(stem) is TRUE <=> stem contains a vowel"""
+ for i in range(len(stem)):
+ if not self._cons(stem, i):
+ return True
+ return False
+
+ def _doublec(self, word):
+ """doublec(word) is TRUE <=> word ends with a double consonant"""
+ if len(word) < 2:
+ return False
+ if (word[-1] != word[-2]):
+ return False
+ return self._cons(word, len(word)-1)
+
+ def _cvc(self, word, i):
+ """cvc(i) is TRUE <=>
+
+ a) ( --NEW--) i == 1, and word[0] word[1] is vowel consonant, or
+
+ b) word[i - 2], word[i - 1], word[i] has the form consonant -
+ vowel - consonant and also if the second c is not w, x or y. this
+ is used when trying to restore an e at the end of a short word.
+ e.g.
+
+ cav(e), lov(e), hop(e), crim(e), but
+ snow, box, tray.
+ """
+ if i == 0: return False # i == 0 never happens perhaps
+ if i == 1: return (not self._cons(word, 0) and self._cons(word, 1))
+ if not self._cons(word, i) or self._cons(word, i-1) or not self._cons(word, i-2): return False
+
+ ch = word[i]
+ if ch == 'w' or ch == 'x' or ch == 'y':
+ return False
+
+ return True
+
+ def _step1ab(self, word):
+ """step1ab() gets rid of plurals and -ed or -ing. e.g.
+
+ caresses -> caress
+ ponies -> poni
+ sties -> sti
+ tie -> tie (--NEW--: see below)
+ caress -> caress
+ cats -> cat
+
+ feed -> feed
+ agreed -> agree
+ disabled -> disable
+
+ matting -> mat
+ mating -> mate
+ meeting -> meet
+ milling -> mill
+ messing -> mess
+
+ meetings -> meet
+ """
+ if word[-1] == 's':
+ if word.endswith("sses"):
+ word = word[:-2]
+ elif word.endswith("ies"):
+ if len(word) == 4:
+ word = word[:-1]
+ # this line extends the original algorithm, so that
+ # 'flies'->'fli' but 'dies'->'die' etc
+ else:
+ word = word[:-2]
+ elif word[-2] != 's':
+ word = word[:-1]
+
+ ed_or_ing_trimmed = False
+ if word.endswith("ied"):
+ if len(word) == 4:
+ word = word[:-1]
+ else:
+ word = word[:-2]
+ # this line extends the original algorithm, so that
+ # 'spied'->'spi' but 'died'->'die' etc
+
+ elif word.endswith("eed"):
+ if self._m(word, len(word)-4) > 0:
+ word = word[:-1]
+
+
+ elif word.endswith("ed") and self._vowelinstem(word[:-2]):
+ word = word[:-2]
+ ed_or_ing_trimmed = True
+ elif word.endswith("ing") and self._vowelinstem(word[:-3]):
+ word = word[:-3]
+ ed_or_ing_trimmed = True
+
+ if ed_or_ing_trimmed:
+ if word.endswith("at") or word.endswith("bl") or word.endswith("iz"):
+ word += 'e'
+ elif self._doublec(word):
+ if word[-1] not in ['l', 's', 'z']:
+ word = word[:-1]
+ elif (self._m(word, len(word)-1) == 1 and self._cvc(word, len(word)-1)):
+ word += 'e'
+
+ return word
+
+ def _step1c(self, word):
+ """step1c() turns terminal y to i when there is another vowel in the stem.
+ --NEW--: This has been modified from the original Porter algorithm so that y->i
+ is only done when y is preceded by a consonant, but not if the stem
+ is only a single consonant, i.e.
+
+ (*c and not c) Y -> I
+
+ So 'happy' -> 'happi', but
+ 'enjoy' -> 'enjoy' etc
+
+ This is a much better rule. Formerly 'enjoy'->'enjoi' and 'enjoyment'->
+ 'enjoy'. Step 1c is perhaps done too soon; but with this modification that
+ no longer really matters.
+
+ Also, the removal of the vowelinstem(z) condition means that 'spy', 'fly',
+ 'try' ... stem to 'spi', 'fli', 'tri' and conflate with 'spied', 'tried',
+ 'flies' ...
+ """
+ if word[-1] == 'y' and len(word) > 2 and self._cons(word, len(word) - 2):
+ return word[:-1] + 'i'
+ else:
+ return word
+
+ def _step2(self, word):
+ """step2() maps double suffices to single ones.
+ so -ization ( = -ize plus -ation) maps to -ize etc. note that the
+ string before the suffix must give m() > 0.
+ """
+ if len(word) <= 1: # Only possible at this stage given unusual inputs to stem_word like 'oed'
+ return word
+
+ ch = word[-2]
+
+ if ch == 'a':
+ if word.endswith("ational"):
+ return word[:-7] + "ate" if self._m(word, len(word)-8) > 0 else word
+ elif word.endswith("tional"):
+ return word[:-2] if self._m(word, len(word)-7) > 0 else word
+ else:
+ return word
+ elif ch == 'c':
+ if word.endswith("enci"):
+ return word[:-4] + "ence" if self._m(word, len(word)-5) > 0 else word
+ elif word.endswith("anci"):
+ return word[:-4] + "ance" if self._m(word, len(word)-5) > 0 else word
+ else:
+ return word
+ elif ch == 'e':
+ if word.endswith("izer"):
+ return word[:-1] if self._m(word, len(word)-5) > 0 else word
+ else:
+ return word
+ elif ch == 'l':
+ if word.endswith("bli"):
+ return word[:-3] + "ble" if self._m(word, len(word)-4) > 0 else word # --DEPARTURE--
+ # To match the published algorithm, replace "bli" with "abli" and "ble" with "able"
+ elif word.endswith("alli"):
+ # --NEW--
+ if self._m(word, len(word)-5) > 0:
+ word = word[:-2]
+ return self._step2(word)
+ else:
+ return word
+ elif word.endswith("fulli"):
+ return word[:-2] if self._m(word, len(word)-6) else word # --NEW--
+ elif word.endswith("entli"):
+ return word[:-2] if self._m(word, len(word)-6) else word
+ elif word.endswith("eli"):
+ return word[:-2] if self._m(word, len(word)-4) else word
+ elif word.endswith("ousli"):
+ return word[:-2] if self._m(word, len(word)-6) else word
+ else:
+ return word
+ elif ch == 'o':
+ if word.endswith("ization"):
+ return word[:-7] + "ize" if self._m(word, len(word)-8) else word
+ elif word.endswith("ation"):
+ return word[:-5] + "ate" if self._m(word, len(word)-6) else word
+ elif word.endswith("ator"):
+ return word[:-4] + "ate" if self._m(word, len(word)-5) else word
+ else:
+ return word
+ elif ch == 's':
+ if word.endswith("alism"):
+ return word[:-3] if self._m(word, len(word)-6) else word
+ elif word.endswith("ness"):
+ if word.endswith("iveness"):
+ return word[:-4] if self._m(word, len(word)-8) else word
+ elif word.endswith("fulness"):
+ return word[:-4] if self._m(word, len(word)-8) else word
+ elif word.endswith("ousness"):
+ return word[:-4] if self._m(word, len(word)-8) else word
+ else:
+ return word
+ else:
+ return word
+ elif ch == 't':
+ if word.endswith("aliti"):
+ return word[:-3] if self._m(word, len(word)-6) else word
+ elif word.endswith("iviti"):
+ return word[:-5] + "ive" if self._m(word, len(word)-6) else word
+ elif word.endswith("biliti"):
+ return word[:-6] + "ble" if self._m(word, len(word)-7) else word
+ else:
+ return word
+ elif ch == 'g': # --DEPARTURE--
+ if word.endswith("logi"):
+ return word[:-1] if self._m(word, len(word) - 4) else word # --NEW-- (Barry Wilkins)
+ # To match the published algorithm, pass len(word)-5 to _m instead of len(word)-4
+ else:
+ return word
+
+ else:
+ return word
+
+ def _step3(self, word):
+ """step3() deals with -ic-, -full, -ness etc. similar strategy to step2."""
+
+ ch = word[-1]
+
+ if ch == 'e':
+ if word.endswith("icate"):
+ return word[:-3] if self._m(word, len(word)-6) else word
+ elif word.endswith("ative"):
+ return word[:-5] if self._m(word, len(word)-6) else word
+ elif word.endswith("alize"):
+ return word[:-3] if self._m(word, len(word)-6) else word
+ else:
+ return word
+ elif ch == 'i':
+ if word.endswith("iciti"):
+ return word[:-3] if self._m(word, len(word)-6) else word
+ else:
+ return word
+ elif ch == 'l':
+ if word.endswith("ical"):
+ return word[:-2] if self._m(word, len(word)-5) else word
+ elif word.endswith("ful"):
+ return word[:-3] if self._m(word, len(word)-4) else word
+ else:
+ return word
+ elif ch == 's':
+ if word.endswith("ness"):
+ return word[:-4] if self._m(word, len(word)-5) else word
+ else:
+ return word
+
+ else:
+ return word
+
+ def _step4(self, word):
+ """step4() takes off -ant, -ence etc., in context <c>vcvc<v>."""
+
+ if len(word) <= 1: # Only possible at this stage given unusual inputs to stem_word like 'oed'
+ return word
+
+ ch = word[-2]
+
+ if ch == 'a':
+ if word.endswith("al"):
+ return word[:-2] if self._m(word, len(word)-3) > 1 else word
+ else:
+ return word
+ elif ch == 'c':
+ if word.endswith("ance"):
+ return word[:-4] if self._m(word, len(word)-5) > 1 else word
+ elif word.endswith("ence"):
+ return word[:-4] if self._m(word, len(word)-5) > 1 else word
+ else:
+ return word
+ elif ch == 'e':
+ if word.endswith("er"):
+ return word[:-2] if self._m(word, len(word)-3) > 1 else word
+ else:
+ return word
+ elif ch == 'i':
+ if word.endswith("ic"):
+ return word[:-2] if self._m(word, len(word)-3) > 1 else word
+ else:
+ return word
+ elif ch == 'l':
+ if word.endswith("able"):
+ return word[:-4] if self._m(word, len(word)-5) > 1 else word
+ elif word.endswith("ible"):
+ return word[:-4] if self._m(word, len(word)-5) > 1 else word
+ else:
+ return word
+ elif ch == 'n':
+ if word.endswith("ant"):
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ elif word.endswith("ement"):
+ return word[:-5] if self._m(word, len(word)-6) > 1 else word
+ elif word.endswith("ment"):
+ return word[:-4] if self._m(word, len(word)-5) > 1 else word
+ elif word.endswith("ent"):
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ else:
+ return word
+ elif ch == 'o':
+ if word.endswith("sion") or word.endswith("tion"): # slightly different logic to all the other cases
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ elif word.endswith("ou"):
+ return word[:-2] if self._m(word, len(word)-3) > 1 else word
+ else:
+ return word
+ elif ch == 's':
+ if word.endswith("ism"):
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ else:
+ return word
+ elif ch == 't':
+ if word.endswith("ate"):
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ elif word.endswith("iti"):
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ else:
+ return word
+ elif ch == 'u':
+ if word.endswith("ous"):
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ else:
+ return word
+ elif ch == 'v':
+ if word.endswith("ive"):
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ else:
+ return word
+ elif ch == 'z':
+ if word.endswith("ize"):
+ return word[:-3] if self._m(word, len(word)-4) > 1 else word
+ else:
+ return word
+ else:
+ return word
+
+ def _step5(self, word):
+ """step5() removes a final -e if m() > 1, and changes -ll to -l if
+ m() > 1.
+ """
+ if word[-1] == 'e':
+ a = self._m(word, len(word)-1)
+ if a > 1 or (a == 1 and not self._cvc(word, len(word)-2)):
+ word = word[:-1]
+ if word.endswith('ll') and self._m(word, len(word)-1) > 1:
+ word = word[:-1]
+
+ return word
+
+ def stem_word(self, p, i=0, j=None):
+ """
+ Returns the stem of p, or, if i and j are given, the stem of p[i:j+1].
+ """
+ ## --NLTK--
+ if j is None and i == 0:
+ word = p
+ else:
+ if j is None:
+ j = len(p) - 1
+ word = p[i:j+1]
+
+ if word in self.pool:
+ return self.pool[word]
+
+ if len(word) <= 2:
+ return word # --DEPARTURE--
+ # With this line, strings of length 1 or 2 don't go through the
+ # stemming process, although no mention is made of this in the
+ # published algorithm. Remove the line to match the published
+ # algorithm.
+
+ word = self._step1ab(word)
+ word = self._step1c(word)
+ word = self._step2(word)
+ word = self._step3(word)
+ word = self._step4(word)
+ word = self._step5(word)
+ return word
+
+ def _adjust_case(self, word, stem):
+ lower = word.lower()
+
+ ret = ""
+ for x in range(len(stem)):
+ if lower[x] == stem[x]:
+ ret += word[x]
+ else:
+ ret += stem[x]
+
+ return ret
+
+ ## --NLTK--
+ ## Don't use this procedure; we want to work with individual
+ ## tokens, instead. (commented out the following procedure)
+ #def stem(self, text):
+ # parts = re.split("(\W+)", text)
+ # numWords = (len(parts) + 1)/2
+ #
+ # ret = ""
+ # for i in xrange(numWords):
+ # word = parts[2 * i]
+ # separator = ""
+ # if ((2 * i) + 1) < len(parts):
+ # separator = parts[(2 * i) + 1]
+ #
+ # stem = self.stem_word(string.lower(word), 0, len(word) - 1)
+ # ret = ret + self.adjust_case(word, stem)
+ # ret = ret + separator
+ # return ret
+
+ ## --NLTK--
+ ## Define a stem() method that implements the StemmerI interface.
+ def stem(self, word):
+ stem = self.stem_word(word.lower(), 0, len(word) - 1)
+ return self._adjust_case(word, stem)
+
+ ## --NLTK--
+ ## Add a string representation function
+ def __repr__(self):
+ return '<PorterStemmer>'
+
+## --NLTK--
+## This test procedure isn't applicable.
+#if __name__ == '__main__':
+# p = PorterStemmer()
+# if len(sys.argv) > 1:
+# for f in sys.argv[1:]:
+# with open(f, 'r') as infile:
+# while 1:
+# w = infile.readline()
+# if w == '':
+# break
+# w = w[:-1]
+# print(p.stem(w))
+
+##--NLTK--
+## Added a demo() function
+
+def demo():
+ """
+ A demonstration of the porter stemmer on a sample from
+ the Penn Treebank corpus.
+ """
+
+ from nltk.corpus import treebank
+ from nltk import stem
+
+ stemmer = stem.PorterStemmer()
+
+ orig = []
+ stemmed = []
+ for item in treebank.files()[:3]:
+ for (word, tag) in treebank.tagged_words(item):
+ orig.append(word)
+ stemmed.append(stemmer.stem(word))
+
+ # Convert the results to a string, and word-wrap them.
+ results = ' '.join(stemmed)
+ results = re.sub(r"(.{,70})\s", r'\1\n', results+' ').rstrip()
+
+ # Convert the original to a string, and word wrap it.
+ original = ' '.join(orig)
+ original = re.sub(r"(.{,70})\s", r'\1\n', original+' ').rstrip()
+
+ # Print the results.
+ print('-Original-'.center(70).replace(' ', '*').replace('-', ' '))
+ print(original)
+ print('-Results-'.center(70).replace(' ', '*').replace('-', ' '))
+ print(results)
+ print('*'*70)
+
+##--NLTK--
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
diff --git a/nltk/stem/regexp.py b/nltk/stem/regexp.py
new file mode 100644
index 0000000..55e3305
--- /dev/null
+++ b/nltk/stem/regexp.py
@@ -0,0 +1,64 @@
+# Natural Language Toolkit: Stemmers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at cs.mu.oz.au>
+# Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import unicode_literals
+import re
+
+from nltk.stem.api import StemmerI
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class RegexpStemmer(StemmerI):
+ """
+ A stemmer that uses regular expressions to identify morphological
+ affixes. Any substrings that match the regular expressions will
+ be removed.
+
+ >>> from nltk.stem import RegexpStemmer
+ >>> st = RegexpStemmer('ing$|s$|e$|able$', min=4)
+ >>> st.stem('cars')
+ 'car'
+ >>> st.stem('mass')
+ 'mas'
+ >>> st.stem('was')
+ 'was'
+ >>> st.stem('bee')
+ 'bee'
+ >>> st.stem('compute')
+ 'comput'
+ >>> st.stem('advisable')
+ 'advis'
+
+ :type regexp: str or regexp
+ :param regexp: The regular expression that should be used to
+ identify morphological affixes.
+ :type min: int
+ :param min: The minimum length of string to stem
+ """
+ def __init__(self, regexp, min=0):
+
+ if not hasattr(regexp, 'pattern'):
+ regexp = re.compile(regexp)
+ self._regexp = regexp
+ self._min = min
+
+ def stem(self, word):
+ if len(word) < self._min:
+ return word
+ else:
+ return self._regexp.sub('', word)
+
+ def __repr__(self):
+ return '<RegexpStemmer: %r>' % self._regexp.pattern
+
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
diff --git a/nltk/stem/rslp.py b/nltk/stem/rslp.py
new file mode 100644
index 0000000..805c44b
--- /dev/null
+++ b/nltk/stem/rslp.py
@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+
+# Natural Language Toolkit: RSLP Stemmer
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Tiago Tresoldi <tresoldi at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# This code is based on the algorithm presented in the paper "A Stemming
+# Algorithm for the Portuguese Language" by Viviane Moreira Orengo and
+# Christian Huyck, which unfortunately I had no access to. The code is a
+# Python version, with some minor modifications of mine, to the description
+# presented at http://www.webcitation.org/5NnvdIzOb and to the C source code
+# available at http://www.inf.ufrgs.br/~arcoelho/rslp/integrando_rslp.html.
+# Please note that this stemmer is intended for demonstration and educational
+# purposes only. Feel free to write me for any comments, including the
+# development of a different and/or better stemmer for Portuguese. I also
+# suggest using NLTK's mailing list for Portuguese for any discussion.
+
+# Este código é baseado no algoritmo apresentado no artigo "A Stemming
+# Algorithm for the Portuguese Language" de Viviane Moreira Orengo e
+# Christian Huyck, o qual infelizmente não tive a oportunidade de ler. O
+# código é uma conversão para Python, com algumas pequenas modificações
+# minhas, daquele apresentado em http://www.webcitation.org/5NnvdIzOb e do
+# código para linguagem C disponível em
+# http://www.inf.ufrgs.br/~arcoelho/rslp/integrando_rslp.html. Por favor,
+# lembre-se de que este stemmer foi desenvolvido com finalidades unicamente
+# de demonstração e didáticas. Sinta-se livre para me escrever para qualquer
+# comentário, inclusive sobre o desenvolvimento de um stemmer diferente
+# e/ou melhor para o português. Também sugiro utilizar-se a lista de discussão
+# do NLTK para o português para qualquer debate.
+from __future__ import print_function, unicode_literals
+from nltk.data import load
+
+from nltk.stem.api import StemmerI
+
+class RSLPStemmer(StemmerI):
+ """
+ A stemmer for Portuguese.
+
+ >>> from nltk.stem import RSLPStemmer
+ >>> st = RSLPStemmer()
+ >>> # opening lines of Erico Verissimo's "Música ao Longe"
+ >>> text = '''
+ ... Clarissa risca com giz no quadro-negro a paisagem que os alunos
+ ... devem copiar . Uma casinha de porta e janela , em cima duma
+ ... coxilha .'''
+ >>> for token in text.split():
+ ... print(st.stem(token))
+ clariss risc com giz no quadro-negr a pais que os alun dev copi .
+ uma cas de port e janel , em cim dum coxilh .
+ """
+
+ def __init__ (self):
+ self._model = []
+
+ self._model.append( self.read_rule("step0.pt") )
+ self._model.append( self.read_rule("step1.pt") )
+ self._model.append( self.read_rule("step2.pt") )
+ self._model.append( self.read_rule("step3.pt") )
+ self._model.append( self.read_rule("step4.pt") )
+ self._model.append( self.read_rule("step5.pt") )
+ self._model.append( self.read_rule("step6.pt") )
+
+ def read_rule (self, filename):
+ rules = load('nltk:stemmers/rslp/' + filename, format='raw').decode("utf8")
+ lines = rules.split("\n")
+
+ lines = [line for line in lines if line != ""] # remove blank lines
+ lines = [line for line in lines if line[0] != "#"] # remove comments
+
+ # NOTE: a simple but ugly hack to make this parser happy with double '\t's
+ lines = [line.replace("\t\t", "\t") for line in lines]
+
+ # parse rules
+ rules = []
+ for line in lines:
+ rule = []
+ tokens = line.split("\t")
+
+ # text to be searched for at the end of the string
+ rule.append( tokens[0][1:-1] ) # remove quotes
+
+ # minimum stem size to perform the replacement
+ rule.append( int(tokens[1]) )
+
+ # text to be replaced into
+ rule.append( tokens[2][1:-1] ) # remove quotes
+
+ # exceptions to this rule
+ rule.append( [token[1:-1] for token in tokens[3].split(",")] )
+
+ # append to the results
+ rules.append(rule)
+
+ return rules
+
+ def stem(self, word):
+ word = word.lower()
+
+ # the word ends in 's'? apply rule for plural reduction
+ if word[-1] == "s":
+ word = self.apply_rule(word, 0)
+
+ # the word ends in 'a'? apply rule for feminine reduction
+ if word[-1] == "a":
+ word = self.apply_rule(word, 1)
+
+ # augmentative reduction
+ word = self.apply_rule(word, 3)
+
+ # adverb reduction
+ word = self.apply_rule(word, 2)
+
+ # noun reduction
+ prev_word = word
+ word = self.apply_rule(word, 4)
+ if word == prev_word:
+ # verb reduction
+ prev_word = word
+ word = self.apply_rule(word, 5)
+ if word == prev_word:
+ # vowel removal
+ word = self.apply_rule(word, 6)
+
+ return word
+
+ def apply_rule(self, word, rule_index):
+ rules = self._model[rule_index]
+ for rule in rules:
+ suffix_length = len(rule[0])
+ if word[-suffix_length:] == rule[0]: # if suffix matches
+ if len(word) >= suffix_length + rule[1]: # if we have minimum size
+ if word not in rule[3]: # if not an exception
+ word = word[:-suffix_length] + rule[2]
+ break
+
+ return word
+
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/stem/snowball.py b/nltk/stem/snowball.py
new file mode 100644
index 0000000..75ba993
--- /dev/null
+++ b/nltk/stem/snowball.py
@@ -0,0 +1,3727 @@
+# -*- coding: utf-8 -*-
+#
+# Natural Language Toolkit: Snowball Stemmer
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Peter Michael Stahl <pemistahl at gmail.com>
+# Peter Ljunglof <peter.ljunglof at heatherleaf.se> (revisions)
+# Algorithms: Dr Martin Porter <martin at tartarus.org>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Snowball stemmers
+
+This module provides a port of the Snowball stemmers
+developed by Martin Porter.
+
+There is also a demo function: `snowball.demo()`.
+
+"""
+from __future__ import unicode_literals, print_function
+
+from nltk import compat
+from nltk.corpus import stopwords
+from nltk.stem import porter
+
+from nltk.stem.api import StemmerI
+
+class SnowballStemmer(StemmerI):
+
+ """
+ Snowball Stemmer
+
+ The following languages are supported:
+ Danish, Dutch, English, Finnish, French, German,
+ Hungarian, Italian, Norwegian, Portuguese, Romanian, Russian,
+ Spanish and Swedish.
+
+ The algorithm for English is documented here:
+
+ Porter, M. \"An algorithm for suffix stripping.\"
+ Program 14.3 (1980): 130-137.
+
+ The algorithms have been developed by Martin Porter.
+ These stemmers are called Snowball, because Porter created
+ a programming language with this name for creating
+ new stemming algorithms. There is more information available
+ at http://snowball.tartarus.org/
+
+ The stemmer is invoked as shown below:
+
+ >>> from nltk.stem import SnowballStemmer
+ >>> print(" ".join(SnowballStemmer.languages)) # See which languages are supported
+ danish dutch english finnish french german hungarian
+ italian norwegian porter portuguese romanian russian
+ spanish swedish
+ >>> stemmer = SnowballStemmer("german") # Choose a language
+ >>> stemmer.stem("Autobahnen") # Stem a word
+ 'autobahn'
+
+ Invoking the stemmers that way is useful if you do not know the
+ language to be stemmed at runtime. Alternatively, if you already know
+ the language, then you can invoke the language specific stemmer directly:
+
+ >>> from nltk.stem.snowball import GermanStemmer
+ >>> stemmer = GermanStemmer()
+ >>> stemmer.stem("Autobahnen")
+ 'autobahn'
+
+ :param language: The language whose subclass is instantiated.
+ :type language: str or unicode
+ :param ignore_stopwords: If set to True, stopwords are
+ not stemmed and returned unchanged.
+ Set to False by default.
+ :type ignore_stopwords: bool
+ :raise ValueError: If there is no stemmer for the specified
+ language, a ValueError is raised.
+ """
+
+ languages = ("danish", "dutch", "english", "finnish", "french", "german",
+ "hungarian", "italian", "norwegian", "porter", "portuguese",
+ "romanian", "russian", "spanish", "swedish")
+
+ def __init__(self, language, ignore_stopwords=False):
+ if language not in self.languages:
+ raise ValueError("The language '%s' is not supported." % language)
+ stemmerclass = globals()[language.capitalize() + "Stemmer"]
+ self.stemmer = stemmerclass(ignore_stopwords)
+ self.stem = self.stemmer.stem
+ self.stopwords = self.stemmer.stopwords
+
+
+ at compat.python_2_unicode_compatible
+class _LanguageSpecificStemmer(StemmerI):
+
+ """
+ This helper subclass offers the possibility
+ to invoke a specific stemmer directly.
+ This is useful if you already know the language to be stemmed at runtime.
+
+ Create an instance of the Snowball stemmer.
+
+ :param ignore_stopwords: If set to True, stopwords are
+ not stemmed and returned unchanged.
+ Set to False by default.
+ :type ignore_stopwords: bool
+ """
+
+ def __init__(self, ignore_stopwords=False):
+ # The language is the name of the class, minus the final "Stemmer".
+ language = type(self).__name__.lower()
+ if language.endswith("stemmer"):
+ language = language[:-7]
+
+ self.stopwords = set()
+ if ignore_stopwords:
+ try:
+ for word in stopwords.words(language):
+ self.stopwords.add(word)
+ except IOError:
+ raise ValueError("%r has no list of stopwords. Please set"
+ " 'ignore_stopwords' to 'False'." % self)
+
+ def __repr__(self):
+ """
+ Print out the string representation of the respective class.
+
+ """
+ return "<%s>" % type(self).__name__
+
+
+class PorterStemmer(_LanguageSpecificStemmer, porter.PorterStemmer):
+ """
+ A word stemmer based on the original Porter stemming algorithm.
+
+ Porter, M. \"An algorithm for suffix stripping.\"
+ Program 14.3 (1980): 130-137.
+
+ A few minor modifications have been made to Porter's basic
+ algorithm. See the source code of the module
+ nltk.stem.porter for more information.
+
+ """
+ def __init__(self, ignore_stopwords=False):
+ _LanguageSpecificStemmer.__init__(self, ignore_stopwords)
+ porter.PorterStemmer.__init__(self)
+
+
+class _ScandinavianStemmer(_LanguageSpecificStemmer):
+
+ """
+ This subclass encapsulates a method for defining the string region R1.
+ It is used by the Danish, Norwegian, and Swedish stemmer.
+
+ """
+
+ def _r1_scandinavian(self, word, vowels):
+ """
+ Return the region R1 that is used by the Scandinavian stemmers.
+
+ R1 is the region after the first non-vowel following a vowel,
+ or is the null region at the end of the word if there is no
+ such non-vowel. But then R1 is adjusted so that the region
+ before it contains at least three letters.
+
+ :param word: The word whose region R1 is determined.
+ :type word: str or unicode
+ :param vowels: The vowels of the respective language that are
+ used to determine the region R1.
+ :type vowels: unicode
+ :return: the region R1 for the respective word.
+ :rtype: unicode
+ :note: This helper method is invoked by the respective stem method of
+ the subclasses DanishStemmer, NorwegianStemmer, and
+ SwedishStemmer. It is not to be invoked directly!
+
+ """
+ r1 = ""
+ for i in range(1, len(word)):
+ if word[i] not in vowels and word[i-1] in vowels:
+ if len(word[:i+1]) < 3 and len(word[:i+1]) > 0:
+ r1 = word[3:]
+ elif len(word[:i+1]) >= 3:
+ r1 = word[i+1:]
+ else:
+ return word
+ break
+
+ return r1
+
+
+
+class _StandardStemmer(_LanguageSpecificStemmer):
+
+ """
+ This subclass encapsulates two methods for defining the standard versions
+ of the string regions R1, R2, and RV.
+
+ """
+
+ def _r1r2_standard(self, word, vowels):
+ """
+ Return the standard interpretations of the string regions R1 and R2.
+
+ R1 is the region after the first non-vowel following a vowel,
+ or is the null region at the end of the word if there is no
+ such non-vowel.
+
+ R2 is the region after the first non-vowel following a vowel
+ in R1, or is the null region at the end of the word if there
+ is no such non-vowel.
+
+ :param word: The word whose regions R1 and R2 are determined.
+ :type word: str or unicode
+ :param vowels: The vowels of the respective language that are
+ used to determine the regions R1 and R2.
+ :type vowels: unicode
+ :return: (r1,r2), the regions R1 and R2 for the respective word.
+ :rtype: tuple
+ :note: This helper method is invoked by the respective stem method of
+ the subclasses DutchStemmer, FinnishStemmer,
+ FrenchStemmer, GermanStemmer, ItalianStemmer,
+ PortugueseStemmer, RomanianStemmer, and SpanishStemmer.
+ It is not to be invoked directly!
+ :note: A detailed description of how to define R1 and R2
+ can be found at http://snowball.tartarus.org/texts/r1r2.html
+
+ """
+ r1 = ""
+ r2 = ""
+ for i in range(1, len(word)):
+ if word[i] not in vowels and word[i-1] in vowels:
+ r1 = word[i+1:]
+ break
+
+ for i in range(1, len(r1)):
+ if r1[i] not in vowels and r1[i-1] in vowels:
+ r2 = r1[i+1:]
+ break
+
+ return (r1, r2)
+
+
+
+ def _rv_standard(self, word, vowels):
+ """
+ Return the standard interpretation of the string region RV.
+
+ If the second letter is a consonant, RV is the region after the
+ next following vowel. If the first two letters are vowels, RV is
+ the region after the next following consonant. Otherwise, RV is
+ the region after the third letter.
+
+ :param word: The word whose region RV is determined.
+ :type word: str or unicode
+ :param vowels: The vowels of the respective language that are
+ used to determine the region RV.
+ :type vowels: unicode
+ :return: the region RV for the respective word.
+ :rtype: unicode
+ :note: This helper method is invoked by the respective stem method of
+ the subclasses ItalianStemmer, PortugueseStemmer,
+ RomanianStemmer, and SpanishStemmer. It is not to be
+ invoked directly!
+
+ """
+ rv = ""
+ if len(word) >= 2:
+ if word[1] not in vowels:
+ for i in range(2, len(word)):
+ if word[i] in vowels:
+ rv = word[i+1:]
+ break
+
+ elif word[:2] in vowels:
+ for i in range(2, len(word)):
+ if word[i] not in vowels:
+ rv = word[i+1:]
+ break
+ else:
+ rv = word[3:]
+
+ return rv
+
+
+
+class DanishStemmer(_ScandinavianStemmer):
+
+ """
+ The Danish Snowball stemmer.
+
+ :cvar __vowels: The Danish vowels.
+ :type __vowels: unicode
+ :cvar __consonants: The Danish consonants.
+ :type __consonants: unicode
+ :cvar __double_consonants: The Danish double consonants.
+ :type __double_consonants: tuple
+ :cvar __s_ending: Letters that may directly appear before a word final 's'.
+ :type __s_ending: unicode
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :note: A detailed description of the Danish
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/danish/stemmer.html
+
+ """
+
+ # The language's vowels and other important characters are defined.
+ __vowels = "aeiouy\xE6\xE5\xF8"
+ __consonants = "bcdfghjklmnpqrstvwxz"
+ __double_consonants = ("bb", "cc", "dd", "ff", "gg", "hh", "jj",
+ "kk", "ll", "mm", "nn", "pp", "qq", "rr",
+ "ss", "tt", "vv", "ww", "xx", "zz")
+ __s_ending = "abcdfghjklmnoprtvyz\xE5"
+
+ # The different suffixes, divided into the algorithm's steps
+ # and organized by length, are listed in tuples.
+ __step1_suffixes = ("erendes", "erende", "hedens", "ethed",
+ "erede", "heden", "heder", "endes",
+ "ernes", "erens", "erets", "ered",
+ "ende", "erne", "eren", "erer", "heds",
+ "enes", "eres", "eret", "hed", "ene", "ere",
+ "ens", "ers", "ets", "en", "er", "es", "et",
+ "e", "s")
+ __step2_suffixes = ("gd", "dt", "gt", "kt")
+ __step3_suffixes = ("elig", "l\xF8st", "lig", "els", "ig")
+
+ def stem(self, word):
+ """
+ Stem a Danish word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ # Every word is put into lower case for normalization.
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ # After this, the required regions are generated
+ # by the respective helper method.
+ r1 = self._r1_scandinavian(word, self.__vowels)
+
+ # Then the actual stemming process starts.
+ # Every new step is explicitly indicated
+ # according to the descriptions on the Snowball website.
+
+ # STEP 1
+ for suffix in self.__step1_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "s":
+ if word[-2] in self.__s_ending:
+ word = word[:-1]
+ r1 = r1[:-1]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ break
+
+ # STEP 2
+ for suffix in self.__step2_suffixes:
+ if r1.endswith(suffix):
+ word = word[:-1]
+ r1 = r1[:-1]
+ break
+
+ # STEP 3
+ if r1.endswith("igst"):
+ word = word[:-2]
+ r1 = r1[:-2]
+
+ for suffix in self.__step3_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "l\xF8st":
+ word = word[:-1]
+ r1 = r1[:-1]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+
+ if r1.endswith(self.__step2_suffixes):
+ word = word[:-1]
+ r1 = r1[:-1]
+ break
+
+ # STEP 4: Undouble
+ for double_cons in self.__double_consonants:
+ if word.endswith(double_cons) and len(word) > 3:
+ word = word[:-1]
+ break
+
+
+ return word
+
+
+class DutchStemmer(_StandardStemmer):
+
+ """
+ The Dutch Snowball stemmer.
+
+ :cvar __vowels: The Dutch vowels.
+ :type __vowels: unicode
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step3b_suffixes: Suffixes to be deleted in step 3b of the algorithm.
+ :type __step3b_suffixes: tuple
+ :note: A detailed description of the Dutch
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/dutch/stemmer.html
+
+ """
+
+ __vowels = "aeiouy\xE8"
+ __step1_suffixes = ("heden", "ene", "en", "se", "s")
+ __step3b_suffixes = ("baar", "lijk", "bar", "end", "ing", "ig")
+
+ def stem(self, word):
+ """
+ Stem a Dutch word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ step2_success = False
+
+ # Vowel accents are removed.
+ word = (word.replace("\xE4", "a").replace("\xE1", "a")
+ .replace("\xEB", "e").replace("\xE9", "e")
+ .replace("\xED", "i").replace("\xEF", "i")
+ .replace("\xF6", "o").replace("\xF3", "o")
+ .replace("\xFC", "u").replace("\xFA", "u"))
+
+ # An initial 'y', a 'y' after a vowel,
+ # and an 'i' between self.__vowels is put into upper case.
+ # As from now these are treated as consonants.
+ if word.startswith("y"):
+ word = "".join(("Y", word[1:]))
+
+ for i in range(1, len(word)):
+ if word[i-1] in self.__vowels and word[i] == "y":
+ word = "".join((word[:i], "Y", word[i+1:]))
+
+ for i in range(1, len(word)-1):
+ if (word[i-1] in self.__vowels and word[i] == "i" and
+ word[i+1] in self.__vowels):
+ word = "".join((word[:i], "I", word[i+1:]))
+
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+
+ # R1 is adjusted so that the region before it
+ # contains at least 3 letters.
+ for i in range(1, len(word)):
+ if word[i] not in self.__vowels and word[i-1] in self.__vowels:
+ if len(word[:i+1]) < 3 and len(word[:i+1]) > 0:
+ r1 = word[3:]
+ elif len(word[:i+1]) == 0:
+ return word
+ break
+
+ # STEP 1
+ for suffix in self.__step1_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "heden":
+ word = "".join((word[:-5], "heid"))
+ r1 = "".join((r1[:-5], "heid"))
+ if r2.endswith("heden"):
+ r2 = "".join((r2[:-5], "heid"))
+
+ elif (suffix in ("ene", "en") and
+ not word.endswith("heden") and
+ word[-len(suffix)-1] not in self.__vowels and
+ word[-len(suffix)-3:-len(suffix)] != "gem"):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ if word.endswith(("kk", "dd", "tt")):
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+
+ elif (suffix in ("se", "s") and
+ word[-len(suffix)-1] not in self.__vowels and
+ word[-len(suffix)-1] != "j"):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ break
+
+ # STEP 2
+ if r1.endswith("e") and word[-2] not in self.__vowels:
+ step2_success = True
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+
+ if word.endswith(("kk", "dd", "tt")):
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+
+ # STEP 3a
+ if r2.endswith("heid") and word[-5] != "c":
+ word = word[:-4]
+ r1 = r1[:-4]
+ r2 = r2[:-4]
+
+ if (r1.endswith("en") and word[-3] not in self.__vowels and
+ word[-5:-2] != "gem"):
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ if word.endswith(("kk", "dd", "tt")):
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+
+ # STEP 3b: Derivational suffixes
+ for suffix in self.__step3b_suffixes:
+ if r2.endswith(suffix):
+ if suffix in ("end", "ing"):
+ word = word[:-3]
+ r2 = r2[:-3]
+
+ if r2.endswith("ig") and word[-3] != "e":
+ word = word[:-2]
+ else:
+ if word.endswith(("kk", "dd", "tt")):
+ word = word[:-1]
+
+ elif suffix == "ig" and word[-3] != "e":
+ word = word[:-2]
+
+ elif suffix == "lijk":
+ word = word[:-4]
+ r1 = r1[:-4]
+
+ if r1.endswith("e") and word[-2] not in self.__vowels:
+ word = word[:-1]
+ if word.endswith(("kk", "dd", "tt")):
+ word = word[:-1]
+
+ elif suffix == "baar":
+ word = word[:-4]
+
+ elif suffix == "bar" and step2_success:
+ word = word[:-3]
+ break
+
+ # STEP 4: Undouble vowel
+ if len(word) >= 4:
+ if word[-1] not in self.__vowels and word[-1] != "I":
+ if word[-3:-1] in ("aa", "ee", "oo", "uu"):
+ if word[-4] not in self.__vowels:
+ word = "".join((word[:-3], word[-3], word[-1]))
+
+ # All occurrences of 'I' and 'Y' are put back into lower case.
+ word = word.replace("I", "i").replace("Y", "y")
+
+
+ return word
+
+
+
+class EnglishStemmer(_StandardStemmer):
+
+ """
+ The English Snowball stemmer.
+
+ :cvar __vowels: The English vowels.
+ :type __vowels: unicode
+ :cvar __double_consonants: The English double consonants.
+ :type __double_consonants: tuple
+ :cvar __li_ending: Letters that may directly appear before a word final 'li'.
+ :type __li_ending: unicode
+ :cvar __step0_suffixes: Suffixes to be deleted in step 0 of the algorithm.
+ :type __step0_suffixes: tuple
+ :cvar __step1a_suffixes: Suffixes to be deleted in step 1a of the algorithm.
+ :type __step1a_suffixes: tuple
+ :cvar __step1b_suffixes: Suffixes to be deleted in step 1b of the algorithm.
+ :type __step1b_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :cvar __step4_suffixes: Suffixes to be deleted in step 4 of the algorithm.
+ :type __step4_suffixes: tuple
+ :cvar __step5_suffixes: Suffixes to be deleted in step 5 of the algorithm.
+ :type __step5_suffixes: tuple
+ :cvar __special_words: A dictionary containing words
+ which have to be stemmed specially.
+ :type __special_words: dict
+ :note: A detailed description of the English
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/english/stemmer.html
+ """
+
+ __vowels = "aeiouy"
+ __double_consonants = ("bb", "dd", "ff", "gg", "mm", "nn",
+ "pp", "rr", "tt")
+ __li_ending = "cdeghkmnrt"
+ __step0_suffixes = ("'s'", "'s", "'")
+ __step1a_suffixes = ("sses", "ied", "ies", "us", "ss", "s")
+ __step1b_suffixes = ("eedly", "ingly", "edly", "eed", "ing", "ed")
+ __step2_suffixes = ('ization', 'ational', 'fulness', 'ousness',
+ 'iveness', 'tional', 'biliti', 'lessli',
+ 'entli', 'ation', 'alism', 'aliti', 'ousli',
+ 'iviti', 'fulli', 'enci', 'anci', 'abli',
+ 'izer', 'ator', 'alli', 'bli', 'ogi', 'li')
+ __step3_suffixes = ('ational', 'tional', 'alize', 'icate', 'iciti',
+ 'ative', 'ical', 'ness', 'ful')
+ __step4_suffixes = ('ement', 'ance', 'ence', 'able', 'ible', 'ment',
+ 'ant', 'ent', 'ism', 'ate', 'iti', 'ous',
+ 'ive', 'ize', 'ion', 'al', 'er', 'ic')
+ __step5_suffixes = ("e", "l")
+ __special_words = {"skis" : "ski",
+ "skies" : "sky",
+ "dying" : "die",
+ "lying" : "lie",
+ "tying" : "tie",
+ "idly" : "idl",
+ "gently" : "gentl",
+ "ugly" : "ugli",
+ "early" : "earli",
+ "only" : "onli",
+ "singly" : "singl",
+ "sky" : "sky",
+ "news" : "news",
+ "howe" : "howe",
+ "atlas" : "atlas",
+ "cosmos" : "cosmos",
+ "bias" : "bias",
+ "andes" : "andes",
+ "inning" : "inning",
+ "innings" : "inning",
+ "outing" : "outing",
+ "outings" : "outing",
+ "canning" : "canning",
+ "cannings" : "canning",
+ "herring" : "herring",
+ "herrings" : "herring",
+ "earring" : "earring",
+ "earrings" : "earring",
+ "proceed" : "proceed",
+ "proceeds" : "proceed",
+ "proceeded" : "proceed",
+ "proceeding" : "proceed",
+ "exceed" : "exceed",
+ "exceeds" : "exceed",
+ "exceeded" : "exceed",
+ "exceeding" : "exceed",
+ "succeed" : "succeed",
+ "succeeds" : "succeed",
+ "succeeded" : "succeed",
+ "succeeding" : "succeed"}
+
+ def stem(self, word):
+
+ """
+ Stem an English word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords or len(word) <= 2:
+ return word
+
+ elif word in self.__special_words:
+ return self.__special_words[word]
+
+ # Map the different apostrophe characters to a single consistent one
+ word = (word.replace("\u2019", "\x27")
+ .replace("\u2018", "\x27")
+ .replace("\u201B", "\x27"))
+
+ if word.startswith("\x27"):
+ word = word[1:]
+
+ if word.startswith("y"):
+ word = "".join(("Y", word[1:]))
+
+ for i in range(1, len(word)):
+ if word[i-1] in self.__vowels and word[i] == "y":
+ word = "".join((word[:i], "Y", word[i+1:]))
+
+ step1a_vowel_found = False
+ step1b_vowel_found = False
+
+ r1 = ""
+ r2 = ""
+
+ if word.startswith(("gener", "commun", "arsen")):
+ if word.startswith(("gener", "arsen")):
+ r1 = word[5:]
+ else:
+ r1 = word[6:]
+
+ for i in range(1, len(r1)):
+ if r1[i] not in self.__vowels and r1[i-1] in self.__vowels:
+ r2 = r1[i+1:]
+ break
+ else:
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+
+
+ # STEP 0
+ for suffix in self.__step0_suffixes:
+ if word.endswith(suffix):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ break
+
+ # STEP 1a
+ for suffix in self.__step1a_suffixes:
+ if word.endswith(suffix):
+
+ if suffix == "sses":
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ elif suffix in ("ied", "ies"):
+ if len(word[:-len(suffix)]) > 1:
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+ else:
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+
+ elif suffix == "s":
+ for letter in word[:-2]:
+ if letter in self.__vowels:
+ step1a_vowel_found = True
+ break
+
+ if step1a_vowel_found:
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+ break
+
+ # STEP 1b
+ for suffix in self.__step1b_suffixes:
+ if word.endswith(suffix):
+ if suffix in ("eed", "eedly"):
+
+ if r1.endswith(suffix):
+ word = "".join((word[:-len(suffix)], "ee"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "ee"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "ee"))
+ else:
+ r2 = ""
+ else:
+ for letter in word[:-len(suffix)]:
+ if letter in self.__vowels:
+ step1b_vowel_found = True
+ break
+
+ if step1b_vowel_found:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+
+ if word.endswith(("at", "bl", "iz")):
+ word = "".join((word, "e"))
+ r1 = "".join((r1, "e"))
+
+ if len(word) > 5 or len(r1) >=3:
+ r2 = "".join((r2, "e"))
+
+ elif word.endswith(self.__double_consonants):
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+
+ elif ((r1 == "" and len(word) >= 3 and
+ word[-1] not in self.__vowels and
+ word[-1] not in "wxY" and
+ word[-2] in self.__vowels and
+ word[-3] not in self.__vowels)
+ or
+ (r1 == "" and len(word) == 2 and
+ word[0] in self.__vowels and
+ word[1] not in self.__vowels)):
+
+ word = "".join((word, "e"))
+
+ if len(r1) > 0:
+ r1 = "".join((r1, "e"))
+
+ if len(r2) > 0:
+ r2 = "".join((r2, "e"))
+ break
+
+ # STEP 1c
+ if len(word) > 2 and word[-1] in "yY" and word[-2] not in self.__vowels:
+ word = "".join((word[:-1], "i"))
+ if len(r1) >= 1:
+ r1 = "".join((r1[:-1], "i"))
+ else:
+ r1 = ""
+
+ if len(r2) >= 1:
+ r2 = "".join((r2[:-1], "i"))
+ else:
+ r2 = ""
+
+ # STEP 2
+ for suffix in self.__step2_suffixes:
+ if word.endswith(suffix):
+ if r1.endswith(suffix):
+ if suffix == "tional":
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ elif suffix in ("enci", "anci", "abli"):
+ word = "".join((word[:-1], "e"))
+
+ if len(r1) >= 1:
+ r1 = "".join((r1[:-1], "e"))
+ else:
+ r1 = ""
+
+ if len(r2) >= 1:
+ r2 = "".join((r2[:-1], "e"))
+ else:
+ r2 = ""
+
+ elif suffix == "entli":
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ elif suffix in ("izer", "ization"):
+ word = "".join((word[:-len(suffix)], "ize"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "ize"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "ize"))
+ else:
+ r2 = ""
+
+ elif suffix in ("ational", "ation", "ator"):
+ word = "".join((word[:-len(suffix)], "ate"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "ate"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "ate"))
+ else:
+ r2 = "e"
+
+ elif suffix in ("alism", "aliti", "alli"):
+ word = "".join((word[:-len(suffix)], "al"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "al"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "al"))
+ else:
+ r2 = ""
+
+ elif suffix == "fulness":
+ word = word[:-4]
+ r1 = r1[:-4]
+ r2 = r2[:-4]
+
+ elif suffix in ("ousli", "ousness"):
+ word = "".join((word[:-len(suffix)], "ous"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "ous"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "ous"))
+ else:
+ r2 = ""
+
+ elif suffix in ("iveness", "iviti"):
+ word = "".join((word[:-len(suffix)], "ive"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "ive"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "ive"))
+ else:
+ r2 = "e"
+
+ elif suffix in ("biliti", "bli"):
+ word = "".join((word[:-len(suffix)], "ble"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "ble"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "ble"))
+ else:
+ r2 = ""
+
+ elif suffix == "ogi" and word[-4] == "l":
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+
+ elif suffix in ("fulli", "lessli"):
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ elif suffix == "li" and word[-3] in self.__li_ending:
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+ break
+
+ # STEP 3
+ for suffix in self.__step3_suffixes:
+ if word.endswith(suffix):
+ if r1.endswith(suffix):
+ if suffix == "tional":
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ elif suffix == "ational":
+ word = "".join((word[:-len(suffix)], "ate"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "ate"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "ate"))
+ else:
+ r2 = ""
+
+ elif suffix == "alize":
+ word = word[:-3]
+ r1 = r1[:-3]
+ r2 = r2[:-3]
+
+ elif suffix in ("icate", "iciti", "ical"):
+ word = "".join((word[:-len(suffix)], "ic"))
+
+ if len(r1) >= len(suffix):
+ r1 = "".join((r1[:-len(suffix)], "ic"))
+ else:
+ r1 = ""
+
+ if len(r2) >= len(suffix):
+ r2 = "".join((r2[:-len(suffix)], "ic"))
+ else:
+ r2 = ""
+
+ elif suffix in ("ful", "ness"):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+
+ elif suffix == "ative" and r2.endswith(suffix):
+ word = word[:-5]
+ r1 = r1[:-5]
+ r2 = r2[:-5]
+ break
+
+ # STEP 4
+ for suffix in self.__step4_suffixes:
+ if word.endswith(suffix):
+ if r2.endswith(suffix):
+ if suffix == "ion":
+ if word[-4] in "st":
+ word = word[:-3]
+ r1 = r1[:-3]
+ r2 = r2[:-3]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ break
+
+ # STEP 5
+ if r2.endswith("l") and word[-2] == "l":
+ word = word[:-1]
+ elif r2.endswith("e"):
+ word = word[:-1]
+ elif r1.endswith("e"):
+ if len(word) >= 4 and (word[-2] in self.__vowels or
+ word[-2] in "wxY" or
+ word[-3] not in self.__vowels or
+ word[-4] in self.__vowels):
+ word = word[:-1]
+
+
+ word = word.replace("Y", "y")
+
+
+ return word
+
+
+
+class FinnishStemmer(_StandardStemmer):
+
+ """
+ The Finnish Snowball stemmer.
+
+ :cvar __vowels: The Finnish vowels.
+ :type __vowels: unicode
+ :cvar __restricted_vowels: A subset of the Finnish vowels.
+ :type __restricted_vowels: unicode
+ :cvar __long_vowels: The Finnish vowels in their long forms.
+ :type __long_vowels: tuple
+ :cvar __consonants: The Finnish consonants.
+ :type __consonants: unicode
+ :cvar __double_consonants: The Finnish double consonants.
+ :type __double_consonants: tuple
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :cvar __step4_suffixes: Suffixes to be deleted in step 4 of the algorithm.
+ :type __step4_suffixes: tuple
+ :note: A detailed description of the Finnish
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/finnish/stemmer.html
+ """
+
+ __vowels = "aeiouy\xE4\xF6"
+ __restricted_vowels = "aeiou\xE4\xF6"
+ __long_vowels = ("aa", "ee", "ii", "oo", "uu", "\xE4\xE4",
+ "\xF6\xF6")
+ __consonants = "bcdfghjklmnpqrstvwxz"
+ __double_consonants = ("bb", "cc", "dd", "ff", "gg", "hh", "jj",
+ "kk", "ll", "mm", "nn", "pp", "qq", "rr",
+ "ss", "tt", "vv", "ww", "xx", "zz")
+ __step1_suffixes = ('kaan', 'k\xE4\xE4n', 'sti', 'kin', 'han',
+ 'h\xE4n', 'ko', 'k\xF6', 'pa', 'p\xE4')
+ __step2_suffixes = ('nsa', 'ns\xE4', 'mme', 'nne', 'si', 'ni',
+ 'an', '\xE4n', 'en')
+ __step3_suffixes = ('siin', 'tten', 'seen', 'han', 'hen', 'hin',
+ 'hon', 'h\xE4n', 'h\xF6n', 'den', 'tta',
+ 'tt\xE4', 'ssa', 'ss\xE4', 'sta',
+ 'st\xE4', 'lla', 'll\xE4', 'lta',
+ 'lt\xE4', 'lle', 'ksi', 'ine', 'ta',
+ 't\xE4', 'na', 'n\xE4', 'a', '\xE4',
+ 'n')
+ __step4_suffixes = ('impi', 'impa', 'imp\xE4', 'immi', 'imma',
+ 'imm\xE4', 'mpi', 'mpa', 'mp\xE4', 'mmi',
+ 'mma', 'mm\xE4', 'eja', 'ej\xE4')
+
+ def stem(self, word):
+ """
+ Stem a Finnish word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ step3_success = False
+
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+
+ # STEP 1: Particles etc.
+ for suffix in self.__step1_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "sti":
+ if suffix in r2:
+ word = word[:-3]
+ r1 = r1[:-3]
+ r2 = r2[:-3]
+ else:
+ if word[-len(suffix)-1] in "ntaeiouy\xE4\xF6":
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ break
+
+ # STEP 2: Possessives
+ for suffix in self.__step2_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "si":
+ if word[-3] != "k":
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ elif suffix == "ni":
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+ if word.endswith("kse"):
+ word = "".join((word[:-3], "ksi"))
+
+ if r1.endswith("kse"):
+ r1 = "".join((r1[:-3], "ksi"))
+
+ if r2.endswith("kse"):
+ r2 = "".join((r2[:-3], "ksi"))
+
+ elif suffix == "an":
+ if (word[-4:-2] in ("ta", "na") or
+ word[-5:-2] in ("ssa", "sta", "lla", "lta")):
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ elif suffix == "\xE4n":
+ if (word[-4:-2] in ("t\xE4", "n\xE4") or
+ word[-5:-2] in ("ss\xE4", "st\xE4",
+ "ll\xE4", "lt\xE4")):
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+
+ elif suffix == "en":
+ if word[-5:-2] in ("lle", "ine"):
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+ else:
+ word = word[:-3]
+ r1 = r1[:-3]
+ r2 = r2[:-3]
+ break
+
+ # STEP 3: Cases
+ for suffix in self.__step3_suffixes:
+ if r1.endswith(suffix):
+ if suffix in ("han", "hen", "hin", "hon", "h\xE4n",
+ "h\xF6n"):
+ if ((suffix == "han" and word[-4] == "a") or
+ (suffix == "hen" and word[-4] == "e") or
+ (suffix == "hin" and word[-4] == "i") or
+ (suffix == "hon" and word[-4] == "o") or
+ (suffix == "h\xE4n" and word[-4] == "\xE4") or
+ (suffix == "h\xF6n" and word[-4] == "\xF6")):
+ word = word[:-3]
+ r1 = r1[:-3]
+ r2 = r2[:-3]
+ step3_success = True
+
+ elif suffix in ("siin", "den", "tten"):
+ if (word[-len(suffix)-1] == "i" and
+ word[-len(suffix)-2] in self.__restricted_vowels):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ step3_success = True
+ else:
+ continue
+
+ elif suffix == "seen":
+ if word[-6:-4] in self.__long_vowels:
+ word = word[:-4]
+ r1 = r1[:-4]
+ r2 = r2[:-4]
+ step3_success = True
+ else:
+ continue
+
+ elif suffix in ("a", "\xE4"):
+ if word[-2] in self.__vowels and word[-3] in self.__consonants:
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+ step3_success = True
+
+ elif suffix in ("tta", "tt\xE4"):
+ if word[-4] == "e":
+ word = word[:-3]
+ r1 = r1[:-3]
+ r2 = r2[:-3]
+ step3_success = True
+
+ elif suffix == "n":
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+ step3_success = True
+
+ if word[-2:] == "ie" or word[-2:] in self.__long_vowels:
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ step3_success = True
+ break
+
+ # STEP 4: Other endings
+ for suffix in self.__step4_suffixes:
+ if r2.endswith(suffix):
+ if suffix in ("mpi", "mpa", "mp\xE4", "mmi", "mma",
+ "mm\xE4"):
+ if word[-5:-3] != "po":
+ word = word[:-3]
+ r1 = r1[:-3]
+ r2 = r2[:-3]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ break
+
+ # STEP 5: Plurals
+ if step3_success and len(r1) >= 1 and r1[-1] in "ij":
+ word = word[:-1]
+ r1 = r1[:-1]
+
+ elif (not step3_success and len(r1) >= 2 and
+ r1[-1] == "t" and r1[-2] in self.__vowels):
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+ if r2.endswith("imma"):
+ word = word[:-4]
+ r1 = r1[:-4]
+ elif r2.endswith("mma") and r2[-5:-3] != "po":
+ word = word[:-3]
+ r1 = r1[:-3]
+
+ # STEP 6: Tidying up
+ if r1[-2:] in self.__long_vowels:
+ word = word[:-1]
+ r1 = r1[:-1]
+
+ if (len(r1) >= 2 and r1[-2] in self.__consonants and
+ r1[-1] in "a\xE4ei"):
+ word = word[:-1]
+ r1 = r1[:-1]
+
+ if r1.endswith(("oj", "uj")):
+ word = word[:-1]
+ r1 = r1[:-1]
+
+ if r1.endswith("jo"):
+ word = word[:-1]
+ r1 = r1[:-1]
+
+ # If the word ends with a double consonant
+ # followed by zero or more vowels, the last consonant is removed.
+ for i in range(1, len(word)):
+ if word[-i] in self.__vowels:
+ continue
+ else:
+ if i == 1:
+ if word[-i-1:] in self.__double_consonants:
+ word = word[:-1]
+ else:
+ if word[-i-1:-i+1] in self.__double_consonants:
+ word = "".join((word[:-i], word[-i+1:]))
+ break
+
+
+ return word
+
+
+
+class FrenchStemmer(_StandardStemmer):
+
+ """
+ The French Snowball stemmer.
+
+ :cvar __vowels: The French vowels.
+ :type __vowels: unicode
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2a_suffixes: Suffixes to be deleted in step 2a of the algorithm.
+ :type __step2a_suffixes: tuple
+ :cvar __step2b_suffixes: Suffixes to be deleted in step 2b of the algorithm.
+ :type __step2b_suffixes: tuple
+ :cvar __step4_suffixes: Suffixes to be deleted in step 4 of the algorithm.
+ :type __step4_suffixes: tuple
+ :note: A detailed description of the French
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/french/stemmer.html
+ """
+
+ __vowels = "aeiouy\xE2\xE0\xEB\xE9\xEA\xE8\xEF\xEE\xF4\xFB\xF9"
+ __step1_suffixes = ('issements', 'issement', 'atrices', 'atrice',
+ 'ateurs', 'ations', 'logies', 'usions',
+ 'utions', 'ements', 'amment', 'emment',
+ 'ances', 'iqUes', 'ismes', 'ables', 'istes',
+ 'ateur', 'ation', 'logie', 'usion', 'ution',
+ 'ences', 'ement', 'euses', 'ments', 'ance',
+ 'iqUe', 'isme', 'able', 'iste', 'ence',
+ 'it\xE9s', 'ives', 'eaux', 'euse', 'ment',
+ 'eux', 'it\xE9', 'ive', 'ifs', 'aux', 'if')
+ __step2a_suffixes = ('issaIent', 'issantes', 'iraIent', 'issante',
+ 'issants', 'issions', 'irions', 'issais',
+ 'issait', 'issant', 'issent', 'issiez', 'issons',
+ 'irais', 'irait', 'irent', 'iriez', 'irons',
+ 'iront', 'isses', 'issez', '\xEEmes',
+ '\xEEtes', 'irai', 'iras', 'irez', 'isse',
+ 'ies', 'ira', '\xEEt', 'ie', 'ir', 'is',
+ 'it', 'i')
+ __step2b_suffixes = ('eraIent', 'assions', 'erions', 'assent',
+ 'assiez', '\xE8rent', 'erais', 'erait',
+ 'eriez', 'erons', 'eront', 'aIent', 'antes',
+ 'asses', 'ions', 'erai', 'eras', 'erez',
+ '\xE2mes', '\xE2tes', 'ante', 'ants',
+ 'asse', '\xE9es', 'era', 'iez', 'ais',
+ 'ait', 'ant', '\xE9e', '\xE9s', 'er',
+ 'ez', '\xE2t', 'ai', 'as', '\xE9', 'a')
+ __step4_suffixes = ('i\xE8re', 'I\xE8re', 'ion', 'ier', 'Ier',
+ 'e', '\xEB')
+
+ def stem(self, word):
+ """
+ Stem a French word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ step1_success = False
+ rv_ending_found = False
+ step2a_success = False
+ step2b_success = False
+
+ # Every occurrence of 'u' after 'q' is put into upper case.
+ for i in range(1, len(word)):
+ if word[i-1] == "q" and word[i] == "u":
+ word = "".join((word[:i], "U", word[i+1:]))
+
+ # Every occurrence of 'u' and 'i'
+ # between vowels is put into upper case.
+ # Every occurrence of 'y' preceded or
+ # followed by a vowel is also put into upper case.
+ for i in range(1, len(word)-1):
+ if word[i-1] in self.__vowels and word[i+1] in self.__vowels:
+ if word[i] == "u":
+ word = "".join((word[:i], "U", word[i+1:]))
+
+ elif word[i] == "i":
+ word = "".join((word[:i], "I", word[i+1:]))
+
+ if word[i-1] in self.__vowels or word[i+1] in self.__vowels:
+ if word[i] == "y":
+ word = "".join((word[:i], "Y", word[i+1:]))
+
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+ rv = self.__rv_french(word, self.__vowels)
+
+ # STEP 1: Standard suffix removal
+ for suffix in self.__step1_suffixes:
+ if word.endswith(suffix):
+ if suffix == "eaux":
+ word = word[:-1]
+ step1_success = True
+
+ elif suffix in ("euse", "euses"):
+ if suffix in r2:
+ word = word[:-len(suffix)]
+ step1_success = True
+
+ elif suffix in r1:
+ word = "".join((word[:-len(suffix)], "eux"))
+ step1_success = True
+
+ elif suffix in ("ement", "ements") and suffix in rv:
+ word = word[:-len(suffix)]
+ step1_success = True
+
+ if word[-2:] == "iv" and "iv" in r2:
+ word = word[:-2]
+
+ if word[-2:] == "at" and "at" in r2:
+ word = word[:-2]
+
+ elif word[-3:] == "eus":
+ if "eus" in r2:
+ word = word[:-3]
+ elif "eus" in r1:
+ word = "".join((word[:-1], "x"))
+
+ elif word[-3:] in ("abl", "iqU"):
+ if "abl" in r2 or "iqU" in r2:
+ word = word[:-3]
+
+ elif word[-3:] in ("i\xE8r", "I\xE8r"):
+ if "i\xE8r" in rv or "I\xE8r" in rv:
+ word = "".join((word[:-3], "i"))
+
+ elif suffix == "amment" and suffix in rv:
+ word = "".join((word[:-6], "ant"))
+ rv = "".join((rv[:-6], "ant"))
+ rv_ending_found = True
+
+ elif suffix == "emment" and suffix in rv:
+ word = "".join((word[:-6], "ent"))
+ rv_ending_found = True
+
+ elif (suffix in ("ment", "ments") and suffix in rv and
+ not rv.startswith(suffix) and
+ rv[rv.rindex(suffix)-1] in self.__vowels):
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ rv_ending_found = True
+
+ elif suffix == "aux" and suffix in r1:
+ word = "".join((word[:-2], "l"))
+ step1_success = True
+
+ elif (suffix in ("issement", "issements") and suffix in r1
+ and word[-len(suffix)-1] not in self.__vowels):
+ word = word[:-len(suffix)]
+ step1_success = True
+
+ elif suffix in ("ance", "iqUe", "isme", "able", "iste",
+ "eux", "ances", "iqUes", "ismes",
+ "ables", "istes") and suffix in r2:
+ word = word[:-len(suffix)]
+ step1_success = True
+
+ elif suffix in ("atrice", "ateur", "ation", "atrices",
+ "ateurs", "ations") and suffix in r2:
+ word = word[:-len(suffix)]
+ step1_success = True
+
+ if word[-2:] == "ic":
+ if "ic" in r2:
+ word = word[:-2]
+ else:
+ word = "".join((word[:-2], "iqU"))
+
+ elif suffix in ("logie", "logies") and suffix in r2:
+ word = "".join((word[:-len(suffix)], "log"))
+ step1_success = True
+
+ elif (suffix in ("usion", "ution", "usions", "utions") and
+ suffix in r2):
+ word = "".join((word[:-len(suffix)], "u"))
+ step1_success = True
+
+ elif suffix in ("ence", "ences") and suffix in r2:
+ word = "".join((word[:-len(suffix)], "ent"))
+ step1_success = True
+
+ elif suffix in ("it\xE9", "it\xE9s") and suffix in r2:
+ word = word[:-len(suffix)]
+ step1_success = True
+
+ if word[-4:] == "abil":
+ if "abil" in r2:
+ word = word[:-4]
+ else:
+ word = "".join((word[:-2], "l"))
+
+ elif word[-2:] == "ic":
+ if "ic" in r2:
+ word = word[:-2]
+ else:
+ word = "".join((word[:-2], "iqU"))
+
+ elif word[-2:] == "iv":
+ if "iv" in r2:
+ word = word[:-2]
+
+ elif (suffix in ("if", "ive", "ifs", "ives") and
+ suffix in r2):
+ word = word[:-len(suffix)]
+ step1_success = True
+
+ if word[-2:] == "at" and "at" in r2:
+ word = word[:-2]
+
+ if word[-2:] == "ic":
+ if "ic" in r2:
+ word = word[:-2]
+ else:
+ word = "".join((word[:-2], "iqU"))
+ break
+
+ # STEP 2a: Verb suffixes beginning 'i'
+ if not step1_success or rv_ending_found:
+ for suffix in self.__step2a_suffixes:
+ if word.endswith(suffix):
+ if (suffix in rv and len(rv) > len(suffix) and
+ rv[rv.rindex(suffix)-1] not in self.__vowels):
+ word = word[:-len(suffix)]
+ step2a_success = True
+ break
+
+ # STEP 2b: Other verb suffixes
+ if not step2a_success:
+ for suffix in self.__step2b_suffixes:
+ if rv.endswith(suffix):
+ if suffix == "ions" and "ions" in r2:
+ word = word[:-4]
+ step2b_success = True
+
+ elif suffix in ('eraIent', 'erions', '\xE8rent',
+ 'erais', 'erait', 'eriez',
+ 'erons', 'eront', 'erai', 'eras',
+ 'erez', '\xE9es', 'era', 'iez',
+ '\xE9e', '\xE9s', 'er', 'ez',
+ '\xE9'):
+ word = word[:-len(suffix)]
+ step2b_success = True
+
+ elif suffix in ('assions', 'assent', 'assiez',
+ 'aIent', 'antes', 'asses',
+ '\xE2mes', '\xE2tes', 'ante',
+ 'ants', 'asse', 'ais', 'ait',
+ 'ant', '\xE2t', 'ai', 'as',
+ 'a'):
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ step2b_success = True
+ if rv.endswith("e"):
+ word = word[:-1]
+ break
+
+ # STEP 3
+ if step1_success or step2a_success or step2b_success:
+ if word[-1] == "Y":
+ word = "".join((word[:-1], "i"))
+ elif word[-1] == "\xE7":
+ word = "".join((word[:-1], "c"))
+
+ # STEP 4: Residual suffixes
+ else:
+ if (len(word) >= 2 and word[-1] == "s" and
+ word[-2] not in "aiou\xE8s"):
+ word = word[:-1]
+
+ for suffix in self.__step4_suffixes:
+ if word.endswith(suffix):
+ if suffix in rv:
+ if (suffix == "ion" and suffix in r2 and
+ rv[-4] in "st"):
+ word = word[:-3]
+
+ elif suffix in ("ier", "i\xE8re", "Ier",
+ "I\xE8re"):
+ word = "".join((word[:-len(suffix)], "i"))
+
+ elif suffix == "e":
+ word = word[:-1]
+
+ elif suffix == "\xEB" and word[-3:-1] == "gu":
+ word = word[:-1]
+ break
+
+ # STEP 5: Undouble
+ if word.endswith(("enn", "onn", "ett", "ell", "eill")):
+ word = word[:-1]
+
+ # STEP 6: Un-accent
+ for i in range(1, len(word)):
+ if word[-i] not in self.__vowels:
+ i += 1
+ else:
+ if i != 1 and word[-i] in ("\xE9", "\xE8"):
+ word = "".join((word[:-i], "e", word[-i+1:]))
+ break
+
+ word = (word.replace("I", "i")
+ .replace("U", "u")
+ .replace("Y", "y"))
+
+
+ return word
+
+
+
+ def __rv_french(self, word, vowels):
+ """
+ Return the region RV that is used by the French stemmer.
+
+ If the word begins with two vowels, RV is the region after
+ the third letter. Otherwise, it is the region after the first
+ vowel not at the beginning of the word, or the end of the word
+ if these positions cannot be found. (Exceptionally, u'par',
+ u'col' or u'tap' at the beginning of a word is also taken to
+ define RV as the region to their right.)
+
+ :param word: The French word whose region RV is determined.
+ :type word: str or unicode
+ :param vowels: The French vowels that are used to determine
+ the region RV.
+ :type vowels: unicode
+ :return: the region RV for the respective French word.
+ :rtype: unicode
+ :note: This helper method is invoked by the stem method of
+ the subclass FrenchStemmer. It is not to be invoked directly!
+
+ """
+ rv = ""
+ if len(word) >= 2:
+ if (word.startswith(("par", "col", "tap")) or
+ (word[0] in vowels and word[1] in vowels)):
+ rv = word[3:]
+ else:
+ for i in range(1, len(word)):
+ if word[i] in vowels:
+ rv = word[i+1:]
+ break
+
+ return rv
+
+
+
+class GermanStemmer(_StandardStemmer):
+
+ """
+ The German Snowball stemmer.
+
+ :cvar __vowels: The German vowels.
+ :type __vowels: unicode
+ :cvar __s_ending: Letters that may directly appear before a word final 's'.
+ :type __s_ending: unicode
+ :cvar __st_ending: Letter that may directly appear before a word final 'st'.
+ :type __st_ending: unicode
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :note: A detailed description of the German
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/german/stemmer.html
+
+ """
+
+ __vowels = "aeiouy\xE4\xF6\xFC"
+ __s_ending = "bdfghklmnrt"
+ __st_ending = "bdfghklmnt"
+
+ __step1_suffixes = ("ern", "em", "er", "en", "es", "e", "s")
+ __step2_suffixes = ("est", "en", "er", "st")
+ __step3_suffixes = ("isch", "lich", "heit", "keit",
+ "end", "ung", "ig", "ik")
+
+ def stem(self, word):
+ """
+ Stem a German word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ word = word.replace("\xDF", "ss")
+
+ # Every occurrence of 'u' and 'y'
+ # between vowels is put into upper case.
+ for i in range(1, len(word)-1):
+ if word[i-1] in self.__vowels and word[i+1] in self.__vowels:
+ if word[i] == "u":
+ word = "".join((word[:i], "U", word[i+1:]))
+
+ elif word[i] == "y":
+ word = "".join((word[:i], "Y", word[i+1:]))
+
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+
+ # R1 is adjusted so that the region before it
+ # contains at least 3 letters.
+ for i in range(1, len(word)):
+ if word[i] not in self.__vowels and word[i-1] in self.__vowels:
+ if len(word[:i+1]) < 3 and len(word[:i+1]) > 0:
+ r1 = word[3:]
+ elif len(word[:i+1]) == 0:
+ return word
+ break
+
+ # STEP 1
+ for suffix in self.__step1_suffixes:
+ if r1.endswith(suffix):
+ if (suffix in ("en", "es", "e") and
+ word[-len(suffix)-4:-len(suffix)] == "niss"):
+ word = word[:-len(suffix)-1]
+ r1 = r1[:-len(suffix)-1]
+ r2 = r2[:-len(suffix)-1]
+
+ elif suffix == "s":
+ if word[-2] in self.__s_ending:
+ word = word[:-1]
+ r1 = r1[:-1]
+ r2 = r2[:-1]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ break
+
+ # STEP 2
+ for suffix in self.__step2_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "st":
+ if word[-3] in self.__st_ending and len(word[:-3]) >= 3:
+ word = word[:-2]
+ r1 = r1[:-2]
+ r2 = r2[:-2]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ break
+
+ # STEP 3: Derivational suffixes
+ for suffix in self.__step3_suffixes:
+ if r2.endswith(suffix):
+ if suffix in ("end", "ung"):
+ if ("ig" in r2[-len(suffix)-2:-len(suffix)] and
+ "e" not in r2[-len(suffix)-3:-len(suffix)-2]):
+ word = word[:-len(suffix)-2]
+ else:
+ word = word[:-len(suffix)]
+
+ elif (suffix in ("ig", "ik", "isch") and
+ "e" not in r2[-len(suffix)-1:-len(suffix)]):
+ word = word[:-len(suffix)]
+
+ elif suffix in ("lich", "heit"):
+ if ("er" in r1[-len(suffix)-2:-len(suffix)] or
+ "en" in r1[-len(suffix)-2:-len(suffix)]):
+ word = word[:-len(suffix)-2]
+ else:
+ word = word[:-len(suffix)]
+
+ elif suffix == "keit":
+ if "lich" in r2[-len(suffix)-4:-len(suffix)]:
+ word = word[:-len(suffix)-4]
+
+ elif "ig" in r2[-len(suffix)-2:-len(suffix)]:
+ word = word[:-len(suffix)-2]
+ else:
+ word = word[:-len(suffix)]
+ break
+
+ # Umlaut accents are removed and
+ # 'u' and 'y' are put back into lower case.
+ word = (word.replace("\xE4", "a").replace("\xF6", "o")
+ .replace("\xFC", "u").replace("U", "u")
+ .replace("Y", "y"))
+
+
+ return word
+
+
+
+class HungarianStemmer(_LanguageSpecificStemmer):
+
+ """
+ The Hungarian Snowball stemmer.
+
+ :cvar __vowels: The Hungarian vowels.
+ :type __vowels: unicode
+ :cvar __digraphs: The Hungarian digraphs.
+ :type __digraphs: tuple
+ :cvar __double_consonants: The Hungarian double consonants.
+ :type __double_consonants: tuple
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :cvar __step4_suffixes: Suffixes to be deleted in step 4 of the algorithm.
+ :type __step4_suffixes: tuple
+ :cvar __step5_suffixes: Suffixes to be deleted in step 5 of the algorithm.
+ :type __step5_suffixes: tuple
+ :cvar __step6_suffixes: Suffixes to be deleted in step 6 of the algorithm.
+ :type __step6_suffixes: tuple
+ :cvar __step7_suffixes: Suffixes to be deleted in step 7 of the algorithm.
+ :type __step7_suffixes: tuple
+ :cvar __step8_suffixes: Suffixes to be deleted in step 8 of the algorithm.
+ :type __step8_suffixes: tuple
+ :cvar __step9_suffixes: Suffixes to be deleted in step 9 of the algorithm.
+ :type __step9_suffixes: tuple
+ :note: A detailed description of the Hungarian
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/hungarian/stemmer.html
+
+ """
+
+ __vowels = "aeiou\xF6\xFC\xE1\xE9\xED\xF3\xF5\xFA\xFB"
+ __digraphs = ("cs", "dz", "dzs", "gy", "ly", "ny", "ty", "zs")
+ __double_consonants = ("bb", "cc", "ccs", "dd", "ff", "gg",
+ "ggy", "jj", "kk", "ll", "lly", "mm",
+ "nn", "nny", "pp", "rr", "ss", "ssz",
+ "tt", "tty", "vv", "zz", "zzs")
+
+ __step1_suffixes = ("al", "el")
+ __step2_suffixes = ('k\xE9ppen', 'onk\xE9nt', 'enk\xE9nt',
+ 'ank\xE9nt', 'k\xE9pp', 'k\xE9nt', 'ban',
+ 'ben', 'nak', 'nek', 'val', 'vel', 't\xF3l',
+ 't\xF5l', 'r\xF3l', 'r\xF5l', 'b\xF3l',
+ 'b\xF5l', 'hoz', 'hez', 'h\xF6z',
+ 'n\xE1l', 'n\xE9l', '\xE9rt', 'kor',
+ 'ba', 'be', 'ra', 're', 'ig', 'at', 'et',
+ 'ot', '\xF6t', 'ul', '\xFCl', 'v\xE1',
+ 'v\xE9', 'en', 'on', 'an', '\xF6n',
+ 'n', 't')
+ __step3_suffixes = ("\xE1nk\xE9nt", "\xE1n", "\xE9n")
+ __step4_suffixes = ('astul', 'est\xFCl', '\xE1stul',
+ '\xE9st\xFCl', 'stul', 'st\xFCl')
+ __step5_suffixes = ("\xE1", "\xE9")
+ __step6_suffixes = ('ok\xE9', '\xF6k\xE9', 'ak\xE9',
+ 'ek\xE9', '\xE1k\xE9', '\xE1\xE9i',
+ '\xE9k\xE9', '\xE9\xE9i', 'k\xE9',
+ '\xE9i', '\xE9\xE9', '\xE9')
+ __step7_suffixes = ('\xE1juk', '\xE9j\xFCk', '\xFCnk',
+ 'unk', 'juk', 'j\xFCk', '\xE1nk',
+ '\xE9nk', 'nk', 'uk', '\xFCk', 'em',
+ 'om', 'am', 'od', 'ed', 'ad', '\xF6d',
+ 'ja', 'je', '\xE1m', '\xE1d', '\xE9m',
+ '\xE9d', 'm', 'd', 'a', 'e', 'o',
+ '\xE1', '\xE9')
+ __step8_suffixes = ('jaitok', 'jeitek', 'jaink', 'jeink', 'aitok',
+ 'eitek', '\xE1itok', '\xE9itek', 'jaim',
+ 'jeim', 'jaid', 'jeid', 'eink', 'aink',
+ 'itek', 'jeik', 'jaik', '\xE1ink',
+ '\xE9ink', 'aim', 'eim', 'aid', 'eid',
+ 'jai', 'jei', 'ink', 'aik', 'eik',
+ '\xE1im', '\xE1id', '\xE1ik', '\xE9im',
+ '\xE9id', '\xE9ik', 'im', 'id', 'ai',
+ 'ei', 'ik', '\xE1i', '\xE9i', 'i')
+ __step9_suffixes = ("\xE1k", "\xE9k", "\xF6k", "ok",
+ "ek", "ak", "k")
+
+ def stem(self, word):
+ """
+ Stem an Hungarian word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ r1 = self.__r1_hungarian(word, self.__vowels, self.__digraphs)
+
+ # STEP 1: Remove instrumental case
+ if r1.endswith(self.__step1_suffixes):
+ for double_cons in self.__double_consonants:
+ if word[-2-len(double_cons):-2] == double_cons:
+ word = "".join((word[:-4], word[-3]))
+
+ if r1[-2-len(double_cons):-2] == double_cons:
+ r1 = "".join((r1[:-4], r1[-3]))
+ break
+
+ # STEP 2: Remove frequent cases
+ for suffix in self.__step2_suffixes:
+ if word.endswith(suffix):
+ if r1.endswith(suffix):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+
+ if r1.endswith("\xE1"):
+ word = "".join((word[:-1], "a"))
+ r1 = "".join((r1[:-1], "a"))
+
+ elif r1.endswith("\xE9"):
+ word = "".join((word[:-1], "e"))
+ r1 = "".join((r1[:-1], "e"))
+ break
+
+ # STEP 3: Remove special cases
+ for suffix in self.__step3_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "\xE9n":
+ word = "".join((word[:-2], "e"))
+ r1 = "".join((r1[:-2], "e"))
+ else:
+ word = "".join((word[:-len(suffix)], "a"))
+ r1 = "".join((r1[:-len(suffix)], "a"))
+ break
+
+ # STEP 4: Remove other cases
+ for suffix in self.__step4_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "\xE1stul":
+ word = "".join((word[:-5], "a"))
+ r1 = "".join((r1[:-5], "a"))
+
+ elif suffix == "\xE9st\xFCl":
+ word = "".join((word[:-5], "e"))
+ r1 = "".join((r1[:-5], "e"))
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ break
+
+ # STEP 5: Remove factive case
+ for suffix in self.__step5_suffixes:
+ if r1.endswith(suffix):
+ for double_cons in self.__double_consonants:
+ if word[-1-len(double_cons):-1] == double_cons:
+ word = "".join((word[:-3], word[-2]))
+
+ if r1[-1-len(double_cons):-1] == double_cons:
+ r1 = "".join((r1[:-3], r1[-2]))
+ break
+
+ # STEP 6: Remove owned
+ for suffix in self.__step6_suffixes:
+ if r1.endswith(suffix):
+ if suffix in ("\xE1k\xE9", "\xE1\xE9i"):
+ word = "".join((word[:-3], "a"))
+ r1 = "".join((r1[:-3], "a"))
+
+ elif suffix in ("\xE9k\xE9", "\xE9\xE9i",
+ "\xE9\xE9"):
+ word = "".join((word[:-len(suffix)], "e"))
+ r1 = "".join((r1[:-len(suffix)], "e"))
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ break
+
+ # STEP 7: Remove singular owner suffixes
+ for suffix in self.__step7_suffixes:
+ if word.endswith(suffix):
+ if r1.endswith(suffix):
+ if suffix in ("\xE1nk", "\xE1juk", "\xE1m",
+ "\xE1d", "\xE1"):
+ word = "".join((word[:-len(suffix)], "a"))
+ r1 = "".join((r1[:-len(suffix)], "a"))
+
+ elif suffix in ("\xE9nk", "\xE9j\xFCk",
+ "\xE9m", "\xE9d", "\xE9"):
+ word = "".join((word[:-len(suffix)], "e"))
+ r1 = "".join((r1[:-len(suffix)], "e"))
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ break
+
+ # STEP 8: Remove plural owner suffixes
+ for suffix in self.__step8_suffixes:
+ if word.endswith(suffix):
+ if r1.endswith(suffix):
+ if suffix in ("\xE1im", "\xE1id", "\xE1i",
+ "\xE1ink", "\xE1itok", "\xE1ik"):
+ word = "".join((word[:-len(suffix)], "a"))
+ r1 = "".join((r1[:-len(suffix)], "a"))
+
+ elif suffix in ("\xE9im", "\xE9id", "\xE9i",
+ "\xE9ink", "\xE9itek", "\xE9ik"):
+ word = "".join((word[:-len(suffix)], "e"))
+ r1 = "".join((r1[:-len(suffix)], "e"))
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ break
+
+ # STEP 9: Remove plural suffixes
+ for suffix in self.__step9_suffixes:
+ if word.endswith(suffix):
+ if r1.endswith(suffix):
+ if suffix == "\xE1k":
+ word = "".join((word[:-2], "a"))
+ elif suffix == "\xE9k":
+ word = "".join((word[:-2], "e"))
+ else:
+ word = word[:-len(suffix)]
+ break
+
+
+ return word
+
+
+
+ def __r1_hungarian(self, word, vowels, digraphs):
+ """
+ Return the region R1 that is used by the Hungarian stemmer.
+
+ If the word begins with a vowel, R1 is defined as the region
+ after the first consonant or digraph (= two letters stand for
+ one phoneme) in the word. If the word begins with a consonant,
+ it is defined as the region after the first vowel in the word.
+ If the word does not contain both a vowel and consonant, R1
+ is the null region at the end of the word.
+
+ :param word: The Hungarian word whose region R1 is determined.
+ :type word: str or unicode
+ :param vowels: The Hungarian vowels that are used to determine
+ the region R1.
+ :type vowels: unicode
+ :param digraphs: The digraphs that are used to determine the
+ region R1.
+ :type digraphs: tuple
+ :return: the region R1 for the respective word.
+ :rtype: unicode
+ :note: This helper method is invoked by the stem method of the subclass
+ HungarianStemmer. It is not to be invoked directly!
+
+ """
+ r1 = ""
+ if word[0] in vowels:
+ for digraph in digraphs:
+ if digraph in word[1:]:
+ r1 = word[word.index(digraph[-1])+1:]
+ return r1
+
+ for i in range(1, len(word)):
+ if word[i] not in vowels:
+ r1 = word[i+1:]
+ break
+ else:
+ for i in range(1, len(word)):
+ if word[i] in vowels:
+ r1 = word[i+1:]
+ break
+
+ return r1
+
+
+
+class ItalianStemmer(_StandardStemmer):
+
+ """
+ The Italian Snowball stemmer.
+
+ :cvar __vowels: The Italian vowels.
+ :type __vowels: unicode
+ :cvar __step0_suffixes: Suffixes to be deleted in step 0 of the algorithm.
+ :type __step0_suffixes: tuple
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :note: A detailed description of the Italian
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/italian/stemmer.html
+
+ """
+
+ __vowels = "aeiou\xE0\xE8\xEC\xF2\xF9"
+ __step0_suffixes = ('gliela', 'gliele', 'glieli', 'glielo',
+ 'gliene', 'sene', 'mela', 'mele', 'meli',
+ 'melo', 'mene', 'tela', 'tele', 'teli',
+ 'telo', 'tene', 'cela', 'cele', 'celi',
+ 'celo', 'cene', 'vela', 'vele', 'veli',
+ 'velo', 'vene', 'gli', 'ci', 'la', 'le',
+ 'li', 'lo', 'mi', 'ne', 'si', 'ti', 'vi')
+ __step1_suffixes = ('atrice', 'atrici', 'azione', 'azioni',
+ 'uzione', 'uzioni', 'usione', 'usioni',
+ 'amento', 'amenti', 'imento', 'imenti',
+ 'amente', 'abile', 'abili', 'ibile', 'ibili',
+ 'mente', 'atore', 'atori', 'logia', 'logie',
+ 'anza', 'anze', 'iche', 'ichi', 'ismo',
+ 'ismi', 'ista', 'iste', 'isti', 'ist\xE0',
+ 'ist\xE8', 'ist\xEC', 'ante', 'anti',
+ 'enza', 'enze', 'ico', 'ici', 'ica', 'ice',
+ 'oso', 'osi', 'osa', 'ose', 'it\xE0',
+ 'ivo', 'ivi', 'iva', 'ive')
+ __step2_suffixes = ('erebbero', 'irebbero', 'assero', 'assimo',
+ 'eranno', 'erebbe', 'eremmo', 'ereste',
+ 'eresti', 'essero', 'iranno', 'irebbe',
+ 'iremmo', 'ireste', 'iresti', 'iscano',
+ 'iscono', 'issero', 'arono', 'avamo', 'avano',
+ 'avate', 'eremo', 'erete', 'erono', 'evamo',
+ 'evano', 'evate', 'iremo', 'irete', 'irono',
+ 'ivamo', 'ivano', 'ivate', 'ammo', 'ando',
+ 'asse', 'assi', 'emmo', 'enda', 'ende',
+ 'endi', 'endo', 'erai', 'erei', 'Yamo',
+ 'iamo', 'immo', 'irai', 'irei', 'isca',
+ 'isce', 'isci', 'isco', 'ano', 'are', 'ata',
+ 'ate', 'ati', 'ato', 'ava', 'avi', 'avo',
+ 'er\xE0', 'ere', 'er\xF2', 'ete', 'eva',
+ 'evi', 'evo', 'ir\xE0', 'ire', 'ir\xF2',
+ 'ita', 'ite', 'iti', 'ito', 'iva', 'ivi',
+ 'ivo', 'ono', 'uta', 'ute', 'uti', 'uto',
+ 'ar', 'ir')
+
+ def stem(self, word):
+ """
+ Stem an Italian word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ step1_success = False
+
+ # All acute accents are replaced by grave accents.
+ word = (word.replace("\xE1", "\xE0")
+ .replace("\xE9", "\xE8")
+ .replace("\xED", "\xEC")
+ .replace("\xF3", "\xF2")
+ .replace("\xFA", "\xF9"))
+
+ # Every occurrence of 'u' after 'q'
+ # is put into upper case.
+ for i in range(1, len(word)):
+ if word[i-1] == "q" and word[i] == "u":
+ word = "".join((word[:i], "U", word[i+1:]))
+
+ # Every occurrence of 'u' and 'i'
+ # between vowels is put into upper case.
+ for i in range(1, len(word)-1):
+ if word[i-1] in self.__vowels and word[i+1] in self.__vowels:
+ if word[i] == "u":
+ word = "".join((word[:i], "U", word[i+1:]))
+
+ elif word [i] == "i":
+ word = "".join((word[:i], "I", word[i+1:]))
+
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+ rv = self._rv_standard(word, self.__vowels)
+
+ # STEP 0: Attached pronoun
+ for suffix in self.__step0_suffixes:
+ if rv.endswith(suffix):
+ if rv[-len(suffix)-4:-len(suffix)] in ("ando", "endo"):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ elif (rv[-len(suffix)-2:-len(suffix)] in
+ ("ar", "er", "ir")):
+ word = "".join((word[:-len(suffix)], "e"))
+ r1 = "".join((r1[:-len(suffix)], "e"))
+ r2 = "".join((r2[:-len(suffix)], "e"))
+ rv = "".join((rv[:-len(suffix)], "e"))
+ break
+
+ # STEP 1: Standard suffix removal
+ for suffix in self.__step1_suffixes:
+ if word.endswith(suffix):
+ if suffix == "amente" and r1.endswith(suffix):
+ step1_success = True
+ word = word[:-6]
+ r2 = r2[:-6]
+ rv = rv[:-6]
+
+ if r2.endswith("iv"):
+ word = word[:-2]
+ r2 = r2[:-2]
+ rv = rv[:-2]
+
+ if r2.endswith("at"):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif r2.endswith(("os", "ic")):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif r2 .endswith("abil"):
+ word = word[:-4]
+ rv = rv[:-4]
+
+ elif (suffix in ("amento", "amenti",
+ "imento", "imenti") and
+ rv.endswith(suffix)):
+ step1_success = True
+ word = word[:-6]
+ rv = rv[:-6]
+
+ elif r2.endswith(suffix):
+ step1_success = True
+ if suffix in ("azione", "azioni", "atore", "atori"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ if r2.endswith("ic"):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif suffix in ("logia", "logie"):
+ word = word[:-2]
+ rv = word[:-2]
+
+ elif suffix in ("uzione", "uzioni",
+ "usione", "usioni"):
+ word = word[:-5]
+ rv = rv[:-5]
+
+ elif suffix in ("enza", "enze"):
+ word = "".join((word[:-2], "te"))
+ rv = "".join((rv[:-2], "te"))
+
+ elif suffix == "it\xE0":
+ word = word[:-3]
+ r2 = r2[:-3]
+ rv = rv[:-3]
+
+ if r2.endswith(("ic", "iv")):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif r2.endswith("abil"):
+ word = word[:-4]
+ rv = rv[:-4]
+
+ elif suffix in ("ivo", "ivi", "iva", "ive"):
+ word = word[:-3]
+ r2 = r2[:-3]
+ rv = rv[:-3]
+
+ if r2.endswith("at"):
+ word = word[:-2]
+ r2 = r2[:-2]
+ rv = rv[:-2]
+
+ if r2.endswith("ic"):
+ word = word[:-2]
+ rv = rv[:-2]
+ else:
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 2: Verb suffixes
+ if not step1_success:
+ for suffix in self.__step2_suffixes:
+ if rv.endswith(suffix):
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 3a
+ if rv.endswith(("a", "e", "i", "o", "\xE0", "\xE8",
+ "\xEC", "\xF2")):
+ word = word[:-1]
+ rv = rv[:-1]
+
+ if rv.endswith("i"):
+ word = word[:-1]
+ rv = rv[:-1]
+
+ # STEP 3b
+ if rv.endswith(("ch", "gh")):
+ word = word[:-1]
+
+ word = word.replace("I", "i").replace("U", "u")
+
+
+ return word
+
+
+
+class NorwegianStemmer(_ScandinavianStemmer):
+
+ """
+ The Norwegian Snowball stemmer.
+
+ :cvar __vowels: The Norwegian vowels.
+ :type __vowels: unicode
+ :cvar __s_ending: Letters that may directly appear before a word final 's'.
+ :type __s_ending: unicode
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :note: A detailed description of the Norwegian
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/norwegian/stemmer.html
+
+ """
+
+ __vowels = "aeiouy\xE6\xE5\xF8"
+ __s_ending = "bcdfghjlmnoprtvyz"
+ __step1_suffixes = ("hetenes", "hetene", "hetens", "heter",
+ "heten", "endes", "ande", "ende", "edes",
+ "enes", "erte", "ede", "ane", "ene", "ens",
+ "ers", "ets", "het", "ast", "ert", "en",
+ "ar", "er", "as", "es", "et", "a", "e", "s")
+
+ __step2_suffixes = ("dt", "vt")
+
+ __step3_suffixes = ("hetslov", "eleg", "elig", "elov", "slov",
+ "leg", "eig", "lig", "els", "lov", "ig")
+
+ def stem(self, word):
+ """
+ Stem a Norwegian word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ r1 = self._r1_scandinavian(word, self.__vowels)
+
+ # STEP 1
+ for suffix in self.__step1_suffixes:
+ if r1.endswith(suffix):
+ if suffix in ("erte", "ert"):
+ word = "".join((word[:-len(suffix)], "er"))
+ r1 = "".join((r1[:-len(suffix)], "er"))
+
+ elif suffix == "s":
+ if (word[-2] in self.__s_ending or
+ (word[-2] == "k" and word[-3] not in self.__vowels)):
+ word = word[:-1]
+ r1 = r1[:-1]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ break
+
+ # STEP 2
+ for suffix in self.__step2_suffixes:
+ if r1.endswith(suffix):
+ word = word[:-1]
+ r1 = r1[:-1]
+ break
+
+ # STEP 3
+ for suffix in self.__step3_suffixes:
+ if r1.endswith(suffix):
+ word = word[:-len(suffix)]
+ break
+
+
+ return word
+
+
+
+class PortugueseStemmer(_StandardStemmer):
+
+ """
+ The Portuguese Snowball stemmer.
+
+ :cvar __vowels: The Portuguese vowels.
+ :type __vowels: unicode
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step4_suffixes: Suffixes to be deleted in step 4 of the algorithm.
+ :type __step4_suffixes: tuple
+ :note: A detailed description of the Portuguese
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/portuguese/stemmer.html
+
+ """
+
+ __vowels = "aeiou\xE1\xE9\xED\xF3\xFA\xE2\xEA\xF4"
+ __step1_suffixes = ('amentos', 'imentos', 'uciones', 'amento',
+ 'imento', 'adoras', 'adores', 'a\xE7o~es',
+ 'log\xEDas', '\xEAncias', 'amente',
+ 'idades', 'ismos', 'istas', 'adora',
+ 'a\xE7a~o', 'antes', '\xE2ncia',
+ 'log\xEDa', 'uci\xF3n', '\xEAncia',
+ 'mente', 'idade', 'ezas', 'icos', 'icas',
+ 'ismo', '\xE1vel', '\xEDvel', 'ista',
+ 'osos', 'osas', 'ador', 'ante', 'ivas',
+ 'ivos', 'iras', 'eza', 'ico', 'ica',
+ 'oso', 'osa', 'iva', 'ivo', 'ira')
+ __step2_suffixes = ('ar\xEDamos', 'er\xEDamos', 'ir\xEDamos',
+ '\xE1ssemos', '\xEAssemos', '\xEDssemos',
+ 'ar\xEDeis', 'er\xEDeis', 'ir\xEDeis',
+ '\xE1sseis', '\xE9sseis', '\xEDsseis',
+ '\xE1ramos', '\xE9ramos', '\xEDramos',
+ '\xE1vamos', 'aremos', 'eremos', 'iremos',
+ 'ariam', 'eriam', 'iriam', 'assem', 'essem',
+ 'issem', 'ara~o', 'era~o', 'ira~o', 'arias',
+ 'erias', 'irias', 'ardes', 'erdes', 'irdes',
+ 'asses', 'esses', 'isses', 'astes', 'estes',
+ 'istes', '\xE1reis', 'areis', '\xE9reis',
+ 'ereis', '\xEDreis', 'ireis', '\xE1veis',
+ '\xEDamos', 'armos', 'ermos', 'irmos',
+ 'aria', 'eria', 'iria', 'asse', 'esse',
+ 'isse', 'aste', 'este', 'iste', 'arei',
+ 'erei', 'irei', 'aram', 'eram', 'iram',
+ 'avam', 'arem', 'erem', 'irem',
+ 'ando', 'endo', 'indo', 'adas', 'idas',
+ 'ar\xE1s', 'aras', 'er\xE1s', 'eras',
+ 'ir\xE1s', 'avas', 'ares', 'eres', 'ires',
+ '\xEDeis', 'ados', 'idos', '\xE1mos',
+ 'amos', 'emos', 'imos', 'iras', 'ada', 'ida',
+ 'ar\xE1', 'ara', 'er\xE1', 'era',
+ 'ir\xE1', 'ava', 'iam', 'ado', 'ido',
+ 'ias', 'ais', 'eis', 'ira', 'ia', 'ei', 'am',
+ 'em', 'ar', 'er', 'ir', 'as',
+ 'es', 'is', 'eu', 'iu', 'ou')
+ __step4_suffixes = ("os", "a", "i", "o", "\xE1",
+ "\xED", "\xF3")
+
+ def stem(self, word):
+ """
+ Stem a Portuguese word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ step1_success = False
+ step2_success = False
+
+ word = (word.replace("\xE3", "a~")
+ .replace("\xF5", "o~"))
+
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+ rv = self._rv_standard(word, self.__vowels)
+
+ # STEP 1: Standard suffix removal
+ for suffix in self.__step1_suffixes:
+ if word.endswith(suffix):
+ if suffix == "amente" and r1.endswith(suffix):
+ step1_success = True
+
+ word = word[:-6]
+ r2 = r2[:-6]
+ rv = rv[:-6]
+
+ if r2.endswith("iv"):
+ word = word[:-2]
+ r2 = r2[:-2]
+ rv = rv[:-2]
+
+ if r2.endswith("at"):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif r2.endswith(("os", "ic", "ad")):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif (suffix in ("ira", "iras") and rv.endswith(suffix) and
+ word[-len(suffix)-1:-len(suffix)] == "e"):
+ step1_success = True
+
+ word = "".join((word[:-len(suffix)], "ir"))
+ rv = "".join((rv[:-len(suffix)], "ir"))
+
+ elif r2.endswith(suffix):
+ step1_success = True
+
+ if suffix in ("log\xEDa", "log\xEDas"):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif suffix in ("uci\xF3n", "uciones"):
+ word = "".join((word[:-len(suffix)], "u"))
+ rv = "".join((rv[:-len(suffix)], "u"))
+
+ elif suffix in ("\xEAncia", "\xEAncias"):
+ word = "".join((word[:-len(suffix)], "ente"))
+ rv = "".join((rv[:-len(suffix)], "ente"))
+
+ elif suffix == "mente":
+ word = word[:-5]
+ r2 = r2[:-5]
+ rv = rv[:-5]
+
+ if r2.endswith(("ante", "avel", "\xEDvel")):
+ word = word[:-4]
+ rv = rv[:-4]
+
+ elif suffix in ("idade", "idades"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ if r2.endswith(("ic", "iv")):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif r2.endswith("abil"):
+ word = word[:-4]
+ rv = rv[:-4]
+
+ elif suffix in ("iva", "ivo", "ivas", "ivos"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ if r2.endswith("at"):
+ word = word[:-2]
+ rv = rv[:-2]
+ else:
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 2: Verb suffixes
+ if not step1_success:
+ for suffix in self.__step2_suffixes:
+ if rv.endswith(suffix):
+ step2_success = True
+
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 3
+ if step1_success or step2_success:
+ if rv.endswith("i") and word[-2] == "c":
+ word = word[:-1]
+ rv = rv[:-1]
+
+ ### STEP 4: Residual suffix
+ if not step1_success and not step2_success:
+ for suffix in self.__step4_suffixes:
+ if rv.endswith(suffix):
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 5
+ if rv.endswith(("e", "\xE9", "\xEA")):
+ word = word[:-1]
+ rv = rv[:-1]
+
+ if ((word.endswith("gu") and rv.endswith("u")) or
+ (word.endswith("ci") and rv.endswith("i"))):
+ word = word[:-1]
+
+ elif word.endswith("\xE7"):
+ word = "".join((word[:-1], "c"))
+
+ word = word.replace("a~", "\xE3").replace("o~", "\xF5")
+
+
+ return word
+
+
+
+class RomanianStemmer(_StandardStemmer):
+
+ """
+ The Romanian Snowball stemmer.
+
+ :cvar __vowels: The Romanian vowels.
+ :type __vowels: unicode
+ :cvar __step0_suffixes: Suffixes to be deleted in step 0 of the algorithm.
+ :type __step0_suffixes: tuple
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :note: A detailed description of the Romanian
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/romanian/stemmer.html
+
+ """
+
+ __vowels = "aeiou\u0103\xE2\xEE"
+ __step0_suffixes = ('iilor', 'ului', 'elor', 'iile', 'ilor',
+ 'atei', 'a\u0163ie', 'a\u0163ia', 'aua',
+ 'ele', 'iua', 'iei', 'ile', 'ul', 'ea',
+ 'ii')
+ __step1_suffixes = ('abilitate', 'abilitati', 'abilit\u0103\u0163i',
+ 'ibilitate', 'abilit\u0103i', 'ivitate',
+ 'ivitati', 'ivit\u0103\u0163i', 'icitate',
+ 'icitati', 'icit\u0103\u0163i', 'icatori',
+ 'ivit\u0103i', 'icit\u0103i', 'icator',
+ 'a\u0163iune', 'atoare', '\u0103toare',
+ 'i\u0163iune', 'itoare', 'iciva', 'icive',
+ 'icivi', 'iciv\u0103', 'icala', 'icale',
+ 'icali', 'ical\u0103', 'ativa', 'ative',
+ 'ativi', 'ativ\u0103', 'atori', '\u0103tori',
+ 'itiva', 'itive', 'itivi', 'itiv\u0103',
+ 'itori', 'iciv', 'ical', 'ativ', 'ator',
+ '\u0103tor', 'itiv', 'itor')
+ __step2_suffixes = ('abila', 'abile', 'abili', 'abil\u0103',
+ 'ibila', 'ibile', 'ibili', 'ibil\u0103',
+ 'atori', 'itate', 'itati', 'it\u0103\u0163i',
+ 'abil', 'ibil', 'oasa', 'oas\u0103', 'oase',
+ 'anta', 'ante', 'anti', 'ant\u0103', 'ator',
+ 'it\u0103i', 'iune', 'iuni', 'isme', 'ista',
+ 'iste', 'isti', 'ist\u0103', 'i\u015Fti',
+ 'ata', 'at\u0103', 'ati', 'ate', 'uta',
+ 'ut\u0103', 'uti', 'ute', 'ita', 'it\u0103',
+ 'iti', 'ite', 'ica', 'ice', 'ici', 'ic\u0103',
+ 'osi', 'o\u015Fi', 'ant', 'iva', 'ive', 'ivi',
+ 'iv\u0103', 'ism', 'ist', 'at', 'ut', 'it',
+ 'ic', 'os', 'iv')
+ __step3_suffixes = ('seser\u0103\u0163i', 'aser\u0103\u0163i',
+ 'iser\u0103\u0163i', '\xE2ser\u0103\u0163i',
+ 'user\u0103\u0163i', 'seser\u0103m',
+ 'aser\u0103m', 'iser\u0103m', '\xE2ser\u0103m',
+ 'user\u0103m', 'ser\u0103\u0163i', 'sese\u015Fi',
+ 'seser\u0103', 'easc\u0103', 'ar\u0103\u0163i',
+ 'ur\u0103\u0163i', 'ir\u0103\u0163i',
+ '\xE2r\u0103\u0163i', 'ase\u015Fi',
+ 'aser\u0103', 'ise\u015Fi', 'iser\u0103',
+ '\xe2se\u015Fi', '\xE2ser\u0103',
+ 'use\u015Fi', 'user\u0103', 'ser\u0103m',
+ 'sesem', 'indu', '\xE2ndu', 'eaz\u0103',
+ 'e\u015Fti', 'e\u015Fte', '\u0103\u015Fti',
+ '\u0103\u015Fte', 'ea\u0163i', 'ia\u0163i',
+ 'ar\u0103m', 'ur\u0103m', 'ir\u0103m',
+ '\xE2r\u0103m', 'asem', 'isem',
+ '\xE2sem', 'usem', 'se\u015Fi', 'ser\u0103',
+ 'sese', 'are', 'ere', 'ire', '\xE2re',
+ 'ind', '\xE2nd', 'eze', 'ezi', 'esc',
+ '\u0103sc', 'eam', 'eai', 'eau', 'iam',
+ 'iai', 'iau', 'a\u015Fi', 'ar\u0103',
+ 'u\u015Fi', 'ur\u0103', 'i\u015Fi', 'ir\u0103',
+ '\xE2\u015Fi', '\xe2r\u0103', 'ase',
+ 'ise', '\xE2se', 'use', 'a\u0163i',
+ 'e\u0163i', 'i\u0163i', '\xe2\u0163i', 'sei',
+ 'ez', 'am', 'ai', 'au', 'ea', 'ia', 'ui',
+ '\xE2i', '\u0103m', 'em', 'im', '\xE2m',
+ 'se')
+
+ def stem(self, word):
+ """
+ Stem a Romanian word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ step1_success = False
+ step2_success = False
+
+ for i in range(1, len(word)-1):
+ if word[i-1] in self.__vowels and word[i+1] in self.__vowels:
+ if word[i] == "u":
+ word = "".join((word[:i], "U", word[i+1:]))
+
+ elif word[i] == "i":
+ word = "".join((word[:i], "I", word[i+1:]))
+
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+ rv = self._rv_standard(word, self.__vowels)
+
+ # STEP 0: Removal of plurals and other simplifications
+ for suffix in self.__step0_suffixes:
+ if word.endswith(suffix):
+ if suffix in r1:
+ if suffix in ("ul", "ului"):
+ word = word[:-len(suffix)]
+
+ if suffix in rv:
+ rv = rv[:-len(suffix)]
+ else:
+ rv = ""
+
+ elif (suffix == "aua" or suffix == "atei" or
+ (suffix == "ile" and word[-5:-3] != "ab")):
+ word = word[:-2]
+
+ elif suffix in ("ea", "ele", "elor"):
+ word = "".join((word[:-len(suffix)], "e"))
+
+ if suffix in rv:
+ rv = "".join((rv[:-len(suffix)], "e"))
+ else:
+ rv = ""
+
+ elif suffix in ("ii", "iua", "iei",
+ "iile", "iilor", "ilor"):
+ word = "".join((word[:-len(suffix)], "i"))
+
+ if suffix in rv:
+ rv = "".join((rv[:-len(suffix)], "i"))
+ else:
+ rv = ""
+
+ elif suffix in ("a\u0163ie", "a\u0163ia"):
+ word = word[:-1]
+ break
+
+ # STEP 1: Reduction of combining suffixes
+ while True:
+
+ replacement_done = False
+
+ for suffix in self.__step1_suffixes:
+ if word.endswith(suffix):
+ if suffix in r1:
+ step1_success = True
+ replacement_done = True
+
+ if suffix in ("abilitate", "abilitati",
+ "abilit\u0103i",
+ "abilit\u0103\u0163i"):
+ word = "".join((word[:-len(suffix)], "abil"))
+
+ elif suffix == "ibilitate":
+ word = word[:-5]
+
+ elif suffix in ("ivitate", "ivitati",
+ "ivit\u0103i",
+ "ivit\u0103\u0163i"):
+ word = "".join((word[:-len(suffix)], "iv"))
+
+ elif suffix in ("icitate", "icitati", "icit\u0103i",
+ "icit\u0103\u0163i", "icator",
+ "icatori", "iciv", "iciva",
+ "icive", "icivi", "iciv\u0103",
+ "ical", "icala", "icale", "icali",
+ "ical\u0103"):
+ word = "".join((word[:-len(suffix)], "ic"))
+
+ elif suffix in ("ativ", "ativa", "ative", "ativi",
+ "ativ\u0103", "a\u0163iune",
+ "atoare", "ator", "atori",
+ "\u0103toare",
+ "\u0103tor", "\u0103tori"):
+ word = "".join((word[:-len(suffix)], "at"))
+
+ if suffix in r2:
+ r2 = "".join((r2[:-len(suffix)], "at"))
+
+ elif suffix in ("itiv", "itiva", "itive", "itivi",
+ "itiv\u0103", "i\u0163iune",
+ "itoare", "itor", "itori"):
+ word = "".join((word[:-len(suffix)], "it"))
+
+ if suffix in r2:
+ r2 = "".join((r2[:-len(suffix)], "it"))
+ else:
+ step1_success = False
+ break
+
+ if not replacement_done:
+ break
+
+ # STEP 2: Removal of standard suffixes
+ for suffix in self.__step2_suffixes:
+ if word.endswith(suffix):
+ if suffix in r2:
+ step2_success = True
+
+ if suffix in ("iune", "iuni"):
+ if word[-5] == "\u0163":
+ word = "".join((word[:-5], "t"))
+
+ elif suffix in ("ism", "isme", "ist", "ista", "iste",
+ "isti", "ist\u0103", "i\u015Fti"):
+ word = "".join((word[:-len(suffix)], "ist"))
+
+ else:
+ word = word[:-len(suffix)]
+ break
+
+ # STEP 3: Removal of verb suffixes
+ if not step1_success and not step2_success:
+ for suffix in self.__step3_suffixes:
+ if word.endswith(suffix):
+ if suffix in rv:
+ if suffix in ('seser\u0103\u0163i', 'seser\u0103m',
+ 'ser\u0103\u0163i', 'sese\u015Fi',
+ 'seser\u0103', 'ser\u0103m', 'sesem',
+ 'se\u015Fi', 'ser\u0103', 'sese',
+ 'a\u0163i', 'e\u0163i', 'i\u0163i',
+ '\xE2\u0163i', 'sei', '\u0103m',
+ 'em', 'im', '\xE2m', 'se'):
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ else:
+ if (not rv.startswith(suffix) and
+ rv[rv.index(suffix)-1] not in
+ "aeio\u0103\xE2\xEE"):
+ word = word[:-len(suffix)]
+ break
+
+ # STEP 4: Removal of final vowel
+ for suffix in ("ie", "a", "e", "i", "\u0103"):
+ if word.endswith(suffix):
+ if suffix in rv:
+ word = word[:-len(suffix)]
+ break
+
+ word = word.replace("I", "i").replace("U", "u")
+
+
+ return word
+
+
+
+class RussianStemmer(_LanguageSpecificStemmer):
+
+ """
+ The Russian Snowball stemmer.
+
+ :cvar __perfective_gerund_suffixes: Suffixes to be deleted.
+ :type __perfective_gerund_suffixes: tuple
+ :cvar __adjectival_suffixes: Suffixes to be deleted.
+ :type __adjectival_suffixes: tuple
+ :cvar __reflexive_suffixes: Suffixes to be deleted.
+ :type __reflexive_suffixes: tuple
+ :cvar __verb_suffixes: Suffixes to be deleted.
+ :type __verb_suffixes: tuple
+ :cvar __noun_suffixes: Suffixes to be deleted.
+ :type __noun_suffixes: tuple
+ :cvar __superlative_suffixes: Suffixes to be deleted.
+ :type __superlative_suffixes: tuple
+ :cvar __derivational_suffixes: Suffixes to be deleted.
+ :type __derivational_suffixes: tuple
+ :note: A detailed description of the Russian
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/russian/stemmer.html
+
+ """
+
+ __perfective_gerund_suffixes = ("ivshis'", "yvshis'", "vshis'",
+ "ivshi", "yvshi", "vshi", "iv",
+ "yv", "v")
+ __adjectival_suffixes = ('ui^ushchi^ui^u', 'ui^ushchi^ai^a',
+ 'ui^ushchimi', 'ui^ushchymi', 'ui^ushchego',
+ 'ui^ushchogo', 'ui^ushchemu', 'ui^ushchomu',
+ 'ui^ushchikh', 'ui^ushchykh',
+ 'ui^ushchui^u', 'ui^ushchaia',
+ 'ui^ushchoi^u', 'ui^ushchei^u',
+ 'i^ushchi^ui^u', 'i^ushchi^ai^a',
+ 'ui^ushchee', 'ui^ushchie',
+ 'ui^ushchye', 'ui^ushchoe', 'ui^ushchei`',
+ 'ui^ushchii`', 'ui^ushchyi`',
+ 'ui^ushchoi`', 'ui^ushchem', 'ui^ushchim',
+ 'ui^ushchym', 'ui^ushchom', 'i^ushchimi',
+ 'i^ushchymi', 'i^ushchego', 'i^ushchogo',
+ 'i^ushchemu', 'i^ushchomu', 'i^ushchikh',
+ 'i^ushchykh', 'i^ushchui^u', 'i^ushchai^a',
+ 'i^ushchoi^u', 'i^ushchei^u', 'i^ushchee',
+ 'i^ushchie', 'i^ushchye', 'i^ushchoe',
+ 'i^ushchei`', 'i^ushchii`',
+ 'i^ushchyi`', 'i^ushchoi`', 'i^ushchem',
+ 'i^ushchim', 'i^ushchym', 'i^ushchom',
+ 'shchi^ui^u', 'shchi^ai^a', 'ivshi^ui^u',
+ 'ivshi^ai^a', 'yvshi^ui^u', 'yvshi^ai^a',
+ 'shchimi', 'shchymi', 'shchego', 'shchogo',
+ 'shchemu', 'shchomu', 'shchikh', 'shchykh',
+ 'shchui^u', 'shchai^a', 'shchoi^u',
+ 'shchei^u', 'ivshimi', 'ivshymi',
+ 'ivshego', 'ivshogo', 'ivshemu', 'ivshomu',
+ 'ivshikh', 'ivshykh', 'ivshui^u',
+ 'ivshai^a', 'ivshoi^u', 'ivshei^u',
+ 'yvshimi', 'yvshymi', 'yvshego', 'yvshogo',
+ 'yvshemu', 'yvshomu', 'yvshikh', 'yvshykh',
+ 'yvshui^u', 'yvshai^a', 'yvshoi^u',
+ 'yvshei^u', 'vshi^ui^u', 'vshi^ai^a',
+ 'shchee', 'shchie', 'shchye', 'shchoe',
+ 'shchei`', 'shchii`', 'shchyi`', 'shchoi`',
+ 'shchem', 'shchim', 'shchym', 'shchom',
+ 'ivshee', 'ivshie', 'ivshye', 'ivshoe',
+ 'ivshei`', 'ivshii`', 'ivshyi`',
+ 'ivshoi`', 'ivshem', 'ivshim', 'ivshym',
+ 'ivshom', 'yvshee', 'yvshie', 'yvshye',
+ 'yvshoe', 'yvshei`', 'yvshii`',
+ 'yvshyi`', 'yvshoi`', 'yvshem',
+ 'yvshim', 'yvshym', 'yvshom', 'vshimi',
+ 'vshymi', 'vshego', 'vshogo', 'vshemu',
+ 'vshomu', 'vshikh', 'vshykh', 'vshui^u',
+ 'vshai^a', 'vshoi^u', 'vshei^u',
+ 'emi^ui^u', 'emi^ai^a', 'nni^ui^u',
+ 'nni^ai^a', 'vshee',
+ 'vshie', 'vshye', 'vshoe', 'vshei`',
+ 'vshii`', 'vshyi`', 'vshoi`',
+ 'vshem', 'vshim', 'vshym', 'vshom',
+ 'emimi', 'emymi', 'emego', 'emogo',
+ 'ememu', 'emomu', 'emikh', 'emykh',
+ 'emui^u', 'emai^a', 'emoi^u', 'emei^u',
+ 'nnimi', 'nnymi', 'nnego', 'nnogo',
+ 'nnemu', 'nnomu', 'nnikh', 'nnykh',
+ 'nnui^u', 'nnai^a', 'nnoi^u', 'nnei^u',
+ 'emee', 'emie', 'emye', 'emoe',
+ 'emei`', 'emii`', 'emyi`',
+ 'emoi`', 'emem', 'emim', 'emym',
+ 'emom', 'nnee', 'nnie', 'nnye', 'nnoe',
+ 'nnei`', 'nnii`', 'nnyi`',
+ 'nnoi`', 'nnem', 'nnim', 'nnym',
+ 'nnom', 'i^ui^u', 'i^ai^a', 'imi', 'ymi',
+ 'ego', 'ogo', 'emu', 'omu', 'ikh',
+ 'ykh', 'ui^u', 'ai^a', 'oi^u', 'ei^u',
+ 'ee', 'ie', 'ye', 'oe', 'ei`',
+ 'ii`', 'yi`', 'oi`', 'em',
+ 'im', 'ym', 'om')
+ __reflexive_suffixes = ("si^a", "s'")
+ __verb_suffixes = ("esh'", 'ei`te', 'ui`te', 'ui^ut',
+ "ish'", 'ete', 'i`te', 'i^ut', 'nno',
+ 'ila', 'yla', 'ena', 'ite', 'ili', 'yli',
+ 'ilo', 'ylo', 'eno', 'i^at', 'uet', 'eny',
+ "it'", "yt'", 'ui^u', 'la', 'na', 'li',
+ 'em', 'lo', 'no', 'et', 'ny', "t'",
+ 'ei`', 'ui`', 'il', 'yl', 'im',
+ 'ym', 'en', 'it', 'yt', 'i^u', 'i`',
+ 'l', 'n')
+ __noun_suffixes = ('ii^ami', 'ii^akh', 'i^ami', 'ii^am', 'i^akh',
+ 'ami', 'iei`', 'i^am', 'iem', 'akh',
+ 'ii^u', "'i^u", 'ii^a', "'i^a", 'ev', 'ov',
+ 'ie', "'e", 'ei', 'ii', 'ei`',
+ 'oi`', 'ii`', 'em', 'am', 'om',
+ 'i^u', 'i^a', 'a', 'e', 'i', 'i`',
+ 'o', 'u', 'y', "'")
+ __superlative_suffixes = ("ei`she", "ei`sh")
+ __derivational_suffixes = ("ost'", "ost")
+
+ def stem(self, word):
+ """
+ Stem a Russian word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ if word in self.stopwords:
+ return word
+
+ chr_exceeded = False
+ for i in range(len(word)):
+ if ord(word[i]) > 255:
+ chr_exceeded = True
+ break
+
+ if chr_exceeded:
+ word = self.__cyrillic_to_roman(word)
+
+ step1_success = False
+ adjectival_removed = False
+ verb_removed = False
+ undouble_success = False
+ superlative_removed = False
+
+ rv, r2 = self.__regions_russian(word)
+
+ # Step 1
+ for suffix in self.__perfective_gerund_suffixes:
+ if rv.endswith(suffix):
+ if suffix in ("v", "vshi", "vshis'"):
+ if (rv[-len(suffix)-3:-len(suffix)] == "i^a" or
+ rv[-len(suffix)-1:-len(suffix)] == "a"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ step1_success = True
+ break
+ else:
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ step1_success = True
+ break
+
+ if not step1_success:
+ for suffix in self.__reflexive_suffixes:
+ if rv.endswith(suffix):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ for suffix in self.__adjectival_suffixes:
+ if rv.endswith(suffix):
+ if suffix in ('i^ushchi^ui^u', 'i^ushchi^ai^a',
+ 'i^ushchui^u', 'i^ushchai^a', 'i^ushchoi^u',
+ 'i^ushchei^u', 'i^ushchimi', 'i^ushchymi',
+ 'i^ushchego', 'i^ushchogo', 'i^ushchemu',
+ 'i^ushchomu', 'i^ushchikh', 'i^ushchykh',
+ 'shchi^ui^u', 'shchi^ai^a', 'i^ushchee',
+ 'i^ushchie', 'i^ushchye', 'i^ushchoe',
+ 'i^ushchei`', 'i^ushchii`', 'i^ushchyi`',
+ 'i^ushchoi`', 'i^ushchem', 'i^ushchim',
+ 'i^ushchym', 'i^ushchom', 'vshi^ui^u',
+ 'vshi^ai^a', 'shchui^u', 'shchai^a',
+ 'shchoi^u', 'shchei^u', 'emi^ui^u',
+ 'emi^ai^a', 'nni^ui^u', 'nni^ai^a',
+ 'shchimi', 'shchymi', 'shchego', 'shchogo',
+ 'shchemu', 'shchomu', 'shchikh', 'shchykh',
+ 'vshui^u', 'vshai^a', 'vshoi^u', 'vshei^u',
+ 'shchee', 'shchie', 'shchye', 'shchoe',
+ 'shchei`', 'shchii`', 'shchyi`', 'shchoi`',
+ 'shchem', 'shchim', 'shchym', 'shchom',
+ 'vshimi', 'vshymi', 'vshego', 'vshogo',
+ 'vshemu', 'vshomu', 'vshikh', 'vshykh',
+ 'emui^u', 'emai^a', 'emoi^u', 'emei^u',
+ 'nnui^u', 'nnai^a', 'nnoi^u', 'nnei^u',
+ 'vshee', 'vshie', 'vshye', 'vshoe',
+ 'vshei`', 'vshii`', 'vshyi`', 'vshoi`',
+ 'vshem', 'vshim', 'vshym', 'vshom',
+ 'emimi', 'emymi', 'emego', 'emogo',
+ 'ememu', 'emomu', 'emikh', 'emykh',
+ 'nnimi', 'nnymi', 'nnego', 'nnogo',
+ 'nnemu', 'nnomu', 'nnikh', 'nnykh',
+ 'emee', 'emie', 'emye', 'emoe', 'emei`',
+ 'emii`', 'emyi`', 'emoi`', 'emem', 'emim',
+ 'emym', 'emom', 'nnee', 'nnie', 'nnye',
+ 'nnoe', 'nnei`', 'nnii`', 'nnyi`', 'nnoi`',
+ 'nnem', 'nnim', 'nnym', 'nnom'):
+ if (rv[-len(suffix)-3:-len(suffix)] == "i^a" or
+ rv[-len(suffix)-1:-len(suffix)] == "a"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ adjectival_removed = True
+ break
+ else:
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ adjectival_removed = True
+ break
+
+ if not adjectival_removed:
+ for suffix in self.__verb_suffixes:
+ if rv.endswith(suffix):
+ if suffix in ("la", "na", "ete", "i`te", "li",
+ "i`", "l", "em", "n", "lo", "no",
+ "et", "i^ut", "ny", "t'", "esh'",
+ "nno"):
+ if (rv[-len(suffix)-3:-len(suffix)] == "i^a" or
+ rv[-len(suffix)-1:-len(suffix)] == "a"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ verb_removed = True
+ break
+ else:
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ verb_removed = True
+ break
+
+ if not adjectival_removed and not verb_removed:
+ for suffix in self.__noun_suffixes:
+ if rv.endswith(suffix):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # Step 2
+ if rv.endswith("i"):
+ word = word[:-1]
+ r2 = r2[:-1]
+
+ # Step 3
+ for suffix in self.__derivational_suffixes:
+ if r2.endswith(suffix):
+ word = word[:-len(suffix)]
+ break
+
+ # Step 4
+ if word.endswith("nn"):
+ word = word[:-1]
+ undouble_success = True
+
+ if not undouble_success:
+ for suffix in self.__superlative_suffixes:
+ if word.endswith(suffix):
+ word = word[:-len(suffix)]
+ superlative_removed = True
+ break
+ if word.endswith("nn"):
+ word = word[:-1]
+
+ if not undouble_success and not superlative_removed:
+ if word.endswith("'"):
+ word = word[:-1]
+
+ if chr_exceeded:
+ word = self.__roman_to_cyrillic(word)
+
+
+ return word
+
+
+
+ def __regions_russian(self, word):
+ """
+ Return the regions RV and R2 which are used by the Russian stemmer.
+
+ In any word, RV is the region after the first vowel,
+ or the end of the word if it contains no vowel.
+
+ R2 is the region after the first non-vowel following
+ a vowel in R1, or the end of the word if there is no such non-vowel.
+
+ R1 is the region after the first non-vowel following a vowel,
+ or the end of the word if there is no such non-vowel.
+
+ :param word: The Russian word whose regions RV and R2 are determined.
+ :type word: str or unicode
+ :return: the regions RV and R2 for the respective Russian word.
+ :rtype: tuple
+ :note: This helper method is invoked by the stem method of the subclass
+ RussianStemmer. It is not to be invoked directly!
+
+ """
+ r1 = ""
+ r2 = ""
+ rv = ""
+
+ vowels = ("A", "U", "E", "a", "e", "i", "o", "u", "y")
+ word = (word.replace("i^a", "A")
+ .replace("i^u", "U")
+ .replace("e`", "E"))
+
+ for i in range(1, len(word)):
+ if word[i] not in vowels and word[i-1] in vowels:
+ r1 = word[i+1:]
+ break
+
+ for i in range(1, len(r1)):
+ if r1[i] not in vowels and r1[i-1] in vowels:
+ r2 = r1[i+1:]
+ break
+
+ for i in range(len(word)):
+ if word[i] in vowels:
+ rv = word[i+1:]
+ break
+
+ r2 = (r2.replace("A", "i^a")
+ .replace("U", "i^u")
+ .replace("E", "e`"))
+ rv = (rv.replace("A", "i^a")
+ .replace("U", "i^u")
+ .replace("E", "e`"))
+
+
+ return (rv, r2)
+
+
+
+ def __cyrillic_to_roman(self, word):
+ """
+ Transliterate a Russian word into the Roman alphabet.
+
+ A Russian word whose letters consist of the Cyrillic
+ alphabet are transliterated into the Roman alphabet
+ in order to ease the forthcoming stemming process.
+
+ :param word: The word that is transliterated.
+ :type word: unicode
+ :return: the transliterated word.
+ :rtype: unicode
+ :note: This helper method is invoked by the stem method of the subclass
+ RussianStemmer. It is not to be invoked directly!
+
+ """
+ word = (word.replace("\u0410", "a").replace("\u0430", "a")
+ .replace("\u0411", "b").replace("\u0431", "b")
+ .replace("\u0412", "v").replace("\u0432", "v")
+ .replace("\u0413", "g").replace("\u0433", "g")
+ .replace("\u0414", "d").replace("\u0434", "d")
+ .replace("\u0415", "e").replace("\u0435", "e")
+ .replace("\u0401", "e").replace("\u0451", "e")
+ .replace("\u0416", "zh").replace("\u0436", "zh")
+ .replace("\u0417", "z").replace("\u0437", "z")
+ .replace("\u0418", "i").replace("\u0438", "i")
+ .replace("\u0419", "i`").replace("\u0439", "i`")
+ .replace("\u041A", "k").replace("\u043A", "k")
+ .replace("\u041B", "l").replace("\u043B", "l")
+ .replace("\u041C", "m").replace("\u043C", "m")
+ .replace("\u041D", "n").replace("\u043D", "n")
+ .replace("\u041E", "o").replace("\u043E", "o")
+ .replace("\u041F", "p").replace("\u043F", "p")
+ .replace("\u0420", "r").replace("\u0440", "r")
+ .replace("\u0421", "s").replace("\u0441", "s")
+ .replace("\u0422", "t").replace("\u0442", "t")
+ .replace("\u0423", "u").replace("\u0443", "u")
+ .replace("\u0424", "f").replace("\u0444", "f")
+ .replace("\u0425", "kh").replace("\u0445", "kh")
+ .replace("\u0426", "t^s").replace("\u0446", "t^s")
+ .replace("\u0427", "ch").replace("\u0447", "ch")
+ .replace("\u0428", "sh").replace("\u0448", "sh")
+ .replace("\u0429", "shch").replace("\u0449", "shch")
+ .replace("\u042A", "''").replace("\u044A", "''")
+ .replace("\u042B", "y").replace("\u044B", "y")
+ .replace("\u042C", "'").replace("\u044C", "'")
+ .replace("\u042D", "e`").replace("\u044D", "e`")
+ .replace("\u042E", "i^u").replace("\u044E", "i^u")
+ .replace("\u042F", "i^a").replace("\u044F", "i^a"))
+
+
+ return word
+
+
+
+ def __roman_to_cyrillic(self, word):
+ """
+ Transliterate a Russian word back into the Cyrillic alphabet.
+
+ A Russian word formerly transliterated into the Roman alphabet
+ in order to ease the stemming process, is transliterated back
+ into the Cyrillic alphabet, its original form.
+
+ :param word: The word that is transliterated.
+ :type word: str or unicode
+ :return: word, the transliterated word.
+ :rtype: unicode
+ :note: This helper method is invoked by the stem method of the subclass
+ RussianStemmer. It is not to be invoked directly!
+
+ """
+ word = (word.replace("i^u", "\u044E").replace("i^a", "\u044F")
+ .replace("shch", "\u0449").replace("kh", "\u0445")
+ .replace("t^s", "\u0446").replace("ch", "\u0447")
+ .replace("e`", "\u044D").replace("i`", "\u0439")
+ .replace("sh", "\u0448").replace("k", "\u043A")
+ .replace("e", "\u0435").replace("zh", "\u0436")
+ .replace("a", "\u0430").replace("b", "\u0431")
+ .replace("v", "\u0432").replace("g", "\u0433")
+ .replace("d", "\u0434").replace("e", "\u0435")
+ .replace("z", "\u0437").replace("i", "\u0438")
+ .replace("l", "\u043B").replace("m", "\u043C")
+ .replace("n", "\u043D").replace("o", "\u043E")
+ .replace("p", "\u043F").replace("r", "\u0440")
+ .replace("s", "\u0441").replace("t", "\u0442")
+ .replace("u", "\u0443").replace("f", "\u0444")
+ .replace("''", "\u044A").replace("y", "\u044B")
+ .replace("'", "\u044C"))
+
+
+ return word
+
+
+
+class SpanishStemmer(_StandardStemmer):
+
+ """
+ The Spanish Snowball stemmer.
+
+ :cvar __vowels: The Spanish vowels.
+ :type __vowels: unicode
+ :cvar __step0_suffixes: Suffixes to be deleted in step 0 of the algorithm.
+ :type __step0_suffixes: tuple
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2a_suffixes: Suffixes to be deleted in step 2a of the algorithm.
+ :type __step2a_suffixes: tuple
+ :cvar __step2b_suffixes: Suffixes to be deleted in step 2b of the algorithm.
+ :type __step2b_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :note: A detailed description of the Spanish
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/spanish/stemmer.html
+
+ """
+
+ __vowels = "aeiou\xE1\xE9\xED\xF3\xFA\xFC"
+ __step0_suffixes = ("selas", "selos", "sela", "selo", "las",
+ "les", "los", "nos", "me", "se", "la", "le",
+ "lo")
+ __step1_suffixes = ('amientos', 'imientos', 'amiento', 'imiento',
+ 'aciones', 'uciones', 'adoras', 'adores',
+ 'ancias', 'log\xEDas', 'encias', 'amente',
+ 'idades', 'anzas', 'ismos', 'ables', 'ibles',
+ 'istas', 'adora', 'aci\xF3n', 'antes',
+ 'ancia', 'log\xEDa', 'uci\xf3n', 'encia',
+ 'mente', 'anza', 'icos', 'icas', 'ismo',
+ 'able', 'ible', 'ista', 'osos', 'osas',
+ 'ador', 'ante', 'idad', 'ivas', 'ivos',
+ 'ico',
+ 'ica', 'oso', 'osa', 'iva', 'ivo')
+ __step2a_suffixes = ('yeron', 'yendo', 'yamos', 'yais', 'yan',
+ 'yen', 'yas', 'yes', 'ya', 'ye', 'yo',
+ 'y\xF3')
+ __step2b_suffixes = ('ar\xEDamos', 'er\xEDamos', 'ir\xEDamos',
+ 'i\xE9ramos', 'i\xE9semos', 'ar\xEDais',
+ 'aremos', 'er\xEDais', 'eremos',
+ 'ir\xEDais', 'iremos', 'ierais', 'ieseis',
+ 'asteis', 'isteis', '\xE1bamos',
+ '\xE1ramos', '\xE1semos', 'ar\xEDan',
+ 'ar\xEDas', 'ar\xE9is', 'er\xEDan',
+ 'er\xEDas', 'er\xE9is', 'ir\xEDan',
+ 'ir\xEDas', 'ir\xE9is',
+ 'ieran', 'iesen', 'ieron', 'iendo', 'ieras',
+ 'ieses', 'abais', 'arais', 'aseis',
+ '\xE9amos', 'ar\xE1n', 'ar\xE1s',
+ 'ar\xEDa', 'er\xE1n', 'er\xE1s',
+ 'er\xEDa', 'ir\xE1n', 'ir\xE1s',
+ 'ir\xEDa', 'iera', 'iese', 'aste', 'iste',
+ 'aban', 'aran', 'asen', 'aron', 'ando',
+ 'abas', 'adas', 'idas', 'aras', 'ases',
+ '\xEDais', 'ados', 'idos', 'amos', 'imos',
+ 'emos', 'ar\xE1', 'ar\xE9', 'er\xE1',
+ 'er\xE9', 'ir\xE1', 'ir\xE9', 'aba',
+ 'ada', 'ida', 'ara', 'ase', '\xEDan',
+ 'ado', 'ido', '\xEDas', '\xE1is',
+ '\xE9is', '\xEDa', 'ad', 'ed', 'id',
+ 'an', 'i\xF3', 'ar', 'er', 'ir', 'as',
+ '\xEDs', 'en', 'es')
+ __step3_suffixes = ("os", "a", "e", "o", "\xE1",
+ "\xE9", "\xED", "\xF3")
+
+ def stem(self, word):
+ """
+ Stem a Spanish word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ step1_success = False
+
+ r1, r2 = self._r1r2_standard(word, self.__vowels)
+ rv = self._rv_standard(word, self.__vowels)
+
+ # STEP 0: Attached pronoun
+ for suffix in self.__step0_suffixes:
+ if word.endswith(suffix):
+ if rv.endswith(suffix):
+ if rv[:-len(suffix)].endswith(("i\xE9ndo",
+ "\xE1ndo",
+ "\xE1r", "\xE9r",
+ "\xEDr")):
+ word = (word[:-len(suffix)].replace("\xE1", "a")
+ .replace("\xE9", "e")
+ .replace("\xED", "i"))
+ r1 = (r1[:-len(suffix)].replace("\xE1", "a")
+ .replace("\xE9", "e")
+ .replace("\xED", "i"))
+ r2 = (r2[:-len(suffix)].replace("\xE1", "a")
+ .replace("\xE9", "e")
+ .replace("\xED", "i"))
+ rv = (rv[:-len(suffix)].replace("\xE1", "a")
+ .replace("\xE9", "e")
+ .replace("\xED", "i"))
+
+ elif rv[:-len(suffix)].endswith(("ando", "iendo",
+ "ar", "er", "ir")):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ elif (rv[:-len(suffix)].endswith("yendo") and
+ word[:-len(suffix)].endswith("uyendo")):
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 1: Standard suffix removal
+ for suffix in self.__step1_suffixes:
+ if word.endswith(suffix):
+ if suffix == "amente" and r1.endswith(suffix):
+ step1_success = True
+ word = word[:-6]
+ r2 = r2[:-6]
+ rv = rv[:-6]
+
+ if r2.endswith("iv"):
+ word = word[:-2]
+ r2 = r2[:-2]
+ rv = rv[:-2]
+
+ if r2.endswith("at"):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif r2.endswith(("os", "ic", "ad")):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif r2.endswith(suffix):
+ step1_success = True
+ if suffix in ("adora", "ador", "aci\xF3n", "adoras",
+ "adores", "aciones", "ante", "antes",
+ "ancia", "ancias"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ if r2.endswith("ic"):
+ word = word[:-2]
+ rv = rv[:-2]
+
+ elif suffix in ("log\xEDa", "log\xEDas"):
+ word = word.replace(suffix, "log")
+ rv = rv.replace(suffix, "log")
+
+ elif suffix in ("uci\xF3n", "uciones"):
+ word = word.replace(suffix, "u")
+ rv = rv.replace(suffix, "u")
+
+ elif suffix in ("encia", "encias"):
+ word = word.replace(suffix, "ente")
+ rv = rv.replace(suffix, "ente")
+
+ elif suffix == "mente":
+ word = word[:-5]
+ r2 = r2[:-5]
+ rv = rv[:-5]
+
+ if r2.endswith(("ante", "able", "ible")):
+ word = word[:-4]
+ rv = rv[:-4]
+
+ elif suffix in ("idad", "idades"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ for pre_suff in ("abil", "ic", "iv"):
+ if r2.endswith(pre_suff):
+ word = word[:-len(pre_suff)]
+ rv = rv[:-len(pre_suff)]
+
+ elif suffix in ("ivo", "iva", "ivos", "ivas"):
+ word = word[:-len(suffix)]
+ r2 = r2[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ if r2.endswith("at"):
+ word = word[:-2]
+ rv = rv[:-2]
+ else:
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 2a: Verb suffixes beginning 'y'
+ if not step1_success:
+ for suffix in self.__step2a_suffixes:
+ if (rv.endswith(suffix) and
+ word[-len(suffix)-1:-len(suffix)] == "u"):
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 2b: Other verb suffixes
+ for suffix in self.__step2b_suffixes:
+ if rv.endswith(suffix):
+ if suffix in ("en", "es", "\xE9is", "emos"):
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ if word.endswith("gu"):
+ word = word[:-1]
+
+ if rv.endswith("gu"):
+ rv = rv[:-1]
+ else:
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+ break
+
+ # STEP 3: Residual suffix
+ for suffix in self.__step3_suffixes:
+ if rv.endswith(suffix):
+ if suffix in ("e", "\xE9"):
+ word = word[:-len(suffix)]
+ rv = rv[:-len(suffix)]
+
+ if word[-2:] == "gu" and rv[-1] == "u":
+ word = word[:-1]
+ else:
+ word = word[:-len(suffix)]
+ break
+
+ word = (word.replace("\xE1", "a").replace("\xE9", "e")
+ .replace("\xED", "i").replace("\xF3", "o")
+ .replace("\xFA", "u"))
+
+
+ return word
+
+
+
+class SwedishStemmer(_ScandinavianStemmer):
+
+ """
+ The Swedish Snowball stemmer.
+
+ :cvar __vowels: The Swedish vowels.
+ :type __vowels: unicode
+ :cvar __s_ending: Letters that may directly appear before a word final 's'.
+ :type __s_ending: unicode
+ :cvar __step1_suffixes: Suffixes to be deleted in step 1 of the algorithm.
+ :type __step1_suffixes: tuple
+ :cvar __step2_suffixes: Suffixes to be deleted in step 2 of the algorithm.
+ :type __step2_suffixes: tuple
+ :cvar __step3_suffixes: Suffixes to be deleted in step 3 of the algorithm.
+ :type __step3_suffixes: tuple
+ :note: A detailed description of the Swedish
+ stemming algorithm can be found under
+ http://snowball.tartarus.org/algorithms/swedish/stemmer.html
+
+ """
+
+ __vowels = "aeiouy\xE4\xE5\xF6"
+ __s_ending = "bcdfghjklmnoprtvy"
+ __step1_suffixes = ("heterna", "hetens", "heter", "heten",
+ "anden", "arnas", "ernas", "ornas", "andes",
+ "andet", "arens", "arna", "erna", "orna",
+ "ande", "arne", "aste", "aren", "ades",
+ "erns", "ade", "are", "ern", "ens", "het",
+ "ast", "ad", "en", "ar", "er", "or", "as",
+ "es", "at", "a", "e", "s")
+ __step2_suffixes = ("dd", "gd", "nn", "dt", "gt", "kt", "tt")
+ __step3_suffixes = ("fullt", "l\xF6st", "els", "lig", "ig")
+
+ def stem(self, word):
+ """
+ Stem a Swedish word and return the stemmed form.
+
+ :param word: The word that is stemmed.
+ :type word: str or unicode
+ :return: The stemmed form.
+ :rtype: unicode
+
+ """
+ word = word.lower()
+
+ if word in self.stopwords:
+ return word
+
+ r1 = self._r1_scandinavian(word, self.__vowels)
+
+ # STEP 1
+ for suffix in self.__step1_suffixes:
+ if r1.endswith(suffix):
+ if suffix == "s":
+ if word[-2] in self.__s_ending:
+ word = word[:-1]
+ r1 = r1[:-1]
+ else:
+ word = word[:-len(suffix)]
+ r1 = r1[:-len(suffix)]
+ break
+
+ # STEP 2
+ for suffix in self.__step2_suffixes:
+ if r1.endswith(suffix):
+ word = word[:-1]
+ r1 = r1[:-1]
+ break
+
+ # STEP 3
+ for suffix in self.__step3_suffixes:
+ if r1.endswith(suffix):
+ if suffix in ("els", "lig", "ig"):
+ word = word[:-len(suffix)]
+ elif suffix in ("fullt", "l\xF6st"):
+ word = word[:-1]
+ break
+
+
+ return word
+
+
+
+def demo():
+ """
+ This function provides a demonstration of the Snowball stemmers.
+
+ After invoking this function and specifying a language,
+ it stems an excerpt of the Universal Declaration of Human Rights
+ (which is a part of the NLTK corpus collection) and then prints
+ out the original and the stemmed text.
+
+ """
+
+ import re
+ from nltk.corpus import udhr
+
+ udhr_corpus = {"danish": "Danish_Dansk-Latin1",
+ "dutch": "Dutch_Nederlands-Latin1",
+ "english": "English-Latin1",
+ "finnish": "Finnish_Suomi-Latin1",
+ "french": "French_Francais-Latin1",
+ "german": "German_Deutsch-Latin1",
+ "hungarian": "Hungarian_Magyar-UTF8",
+ "italian": "Italian_Italiano-Latin1",
+ "norwegian": "Norwegian-Latin1",
+ "porter": "English-Latin1",
+ "portuguese": "Portuguese_Portugues-Latin1",
+ "romanian": "Romanian_Romana-Latin2",
+ "russian": "Russian-UTF8",
+ "spanish": "Spanish-Latin1",
+ "swedish": "Swedish_Svenska-Latin1",
+ }
+
+ print("\n")
+ print("******************************")
+ print("Demo for the Snowball stemmers")
+ print("******************************")
+
+ while True:
+
+ language = compat.raw_input("Please enter the name of the language " +
+ "to be demonstrated\n" +
+ "/".join(SnowballStemmer.languages) +
+ "\n" +
+ "(enter 'exit' in order to leave): ")
+
+ if language == "exit":
+ break
+
+ if language not in SnowballStemmer.languages:
+ print(("\nOops, there is no stemmer for this language. " +
+ "Please try again.\n"))
+ continue
+
+ stemmer = SnowballStemmer(language)
+ excerpt = udhr.words(udhr_corpus[language]) [:300]
+
+ stemmed = " ".join(stemmer.stem(word) for word in excerpt)
+ stemmed = re.sub(r"(.{,70})\s", r'\1\n', stemmed+' ').rstrip()
+ excerpt = " ".join(excerpt)
+ excerpt = re.sub(r"(.{,70})\s", r'\1\n', excerpt+' ').rstrip()
+
+ print("\n")
+ print('-' * 70)
+ print('ORIGINAL'.center(70))
+ print(excerpt)
+ print("\n\n")
+ print('STEMMED RESULTS'.center(70))
+ print(stemmed)
+ print('-' * 70)
+ print("\n")
+
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
diff --git a/nltk/stem/wordnet.py b/nltk/stem/wordnet.py
new file mode 100644
index 0000000..7df2fee
--- /dev/null
+++ b/nltk/stem/wordnet.py
@@ -0,0 +1,54 @@
+# Natural Language Toolkit: WordNet stemmer interface
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import unicode_literals
+
+from nltk.corpus.reader.wordnet import NOUN
+from nltk.corpus import wordnet
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class WordNetLemmatizer(object):
+ """
+ WordNet Lemmatizer
+
+ Lemmatize using WordNet's built-in morphy function.
+ Returns the input word unchanged if it cannot be found in WordNet.
+
+ >>> from nltk.stem import WordNetLemmatizer
+ >>> wnl = WordNetLemmatizer()
+ >>> print(wnl.lemmatize('dogs'))
+ dog
+ >>> print(wnl.lemmatize('churches'))
+ church
+ >>> print(wnl.lemmatize('aardwolves'))
+ aardwolf
+ >>> print(wnl.lemmatize('abaci'))
+ abacus
+ >>> print(wnl.lemmatize('hardrock'))
+ hardrock
+ """
+
+ def __init__(self):
+ pass
+
+ def lemmatize(self, word, pos=NOUN):
+ lemmas = wordnet._morphy(word, pos)
+ return min(lemmas, key=len) if lemmas else word
+
+ def __repr__(self):
+ return '<WordNetLemmatizer>'
+
+
+# unload wordnet
+def teardown_module(module=None):
+ from nltk.corpus import wordnet
+ wordnet._unload()
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/__init__.py b/nltk/tag/__init__.py
new file mode 100644
index 0000000..6262eb5
--- /dev/null
+++ b/nltk/tag/__init__.py
@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Taggers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+"""
+NLTK Taggers
+
+This package contains classes and interfaces for part-of-speech
+tagging, or simply "tagging".
+
+A "tag" is a case-sensitive string that specifies some property of a token,
+such as its part of speech. Tagged tokens are encoded as tuples
+``(tag, token)``. For example, the following tagged token combines
+the word ``'fly'`` with a noun part of speech tag (``'NN'``):
+
+ >>> tagged_tok = ('fly', 'NN')
+
+An off-the-shelf tagger is available. It uses the Penn Treebank tagset:
+
+ >>> from nltk.tag import pos_tag # doctest: +SKIP
+ >>> from nltk.tokenize import word_tokenize # doctest: +SKIP
+ >>> pos_tag(word_tokenize("John's big idea isn't all that bad.")) # doctest: +SKIP
+ [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'), ('idea', 'NN'), ('is',
+ 'VBZ'), ("n't", 'RB'), ('all', 'DT'), ('that', 'DT'), ('bad', 'JJ'),
+ ('.', '.')]
+
+This package defines several taggers, which take a token list (typically a
+sentence), assign a tag to each token, and return the resulting list of
+tagged tokens. Most of the taggers are built automatically based on a
+training corpus. For example, the unigram tagger tags each word *w*
+by checking what the most frequent tag for *w* was in a training corpus:
+
+ >>> from nltk.corpus import brown
+ >>> from nltk.tag import UnigramTagger
+ >>> tagger = UnigramTagger(brown.tagged_sents(categories='news')[:500])
+ >>> sent = ['Mitchell', 'decried', 'the', 'high', 'rate', 'of', 'unemployment']
+ >>> for word, tag in tagger.tag(sent):
+ ... print(word, '->', tag)
+ Mitchell -> NP
+ decried -> None
+ the -> AT
+ high -> JJ
+ rate -> NN
+ of -> IN
+ unemployment -> None
+
+Note that words that the tagger has not seen during training receive a tag
+of ``None``.
+
+We evaluate a tagger on data that was not seen during training:
+
+ >>> tagger.evaluate(brown.tagged_sents(categories='news')[500:600])
+ 0.73...
+
+For more information, please consult chapter 5 of the NLTK Book.
+"""
+from __future__ import print_function
+
+from nltk.tag.api import TaggerI
+from nltk.tag.util import str2tuple, tuple2str, untag
+from nltk.tag.sequential import (SequentialBackoffTagger, ContextTagger,
+ DefaultTagger, NgramTagger, UnigramTagger,
+ BigramTagger, TrigramTagger, AffixTagger,
+ RegexpTagger, ClassifierBasedTagger,
+ ClassifierBasedPOSTagger)
+from nltk.tag.brill import BrillTagger
+from nltk.tag.brill_trainer import BrillTaggerTrainer
+from nltk.tag.tnt import TnT
+from nltk.tag.hunpos import HunposTagger
+from nltk.tag.stanford import StanfordTagger
+from nltk.tag.hmm import HiddenMarkovModelTagger, HiddenMarkovModelTrainer
+from nltk.tag.mapping import tagset_mapping, map_tag
+
+from nltk.data import load
+
+
+# Standard treebank POS tagger
+_POS_TAGGER = 'taggers/maxent_treebank_pos_tagger/english.pickle'
+def pos_tag(tokens):
+ """
+ Use NLTK's currently recommended part of speech tagger to
+ tag the given list of tokens.
+
+ >>> from nltk.tag import pos_tag # doctest: +SKIP
+ >>> from nltk.tokenize import word_tokenize # doctest: +SKIP
+ >>> pos_tag(word_tokenize("John's big idea isn't all that bad.")) # doctest: +SKIP
+ [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'), ('idea', 'NN'), ('is',
+ 'VBZ'), ("n't", 'RB'), ('all', 'DT'), ('that', 'DT'), ('bad', 'JJ'),
+ ('.', '.')]
+
+ :param tokens: Sequence of tokens to be tagged
+ :type tokens: list(str)
+ :return: The tagged tokens
+ :rtype: list(tuple(str, str))
+ """
+ tagger = load(_POS_TAGGER)
+ return tagger.tag(tokens)
+
+def pos_tag_sents(sentences):
+ """
+ Use NLTK's currently recommended part of speech tagger to tag the
+ given list of sentences, each consisting of a list of tokens.
+ """
+ tagger = load(_POS_TAGGER)
+ return tagger.tag_sents(sentences)
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/api.py b/nltk/tag/api.py
new file mode 100644
index 0000000..8fc3efd
--- /dev/null
+++ b/nltk/tag/api.py
@@ -0,0 +1,84 @@
+# Natural Language Toolkit: Tagger Interface
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interface for tagging each token in a sentence with supplementary
+information, such as its part of speech.
+"""
+from nltk.internals import overridden
+from nltk.metrics import accuracy
+
+from nltk.tag.util import untag
+
+class TaggerI(object):
+ """
+ A processing interface for assigning a tag to each token in a list.
+ Tags are case sensitive strings that identify some property of each
+ token, such as its part of speech or its sense.
+
+ Some taggers require specific types for their tokens. This is
+ generally indicated by the use of a sub-interface to ``TaggerI``.
+ For example, featureset taggers, which are subclassed from
+ ``FeaturesetTagger``, require that each token be a ``featureset``.
+
+ Subclasses must define:
+ - either ``tag()`` or ``tag_sents()`` (or both)
+ """
+ def tag(self, tokens):
+ """
+ Determine the most appropriate tag sequence for the given
+ token sequence, and return a corresponding list of tagged
+ tokens. A tagged token is encoded as a tuple ``(token, tag)``.
+
+ :rtype: list(tuple(str, str))
+ """
+ if overridden(self.tag_sents):
+ return self.tag_sents([tokens])[0]
+ else:
+ raise NotImplementedError()
+
+ def tag_sents(self, sentences):
+ """
+ Apply ``self.tag()`` to each element of *sentences*. I.e.:
+
+ return [self.tag(sent) for sent in sentences]
+ """
+ return [self.tag(sent) for sent in sentences]
+
+ def evaluate(self, gold):
+ """
+ Score the accuracy of the tagger against the gold standard.
+ Strip the tags from the gold standard text, retag it using
+ the tagger, then compute the accuracy score.
+
+ :type gold: list(list(tuple(str, str)))
+ :param gold: The list of tagged sentences to score the tagger on.
+ :rtype: float
+ """
+
+ tagged_sents = self.tag_sents(untag(sent) for sent in gold)
+ gold_tokens = sum(gold, [])
+ test_tokens = sum(tagged_sents, [])
+ return accuracy(gold_tokens, test_tokens)
+
+ def _check_params(self, train, model):
+ if (train and model) or (not train and not model):
+ raise ValueError('Must specify either training data or trained model.')
+
+class FeaturesetTaggerI(TaggerI):
+ """
+ A tagger that requires tokens to be ``featuresets``. A featureset
+ is a dictionary that maps from feature names to feature
+ values. See ``nltk.classify`` for more information about features
+ and featuresets.
+ """
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/brill.py b/nltk/tag/brill.py
new file mode 100644
index 0000000..ea236fa
--- /dev/null
+++ b/nltk/tag/brill.py
@@ -0,0 +1,421 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function, division
+
+from collections import defaultdict
+
+from nltk.compat import Counter
+from nltk.tag import TaggerI
+from nltk.tbl import Feature, Template
+from nltk import jsontags
+
+
+######################################################################
+## Brill Templates
+######################################################################
+
+ at jsontags.register_tag
+class Word(Feature):
+ """
+ Feature which examines the text (word) of nearby tokens.
+ """
+
+ json_tag = 'nltk.tag.brill.Word'
+
+ @staticmethod
+ def extract_property(tokens, index):
+ """@return: The given token's text."""
+ return tokens[index][0]
+
+
+ at jsontags.register_tag
+class Pos(Feature):
+ """
+ Feature which examines the tags of nearby tokens.
+ """
+
+ json_tag = 'nltk.tag.brill.Pos'
+
+ @staticmethod
+ def extract_property(tokens, index):
+ """@return: The given token's tag."""
+ return tokens[index][1]
+
+
+def nltkdemo18():
+ """
+ Return 18 templates, from the original nltk demo, in multi-feature syntax
+ """
+ return [
+ Template(Pos([-1])),
+ Template(Pos([1])),
+ Template(Pos([-2])),
+ Template(Pos([2])),
+ Template(Pos([-2, -1])),
+ Template(Pos([1, 2])),
+ Template(Pos([-3, -2, -1])),
+ Template(Pos([1, 2, 3])),
+ Template(Pos([-1]), Pos([1])),
+ Template(Word([-1])),
+ Template(Word([1])),
+ Template(Word([-2])),
+ Template(Word([2])),
+ Template(Word([-2, -1])),
+ Template(Word([1, 2])),
+ Template(Word([-3, -2, -1])),
+ Template(Word([1, 2, 3])),
+ Template(Word([-1]), Word([1])),
+ ]
+
+def nltkdemo18plus():
+ """
+ Return 18 templates, from the original nltk demo, and additionally a few
+ multi-feature ones (the motivation is easy comparison with nltkdemo18)
+ """
+ return nltkdemo18() + [
+ Template(Word([-1]), Pos([1])),
+ Template(Pos([-1]), Word([1])),
+ Template(Word([-1]), Word([0]), Pos([1])),
+ Template(Pos([-1]), Word([0]), Word([1])),
+ Template(Pos([-1]), Word([0]), Pos([1])),
+ ]
+
+def fntbl37():
+ """
+ Return 37 templates taken from the postagging task of the
+ fntbl distribution http://www.cs.jhu.edu/~rflorian/fntbl/
+ (37 is after excluding a handful which do not condition on Pos[0];
+ fntbl can do that but the current nltk implementation cannot.)
+ """
+ return [
+ Template(Word([0]), Word([1]), Word([2])),
+ Template(Word([-1]), Word([0]), Word([1])),
+ Template(Word([0]), Word([-1])),
+ Template(Word([0]), Word([1])),
+ Template(Word([0]), Word([2])),
+ Template(Word([0]), Word([-2])),
+ Template(Word([1, 2])),
+ Template(Word([-2, -1])),
+ Template(Word([1, 2, 3])),
+ Template(Word([-3, -2, -1])),
+ Template(Word([0]), Pos([2])),
+ Template(Word([0]), Pos([-2])),
+ Template(Word([0]), Pos([1])),
+ Template(Word([0]), Pos([-1])),
+ Template(Word([0])),
+ Template(Word([-2])),
+ Template(Word([2])),
+ Template(Word([1])),
+ Template(Word([-1])),
+ Template(Pos([-1]), Pos([1])),
+ Template(Pos([1]), Pos([2])),
+ Template(Pos([-1]), Pos([-2])),
+ Template(Pos([1])),
+ Template(Pos([-1])),
+ Template(Pos([-2])),
+ Template(Pos([2])),
+ Template(Pos([1, 2, 3])),
+ Template(Pos([1, 2])),
+ Template(Pos([-3, -2, -1])),
+ Template(Pos([-2, -1])),
+ Template(Pos([1]), Word([0]), Word([1])),
+ Template(Pos([1]), Word([0]), Word([-1])),
+ Template(Pos([-1]), Word([-1]), Word([0])),
+ Template(Pos([-1]), Word([0]), Word([1])),
+ Template(Pos([-2]), Pos([-1])),
+ Template(Pos([1]), Pos([2])),
+ Template(Pos([1]), Pos([2]), Word([1]))
+ ]
+
+def brill24():
+ """
+ Return 24 templates of the seminal TBL paper, Brill (1995)
+ """
+ return [
+ Template(Pos([-1])),
+ Template(Pos([1])),
+ Template(Pos([-2])),
+ Template(Pos([2])),
+ Template(Pos([-2, -1])),
+ Template(Pos([1, 2])),
+ Template(Pos([-3, -2, -1])),
+ Template(Pos([1, 2, 3])),
+ Template(Pos([-1]), Pos([1])),
+ Template(Pos([-2]), Pos([-1])),
+ Template(Pos([1]), Pos([2])),
+ Template(Word([-1])),
+ Template(Word([1])),
+ Template(Word([-2])),
+ Template(Word([2])),
+ Template(Word([-2, -1])),
+ Template(Word([1, 2])),
+ Template(Word([-1, 0])),
+ Template(Word([0, 1])),
+ Template(Word([0])),
+ Template(Word([-1]), Pos([-1])),
+ Template(Word([1]), Pos([1])),
+ Template(Word([0]), Word([-1]), Pos([-1])),
+ Template(Word([0]), Word([1]), Pos([1])),
+ ]
+
+
+def describe_template_sets():
+ """
+ Print the available template sets in this demo, with a short description"
+ """
+ import inspect, sys
+
+ #a bit of magic to get all functions in this module
+ templatesets = inspect.getmembers(sys.modules[__name__], inspect.isfunction)
+ for (name, obj) in templatesets:
+ if name == "describe_template_sets":
+ continue
+ print(name, obj.__doc__, "\n")
+
+
+######################################################################
+## The Brill Tagger
+######################################################################
+
+ at jsontags.register_tag
+class BrillTagger(TaggerI):
+ """
+ Brill's transformational rule-based tagger. Brill taggers use an
+ initial tagger (such as ``tag.DefaultTagger``) to assign an initial
+ tag sequence to a text; and then apply an ordered list of
+ transformational rules to correct the tags of individual tokens.
+ These transformation rules are specified by the ``TagRule``
+ interface.
+
+ Brill taggers can be created directly, from an initial tagger and
+ a list of transformational rules; but more often, Brill taggers
+ are created by learning rules from a training corpus, using one
+ of the TaggerTrainers available.
+ """
+
+ json_tag='nltk.tag.BrillTagger'
+
+ def __init__(self, initial_tagger, rules, training_stats=None):
+ """
+ :param initial_tagger: The initial tagger
+ :type initial_tagger: TaggerI
+
+ :param rules: An ordered list of transformation rules that
+ should be used to correct the initial tagging.
+ :type rules: list(TagRule)
+
+ :param training_stats: A dictionary of statistics collected
+ during training, for possible later use
+ :type training_stats: dict
+
+ """
+ self._initial_tagger = initial_tagger
+ self._rules = tuple(rules)
+ self._training_stats = training_stats
+
+ def encode_json_obj(self):
+ return self._initial_tagger, self._rules, self._training_stats
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ _initial_tagger, _rules, _training_stats = obj
+ return cls(_initial_tagger, _rules, _training_stats)
+
+ def rules(self):
+ """
+ Return the ordered list of transformation rules that this tagger has learnt
+
+ :return: the ordered list of transformation rules that correct the initial tagging
+ :rtype: list of Rules
+ """
+ return self._rules
+
+ def train_stats(self, statistic=None):
+ """
+ Return a named statistic collected during training, or a dictionary of all
+ available statistics if no name given
+
+ :param statistic: name of statistic
+ :type statistic: str
+ :return: some statistic collected during training of this tagger
+ :rtype: any (but usually a number)
+ """
+ if statistic is None:
+ return self._training_stats
+ else:
+ return self._training_stats.get(statistic)
+
+ def tag(self, tokens):
+ # Inherit documentation from TaggerI
+
+ # Run the initial tagger.
+ tagged_tokens = self._initial_tagger.tag(tokens)
+
+ # Create a dictionary that maps each tag to a list of the
+ # indices of tokens that have that tag.
+ tag_to_positions = defaultdict(set)
+ for i, (token, tag) in enumerate(tagged_tokens):
+ tag_to_positions[tag].add(i)
+
+ # Apply each rule, in order. Only try to apply rules at
+ # positions that have the desired original tag.
+ for rule in self._rules:
+ # Find the positions where it might apply
+ positions = tag_to_positions.get(rule.original_tag, [])
+ # Apply the rule at those positions.
+ changed = rule.apply(tagged_tokens, positions)
+ # Update tag_to_positions with the positions of tags that
+ # were modified.
+ for i in changed:
+ tag_to_positions[rule.original_tag].remove(i)
+ tag_to_positions[rule.replacement_tag].add(i)
+
+ return tagged_tokens
+
+ def print_template_statistics(self, test_stats=None, printunused=True):
+ """
+ Print a list of all templates, ranked according to efficiency.
+
+ If test_stats is available, the templates are ranked according to their
+ relative contribution (summed for all rules created from a given template,
+ weighted by score) to the performance on the test set. If no test_stats, then
+ statistics collected during training are used instead. There is also
+ an unweighted measure (just counting the rules). This is less informative,
+ though, as many low-score rules will appear towards end of training.
+
+ :param test_stats: dictionary of statistics collected during testing
+ :type test_stats: dict of str -> any (but usually numbers)
+ :param printunused: if True, print a list of all unused templates
+ :type printunused: bool
+ :return: None
+ :rtype: None
+ """
+ tids = [r.templateid for r in self._rules]
+ train_stats = self.train_stats()
+
+ trainscores = train_stats['rulescores']
+ assert len(trainscores) == len(tids), "corrupt statistics: " \
+ "{0} train scores for {1} rules".format(trainscores, tids)
+ template_counts = Counter(tids)
+ weighted_traincounts = Counter()
+ for (tid, score) in zip(tids, trainscores):
+ weighted_traincounts[tid] += score
+ tottrainscores = sum(trainscores)
+
+ #det_tplsort() is for deterministic sorting;
+ #the otherwise convenient Counter.most_common() unfortunately
+ #does not break ties deterministically
+ #between python versions and will break cross-version tests
+ def det_tplsort(tpl_value):
+ return (tpl_value[1], repr(tpl_value[0]))
+
+ def print_train_stats():
+ print("TEMPLATE STATISTICS (TRAIN) {0} templates, {1} rules)".format(
+ len(template_counts),len(tids)))
+ print("TRAIN ({tokencount:7d} tokens) initial {initialerrors:5d} {initialacc:.4f} "
+ "final: {finalerrors:5d} {finalacc:.4f} ".format(**train_stats))
+ head = "#ID | Score (train) | #Rules | Template"
+ print(head, "\n", "-" * len(head), sep="")
+ train_tplscores = sorted(weighted_traincounts.items(), key=det_tplsort, reverse=True)
+ for (tid, trainscore) in train_tplscores:
+ s = "{0:s} | {1:5d} {2:5.3f} |{3:4d} {4:.3f} | {5:s}".format(
+ tid,
+ trainscore,
+ trainscore/tottrainscores,
+ template_counts[tid],
+ template_counts[tid]/len(tids),
+ Template.ALLTEMPLATES[int(tid)])
+ print(s)
+
+ def print_testtrain_stats():
+ testscores = test_stats['rulescores']
+ print("TEMPLATE STATISTICS (TEST AND TRAIN) ({0} templates, {1} rules)".format(
+ len(template_counts),len(tids)))
+ print("TEST ({tokencount:7d} tokens) initial {initialerrors:5d} {initialacc:.4f} "
+ "final: {finalerrors:5d} {finalacc:.4f} ".format(**test_stats))
+ print("TRAIN ({tokencount:7d} tokens) initial {initialerrors:5d} {initialacc:.4f} "
+ "final: {finalerrors:5d} {finalacc:.4f} ".format(**train_stats))
+ weighted_testcounts = Counter()
+ for (tid, score) in zip(tids, testscores):
+ weighted_testcounts[tid] += score
+ tottestscores = sum(testscores)
+ head = "#ID | Score (test) | Score (train) | #Rules | Template"
+ print(head, "\n", "-" * len(head), sep="")
+ test_tplscores = sorted(weighted_testcounts.items(), key=det_tplsort, reverse=True)
+ for (tid, testscore) in test_tplscores:
+ s = "{0:s} |{1:5d} {2:6.3f} | {3:4d} {4:.3f} |{5:4d} {6:.3f} | {7:s}".format(
+ tid,
+ testscore,
+ testscore/tottestscores,
+ weighted_traincounts[tid],
+ weighted_traincounts[tid]/tottrainscores,
+ template_counts[tid],
+ template_counts[tid]/len(tids),
+ Template.ALLTEMPLATES[int(tid)])
+ print(s)
+
+ def print_unused_templates():
+ usedtpls = set([int(tid) for tid in tids])
+ unused = [(tid, tpl) for (tid, tpl) in enumerate(Template.ALLTEMPLATES) if tid not in usedtpls]
+ print("UNUSED TEMPLATES ({0})".format(len(unused)))
+ for (tid, tpl) in unused:
+ print("{0:03d} {1:s}".format(tid, tpl))
+
+ if test_stats is None:
+ print_train_stats()
+ else:
+ print_testtrain_stats()
+ print()
+ if printunused:
+ print_unused_templates()
+ print()
+
+ def batch_tag_incremental(self, sequences, gold):
+ """
+ Tags by applying each rule to the entire corpus (rather than all rules to a
+ single sequence). The point is to collect statistics on the test set for
+ individual rules.
+
+ NOTE: This is inefficient (does not build any index, so will traverse the entire
+ corpus N times for N rules) -- usually you would not care about statistics for
+ individual rules and thus use batch_tag() instead
+
+ :param sequences: lists of token sequences (sentences, in some applications) to be tagged
+ :type sequences: list of list of strings
+ :param gold: the gold standard
+ :type gold: list of list of strings
+ :returns: tuple of (tagged_sequences, ordered list of rule scores (one for each rule))
+ """
+ def counterrors(xs):
+ return sum(t[1] != g[1]
+ for pair in zip(xs, gold)
+ for (t, g) in zip(*pair))
+ testing_stats = {}
+ testing_stats['tokencount'] = sum(len(t) for t in sequences)
+ testing_stats['sequencecount'] = len(sequences)
+ tagged_tokenses = [self._initial_tagger.tag(tokens) for tokens in sequences]
+ testing_stats['initialerrors'] = counterrors(tagged_tokenses)
+ testing_stats['initialacc'] = 1- testing_stats['initialerrors']/testing_stats['tokencount']
+ # Apply each rule to the entire corpus, in order
+ errors = [testing_stats['initialerrors']]
+ for rule in self._rules:
+ for tagged_tokens in tagged_tokenses:
+ rule.apply(tagged_tokens)
+ errors.append(counterrors(tagged_tokenses))
+ testing_stats['rulescores'] = [err0 - err1 for (err0, err1) in zip(errors, errors[1:])]
+ testing_stats['finalerrors'] = errors[-1]
+ testing_stats['finalacc'] = 1 - testing_stats['finalerrors']/testing_stats['tokencount']
+ return (tagged_tokenses, testing_stats)
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/brill_trainer.py b/nltk/tag/brill_trainer.py
new file mode 100644
index 0000000..7e835ed
--- /dev/null
+++ b/nltk/tag/brill_trainer.py
@@ -0,0 +1,630 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function, division
+
+import bisect
+from collections import defaultdict
+import os.path
+from codecs import open
+import textwrap
+
+from nltk.tag.util import untag
+from nltk.tag.brill import BrillTagger
+
+######################################################################
+## Brill Tagger Trainer
+######################################################################
+
+class BrillTaggerTrainer(object):
+ """
+ A trainer for tbl taggers.
+ """
+ def __init__(self, initial_tagger, templates, trace=0,
+ deterministic=None, ruleformat="str"):
+ """
+ Construct a Brill tagger from a baseline tagger and a
+ set of templates
+
+ :param initial_tagger: the baseline tagger
+ :type initial_tagger: Tagger
+ :param templates: templates to be used in training
+ :type templates: list of Templates
+ :param trace: verbosity level
+ :type trace: int
+ :param deterministic: if True, adjudicate ties deterministically
+ :type deterministic: bool
+ :param ruleformat: format of reported Rules
+ :type ruleformat: str
+ :return: An untrained BrillTagger
+ :rtype: BrillTagger
+ """
+
+ if deterministic is None:
+ deterministic = (trace > 0)
+ self._initial_tagger = initial_tagger
+ self._templates = templates
+ self._trace = trace
+ self._deterministic = deterministic
+ self._ruleformat = ruleformat
+
+ self._tag_positions = None
+ """Mapping from tags to lists of positions that use that tag."""
+
+ self._rules_by_position = None
+ """Mapping from positions to the set of rules that are known
+ to occur at that position. Position is (sentnum, wordnum).
+ Initially, this will only contain positions where each rule
+ applies in a helpful way; but when we examine a rule, we'll
+ extend this list to also include positions where each rule
+ applies in a harmful or neutral way."""
+
+ self._positions_by_rule = None
+ """Mapping from rule to position to effect, specifying the
+ effect that each rule has on the overall score, at each
+ position. Position is (sentnum, wordnum); and effect is
+ -1, 0, or 1. As with _rules_by_position, this mapping starts
+ out only containing rules with positive effects; but when
+ we examine a rule, we'll extend this mapping to include
+ the positions where the rule is harmful or neutral."""
+
+ self._rules_by_score = None
+ """Mapping from scores to the set of rules whose effect on the
+ overall score is upper bounded by that score. Invariant:
+ rulesByScore[s] will contain r iff the sum of
+ _positions_by_rule[r] is s."""
+
+ self._rule_scores = None
+ """Mapping from rules to upper bounds on their effects on the
+ overall score. This is the inverse mapping to _rules_by_score.
+ Invariant: ruleScores[r] = sum(_positions_by_rule[r])"""
+
+ self._first_unknown_position = None
+ """Mapping from rules to the first position where we're unsure
+ if the rule applies. This records the next position we
+ need to check to see if the rule messed anything up."""
+
+
+ #////////////////////////////////////////////////////////////
+ # Training
+ #////////////////////////////////////////////////////////////
+
+ def train(self, train_sents, max_rules=200, min_score=2, min_acc=None):
+
+ """
+ Trains the Brill tagger on the corpus *train_sents*,
+ producing at most *max_rules* transformations, each of which
+ reduces the net number of errors in the corpus by at least
+ *min_score*, and each of which has accuracy not lower than
+ *min_acc*.
+
+ #imports
+ >>> from nltk.tbl.template import Template
+ >>> from nltk.tag.brill import Pos, Word
+ >>> from nltk.tag import RegexpTagger, BrillTaggerTrainer
+
+ #some data
+ >>> from nltk.corpus import treebank
+ >>> training_data = treebank.tagged_sents()[:100]
+ >>> baseline_data = treebank.tagged_sents()[100:200]
+ >>> gold_data = treebank.tagged_sents()[200:300]
+ >>> testing_data = [untag(s) for s in gold_data]
+
+ >>> backoff = RegexpTagger([
+ ... (r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
+ ... (r'(The|the|A|a|An|an)$', 'AT'), # articles
+ ... (r'.*able$', 'JJ'), # adjectives
+ ... (r'.*ness$', 'NN'), # nouns formed from adjectives
+ ... (r'.*ly$', 'RB'), # adverbs
+ ... (r'.*s$', 'NNS'), # plural nouns
+ ... (r'.*ing$', 'VBG'), # gerunds
+ ... (r'.*ed$', 'VBD'), # past tense verbs
+ ... (r'.*', 'NN') # nouns (default)
+ ... ])
+
+ >>> baseline = backoff #see NOTE1
+
+ >>> baseline.evaluate(gold_data) #doctest: +ELLIPSIS
+ 0.2450142...
+
+ #templates
+ >>> Template._cleartemplates() #clear any templates created in earlier tests
+ >>> templates = [Template(Pos([-1])), Template(Pos([-1]), Word([0]))]
+
+ #construct a BrillTaggerTrainer
+ >>> tt = BrillTaggerTrainer(baseline, templates, trace=3)
+ >>> tagger1 = tt.train(training_data, max_rules=10)
+ TBL train (fast) (seqs: 100; tokens: 2417; tpls: 2; min score: 2; min acc: None)
+ Finding initial useful rules...
+ Found 845 useful rules.
+ <BLANKLINE>
+ B |
+ S F r O | Score = Fixed - Broken
+ c i o t | R Fixed = num tags changed incorrect -> correct
+ o x k h | u Broken = num tags changed correct -> incorrect
+ r e e e | l Other = num tags changed incorrect -> incorrect
+ e d n r | e
+ ------------------+-------------------------------------------------------
+ 132 132 0 0 | AT->DT if Pos:NN@[-1]
+ 85 85 0 0 | NN->, if Pos:NN@[-1] & Word:,@[0]
+ 69 69 0 0 | NN->. if Pos:NN@[-1] & Word:.@[0]
+ 51 51 0 0 | NN->IN if Pos:NN@[-1] & Word:of@[0]
+ 47 63 16 161 | NN->IN if Pos:NNS@[-1]
+ 33 33 0 0 | NN->TO if Pos:NN@[-1] & Word:to@[0]
+ 26 26 0 0 | IN->. if Pos:NNS@[-1] & Word:.@[0]
+ 24 24 0 0 | IN->, if Pos:NNS@[-1] & Word:,@[0]
+ 22 27 5 24 | NN->-NONE- if Pos:VBD@[-1]
+ 17 17 0 0 | NN->CC if Pos:NN@[-1] & Word:and@[0]
+
+
+
+ >>> tagger1.rules()[1:3]
+ (Rule('001', 'NN', ',', [(Pos([-1]),'NN'), (Word([0]),',')]), Rule('001', 'NN', '.', [(Pos([-1]),'NN'), (Word([0]),'.')]))
+
+
+ >>> train_stats = tagger1.train_stats()
+ >>> [train_stats[stat] for stat in ['initialerrors', 'finalerrors', 'rulescores']]
+ [1775, 1269, [132, 85, 69, 51, 47, 33, 26, 24, 22, 17]]
+
+
+ ##FIXME: the following test fails -- why?
+ #
+ #>>> tagger1.print_template_statistics(printunused=False)
+ #TEMPLATE STATISTICS (TRAIN) 2 templates, 10 rules)
+ #TRAIN ( 3163 tokens) initial 2358 0.2545 final: 1719 0.4565
+ ##ID | Score (train) | #Rules | Template
+ #--------------------------------------------
+ #001 | 404 0.632 | 7 0.700 | Template(Pos([-1]),Word([0]))
+ #000 | 235 0.368 | 3 0.300 | Template(Pos([-1]))
+ #<BLANKLINE>
+ #<BLANKLINE>
+
+ >>> tagger1.evaluate(gold_data) # doctest: +ELLIPSIS
+ 0.43996...
+
+ >>> (tagged, test_stats) = tagger1.batch_tag_incremental(testing_data, gold_data)
+
+
+ >>> tagged[33][12:] == [('foreign', 'IN'), ('debt', 'NN'), ('of', 'IN'), ('$', 'NN'), ('64', 'CD'),
+ ... ('billion', 'NN'), ('*U*', 'NN'), ('--', 'NN'), ('the', 'DT'), ('third-highest', 'NN'), ('in', 'NN'),
+ ... ('the', 'DT'), ('developing', 'VBG'), ('world', 'NN'), ('.', '.')]
+ True
+
+
+ >>> [test_stats[stat] for stat in ['initialerrors', 'finalerrors', 'rulescores']]
+ [1855, 1376, [100, 85, 67, 58, 27, 36, 27, 16, 31, 32]]
+
+ ##a high-accuracy tagger
+ >>> tagger2 = tt.train(training_data, max_rules=10, min_acc=0.99)
+ TBL train (fast) (seqs: 100; tokens: 2417; tpls: 2; min score: 2; min acc: 0.99)
+ Finding initial useful rules...
+ Found 845 useful rules.
+ <BLANKLINE>
+ B |
+ S F r O | Score = Fixed - Broken
+ c i o t | R Fixed = num tags changed incorrect -> correct
+ o x k h | u Broken = num tags changed correct -> incorrect
+ r e e e | l Other = num tags changed incorrect -> incorrect
+ e d n r | e
+ ------------------+-------------------------------------------------------
+ 132 132 0 0 | AT->DT if Pos:NN@[-1]
+ 85 85 0 0 | NN->, if Pos:NN@[-1] & Word:,@[0]
+ 69 69 0 0 | NN->. if Pos:NN@[-1] & Word:.@[0]
+ 51 51 0 0 | NN->IN if Pos:NN@[-1] & Word:of@[0]
+ 36 36 0 0 | NN->TO if Pos:NN@[-1] & Word:to@[0]
+ 26 26 0 0 | NN->. if Pos:NNS@[-1] & Word:.@[0]
+ 24 24 0 0 | NN->, if Pos:NNS@[-1] & Word:,@[0]
+ 19 19 0 6 | NN->VB if Pos:TO@[-1]
+ 18 18 0 0 | CD->-NONE- if Pos:NN@[-1] & Word:0@[0]
+ 18 18 0 0 | NN->CC if Pos:NN@[-1] & Word:and@[0]
+
+
+ >>> tagger2.evaluate(gold_data) # doctest: +ELLIPSIS
+ 0.44159544...
+
+ >>> tagger2.rules()[2:4]
+ (Rule('001', 'NN', '.', [(Pos([-1]),'NN'), (Word([0]),'.')]), Rule('001', 'NN', 'IN', [(Pos([-1]),'NN'), (Word([0]),'of')]))
+
+ #NOTE1: (!!FIXME) A far better baseline uses nltk.tag.UnigramTagger,
+ #with a RegexpTagger only as backoff. For instance,
+ #>>> baseline = UnigramTagger(baseline_data, backoff=backoff)
+ #However, as of Nov 2013, nltk.tag.UnigramTagger does not yield consistent results
+ #between python versions. The simplistic backoff above is a workaround to make doctests
+ #get consistent input.
+
+
+ :param train_sents: training data
+ :type train_sents: list(list(tuple))
+ :param max_rules: output at most max_rules rules
+ :type max_rules: int
+ :param min_score: stop training when no rules better than min_score can be found
+ :type min_score: int
+ :param min_acc: discard any rule with lower accuracy than min_acc
+ :type min_acc: float or None
+ :return: the learned tagger
+ :rtype: BrillTagger
+
+ """
+ #!! FIXME: several tests are a bit too dependent on tracing format
+ #!! FIXME: tests in trainer.fast and trainer.brillorig are exact duplicates
+
+
+ # Basic idea: Keep track of the rules that apply at each position.
+ # And keep track of the positions to which each rule applies.
+
+ # Create a new copy of the training corpus, and run the
+ # initial tagger on it. We will progressively update this
+ # test corpus to look more like the training corpus.
+ test_sents = [list(self._initial_tagger.tag(untag(sent)))
+ for sent in train_sents]
+
+ # Collect some statistics on the training process
+ trainstats = {}
+ trainstats['min_acc'] = min_acc
+ trainstats['min_score'] = min_score
+ trainstats['tokencount'] = sum(len(t) for t in test_sents)
+ trainstats['sequencecount'] = len(test_sents)
+ trainstats['templatecount'] = len(self._templates)
+ trainstats['rulescores'] = []
+ trainstats['initialerrors'] = sum(tag[1] != truth[1]
+ for paired in zip(test_sents, train_sents)
+ for (tag, truth) in zip(*paired))
+ trainstats['initialacc'] = 1 - trainstats['initialerrors']/trainstats['tokencount']
+ if self._trace > 0:
+ print("TBL train (fast) (seqs: {sequencecount}; tokens: {tokencount}; "
+ "tpls: {templatecount}; min score: {min_score}; min acc: {min_acc})".format(**trainstats))
+
+ # Initialize our mappings. This will find any errors made
+ # by the initial tagger, and use those to generate repair
+ # rules, which are added to the rule mappings.
+ if self._trace > 0: print("Finding initial useful rules...")
+ self._init_mappings(test_sents, train_sents)
+ if self._trace > 0: print((" Found %d useful rules." %
+ len(self._rule_scores)))
+
+ # Let the user know what we're up to.
+ if self._trace > 2: self._trace_header()
+ elif self._trace == 1: print("Selecting rules...")
+
+ # Repeatedly select the best rule, and add it to `rules`.
+ rules = []
+ try:
+ while (len(rules) < max_rules):
+ # Find the best rule, and add it to our rule list.
+ rule = self._best_rule(train_sents, test_sents, min_score, min_acc)
+ if rule:
+ rules.append(rule)
+ score = self._rule_scores[rule]
+ trainstats['rulescores'].append(score)
+ else:
+ break # No more good rules left!
+
+ # Report the rule that we found.
+ if self._trace > 1: self._trace_rule(rule)
+
+ # Apply the new rule at the relevant sites
+ self._apply_rule(rule, test_sents)
+
+ # Update _tag_positions[rule.original_tag] and
+ # _tag_positions[rule.replacement_tag] for the affected
+ # positions (i.e., self._positions_by_rule[rule]).
+ self._update_tag_positions(rule)
+
+ # Update rules that were affected by the change.
+ self._update_rules(rule, train_sents, test_sents)
+
+ # The user can cancel training manually:
+ except KeyboardInterrupt:
+ print("Training stopped manually -- %d rules found" % len(rules))
+
+ # Discard our tag position mapping & rule mappings.
+ self._clean()
+ trainstats['finalerrors'] = trainstats['initialerrors'] - sum(trainstats['rulescores'])
+ trainstats['finalacc'] = 1 - trainstats['finalerrors']/trainstats['tokencount']
+ # Create and return a tagger from the rules we found.
+ return BrillTagger(self._initial_tagger, rules, trainstats)
+
+ def _init_mappings(self, test_sents, train_sents):
+ """
+ Initialize the tag position mapping & the rule related
+ mappings. For each error in test_sents, find new rules that
+ would correct them, and add them to the rule mappings.
+ """
+ self._tag_positions = defaultdict(list)
+ self._rules_by_position = defaultdict(set)
+ self._positions_by_rule = defaultdict(dict)
+ self._rules_by_score = defaultdict(set)
+ self._rule_scores = defaultdict(int)
+ self._first_unknown_position = defaultdict(int)
+ # Scan through the corpus, initializing the tag_positions
+ # mapping and all the rule-related mappings.
+ for sentnum, sent in enumerate(test_sents):
+ for wordnum, (word, tag) in enumerate(sent):
+
+ # Initialize tag_positions
+ self._tag_positions[tag].append( (sentnum,wordnum) )
+
+ # If it's an error token, update the rule-related mappings.
+ correct_tag = train_sents[sentnum][wordnum][1]
+ if tag != correct_tag:
+ for rule in self._find_rules(sent, wordnum, correct_tag):
+ self._update_rule_applies(rule, sentnum, wordnum,
+ train_sents)
+
+ def _clean(self):
+ self._tag_positions = None
+ self._rules_by_position = None
+ self._positions_by_rule = None
+ self._rules_by_score = None
+ self._rule_scores = None
+ self._first_unknown_position = None
+
+ def _find_rules(self, sent, wordnum, new_tag):
+ """
+ Use the templates to find rules that apply at index *wordnum*
+ in the sentence *sent* and generate the tag *new_tag*.
+ """
+ for template in self._templates:
+ for rule in template.applicable_rules(sent, wordnum, new_tag):
+ yield rule
+
+ def _update_rule_applies(self, rule, sentnum, wordnum, train_sents):
+ """
+ Update the rule data tables to reflect the fact that
+ *rule* applies at the position *(sentnum, wordnum)*.
+ """
+ pos = sentnum, wordnum
+
+ # If the rule is already known to apply here, ignore.
+ # (This only happens if the position's tag hasn't changed.)
+ if pos in self._positions_by_rule[rule]:
+ return
+
+ # Update self._positions_by_rule.
+ correct_tag = train_sents[sentnum][wordnum][1]
+ if rule.replacement_tag == correct_tag:
+ self._positions_by_rule[rule][pos] = 1
+ elif rule.original_tag == correct_tag:
+ self._positions_by_rule[rule][pos] = -1
+ else: # was wrong, remains wrong
+ self._positions_by_rule[rule][pos] = 0
+
+ # Update _rules_by_position
+ self._rules_by_position[pos].add(rule)
+
+ # Update _rule_scores.
+ old_score = self._rule_scores[rule]
+ self._rule_scores[rule] += self._positions_by_rule[rule][pos]
+
+ # Update _rules_by_score.
+ self._rules_by_score[old_score].discard(rule)
+ self._rules_by_score[self._rule_scores[rule]].add(rule)
+
+ def _update_rule_not_applies(self, rule, sentnum, wordnum):
+ """
+ Update the rule data tables to reflect the fact that *rule*
+ does not apply at the position *(sentnum, wordnum)*.
+ """
+ pos = sentnum, wordnum
+
+ # Update _rule_scores.
+ old_score = self._rule_scores[rule]
+ self._rule_scores[rule] -= self._positions_by_rule[rule][pos]
+
+ # Update _rules_by_score.
+ self._rules_by_score[old_score].discard(rule)
+ self._rules_by_score[self._rule_scores[rule]].add(rule)
+
+ # Update _positions_by_rule
+ del self._positions_by_rule[rule][pos]
+ self._rules_by_position[pos].remove(rule)
+
+ # Optional addition: if the rule now applies nowhere, delete
+ # all its dictionary entries.
+
+ def _best_rule(self, train_sents, test_sents, min_score, min_acc):
+ """
+ Find the next best rule. This is done by repeatedly taking a
+ rule with the highest score and stepping through the corpus to
+ see where it applies. When it makes an error (decreasing its
+ score) it's bumped down, and we try a new rule with the
+ highest score. When we find a rule which has the highest
+ score *and* which has been tested against the entire corpus, we
+ can conclude that it's the next best rule.
+ """
+ for max_score in sorted(self._rules_by_score.keys(), reverse=True):
+ if len(self._rules_by_score) == 0:
+ return None
+ if max_score < min_score or max_score <= 0:
+ return None
+ best_rules = list(self._rules_by_score[max_score])
+ if self._deterministic:
+ best_rules.sort(key=repr)
+ for rule in best_rules:
+ positions = self._tag_positions[rule.original_tag]
+
+ unk = self._first_unknown_position.get(rule, (0,-1))
+ start = bisect.bisect_left(positions, unk)
+
+ for i in range(start, len(positions)):
+ sentnum, wordnum = positions[i]
+ if rule.applies(test_sents[sentnum], wordnum):
+ self._update_rule_applies(rule, sentnum, wordnum,
+ train_sents)
+ if self._rule_scores[rule] < max_score:
+ self._first_unknown_position[rule] = (sentnum,
+ wordnum+1)
+ break # The update demoted the rule.
+
+ if self._rule_scores[rule] == max_score:
+ self._first_unknown_position[rule] = (len(train_sents)+1,0)
+ #optimization: if no min_acc threshold given, don't bother computing accuracy
+ if min_acc is None:
+ return rule
+ else:
+ changes = self._positions_by_rule[rule].values()
+ num_fixed = len([c for c in changes if c==1])
+ num_broken = len([c for c in changes if c==-1])
+ #acc here is fixed/(fixed+broken); could also be
+ #fixed/(fixed+broken+other) == num_fixed/len(changes)
+ acc = num_fixed/(num_fixed+num_broken)
+ if acc >= min_acc:
+ return rule
+ #else: rule too inaccurate, discard and try next
+
+ # We demoted (or skipped due to < min_acc, if that was given)
+ # all the rules with score==max_score.
+
+ assert min_acc is not None or not self._rules_by_score[max_score]
+ if not self._rules_by_score[max_score]:
+ del self._rules_by_score[max_score]
+
+
+
+
+ def _apply_rule(self, rule, test_sents):
+ """
+ Update *test_sents* by applying *rule* everywhere where its
+ conditions are met.
+ """
+ update_positions = set(self._positions_by_rule[rule])
+ old_tag = rule.original_tag
+ new_tag = rule.replacement_tag
+
+ if self._trace > 3: self._trace_apply(len(update_positions))
+
+ # Update test_sents.
+ for (sentnum, wordnum) in update_positions:
+ text = test_sents[sentnum][wordnum][0]
+ test_sents[sentnum][wordnum] = (text, new_tag)
+
+ def _update_tag_positions(self, rule):
+ """
+ Update _tag_positions to reflect the changes to tags that are
+ made by *rule*.
+ """
+ # Update the tag index.
+ for pos in self._positions_by_rule[rule]:
+ # Delete the old tag.
+ old_tag_positions = self._tag_positions[rule.original_tag]
+ old_index = bisect.bisect_left(old_tag_positions, pos)
+ del old_tag_positions[old_index]
+ # Insert the new tag.
+ new_tag_positions = self._tag_positions[rule.replacement_tag]
+ bisect.insort_left(new_tag_positions, pos)
+
+ def _update_rules(self, rule, train_sents, test_sents):
+ """
+ Check if we should add or remove any rules from consideration,
+ given the changes made by *rule*.
+ """
+ # Collect a list of all positions that might be affected.
+ neighbors = set()
+ for sentnum, wordnum in self._positions_by_rule[rule]:
+ for template in self._templates:
+ n = template.get_neighborhood(test_sents[sentnum], wordnum)
+ neighbors.update([(sentnum, i) for i in n])
+
+ # Update the rules at each position.
+ num_obsolete = num_new = num_unseen = 0
+ for sentnum, wordnum in neighbors:
+ test_sent = test_sents[sentnum]
+ correct_tag = train_sents[sentnum][wordnum][1]
+
+ # Check if the change causes any rule at this position to
+ # stop matching; if so, then update our rule mappings
+ # accordingly.
+ old_rules = set(self._rules_by_position[sentnum, wordnum])
+ for old_rule in old_rules:
+ if not old_rule.applies(test_sent, wordnum):
+ num_obsolete += 1
+ self._update_rule_not_applies(old_rule, sentnum, wordnum)
+
+ # Check if the change causes our templates to propose any
+ # new rules for this position.
+ site_rules = set()
+ for template in self._templates:
+ for new_rule in template.applicable_rules(test_sent, wordnum,
+ correct_tag):
+ if new_rule not in old_rules:
+ num_new += 1
+ if new_rule not in self._rule_scores:
+ num_unseen += 1
+ old_rules.add(new_rule)
+ self._update_rule_applies(new_rule, sentnum,
+ wordnum, train_sents)
+
+ # We may have caused other rules to match here, that are
+ # not proposed by our templates -- in particular, rules
+ # that are harmful or neutral. We therefore need to
+ # update any rule whose first_unknown_position is past
+ # this rule.
+ for new_rule, pos in self._first_unknown_position.items():
+ if pos > (sentnum, wordnum):
+ if new_rule not in old_rules:
+ num_new += 1
+ if new_rule.applies(test_sent, wordnum):
+ self._update_rule_applies(new_rule, sentnum,
+ wordnum, train_sents)
+
+ if self._trace > 3:
+ self._trace_update_rules(num_obsolete, num_new, num_unseen)
+
+ #////////////////////////////////////////////////////////////
+ # Tracing
+ #////////////////////////////////////////////////////////////
+
+ def _trace_header(self):
+ print("""
+ B |
+ S F r O | Score = Fixed - Broken
+ c i o t | R Fixed = num tags changed incorrect -> correct
+ o x k h | u Broken = num tags changed correct -> incorrect
+ r e e e | l Other = num tags changed incorrect -> incorrect
+ e d n r | e
+------------------+-------------------------------------------------------
+ """.rstrip())
+
+ def _trace_rule(self, rule):
+ assert self._rule_scores[rule] == \
+ sum(self._positions_by_rule[rule].values())
+
+ changes = self._positions_by_rule[rule].values()
+ num_changed = len(changes)
+ num_fixed = len([c for c in changes if c==1])
+ num_broken = len([c for c in changes if c==-1])
+ num_other = len([c for c in changes if c==0])
+ score = self._rule_scores[rule]
+
+ rulestr = rule.format(self._ruleformat)
+ if self._trace > 2:
+ print('%4d%4d%4d%4d |' % (score,num_fixed,num_broken,num_other), end=' ')
+ print(textwrap.fill(rulestr, initial_indent=' '*20, width=79,
+ subsequent_indent=' '*18+'| ').strip())
+ else:
+ print(rulestr)
+
+ def _trace_apply(self, num_updates):
+ prefix = ' '*18+'|'
+ print(prefix)
+ print(prefix, 'Applying rule to %d positions.' % num_updates)
+
+ def _trace_update_rules(self, num_obsolete, num_new, num_unseen):
+ prefix = ' '*18+'|'
+ print(prefix, 'Updated rule tables:')
+ print(prefix, (' - %d rule applications removed' % num_obsolete))
+ print(prefix, (' - %d rule applications added (%d novel)' %
+ (num_new, num_unseen)))
+ print(prefix)
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/brill_trainer_orig.py b/nltk/tag/brill_trainer_orig.py
new file mode 100644
index 0000000..b8a4423
--- /dev/null
+++ b/nltk/tag/brill_trainer_orig.py
@@ -0,0 +1,414 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2013 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function, division
+
+from collections import defaultdict
+import textwrap
+
+from nltk.tag.util import untag
+from nltk.tag.brill import BrillTagger
+
+######################################################################
+## Original Brill Tagger Trainer
+######################################################################
+
+class BrillTaggerTrainer(object):
+ """
+ A trainer for tbl taggers, superseded by nltk.tag.brill_trainer.BrillTaggerTrainer
+
+ :param deterministic: If true, then choose between rules that
+ have the same score by picking the one whose __repr__
+ is lexicographically smaller. If false, then just pick the
+ first rule we find with a given score -- this will depend
+ on the order in which keys are returned from dictionaries,
+ and so may not be the same from one run to the next. If
+ not specified, treat as true iff trace > 0.
+ """
+
+ def __init__(self, initial_tagger, templates, trace=0,
+ deterministic=None, ruleformat="str"):
+ if deterministic is None:
+ deterministic = (trace > 0)
+ self._initial_tagger = initial_tagger
+ self._templates = templates
+ self._trace = trace
+ self._deterministic = deterministic
+ self._ruleformat = ruleformat
+
+ #////////////////////////////////////////////////////////////
+ # Training
+ #////////////////////////////////////////////////////////////
+
+ def train(self, train_sents, max_rules=200, min_score=2, min_acc=None):
+ """
+ Trains the Brill tagger on the corpus *train_sents*,
+ producing at most *max_rules* transformations, each of which
+ reduces the net number of errors in the corpus by at least
+ *min_score*, and each of which has accuracy not lower than
+ *min_acc*.
+
+ #imports
+ >>> from nltk.tbl.template import Template
+ >>> from nltk.tag.brill import Pos, Word
+ >>> from nltk.tag import RegexpTagger
+ >>> from nltk.tag.brill_trainer_orig import BrillTaggerTrainer
+
+ #some data
+ >>> from nltk.corpus import treebank
+ >>> training_data = treebank.tagged_sents()[:100]
+ >>> baseline_data = treebank.tagged_sents()[100:200]
+ >>> gold_data = treebank.tagged_sents()[200:300]
+ >>> testing_data = [untag(s) for s in gold_data]
+
+ >>> backoff = RegexpTagger([
+ ... (r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
+ ... (r'(The|the|A|a|An|an)$', 'AT'), # articles
+ ... (r'.*able$', 'JJ'), # adjectives
+ ... (r'.*ness$', 'NN'), # nouns formed from adjectives
+ ... (r'.*ly$', 'RB'), # adverbs
+ ... (r'.*s$', 'NNS'), # plural nouns
+ ... (r'.*ing$', 'VBG'), # gerunds
+ ... (r'.*ed$', 'VBD'), # past tense verbs
+ ... (r'.*', 'NN') # nouns (default)
+ ... ])
+
+ >>> baseline = backoff #see NOTE1
+
+ >>> baseline.evaluate(gold_data) #doctest: +ELLIPSIS
+ 0.2450142...
+
+ #templates
+ >>> Template._cleartemplates() #clear any templates created in earlier tests
+ >>> templates = [Template(Pos([-1])), Template(Pos([-1]), Word([0]))]
+
+ #construct a BrillTaggerTrainer
+ >>> tt = BrillTaggerTrainer(baseline, templates, trace=3)
+ >>> tagger1 = tt.train(training_data, max_rules=10)
+ TBL train (orig) (seqs: 100; tokens: 2417; tpls: 2; min score: 2; min acc: None)
+ <BLANKLINE>
+ B |
+ S F r O | Score = Fixed - Broken
+ c i o t | R Fixed = num tags changed incorrect -> correct
+ o x k h | u Broken = num tags changed correct -> incorrect
+ r e e e | l Other = num tags changed incorrect -> incorrect
+ e d n r | e
+ ------------------+-------------------------------------------------------
+ 132 132 0 0 | AT->DT if Pos:NN@[-1]
+ 85 85 0 0 | NN->, if Pos:NN@[-1] & Word:,@[0]
+ 69 69 0 0 | NN->. if Pos:NN@[-1] & Word:.@[0]
+ 51 51 0 0 | NN->IN if Pos:NN@[-1] & Word:of@[0]
+ 47 63 16 161 | NN->IN if Pos:NNS@[-1]
+ 33 33 0 0 | NN->TO if Pos:NN@[-1] & Word:to@[0]
+ 26 26 0 0 | IN->. if Pos:NNS@[-1] & Word:.@[0]
+ 24 24 0 0 | IN->, if Pos:NNS@[-1] & Word:,@[0]
+ 22 27 5 24 | NN->-NONE- if Pos:VBD@[-1]
+ 17 17 0 0 | NN->CC if Pos:NN@[-1] & Word:and@[0]
+
+
+
+ >>> tagger1.rules()[1:3]
+ (Rule('001', 'NN', ',', [(Pos([-1]),'NN'), (Word([0]),',')]), Rule('001', 'NN', '.', [(Pos([-1]),'NN'), (Word([0]),'.')]))
+
+
+ >>> train_stats = tagger1.train_stats()
+ >>> [train_stats[stat] for stat in ['initialerrors', 'finalerrors', 'rulescores']]
+ [1775, 1269, [132, 85, 69, 51, 47, 33, 26, 24, 22, 17]]
+
+
+ ##FIXME: the following test fails -- why?
+ #
+ #>>> tagger1.print_template_statistics(printunused=False)
+ #TEMPLATE STATISTICS (TRAIN) 2 templates, 10 rules)
+ #TRAIN ( 3163 tokens) initial 2358 0.2545 final: 1719 0.4565
+ ##ID | Score (train) | #Rules | Template
+ #--------------------------------------------
+ #001 | 404 0.632 | 7 0.700 | Template(Pos([-1]),Word([0]))
+ #000 | 235 0.368 | 3 0.300 | Template(Pos([-1]))
+ #<BLANKLINE>
+ #<BLANKLINE>
+
+ >>> tagger1.evaluate(gold_data) # doctest: +ELLIPSIS
+ 0.43996...
+
+ >>> (tagged, test_stats) = tagger1.batch_tag_incremental(testing_data, gold_data)
+
+
+ >>> tagged[33][12:] == [('foreign', 'IN'), ('debt', 'NN'), ('of', 'IN'), ('$', 'NN'), ('64', 'CD'),
+ ... ('billion', 'NN'), ('*U*', 'NN'), ('--', 'NN'), ('the', 'DT'), ('third-highest', 'NN'), ('in', 'NN'),
+ ... ('the', 'DT'), ('developing', 'VBG'), ('world', 'NN'), ('.', '.')]
+ True
+
+
+ >>> [test_stats[stat] for stat in ['initialerrors', 'finalerrors', 'rulescores']]
+ [1855, 1376, [100, 85, 67, 58, 27, 36, 27, 16, 31, 32]]
+
+ ##a high-accuracy tagger
+ >>> tagger2 = tt.train(training_data, max_rules=10, min_acc=0.99)
+ TBL train (orig) (seqs: 100; tokens: 2417; tpls: 2; min score: 2; min acc: 0.99)
+ <BLANKLINE>
+ B |
+ S F r O | Score = Fixed - Broken
+ c i o t | R Fixed = num tags changed incorrect -> correct
+ o x k h | u Broken = num tags changed correct -> incorrect
+ r e e e | l Other = num tags changed incorrect -> incorrect
+ e d n r | e
+ ------------------+-------------------------------------------------------
+ 132 132 0 0 | AT->DT if Pos:NN@[-1]
+ 85 85 0 0 | NN->, if Pos:NN@[-1] & Word:,@[0]
+ 69 69 0 0 | NN->. if Pos:NN@[-1] & Word:.@[0]
+ 51 51 0 0 | NN->IN if Pos:NN@[-1] & Word:of@[0]
+ 36 36 0 0 | NN->TO if Pos:NN@[-1] & Word:to@[0]
+ 26 26 0 0 | NN->. if Pos:NNS@[-1] & Word:.@[0]
+ 24 24 0 0 | NN->, if Pos:NNS@[-1] & Word:,@[0]
+ 19 19 0 6 | NN->VB if Pos:TO@[-1]
+ 18 18 0 0 | CD->-NONE- if Pos:NN@[-1] & Word:0@[0]
+ 18 18 0 0 | NN->CC if Pos:NN@[-1] & Word:and@[0]
+
+
+ >>> tagger2.evaluate(gold_data) # doctest: +ELLIPSIS
+ 0.44159544...
+
+ >>> tagger2.rules()[2:4]
+ (Rule('001', 'NN', '.', [(Pos([-1]),'NN'), (Word([0]),'.')]), Rule('001', 'NN', 'IN', [(Pos([-1]),'NN'), (Word([0]),'of')]))
+
+ #NOTE1: (!!FIXME) A far better baseline uses nltk.tag.UnigramTagger,
+ #with a RegexpTagger only as backoff. For instance,
+ #>>> baseline = UnigramTagger(baseline_data, backoff=backoff)
+ #However, as of Nov 2013, nltk.tag.UnigramTagger does not yield consistent results
+ #between python versions. The simplistic backoff above is a workaround to make doctests
+ #get consistent input.
+
+ :param train_sents: training data
+ :type train_sents: list(list(tuple))
+ :param max_rules: output at most max_rules rules
+ :type max_rules: int
+ :param min_score: stop training when no rules better than min_score can be found
+ :type min_score: int
+ :param min_acc: discard any rule with lower accuracy than min_acc
+ :type min_acc: float or None
+ :return: the learned tagger
+ :rtype: BrillTagger
+
+
+ :param train_sents: training data
+ :type train_sents: list(list(tuple))
+ :param max_rules: output at most max_rules rules
+ :type max_rules: int
+ :param min_score: stop training when no rules better than min_score can be found
+ :type min_score: int
+ :param min_acc: discard any rule with lower accuracy than min_acc
+ :type min_acc: float or None
+ :return: the learned tagger
+ :rtype: BrillTagger
+
+ """
+
+ # Create a new copy of the training corpus, and run the
+ # initial tagger on it. We will progressively update this
+ # test corpus to look more like the training corpus.
+ test_sents = [self._initial_tagger.tag(untag(sent))
+ for sent in train_sents]
+ trainstats = {}
+ trainstats['min_acc'] = min_acc
+ trainstats['min_score'] = min_score
+ trainstats['tokencount'] = sum(len(t) for t in test_sents)
+ trainstats['sequencecount'] = len(test_sents)
+ trainstats['templatecount'] = len(self._templates)
+ trainstats['rulescores'] = []
+ trainstats['initialerrors'] = sum(tag[1] != truth[1]
+ for paired in zip(test_sents, train_sents)
+ for (tag, truth) in zip(*paired))
+ trainstats['initialacc'] = 1 - trainstats['initialerrors']/trainstats['tokencount']
+ if self._trace > 0:
+ print("TBL train (orig) (seqs: {sequencecount}; tokens: {tokencount}; "
+ "tpls: {templatecount}; min score: {min_score}; min acc: {min_acc})".format(**trainstats))
+
+ if self._trace > 2:
+ self._trace_header()
+
+ # Look for useful rules.
+ rules = []
+ try:
+ while len(rules) < max_rules:
+ (rule, score, fixscore) = self._best_rule(test_sents,
+ train_sents, min_acc=min_acc)
+ if rule is None or score < min_score:
+ if self._trace > 1:
+ print('Insufficient improvement; stopping')
+ break
+ else:
+ # Add the rule to our list of rules.
+ rules.append(rule)
+ trainstats['rulescores'].append(score)
+ # Use the rules to update the test corpus. Keep
+ # track of how many times the rule applied (k).
+ k = 0
+ for sent in test_sents:
+ k += len(rule.apply(sent))
+ # Display trace output.
+ if self._trace > 1:
+ self._trace_rule(rule, score, fixscore, k)
+ # The user can also cancel training manually:
+ except KeyboardInterrupt:
+ print("Training stopped manually -- %d rules found" % len(rules))
+
+ trainstats['finalerrors'] = trainstats['initialerrors'] - sum(trainstats['rulescores'])
+ trainstats['finalacc'] = 1 - trainstats['finalerrors']/trainstats['tokencount']
+ # Create and return a tagger from the rules we found.
+ return BrillTagger(self._initial_tagger, rules, trainstats)
+
+ #////////////////////////////////////////////////////////////
+ # Finding the best rule
+ #////////////////////////////////////////////////////////////
+
+ # Finds the rule that makes the biggest net improvement in the corpus.
+ # Returns a (rule, score) pair.
+ def _best_rule(self, test_sents, train_sents, min_acc):
+ # Create a dictionary mapping from each tag to a list of the
+ # indices that have that tag in both test_sents and
+ # train_sents (i.e., where it is correctly tagged).
+ correct_indices = defaultdict(list)
+ for sentnum, sent in enumerate(test_sents):
+ for wordnum, tagged_word in enumerate(sent):
+ if tagged_word[1] == train_sents[sentnum][wordnum][1]:
+ tag = tagged_word[1]
+ correct_indices[tag].append( (sentnum, wordnum) )
+
+ # Find all the rules that correct at least one token's tag,
+ # and the number of tags that each rule corrects (in
+ # descending order of number of tags corrected).
+ rules = self._find_rules(test_sents, train_sents)
+
+ # Keep track of the current best rule, and its score.
+ best_rule, best_score, best_fixscore = None, 0, 0
+
+ # Consider each rule, in descending order of fixscore (the
+ # number of tags that the rule corrects, not including the
+ # number that it breaks).
+ for (rule, fixscore) in rules:
+ # The actual score must be <= fixscore; so if best_score
+ # is bigger than fixscore, then we already have the best
+ # rule.
+ if best_score > fixscore or (best_score == fixscore and
+ not self._deterministic):
+ return best_rule, best_score, best_fixscore
+
+ # Calculate the actual score, by decrementing score (initialized
+ # to fixscore once for each tag that the rule changes to an incorrect
+ # value.
+ score = fixscore
+ if rule.original_tag in correct_indices:
+ for (sentnum, wordnum) in correct_indices[rule.original_tag]:
+ if rule.applies(test_sents[sentnum], wordnum):
+ score -= 1
+ # If the rule accuracy goes below min_acc,
+ # this rule is not eligible; so move on
+
+ if min_acc is not None and fixscore/(2*fixscore-score) < min_acc:
+ break
+ # If the score goes below best_score, then we know
+ # that this isn't the best rule; so move on
+ if score < best_score or (score == best_score and
+ not self._deterministic):
+ break
+
+ # If the actual score is better than the best score, then
+ # update best_score and best_rule.
+ if (( min_acc is None or #IF: either no threshold for accuracy,
+ fixscore/(2*fixscore-score) >= min_acc) and #or accuracy good enough AND
+ ( score > best_score or #(score is higher than current leader OR
+ (score == best_score and #score is same as leader, but this
+ self._deterministic and #rule sorts before it when determinism
+ repr(rule) < repr(best_rule)))): # is asked for): THEN update...
+ best_rule, best_score, best_fixscore = rule, score, fixscore
+
+ # Return the best rule, and its score.
+ return best_rule, best_score, best_fixscore
+
+ def _find_rules(self, test_sents, train_sents):
+ """
+ Find all rules that correct at least one token's tag in *test_sents*.
+
+ :return: A list of tuples ``(rule, fixscore)``, where rule
+ is a tbl rule and ``fixscore`` is the number of tokens
+ whose tag the rule corrects. Note that ``fixscore`` does
+ *not* include the number of tokens whose tags are changed
+ to incorrect values.
+ """
+
+ # Create a list of all indices that are incorrectly tagged.
+ error_indices = []
+ for sentnum, sent in enumerate(test_sents):
+ for wordnum, tagged_word in enumerate(sent):
+ if tagged_word[1] != train_sents[sentnum][wordnum][1]:
+ error_indices.append( (sentnum, wordnum) )
+
+ # Create a dictionary mapping from rules to their positive-only
+ # scores.
+ rule_score_dict = defaultdict(int)
+ for (sentnum, wordnum) in error_indices:
+ test_sent = test_sents[sentnum]
+ train_sent = train_sents[sentnum]
+ for rule in self._find_rules_at(test_sent, train_sent, wordnum):
+ rule_score_dict[rule] += 1
+
+ # Convert the dictionary into a list of (rule, score) tuples,
+ # sorted in descending order of score.
+ return sorted(rule_score_dict.items(),
+ key=lambda rule_score: -rule_score[1])
+
+ def _find_rules_at(self, test_sent, train_sent, i):
+ """
+ :rtype: set
+ :return: the set of all rules (based on the templates) that
+ correct token *i*'s tag in *test_sent*.
+ """
+ applicable_rules = set()
+ if test_sent[i][1] != train_sent[i][1]:
+ correct_tag = train_sent[i][1]
+ for template in self._templates:
+ new_rules = template.applicable_rules(test_sent, i,
+ correct_tag)
+ applicable_rules.update(new_rules)
+
+ return applicable_rules
+
+ #////////////////////////////////////////////////////////////
+ # Tracing
+ #////////////////////////////////////////////////////////////
+
+ def _trace_header(self):
+ print("""
+ B |
+ S F r O | Score = Fixed - Broken
+ c i o t | R Fixed = num tags changed incorrect -> correct
+ o x k h | u Broken = num tags changed correct -> incorrect
+ r e e e | l Other = num tags changed incorrect -> incorrect
+ e d n r | e
+------------------+-------------------------------------------------------
+ """.rstrip())
+
+ def _trace_rule(self, rule, score, fixscore, numchanges):
+ rulestr = rule.format(self._ruleformat)
+
+ if self._trace > 2:
+ print(('%4d%4d%4d%4d ' % (score, fixscore, fixscore-score,
+ numchanges-fixscore*2+score)), '|', end=' ')
+ print(textwrap.fill(rulestr, initial_indent=' '*20, width=79,
+ subsequent_indent=' '*18+'| ').strip())
+ else:
+ print(rulestr)
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
diff --git a/nltk/tag/hmm.py b/nltk/tag/hmm.py
new file mode 100644
index 0000000..a4401ab
--- /dev/null
+++ b/nltk/tag/hmm.py
@@ -0,0 +1,1280 @@
+# Natural Language Toolkit: Hidden Markov Model
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Trevor Cohn <tacohn at csse.unimelb.edu.au>
+# Philip Blunsom <pcbl at csse.unimelb.edu.au>
+# Tiago Tresoldi <tiago at tresoldi.pro.br> (fixes)
+# Steven Bird <stevenbird1 at gmail.com> (fixes)
+# Joseph Frazee <jfrazee at mail.utexas.edu> (fixes)
+# Steven Xu <xxu at student.unimelb.edu.au> (fixes)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Hidden Markov Models (HMMs) largely used to assign the correct label sequence
+to sequential data or assess the probability of a given label and data
+sequence. These models are finite state machines characterised by a number of
+states, transitions between these states, and output symbols emitted while in
+each state. The HMM is an extension to the Markov chain, where each state
+corresponds deterministically to a given event. In the HMM the observation is
+a probabilistic function of the state. HMMs share the Markov chain's
+assumption, being that the probability of transition from one state to another
+only depends on the current state - i.e. the series of states that led to the
+current state are not used. They are also time invariant.
+
+The HMM is a directed graph, with probability weighted edges (representing the
+probability of a transition between the source and sink states) where each
+vertex emits an output symbol when entered. The symbol (or observation) is
+non-deterministically generated. For this reason, knowing that a sequence of
+output observations was generated by a given HMM does not mean that the
+corresponding sequence of states (and what the current state is) is known.
+This is the 'hidden' in the hidden markov model.
+
+Formally, a HMM can be characterised by:
+
+- the output observation alphabet. This is the set of symbols which may be
+ observed as output of the system.
+- the set of states.
+- the transition probabilities *a_{ij} = P(s_t = j | s_{t-1} = i)*. These
+ represent the probability of transition to each state from a given state.
+- the output probability matrix *b_i(k) = P(X_t = o_k | s_t = i)*. These
+ represent the probability of observing each symbol in a given state.
+- the initial state distribution. This gives the probability of starting
+ in each state.
+
+To ground this discussion, take a common NLP application, part-of-speech (POS)
+tagging. An HMM is desirable for this task as the highest probability tag
+sequence can be calculated for a given sequence of word forms. This differs
+from other tagging techniques which often tag each word individually, seeking
+to optimise each individual tagging greedily without regard to the optimal
+combination of tags for a larger unit, such as a sentence. The HMM does this
+with the Viterbi algorithm, which efficiently computes the optimal path
+through the graph given the sequence of words forms.
+
+In POS tagging the states usually have a 1:1 correspondence with the tag
+alphabet - i.e. each state represents a single tag. The output observation
+alphabet is the set of word forms (the lexicon), and the remaining three
+parameters are derived by a training regime. With this information the
+probability of a given sentence can be easily derived, by simply summing the
+probability of each distinct path through the model. Similarly, the highest
+probability tagging sequence can be derived with the Viterbi algorithm,
+yielding a state sequence which can be mapped into a tag sequence.
+
+This discussion assumes that the HMM has been trained. This is probably the
+most difficult task with the model, and requires either MLE estimates of the
+parameters or unsupervised learning using the Baum-Welch algorithm, a variant
+of EM.
+
+For more information, please consult the source code for this module,
+which includes extensive demonstration code.
+"""
+from __future__ import print_function, unicode_literals, division
+
+import re
+import itertools
+
+try:
+ import numpy as np
+except ImportError:
+ pass
+
+from nltk.probability import (FreqDist, ConditionalFreqDist,
+ ConditionalProbDist, DictionaryProbDist,
+ DictionaryConditionalProbDist,
+ LidstoneProbDist, MutableProbDist,
+ MLEProbDist, RandomProbDist)
+from nltk.metrics import accuracy
+from nltk.util import LazyMap, unique_list
+from nltk.compat import python_2_unicode_compatible, izip, imap
+from nltk.tag.api import TaggerI
+
+
+_TEXT = 0 # index of text in a tuple
+_TAG = 1 # index of tag in a tuple
+
+def _identity(labeled_symbols):
+ return labeled_symbols
+
+ at python_2_unicode_compatible
+class HiddenMarkovModelTagger(TaggerI):
+ """
+ Hidden Markov model class, a generative model for labelling sequence data.
+ These models define the joint probability of a sequence of symbols and
+ their labels (state transitions) as the product of the starting state
+ probability, the probability of each state transition, and the probability
+ of each observation being generated from each state. This is described in
+ more detail in the module documentation.
+
+ This implementation is based on the HMM description in Chapter 8, Huang,
+ Acero and Hon, Spoken Language Processing and includes an extension for
+ training shallow HMM parsers or specialized HMMs as in Molina et.
+ al, 2002. A specialized HMM modifies training data by applying a
+ specialization function to create a new training set that is more
+ appropriate for sequential tagging with an HMM. A typical use case is
+ chunking.
+
+ :param symbols: the set of output symbols (alphabet)
+ :type symbols: seq of any
+ :param states: a set of states representing state space
+ :type states: seq of any
+ :param transitions: transition probabilities; Pr(s_i | s_j) is the
+ probability of transition from state i given the model is in
+ state_j
+ :type transitions: ConditionalProbDistI
+ :param outputs: output probabilities; Pr(o_k | s_i) is the probability
+ of emitting symbol k when entering state i
+ :type outputs: ConditionalProbDistI
+ :param priors: initial state distribution; Pr(s_i) is the probability
+ of starting in state i
+ :type priors: ProbDistI
+ :param transform: an optional function for transforming training
+ instances, defaults to the identity function.
+ :type transform: callable
+ """
+ def __init__(self, symbols, states, transitions, outputs, priors,
+ transform=_identity):
+ self._symbols = unique_list(symbols)
+ self._states = unique_list(states)
+ self._transitions = transitions
+ self._outputs = outputs
+ self._priors = priors
+ self._cache = None
+ self._transform = transform
+
+ @classmethod
+ def _train(cls, labeled_sequence, test_sequence=None,
+ unlabeled_sequence=None, transform=_identity,
+ estimator=None, **kwargs):
+
+ if estimator is None:
+ def estimator(fd, bins):
+ return LidstoneProbDist(fd, 0.1, bins)
+
+ labeled_sequence = LazyMap(transform, labeled_sequence)
+ symbols = unique_list(word for sent in labeled_sequence
+ for word, tag in sent)
+ tag_set = unique_list(tag for sent in labeled_sequence
+ for word, tag in sent)
+
+ trainer = HiddenMarkovModelTrainer(tag_set, symbols)
+ hmm = trainer.train_supervised(labeled_sequence, estimator=estimator)
+ hmm = cls(hmm._symbols, hmm._states, hmm._transitions, hmm._outputs,
+ hmm._priors, transform=transform)
+
+ if test_sequence:
+ hmm.test(test_sequence, verbose=kwargs.get('verbose', False))
+
+ if unlabeled_sequence:
+ max_iterations = kwargs.get('max_iterations', 5)
+ hmm = trainer.train_unsupervised(unlabeled_sequence, model=hmm,
+ max_iterations=max_iterations)
+ if test_sequence:
+ hmm.test(test_sequence, verbose=kwargs.get('verbose', False))
+
+ return hmm
+
+ @classmethod
+ def train(cls, labeled_sequence, test_sequence=None,
+ unlabeled_sequence=None, **kwargs):
+ """
+ Train a new HiddenMarkovModelTagger using the given labeled and
+ unlabeled training instances. Testing will be performed if test
+ instances are provided.
+
+ :return: a hidden markov model tagger
+ :rtype: HiddenMarkovModelTagger
+ :param labeled_sequence: a sequence of labeled training instances,
+ i.e. a list of sentences represented as tuples
+ :type labeled_sequence: list(list)
+ :param test_sequence: a sequence of labeled test instances
+ :type test_sequence: list(list)
+ :param unlabeled_sequence: a sequence of unlabeled training instances,
+ i.e. a list of sentences represented as words
+ :type unlabeled_sequence: list(list)
+ :param transform: an optional function for transforming training
+ instances, defaults to the identity function, see ``transform()``
+ :type transform: function
+ :param estimator: an optional function or class that maps a
+ condition's frequency distribution to its probability
+ distribution, defaults to a Lidstone distribution with gamma = 0.1
+ :type estimator: class or function
+ :param verbose: boolean flag indicating whether training should be
+ verbose or include printed output
+ :type verbose: bool
+ :param max_iterations: number of Baum-Welch interations to perform
+ :type max_iterations: int
+ """
+ return cls._train(labeled_sequence, test_sequence,
+ unlabeled_sequence, **kwargs)
+
+ def probability(self, sequence):
+ """
+ Returns the probability of the given symbol sequence. If the sequence
+ is labelled, then returns the joint probability of the symbol, state
+ sequence. Otherwise, uses the forward algorithm to find the
+ probability over all label sequences.
+
+ :return: the probability of the sequence
+ :rtype: float
+ :param sequence: the sequence of symbols which must contain the TEXT
+ property, and optionally the TAG property
+ :type sequence: Token
+ """
+ return 2**(self.log_probability(self._transform(sequence)))
+
+ def log_probability(self, sequence):
+ """
+ Returns the log-probability of the given symbol sequence. If the
+ sequence is labelled, then returns the joint log-probability of the
+ symbol, state sequence. Otherwise, uses the forward algorithm to find
+ the log-probability over all label sequences.
+
+ :return: the log-probability of the sequence
+ :rtype: float
+ :param sequence: the sequence of symbols which must contain the TEXT
+ property, and optionally the TAG property
+ :type sequence: Token
+ """
+ sequence = self._transform(sequence)
+
+ T = len(sequence)
+
+ if T > 0 and sequence[0][_TAG]:
+ last_state = sequence[0][_TAG]
+ p = self._priors.logprob(last_state) + \
+ self._output_logprob(last_state, sequence[0][_TEXT])
+ for t in range(1, T):
+ state = sequence[t][_TAG]
+ p += self._transitions[last_state].logprob(state) + \
+ self._output_logprob(state, sequence[t][_TEXT])
+ last_state = state
+ return p
+ else:
+ alpha = self._forward_probability(sequence)
+ p = logsumexp2(alpha[T-1])
+ return p
+
+ def tag(self, unlabeled_sequence):
+ """
+ Tags the sequence with the highest probability state sequence. This
+ uses the best_path method to find the Viterbi path.
+
+ :return: a labelled sequence of symbols
+ :rtype: list
+ :param unlabeled_sequence: the sequence of unlabeled symbols
+ :type unlabeled_sequence: list
+ """
+ unlabeled_sequence = self._transform(unlabeled_sequence)
+ return self._tag(unlabeled_sequence)
+
+ def _tag(self, unlabeled_sequence):
+ path = self._best_path(unlabeled_sequence)
+ return list(izip(unlabeled_sequence, path))
+
+ def _output_logprob(self, state, symbol):
+ """
+ :return: the log probability of the symbol being observed in the given
+ state
+ :rtype: float
+ """
+ return self._outputs[state].logprob(symbol)
+
+ def _create_cache(self):
+ """
+ The cache is a tuple (P, O, X, S) where:
+
+ - S maps symbols to integers. I.e., it is the inverse
+ mapping from self._symbols; for each symbol s in
+ self._symbols, the following is true::
+
+ self._symbols[S[s]] == s
+
+ - O is the log output probabilities::
+
+ O[i,k] = log( P(token[t]=sym[k]|tag[t]=state[i]) )
+
+ - X is the log transition probabilities::
+
+ X[i,j] = log( P(tag[t]=state[j]|tag[t-1]=state[i]) )
+
+ - P is the log prior probabilities::
+
+ P[i] = log( P(tag[0]=state[i]) )
+ """
+ if not self._cache:
+ N = len(self._states)
+ M = len(self._symbols)
+ P = np.zeros(N, np.float32)
+ X = np.zeros((N, N), np.float32)
+ O = np.zeros((N, M), np.float32)
+ for i in range(N):
+ si = self._states[i]
+ P[i] = self._priors.logprob(si)
+ for j in range(N):
+ X[i, j] = self._transitions[si].logprob(self._states[j])
+ for k in range(M):
+ O[i, k] = self._output_logprob(si, self._symbols[k])
+ S = {}
+ for k in range(M):
+ S[self._symbols[k]] = k
+ self._cache = (P, O, X, S)
+
+ def _update_cache(self, symbols):
+ # add new symbols to the symbol table and repopulate the output
+ # probabilities and symbol table mapping
+ if symbols:
+ self._create_cache()
+ P, O, X, S = self._cache
+ for symbol in symbols:
+ if symbol not in self._symbols:
+ self._cache = None
+ self._symbols.append(symbol)
+ # don't bother with the work if there aren't any new symbols
+ if not self._cache:
+ N = len(self._states)
+ M = len(self._symbols)
+ Q = O.shape[1]
+ # add new columns to the output probability table without
+ # destroying the old probabilities
+ O = np.hstack([O, np.zeros((N, M - Q), np.float32)])
+ for i in range(N):
+ si = self._states[i]
+ # only calculate probabilities for new symbols
+ for k in range(Q, M):
+ O[i, k] = self._output_logprob(si, self._symbols[k])
+ # only create symbol mappings for new symbols
+ for k in range(Q, M):
+ S[self._symbols[k]] = k
+ self._cache = (P, O, X, S)
+
+ def reset_cache(self):
+ self._cache = None
+
+ def best_path(self, unlabeled_sequence):
+ """
+ Returns the state sequence of the optimal (most probable) path through
+ the HMM. Uses the Viterbi algorithm to calculate this part by dynamic
+ programming.
+
+ :return: the state sequence
+ :rtype: sequence of any
+ :param unlabeled_sequence: the sequence of unlabeled symbols
+ :type unlabeled_sequence: list
+ """
+ unlabeled_sequence = self._transform(unlabeled_sequence)
+ return self._best_path(unlabeled_sequence)
+
+ def _best_path(self, unlabeled_sequence):
+ T = len(unlabeled_sequence)
+ N = len(self._states)
+ self._create_cache()
+ self._update_cache(unlabeled_sequence)
+ P, O, X, S = self._cache
+
+ V = np.zeros((T, N), np.float32)
+ B = -np.ones((T, N), np.int)
+
+ V[0] = P + O[:, S[unlabeled_sequence[0]]]
+ for t in range(1, T):
+ for j in range(N):
+ vs = V[t-1, :] + X[:, j]
+ best = np.argmax(vs)
+ V[t, j] = vs[best] + O[j, S[unlabeled_sequence[t]]]
+ B[t, j] = best
+
+ current = np.argmax(V[T-1,:])
+ sequence = [current]
+ for t in range(T-1, 0, -1):
+ last = B[t, current]
+ sequence.append(last)
+ current = last
+
+ sequence.reverse()
+ return list(map(self._states.__getitem__, sequence))
+
+ def best_path_simple(self, unlabeled_sequence):
+ """
+ Returns the state sequence of the optimal (most probable) path through
+ the HMM. Uses the Viterbi algorithm to calculate this part by dynamic
+ programming. This uses a simple, direct method, and is included for
+ teaching purposes.
+
+ :return: the state sequence
+ :rtype: sequence of any
+ :param unlabeled_sequence: the sequence of unlabeled symbols
+ :type unlabeled_sequence: list
+ """
+ unlabeled_sequence = self._transform(unlabeled_sequence)
+ return self._best_path_simple(unlabeled_sequence)
+
+ def _best_path_simple(self, unlabeled_sequence):
+ T = len(unlabeled_sequence)
+ N = len(self._states)
+ V = np.zeros((T, N), np.float64)
+ B = {}
+
+ # find the starting log probabilities for each state
+ symbol = unlabeled_sequence[0]
+ for i, state in enumerate(self._states):
+ V[0, i] = self._priors.logprob(state) + \
+ self._output_logprob(state, symbol)
+ B[0, state] = None
+
+ # find the maximum log probabilities for reaching each state at time t
+ for t in range(1, T):
+ symbol = unlabeled_sequence[t]
+ for j in range(N):
+ sj = self._states[j]
+ best = None
+ for i in range(N):
+ si = self._states[i]
+ va = V[t-1, i] + self._transitions[si].logprob(sj)
+ if not best or va > best[0]:
+ best = (va, si)
+ V[t, j] = best[0] + self._output_logprob(sj, symbol)
+ B[t, sj] = best[1]
+
+ # find the highest probability final state
+ best = None
+ for i in range(N):
+ val = V[T-1, i]
+ if not best or val > best[0]:
+ best = (val, self._states[i])
+
+ # traverse the back-pointers B to find the state sequence
+ current = best[1]
+ sequence = [current]
+ for t in range(T-1, 0, -1):
+ last = B[t, current]
+ sequence.append(last)
+ current = last
+
+ sequence.reverse()
+ return sequence
+
+ def random_sample(self, rng, length):
+ """
+ Randomly sample the HMM to generate a sentence of a given length. This
+ samples the prior distribution then the observation distribution and
+ transition distribution for each subsequent observation and state.
+ This will mostly generate unintelligible garbage, but can provide some
+ amusement.
+
+ :return: the randomly created state/observation sequence,
+ generated according to the HMM's probability
+ distributions. The SUBTOKENS have TEXT and TAG
+ properties containing the observation and state
+ respectively.
+ :rtype: list
+ :param rng: random number generator
+ :type rng: Random (or any object with a random() method)
+ :param length: desired output length
+ :type length: int
+ """
+
+ # sample the starting state and symbol prob dists
+ tokens = []
+ state = self._sample_probdist(self._priors, rng.random(), self._states)
+ symbol = self._sample_probdist(self._outputs[state],
+ rng.random(), self._symbols)
+ tokens.append((symbol, state))
+
+ for i in range(1, length):
+ # sample the state transition and symbol prob dists
+ state = self._sample_probdist(self._transitions[state],
+ rng.random(), self._states)
+ symbol = self._sample_probdist(self._outputs[state],
+ rng.random(), self._symbols)
+ tokens.append((symbol, state))
+
+ return tokens
+
+ def _sample_probdist(self, probdist, p, samples):
+ cum_p = 0
+ for sample in samples:
+ add_p = probdist.prob(sample)
+ if cum_p <= p <= cum_p + add_p:
+ return sample
+ cum_p += add_p
+ raise Exception('Invalid probability distribution - '
+ 'does not sum to one')
+
+ def entropy(self, unlabeled_sequence):
+ """
+ Returns the entropy over labellings of the given sequence. This is
+ given by::
+
+ H(O) = - sum_S Pr(S | O) log Pr(S | O)
+
+ where the summation ranges over all state sequences, S. Let
+ *Z = Pr(O) = sum_S Pr(S, O)}* where the summation ranges over all state
+ sequences and O is the observation sequence. As such the entropy can
+ be re-expressed as::
+
+ H = - sum_S Pr(S | O) log [ Pr(S, O) / Z ]
+ = log Z - sum_S Pr(S | O) log Pr(S, 0)
+ = log Z - sum_S Pr(S | O) [ log Pr(S_0) + sum_t Pr(S_t | S_{t-1}) + sum_t Pr(O_t | S_t) ]
+
+ The order of summation for the log terms can be flipped, allowing
+ dynamic programming to be used to calculate the entropy. Specifically,
+ we use the forward and backward probabilities (alpha, beta) giving::
+
+ H = log Z - sum_s0 alpha_0(s0) beta_0(s0) / Z * log Pr(s0)
+ + sum_t,si,sj alpha_t(si) Pr(sj | si) Pr(O_t+1 | sj) beta_t(sj) / Z * log Pr(sj | si)
+ + sum_t,st alpha_t(st) beta_t(st) / Z * log Pr(O_t | st)
+
+ This simply uses alpha and beta to find the probabilities of partial
+ sequences, constrained to include the given state(s) at some point in
+ time.
+ """
+ unlabeled_sequence = self._transform(unlabeled_sequence)
+
+ T = len(unlabeled_sequence)
+ N = len(self._states)
+
+ alpha = self._forward_probability(unlabeled_sequence)
+ beta = self._backward_probability(unlabeled_sequence)
+ normalisation = logsumexp2(alpha[T-1])
+
+ entropy = normalisation
+
+ # starting state, t = 0
+ for i, state in enumerate(self._states):
+ p = 2**(alpha[0, i] + beta[0, i] - normalisation)
+ entropy -= p * self._priors.logprob(state)
+ #print 'p(s_0 = %s) =' % state, p
+
+ # state transitions
+ for t0 in range(T - 1):
+ t1 = t0 + 1
+ for i0, s0 in enumerate(self._states):
+ for i1, s1 in enumerate(self._states):
+ p = 2**(alpha[t0, i0] + self._transitions[s0].logprob(s1) +
+ self._outputs[s1].logprob(
+ unlabeled_sequence[t1][_TEXT]) +
+ beta[t1, i1] - normalisation)
+ entropy -= p * self._transitions[s0].logprob(s1)
+ #print 'p(s_%d = %s, s_%d = %s) =' % (t0, s0, t1, s1), p
+
+ # symbol emissions
+ for t in range(T):
+ for i, state in enumerate(self._states):
+ p = 2**(alpha[t, i] + beta[t, i] - normalisation)
+ entropy -= p * self._outputs[state].logprob(
+ unlabeled_sequence[t][_TEXT])
+ #print 'p(s_%d = %s) =' % (t, state), p
+
+ return entropy
+
+ def point_entropy(self, unlabeled_sequence):
+ """
+ Returns the pointwise entropy over the possible states at each
+ position in the chain, given the observation sequence.
+ """
+ unlabeled_sequence = self._transform(unlabeled_sequence)
+
+ T = len(unlabeled_sequence)
+ N = len(self._states)
+
+ alpha = self._forward_probability(unlabeled_sequence)
+ beta = self._backward_probability(unlabeled_sequence)
+ normalisation = logsumexp2(alpha[T-1])
+
+ entropies = np.zeros(T, np.float64)
+ probs = np.zeros(N, np.float64)
+ for t in range(T):
+ for s in range(N):
+ probs[s] = alpha[t, s] + beta[t, s] - normalisation
+
+ for s in range(N):
+ entropies[t] -= 2**(probs[s]) * probs[s]
+
+ return entropies
+
+ def _exhaustive_entropy(self, unlabeled_sequence):
+ unlabeled_sequence = self._transform(unlabeled_sequence)
+
+ T = len(unlabeled_sequence)
+ N = len(self._states)
+
+ labellings = [[state] for state in self._states]
+ for t in range(T - 1):
+ current = labellings
+ labellings = []
+ for labelling in current:
+ for state in self._states:
+ labellings.append(labelling + [state])
+
+ log_probs = []
+ for labelling in labellings:
+ labeled_sequence = unlabeled_sequence[:]
+ for t, label in enumerate(labelling):
+ labeled_sequence[t] = (labeled_sequence[t][_TEXT], label)
+ lp = self.log_probability(labeled_sequence)
+ log_probs.append(lp)
+ normalisation = _log_add(*log_probs)
+
+ #ps = zeros((T, N), float64)
+ #for labelling, lp in zip(labellings, log_probs):
+ #for t in range(T):
+ #ps[t, self._states.index(labelling[t])] += \
+ # 2**(lp - normalisation)
+
+ #for t in range(T):
+ #print 'prob[%d] =' % t, ps[t]
+
+ entropy = 0
+ for lp in log_probs:
+ lp -= normalisation
+ entropy -= 2**(lp) * lp
+
+ return entropy
+
+ def _exhaustive_point_entropy(self, unlabeled_sequence):
+ unlabeled_sequence = self._transform(unlabeled_sequence)
+
+ T = len(unlabeled_sequence)
+ N = len(self._states)
+
+ labellings = [[state] for state in self._states]
+ for t in range(T - 1):
+ current = labellings
+ labellings = []
+ for labelling in current:
+ for state in self._states:
+ labellings.append(labelling + [state])
+
+ log_probs = []
+ for labelling in labellings:
+ labelled_sequence = unlabeled_sequence[:]
+ for t, label in enumerate(labelling):
+ labelled_sequence[t] = (labelled_sequence[t][_TEXT], label)
+ lp = self.log_probability(labelled_sequence)
+ log_probs.append(lp)
+
+ normalisation = _log_add(*log_probs)
+
+ probabilities = _ninf_array((T,N))
+
+ for labelling, lp in zip(labellings, log_probs):
+ lp -= normalisation
+ for t, label in enumerate(labelling):
+ index = self._states.index(label)
+ probabilities[t, index] = _log_add(probabilities[t, index], lp)
+
+ entropies = np.zeros(T, np.float64)
+ for t in range(T):
+ for s in range(N):
+ entropies[t] -= 2**(probabilities[t, s]) * probabilities[t, s]
+
+ return entropies
+
+ def _transitions_matrix(self):
+ """ Return a matrix of transition log probabilities. """
+ trans_iter = (self._transitions[sj].logprob(si)
+ for sj in self._states
+ for si in self._states)
+
+ transitions_logprob = np.fromiter(trans_iter, dtype=np.float64)
+ N = len(self._states)
+ return transitions_logprob.reshape((N, N)).T
+
+ def _outputs_vector(self, symbol):
+ """
+ Return a vector with log probabilities of emitting a symbol
+ when entering states.
+ """
+ out_iter = (self._output_logprob(sj, symbol) for sj in self._states)
+ return np.fromiter(out_iter, dtype=np.float64)
+
+ def _forward_probability(self, unlabeled_sequence):
+ """
+ Return the forward probability matrix, a T by N array of
+ log-probabilities, where T is the length of the sequence and N is the
+ number of states. Each entry (t, s) gives the probability of being in
+ state s at time t after observing the partial symbol sequence up to
+ and including t.
+
+ :param unlabeled_sequence: the sequence of unlabeled symbols
+ :type unlabeled_sequence: list
+ :return: the forward log probability matrix
+ :rtype: array
+ """
+ T = len(unlabeled_sequence)
+ N = len(self._states)
+ alpha = _ninf_array((T, N))
+
+ transitions_logprob = self._transitions_matrix()
+
+ # Initialization
+ symbol = unlabeled_sequence[0][_TEXT]
+ for i, state in enumerate(self._states):
+ alpha[0, i] = self._priors.logprob(state) + \
+ self._output_logprob(state, symbol)
+
+ # Induction
+ for t in range(1, T):
+ symbol = unlabeled_sequence[t][_TEXT]
+ output_logprob = self._outputs_vector(symbol)
+
+ for i in range(N):
+ summand = alpha[t-1] + transitions_logprob[i]
+ alpha[t, i] = logsumexp2(summand) + output_logprob[i]
+
+ return alpha
+
+ def _backward_probability(self, unlabeled_sequence):
+ """
+ Return the backward probability matrix, a T by N array of
+ log-probabilities, where T is the length of the sequence and N is the
+ number of states. Each entry (t, s) gives the probability of being in
+ state s at time t after observing the partial symbol sequence from t
+ .. T.
+
+ :return: the backward log probability matrix
+ :rtype: array
+ :param unlabeled_sequence: the sequence of unlabeled symbols
+ :type unlabeled_sequence: list
+ """
+ T = len(unlabeled_sequence)
+ N = len(self._states)
+ beta = _ninf_array((T, N))
+
+ transitions_logprob = self._transitions_matrix().T
+
+ # initialise the backward values;
+ # "1" is an arbitrarily chosen value from Rabiner tutorial
+ beta[T-1, :] = np.log2(1)
+
+ # inductively calculate remaining backward values
+ for t in range(T-2, -1, -1):
+ symbol = unlabeled_sequence[t+1][_TEXT]
+ outputs = self._outputs_vector(symbol)
+
+ for i in range(N):
+ summand = transitions_logprob[i] + beta[t+1] + outputs
+ beta[t, i] = logsumexp2(summand)
+
+ return beta
+
+ def test(self, test_sequence, verbose=False, **kwargs):
+ """
+ Tests the HiddenMarkovModelTagger instance.
+
+ :param test_sequence: a sequence of labeled test instances
+ :type test_sequence: list(list)
+ :param verbose: boolean flag indicating whether training should be
+ verbose or include printed output
+ :type verbose: bool
+ """
+
+ def words(sent):
+ return [word for (word, tag) in sent]
+
+ def tags(sent):
+ return [tag for (word, tag) in sent]
+
+ def flatten(seq):
+ return list(itertools.chain(*seq))
+
+ test_sequence = self._transform(test_sequence)
+ predicted_sequence = list(imap(self._tag, imap(words, test_sequence)))
+
+ if verbose:
+ for test_sent, predicted_sent in izip(test_sequence, predicted_sequence):
+ print('Test:',
+ ' '.join('%s/%s' % (token, tag)
+ for (token, tag) in test_sent))
+ print()
+ print('Untagged:',
+ ' '.join("%s" % token for (token, tag) in test_sent))
+ print()
+ print('HMM-tagged:',
+ ' '.join('%s/%s' % (token, tag)
+ for (token, tag) in predicted_sent))
+ print()
+ print('Entropy:',
+ self.entropy([(token, None) for
+ (token, tag) in predicted_sent]))
+ print()
+ print('-' * 60)
+
+ test_tags = flatten(imap(tags, test_sequence))
+ predicted_tags = flatten(imap(tags, predicted_sequence))
+
+ acc = accuracy(test_tags, predicted_tags)
+ count = sum(len(sent) for sent in test_sequence)
+ print('accuracy over %d tokens: %.2f' % (count, acc * 100))
+
+ def __repr__(self):
+ return ('<HiddenMarkovModelTagger %d states and %d output symbols>'
+ % (len(self._states), len(self._symbols)))
+
+
+class HiddenMarkovModelTrainer(object):
+ """
+ Algorithms for learning HMM parameters from training data. These include
+ both supervised learning (MLE) and unsupervised learning (Baum-Welch).
+
+ Creates an HMM trainer to induce an HMM with the given states and
+ output symbol alphabet. A supervised and unsupervised training
+ method may be used. If either of the states or symbols are not given,
+ these may be derived from supervised training.
+
+ :param states: the set of state labels
+ :type states: sequence of any
+ :param symbols: the set of observation symbols
+ :type symbols: sequence of any
+ """
+ def __init__(self, states=None, symbols=None):
+ self._states = (states if states else [])
+ self._symbols = (symbols if symbols else [])
+
+ def train(self, labeled_sequences=None, unlabeled_sequences=None,
+ **kwargs):
+ """
+ Trains the HMM using both (or either of) supervised and unsupervised
+ techniques.
+
+ :return: the trained model
+ :rtype: HiddenMarkovModelTagger
+ :param labelled_sequences: the supervised training data, a set of
+ labelled sequences of observations
+ :type labelled_sequences: list
+ :param unlabeled_sequences: the unsupervised training data, a set of
+ sequences of observations
+ :type unlabeled_sequences: list
+ :param kwargs: additional arguments to pass to the training methods
+ """
+ assert labeled_sequences or unlabeled_sequences
+ model = None
+ if labeled_sequences:
+ model = self.train_supervised(labeled_sequences, **kwargs)
+ if unlabeled_sequences:
+ if model: kwargs['model'] = model
+ model = self.train_unsupervised(unlabeled_sequences, **kwargs)
+ return model
+
+
+ def _baum_welch_step(self, sequence, model, symbol_to_number):
+
+ N = len(model._states)
+ M = len(model._symbols)
+ T = len(sequence)
+
+ # compute forward and backward probabilities
+ alpha = model._forward_probability(sequence)
+ beta = model._backward_probability(sequence)
+
+ # find the log probability of the sequence
+ lpk = logsumexp2(alpha[T-1])
+
+ A_numer = _ninf_array((N, N))
+ B_numer = _ninf_array((N, M))
+ A_denom = _ninf_array(N)
+ B_denom = _ninf_array(N)
+
+ transitions_logprob = model._transitions_matrix().T
+
+ for t in range(T):
+ symbol = sequence[t][_TEXT] # not found? FIXME
+ next_symbol = None
+ if t < T - 1:
+ next_symbol = sequence[t+1][_TEXT] # not found? FIXME
+ xi = symbol_to_number[symbol]
+
+ next_outputs_logprob = model._outputs_vector(next_symbol)
+ alpha_plus_beta = alpha[t] + beta[t]
+
+ if t < T - 1:
+ numer_add = transitions_logprob + next_outputs_logprob + \
+ beta[t+1] + alpha[t].reshape(N, 1)
+ A_numer = np.logaddexp2(A_numer, numer_add)
+ A_denom = np.logaddexp2(A_denom, alpha_plus_beta)
+ else:
+ B_denom = np.logaddexp2(A_denom, alpha_plus_beta)
+
+ B_numer[:,xi] = np.logaddexp2(B_numer[:,xi], alpha_plus_beta)
+
+ return lpk, A_numer, A_denom, B_numer, B_denom
+
+ def train_unsupervised(self, unlabeled_sequences, update_outputs=True,
+ **kwargs):
+ """
+ Trains the HMM using the Baum-Welch algorithm to maximise the
+ probability of the data sequence. This is a variant of the EM
+ algorithm, and is unsupervised in that it doesn't need the state
+ sequences for the symbols. The code is based on 'A Tutorial on Hidden
+ Markov Models and Selected Applications in Speech Recognition',
+ Lawrence Rabiner, IEEE, 1989.
+
+ :return: the trained model
+ :rtype: HiddenMarkovModelTagger
+ :param unlabeled_sequences: the training data, a set of
+ sequences of observations
+ :type unlabeled_sequences: list
+
+ kwargs may include following parameters:
+
+ :param model: a HiddenMarkovModelTagger instance used to begin
+ the Baum-Welch algorithm
+ :param max_iterations: the maximum number of EM iterations
+ :param convergence_logprob: the maximum change in log probability to
+ allow convergence
+ """
+
+ # create a uniform HMM, which will be iteratively refined, unless
+ # given an existing model
+ model = kwargs.get('model')
+ if not model:
+ priors = RandomProbDist(self._states)
+ transitions = DictionaryConditionalProbDist(
+ dict((state, RandomProbDist(self._states))
+ for state in self._states))
+ outputs = DictionaryConditionalProbDist(
+ dict((state, RandomProbDist(self._symbols))
+ for state in self._states))
+ model = HiddenMarkovModelTagger(self._symbols, self._states,
+ transitions, outputs, priors)
+
+ self._states = model._states
+ self._symbols = model._symbols
+
+ N = len(self._states)
+ M = len(self._symbols)
+ symbol_numbers = dict((sym, i) for i, sym in enumerate(self._symbols))
+
+ # update model prob dists so that they can be modified
+ # model._priors = MutableProbDist(model._priors, self._states)
+
+ model._transitions = DictionaryConditionalProbDist(
+ dict((s, MutableProbDist(model._transitions[s], self._states))
+ for s in self._states))
+
+ if update_outputs:
+ model._outputs = DictionaryConditionalProbDist(
+ dict((s, MutableProbDist(model._outputs[s], self._symbols))
+ for s in self._states))
+
+ model.reset_cache()
+
+ # iterate until convergence
+ converged = False
+ last_logprob = None
+ iteration = 0
+ max_iterations = kwargs.get('max_iterations', 1000)
+ epsilon = kwargs.get('convergence_logprob', 1e-6)
+
+ while not converged and iteration < max_iterations:
+ A_numer = _ninf_array((N, N))
+ B_numer = _ninf_array((N, M))
+ A_denom = _ninf_array(N)
+ B_denom = _ninf_array(N)
+
+ logprob = 0
+ for sequence in unlabeled_sequences:
+ sequence = list(sequence)
+ if not sequence:
+ continue
+
+ (lpk, seq_A_numer, seq_A_denom,
+ seq_B_numer, seq_B_denom) = self._baum_welch_step(sequence, model, symbol_numbers)
+
+ # add these sums to the global A and B values
+ for i in range(N):
+ A_numer[i] = np.logaddexp2(A_numer[i], seq_A_numer[i]-lpk)
+ B_numer[i] = np.logaddexp2(B_numer[i], seq_B_numer[i]-lpk)
+
+ A_denom = np.logaddexp2(A_denom, seq_A_denom-lpk)
+ B_denom = np.logaddexp2(B_denom, seq_B_denom-lpk)
+
+ logprob += lpk
+
+ # use the calculated values to update the transition and output
+ # probability values
+ for i in range(N):
+ logprob_Ai = A_numer[i] - A_denom[i]
+ logprob_Bi = B_numer[i] - B_denom[i]
+
+ # We should normalize all probabilities (see p.391 Huang et al)
+ # Let sum(P) be K.
+ # We can divide each Pi by K to make sum(P) == 1.
+ # Pi' = Pi/K
+ # log2(Pi') = log2(Pi) - log2(K)
+ logprob_Ai -= logsumexp2(logprob_Ai)
+ logprob_Bi -= logsumexp2(logprob_Bi)
+
+ # update output and transition probabilities
+ si = self._states[i]
+
+ for j in range(N):
+ sj = self._states[j]
+ model._transitions[si].update(sj, logprob_Ai[j])
+
+ if update_outputs:
+ for k in range(M):
+ ok = self._symbols[k]
+ model._outputs[si].update(ok, logprob_Bi[k])
+
+ # Rabiner says the priors don't need to be updated. I don't
+ # believe him. FIXME
+
+ # test for convergence
+ if iteration > 0 and abs(logprob - last_logprob) < epsilon:
+ converged = True
+
+ print('iteration', iteration, 'logprob', logprob)
+ iteration += 1
+ last_logprob = logprob
+
+ return model
+
+ def train_supervised(self, labelled_sequences, estimator=None):
+ """
+ Supervised training maximising the joint probability of the symbol and
+ state sequences. This is done via collecting frequencies of
+ transitions between states, symbol observations while within each
+ state and which states start a sentence. These frequency distributions
+ are then normalised into probability estimates, which can be
+ smoothed if desired.
+
+ :return: the trained model
+ :rtype: HiddenMarkovModelTagger
+ :param labelled_sequences: the training data, a set of
+ labelled sequences of observations
+ :type labelled_sequences: list
+ :param estimator: a function taking
+ a FreqDist and a number of bins and returning a CProbDistI;
+ otherwise a MLE estimate is used
+ """
+
+ # default to the MLE estimate
+ if estimator is None:
+ estimator = lambda fdist, bins: MLEProbDist(fdist)
+
+ # count occurrences of starting states, transitions out of each state
+ # and output symbols observed in each state
+ known_symbols = set(self._symbols)
+ known_states = set(self._states)
+
+ starting = FreqDist()
+ transitions = ConditionalFreqDist()
+ outputs = ConditionalFreqDist()
+ for sequence in labelled_sequences:
+ lasts = None
+ for token in sequence:
+ state = token[_TAG]
+ symbol = token[_TEXT]
+ if lasts is None:
+ starting[state] += 1
+ else:
+ transitions[lasts][state] += 1
+ outputs[state][symbol] += 1
+ lasts = state
+
+ # update the state and symbol lists
+ if state not in known_states:
+ self._states.append(state)
+ known_states.add(state)
+
+ if symbol not in known_symbols:
+ self._symbols.append(symbol)
+ known_symbols.add(symbol)
+
+ # create probability distributions (with smoothing)
+ N = len(self._states)
+ pi = estimator(starting, N)
+ A = ConditionalProbDist(transitions, estimator, N)
+ B = ConditionalProbDist(outputs, estimator, len(self._symbols))
+
+ return HiddenMarkovModelTagger(self._symbols, self._states, A, B, pi)
+
+
+def _ninf_array(shape):
+ res = np.empty(shape, np.float64)
+ res.fill(-np.inf)
+ return res
+
+
+def logsumexp2(arr):
+ max_ = arr.max()
+ return np.log2(np.sum(2**(arr - max_))) + max_
+
+
+def _log_add(*values):
+ """
+ Adds the logged values, returning the logarithm of the addition.
+ """
+ x = max(values)
+ if x > -np.inf:
+ sum_diffs = 0
+ for value in values:
+ sum_diffs += 2**(value - x)
+ return x + np.log2(sum_diffs)
+ else:
+ return x
+
+
+def _create_hmm_tagger(states, symbols, A, B, pi):
+ def pd(values, samples):
+ d = dict(zip(samples, values))
+ return DictionaryProbDist(d)
+
+ def cpd(array, conditions, samples):
+ d = {}
+ for values, condition in zip(array, conditions):
+ d[condition] = pd(values, samples)
+ return DictionaryConditionalProbDist(d)
+
+ A = cpd(A, states, states)
+ B = cpd(B, states, symbols)
+ pi = pd(pi, states)
+ return HiddenMarkovModelTagger(symbols=symbols, states=states,
+ transitions=A, outputs=B, priors=pi)
+
+
+def _market_hmm_example():
+ """
+ Return an example HMM (described at page 381, Huang et al)
+ """
+ states = ['bull', 'bear', 'static']
+ symbols = ['up', 'down', 'unchanged']
+ A = np.array([[0.6, 0.2, 0.2], [0.5, 0.3, 0.2], [0.4, 0.1, 0.5]], np.float64)
+ B = np.array([[0.7, 0.1, 0.2], [0.1, 0.6, 0.3], [0.3, 0.3, 0.4]], np.float64)
+ pi = np.array([0.5, 0.2, 0.3], np.float64)
+
+ model = _create_hmm_tagger(states, symbols, A, B, pi)
+ return model, states, symbols
+
+
+def demo():
+ # demonstrates HMM probability calculation
+
+ print()
+ print("HMM probability calculation demo")
+ print()
+
+ model, states, symbols = _market_hmm_example()
+
+ print('Testing', model)
+
+ for test in [['up', 'up'], ['up', 'down', 'up'],
+ ['down'] * 5, ['unchanged'] * 5 + ['up']]:
+
+ sequence = [(t, None) for t in test]
+
+ print('Testing with state sequence', test)
+ print('probability =', model.probability(sequence))
+ print('tagging = ', model.tag([word for (word,tag) in sequence]))
+ print('p(tagged) = ', model.probability(sequence))
+ print('H = ', model.entropy(sequence))
+ print('H_exh = ', model._exhaustive_entropy(sequence))
+ print('H(point) = ', model.point_entropy(sequence))
+ print('H_exh(point)=', model._exhaustive_point_entropy(sequence))
+ print()
+
+def load_pos(num_sents):
+ from nltk.corpus import brown
+
+ sentences = brown.tagged_sents(categories='news')[:num_sents]
+
+ tag_re = re.compile(r'[*]|--|[^+*-]+')
+ tag_set = set()
+ symbols = set()
+
+ cleaned_sentences = []
+ for sentence in sentences:
+ for i in range(len(sentence)):
+ word, tag = sentence[i]
+ word = word.lower() # normalize
+ symbols.add(word) # log this word
+ # Clean up the tag.
+ tag = tag_re.match(tag).group()
+ tag_set.add(tag)
+ sentence[i] = (word, tag) # store cleaned-up tagged token
+ cleaned_sentences += [sentence]
+
+ return cleaned_sentences, list(tag_set), list(symbols)
+
+def demo_pos():
+ # demonstrates POS tagging using supervised training
+
+ print()
+ print("HMM POS tagging demo")
+ print()
+
+ print('Training HMM...')
+ labelled_sequences, tag_set, symbols = load_pos(20000)
+ trainer = HiddenMarkovModelTrainer(tag_set, symbols)
+ hmm = trainer.train_supervised(labelled_sequences[10:],
+ estimator=lambda fd, bins: LidstoneProbDist(fd, 0.1, bins))
+
+ print('Testing...')
+ hmm.test(labelled_sequences[:10], verbose=True)
+
+def _untag(sentences):
+ unlabeled = []
+ for sentence in sentences:
+ unlabeled.append([(token[_TEXT], None) for token in sentence])
+ return unlabeled
+
+def demo_pos_bw(test=10, supervised=20, unsupervised=10, verbose=True,
+ max_iterations=5):
+ # demonstrates the Baum-Welch algorithm in POS tagging
+
+ print()
+ print("Baum-Welch demo for POS tagging")
+ print()
+
+ print('Training HMM (supervised, %d sentences)...' % supervised)
+
+ sentences, tag_set, symbols = load_pos(test + supervised + unsupervised)
+
+ symbols = set()
+ for sentence in sentences:
+ for token in sentence:
+ symbols.add(token[_TEXT])
+
+ trainer = HiddenMarkovModelTrainer(tag_set, list(symbols))
+ hmm = trainer.train_supervised(sentences[test:test+supervised],
+ estimator=lambda fd, bins: LidstoneProbDist(fd, 0.1, bins))
+
+ hmm.test(sentences[:test], verbose=verbose)
+
+ print('Training (unsupervised, %d sentences)...' % unsupervised)
+ # it's rather slow - so only use 10 samples by default
+ unlabeled = _untag(sentences[test+supervised:])
+ hmm = trainer.train_unsupervised(unlabeled, model=hmm,
+ max_iterations=max_iterations)
+ hmm.test(sentences[:test], verbose=verbose)
+
+def demo_bw():
+ # demo Baum Welch by generating some sequences and then performing
+ # unsupervised training on them
+
+ print()
+ print("Baum-Welch demo for market example")
+ print()
+
+ model, states, symbols = _market_hmm_example()
+
+ # generate some random sequences
+ training = []
+ import random
+ rng = random.Random()
+ rng.seed(0)
+ for i in range(10):
+ item = model.random_sample(rng, 5)
+ training.append([(i[0], None) for i in item])
+
+ # train on those examples, starting with the model that generated them
+ trainer = HiddenMarkovModelTrainer(states, symbols)
+ hmm = trainer.train_unsupervised(training, model=model,
+ max_iterations=1000)
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
+
diff --git a/nltk/tag/hunpos.py b/nltk/tag/hunpos.py
new file mode 100644
index 0000000..1e00753
--- /dev/null
+++ b/nltk/tag/hunpos.py
@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the HunPos POS-tagger
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Peter Ljunglöf <peter.ljunglof at heatherleaf.se>
+# David Nemeskey <nemeskeyd at gmail.com> (modifications)
+# Attila Zseder <zseder at gmail.com> (modifications)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A module for interfacing with the HunPos open-source POS-tagger.
+"""
+
+import os
+from subprocess import Popen, PIPE
+
+from nltk.internals import find_binary, find_file
+from nltk.tag.api import TaggerI
+from nltk import compat
+
+_hunpos_url = 'http://code.google.com/p/hunpos/'
+
+_hunpos_charset = 'ISO-8859-1'
+"""The default encoding used by hunpos: ISO-8859-1."""
+
+class HunposTagger(TaggerI):
+ """
+ A class for pos tagging with HunPos. The input is the paths to:
+ - a model trained on training data
+ - (optionally) the path to the hunpos-tag binary
+ - (optionally) the encoding of the training data (default: ISO-8859-1)
+
+ Example:
+
+ >>> from nltk.tag.hunpos import HunposTagger
+ >>> ht = HunposTagger('english.model')
+ >>> ht.tag('What is the airspeed of an unladen swallow ?'.split())
+ [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]
+ >>> ht.close()
+
+ This class communicates with the hunpos-tag binary via pipes. When the
+ tagger object is no longer needed, the close() method should be called to
+ free system resources. The class supports the context manager interface; if
+ used in a with statement, the close() method is invoked automatically:
+
+ >>> with HunposTagger('english.model') as ht:
+ ... ht.tag('What is the airspeed of an unladen swallow ?'.split())
+ ...
+ [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'NN'), ('swallow', 'VB'), ('?', '.')]
+ """
+
+ def __init__(self, path_to_model, path_to_bin=None,
+ encoding=_hunpos_charset, verbose=False):
+ """
+ Starts the hunpos-tag executable and establishes a connection with it.
+
+ :param path_to_model: The model file.
+ :param path_to_bin: The hunpos-tag binary.
+ :param encoding: The encoding used by the model. Unicode tokens
+ passed to the tag() and tag_sents() methods are converted to
+ this charset when they are sent to hunpos-tag.
+ The default is ISO-8859-1 (Latin-1).
+
+ This parameter is ignored for str tokens, which are sent as-is.
+ The caller must ensure that tokens are encoded in the right charset.
+ """
+ self._closed = True
+ hunpos_paths = ['.', '/usr/bin', '/usr/local/bin', '/opt/local/bin',
+ '/Applications/bin', '~/bin', '~/Applications/bin']
+ hunpos_paths = list(map(os.path.expanduser, hunpos_paths))
+
+ self._hunpos_bin = find_binary(
+ 'hunpos-tag', path_to_bin,
+ env_vars=('HUNPOS', 'HUNPOS_HOME'),
+ searchpath=hunpos_paths,
+ url=_hunpos_url,
+ verbose=verbose)
+
+ self._hunpos_model = find_file(path_to_model,
+ env_vars=('HUNPOS', 'HUNPOS_HOME'), verbose=verbose)
+ self._encoding = encoding
+ self._hunpos = Popen([self._hunpos_bin, self._hunpos_model],
+ shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ self._closed = False
+
+ def __del__(self):
+ self.close()
+
+ def close(self):
+ """Closes the pipe to the hunpos executable."""
+ if not self._closed:
+ self._hunpos.communicate()
+ self._closed = True
+
+ def __enter__(self):
+ return self
+ def __exit__(self, exc_type, exc_value, traceback):
+ self.close()
+
+ def tag(self, tokens):
+ """Tags a single sentence: a list of words.
+ The tokens should not contain any newline characters.
+ """
+ for token in tokens:
+ assert "\n" not in token, "Tokens should not contain newlines"
+ if isinstance(token, compat.text_type):
+ token = token.encode(self._encoding)
+ self._hunpos.stdin.write(token + "\n")
+ # We write a final empty line to tell hunpos that the sentence is finished:
+ self._hunpos.stdin.write("\n")
+ self._hunpos.stdin.flush()
+
+ tagged_tokens = []
+ for token in tokens:
+ tagged = self._hunpos.stdout.readline().strip().split("\t")
+ tag = (tagged[1] if len(tagged) > 1 else None)
+ tagged_tokens.append((token, tag))
+ # We have to read (and dismiss) the final empty line:
+ self._hunpos.stdout.readline()
+
+ return tagged_tokens
+
+# skip doctests if Hunpos tagger is not installed
+def setup_module(module):
+ from nose import SkipTest
+ try:
+ HunposTagger('english.model')
+ except LookupError:
+ raise SkipTest("HunposTagger is not available")
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/mapping.py b/nltk/tag/mapping.py
new file mode 100644
index 0000000..cf458ee
--- /dev/null
+++ b/nltk/tag/mapping.py
@@ -0,0 +1,100 @@
+# Natural Language Toolkit: Tagset Mapping
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Nathan Schneider <nathan at cmu.edu>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Interface for converting POS tags from various treebanks
+to the universal tagset of Petrov, Das, & McDonald.
+
+The tagset consists of the following 12 coarse tags:
+
+VERB - verbs (all tenses and modes)
+NOUN - nouns (common and proper)
+PRON - pronouns
+ADJ - adjectives
+ADV - adverbs
+ADP - adpositions (prepositions and postpositions)
+CONJ - conjunctions
+DET - determiners
+NUM - cardinal numbers
+PRT - particles or other function words
+X - other: foreign words, typos, abbreviations
+. - punctuation
+
+ at see: http://arxiv.org/abs/1104.2086 and http://code.google.com/p/universal-pos-tags/
+
+"""
+
+from __future__ import print_function, unicode_literals, division
+from collections import defaultdict
+from os.path import join
+
+from nltk.data import load
+
+_UNIVERSAL_DATA = "taggers/universal_tagset"
+_UNIVERSAL_TAGS = ('VERB','NOUN','PRON','ADJ','ADV','ADP','CONJ','DET','NUM','PRT','X','.')
+
+# _MAPPINGS = defaultdict(lambda: defaultdict(dict))
+# the mapping between tagset T1 and T2 returns UNK if appied to an unrecognized tag
+_MAPPINGS = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: 'UNK')))
+
+
+def _load_universal_map(fileid):
+ mapping = {}
+ contents = load(join(_UNIVERSAL_DATA, fileid+'.map'), format="text")
+ for line in contents.splitlines():
+ line = line.strip()
+ if line == '':
+ continue
+ fine, coarse = line.split('\t')
+
+ assert coarse in _UNIVERSAL_TAGS, 'Unexpected coarse tag: {}'.format(coarse)
+ assert fine not in _MAPPINGS[fileid]['universal'], 'Multiple entries for original tag: {}'.format(fine)
+
+ _MAPPINGS[fileid]['universal'][fine] = coarse
+
+
+def tagset_mapping(source, target):
+ """
+ Retrieve the mapping dictionary between tagsets.
+
+ >>> tagset_mapping('ru-rnc', 'universal') == {'!': '.', 'A': 'ADJ', 'C': 'CONJ', 'AD': 'ADV',\
+ 'NN': 'NOUN', 'VG': 'VERB', 'COMP': 'CONJ', 'NC': 'NUM', 'VP': 'VERB', 'P': 'ADP',\
+ 'IJ': 'X', 'V': 'VERB', 'Z': 'X', 'VI': 'VERB', 'YES_NO_SENT': 'X', 'PTCL': 'PRT'}
+ True
+ """
+
+ if source not in _MAPPINGS or target not in _MAPPINGS[source]:
+ if target == 'universal':
+ _load_universal_map(source)
+ return _MAPPINGS[source][target]
+
+def map_tag(source, target, source_tag):
+ """
+ Maps the tag from the source tagset to the target tagset.
+
+ >>> map_tag('en-ptb', 'universal', 'VBZ')
+ 'VERB'
+ >>> map_tag('en-ptb', 'universal', 'VBP')
+ 'VERB'
+ >>> map_tag('en-ptb', 'universal', '``')
+ '.'
+ """
+
+ # we need a systematic approach to naming
+ if target == 'universal':
+ if source == 'wsj':
+ source = 'en-ptb'
+ if source == 'brown':
+ source = 'en-brown'
+
+ return tagset_mapping(source, target)[source_tag]
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/senna.py b/nltk/tag/senna.py
new file mode 100644
index 0000000..e7897a1
--- /dev/null
+++ b/nltk/tag/senna.py
@@ -0,0 +1,286 @@
+# encoding: utf-8
+# Natural Language Toolkit: Interface to the Senna tagger
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Rami Al-Rfou' <ralrfou at cs.stonybrook.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A module for interfacing with the SENNA pipeline.
+"""
+
+from os import path, sep
+from subprocess import Popen, PIPE
+from platform import architecture, system
+from nltk.tag.api import TaggerI
+from nltk import compat
+
+_senna_url = 'http://ml.nec-labs.com/senna/'
+
+
+class Error(Exception):
+ """Basic error handling class to be extended by the module specific
+ exceptions"""
+
+
+class ExecutableNotFound(Error):
+ """Raised if the senna executable does not exist"""
+
+
+class RunFailure(Error):
+ """Raised if the pipeline fails to execute"""
+
+
+class SentenceMisalignment(Error):
+ """Raised if the new sentence is shorter than the original one or the number
+ of sentences in the result is less than the input."""
+
+
+class SennaTagger(TaggerI):
+ r"""
+ A general interface of the SENNA pipeline that supports any of the
+ operations specified in SUPPORTED_OPERATIONS.
+
+ Applying multiple operations at once has the speed advantage. For example,
+ senna v2.0 will calculate the POS tags in case you are extracting the named
+ entities. Applying both of the operations will cost only the time of
+ extracting the named entities.
+
+ SENNA pipeline has a fixed maximum size of the sentences that it can read.
+ By default it is 1024 token/sentence. If you have larger sentences, changing
+ the MAX_SENTENCE_SIZE value in SENNA_main.c should be considered and your
+ system specific binary should be rebuilt. Otherwise this could introduce
+ misalignment errors.
+
+ The input is:
+ - path to the directory that contains SENNA executables.
+ - List of the operations needed to be performed.
+ - (optionally) the encoding of the input data (default:utf-8)
+
+ Example:
+
+ >>> from nltk.tag.senna import SennaTagger
+ >>> pipeline = SennaTagger('/usr/share/senna-v2.0', ['pos', 'chk', 'ner'])
+ >>> sent = u'Düsseldorf is an international business center'.split()
+ >>> pipeline.tag(sent)
+ [{'word': u'D\xfcsseldorf', 'chk': u'B-NP', 'ner': u'B-PER', 'pos': u'NNP'},
+ {'word': u'is', 'chk': u'B-VP', 'ner': u'O', 'pos': u'VBZ'},
+ {'word': u'an', 'chk': u'B-NP', 'ner': u'O', 'pos': u'DT'},
+ {'word': u'international', 'chk': u'I-NP', 'ner': u'O', 'pos': u'JJ'},
+ {'word': u'business', 'chk': u'I-NP', 'ner': u'O', 'pos': u'NN'},
+ {'word': u'center', 'chk': u'I-NP', 'ner': u'O','pos': u'NN'}]
+ """
+
+ SUPPORTED_OPERATIONS = ['pos', 'chk', 'ner']
+
+ def __init__(self, senna_path, operations, encoding='utf-8'):
+ self._encoding = encoding
+ self._path = path.normpath(senna_path) + sep
+ self.operations = operations
+
+ @property
+ def executable(self):
+ """
+ A property that determines the system specific binary that should be
+ used in the pipeline. In case, the system is not known the senna binary will
+ be used.
+ """
+ os_name = system()
+ if os_name == 'Linux':
+ bits = architecture()[0]
+ if bits == '64bit':
+ return path.join(self._path, 'senna-linux64')
+ return path.join(self._path, 'senna-linux32')
+ if os_name == 'Windows':
+ return path.join(self._path, 'senna-win32.exe')
+ if os_name == 'Darwin':
+ return path.join(self._path, 'senna-osx')
+ return path.join(self._path, 'senna')
+
+ def _map(self):
+ """
+ A method that calculates the order of the columns that SENNA pipeline
+ will output the tags into. This depends on the operations being ordered.
+ """
+ _map = {}
+ i = 1
+ for operation in SennaTagger.SUPPORTED_OPERATIONS:
+ if operation in self.operations:
+ _map[operation] = i
+ i+= 1
+ return _map
+
+ def tag(self, tokens):
+ """
+ Applies the specified operation(s) on a list of tokens.
+ """
+ return self.tag_sents([tokens])[0]
+
+ def tag_sents(self, sentences):
+ """
+ Applies the tag method over a list of sentences. This method will return a
+ list of dictionaries. Every dictionary will contain a word with its
+ calculated annotations/tags.
+ """
+ encoding = self._encoding
+
+ # Verifies the existence of the executable
+ if not path.isfile(self.executable):
+ raise ExecutableNotFound("Senna executable expected at %s but not found" %
+ self.executable)
+
+ # Build the senna command to run the tagger
+ _senna_cmd = [self.executable, '-path', self._path, '-usrtokens', '-iobtags']
+ _senna_cmd.extend(['-'+op for op in self.operations])
+
+ # Serialize the actual sentences to a temporary string
+ _input = '\n'.join((' '.join(x) for x in sentences))+'\n'
+ if isinstance(_input, compat.text_type) and encoding:
+ _input = _input.encode(encoding)
+
+ # Run the tagger and get the output
+ p = Popen(_senna_cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+ (stdout, stderr) = p.communicate(input=_input)
+ senna_output = stdout
+
+ # Check the return code.
+ if p.returncode != 0:
+ raise RunFailure('Senna command failed! Details: %s' % stderr)
+
+ if encoding:
+ senna_output = stdout.decode(encoding)
+
+ # Output the tagged sentences
+ map_ = self._map()
+ tagged_sentences = [[]]
+ sentence_index = 0
+ token_index = 0
+ for tagged_word in senna_output.strip().split("\n"):
+ if not tagged_word:
+ tagged_sentences.append([])
+ sentence_index += 1
+ token_index = 0
+ continue
+ tags = tagged_word.split('\t')
+ result = {}
+ for tag in map_:
+ result[tag] = tags[map_[tag]].strip()
+ try:
+ result['word'] = sentences[sentence_index][token_index]
+ except IndexError:
+ raise SentenceMisalignment(
+ "Misalignment error occurred at sentence number %d. Possible reason"
+ " is that the sentence size exceeded the maximum size. Check the "
+ "documentation of SennaTagger class for more information."
+ % sentence_index)
+ tagged_sentences[-1].append(result)
+ token_index += 1
+ return tagged_sentences
+
+
+class POSTagger(SennaTagger):
+ """
+ A Part of Speech tagger.
+
+ The input is:
+ - path to the directory that contains SENNA executables.
+ - (optionally) the encoding of the input data (default:utf-8)
+
+ Example:
+
+ >>> from nltk.tag.senna import POSTagger
+ >>> postagger = POSTagger('/usr/share/senna-v2.0')
+ >>> postagger.tag('What is the airspeed of an unladen swallow ?'.split())
+ [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'),
+ ('of', 'IN'), ('an', 'DT'), ('unladen', 'JJ'), ('swallow', 'VB'), ('?', '.')]
+ """
+ def __init__(self, path, encoding='utf-8'):
+ super(POSTagger, self).__init__(path, ['pos'], encoding)
+
+ def tag_sents(self, sentences):
+ """
+ Applies the tag method over a list of sentences. This method will return
+ for each sentence a list of tuples of (word, tag).
+ """
+ tagged_sents = super(POSTagger, self).tag_sents(sentences)
+ for i in range(len(tagged_sents)):
+ for j in range(len(tagged_sents[i])):
+ annotations = tagged_sents[i][j]
+ tagged_sents[i][j] = (annotations['word'], annotations['pos'])
+ return tagged_sents
+
+
+class NERTagger(SennaTagger):
+ """
+ A named entity extractor.
+
+ The input is:
+ - path to the directory that contains SENNA executables.
+ - (optionally) the encoding of the input data (default:utf-8)
+
+ Example:
+
+ >>> from nltk.tag.senna import NERTagger
+ >>> nertagger = NERTagger('/usr/share/senna-v2.0')
+ >>> nertagger.tag('Shakespeare theatre was in London .'.split())
+ [('Shakespeare', u'B-PER'), ('theatre', u'O'), ('was', u'O'), ('in', u'O'),
+ ('London', u'B-LOC'), ('.', u'O')]
+ >>> nertagger.tag('UN headquarters are in NY , USA .'.split())
+ [('UN', u'B-ORG'), ('headquarters', u'O'), ('are', u'O'), ('in', u'O'),
+ ('NY', u'B-LOC'), (',', u'O'), ('USA', u'B-LOC'), ('.', u'O')]
+ """
+ def __init__(self, path, encoding='utf-8'):
+ super(NERTagger, self).__init__(path, ['ner'], encoding)
+
+ def tag_sents(self, sentences):
+ """
+ Applies the tag method over a list of sentences. This method will return
+ for each sentence a list of tuples of (word, tag).
+ """
+ tagged_sents = super(NERTagger, self).tag_sents(sentences)
+ for i in range(len(tagged_sents)):
+ for j in range(len(tagged_sents[i])):
+ annotations = tagged_sents[i][j]
+ tagged_sents[i][j] = (annotations['word'], annotations['ner'])
+ return tagged_sents
+
+
+class CHKTagger(SennaTagger):
+ """
+ A chunker.
+
+ The input is:
+ - path to the directory that contains SENNA executables.
+ - (optionally) the encoding of the input data (default:utf-8)
+
+ Example:
+
+ >>> from nltk.tag.senna import CHKTagger
+ >>> chktagger = CHKTagger('/usr/share/senna-v2.0')
+ >>> chktagger.tag('What is the airspeed of an unladen swallow ?'.split())
+ [('What', u'B-NP'), ('is', u'B-VP'), ('the', u'B-NP'), ('airspeed', u'I-NP'),
+ ('of', u'B-PP'), ('an', u'B-NP'), ('unladen', u'I-NP'), ('swallow',u'I-NP'),
+ ('?', u'O')]
+ """
+ def __init__(self, path, encoding='utf-8'):
+ super(CHKTagger, self).__init__(path, ['chk'], encoding)
+
+ def tag_sents(self, sentences):
+ """
+ Applies the tag method over a list of sentences. This method will return
+ for each sentence a list of tuples of (word, tag).
+ """
+ tagged_sents = super(CHKTagger, self).tag_sents(sentences)
+ for i in range(len(tagged_sents)):
+ for j in range(len(tagged_sents[i])):
+ annotations = tagged_sents[i][j]
+ tagged_sents[i][j] = (annotations['word'], annotations['chk'])
+ return tagged_sents
+
+# skip doctests if Senna is not installed
+def setup_module(module):
+ from nose import SkipTest
+ tagger = POSTagger('/usr/share/senna-v2.0')
+ if not path.isfile(tagger.executable):
+ raise SkipTest("Senna executable expected at /usr/share/senna-v2.0/senna-osx but not found")
diff --git a/nltk/tag/sequential.py b/nltk/tag/sequential.py
new file mode 100644
index 0000000..a93a5bd
--- /dev/null
+++ b/nltk/tag/sequential.py
@@ -0,0 +1,746 @@
+# Natural Language Toolkit: Sequential Backoff Taggers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# Tiago Tresoldi <tresoldi at users.sf.net> (original affix tagger)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Classes for tagging sentences sequentially, left to right. The
+abstract base class SequentialBackoffTagger serves as the base
+class for all the taggers in this module. Tagging of individual words
+is performed by the method ``choose_tag()``, which is defined by
+subclasses of SequentialBackoffTagger. If a tagger is unable to
+determine a tag for the specified token, then its backoff tagger is
+consulted instead. Any SequentialBackoffTagger may serve as a
+backoff tagger for any other SequentialBackoffTagger.
+"""
+from __future__ import print_function, unicode_literals
+
+import re
+
+from nltk.probability import ConditionalFreqDist
+from nltk.classify.naivebayes import NaiveBayesClassifier
+from nltk.compat import python_2_unicode_compatible
+
+from nltk.tag.api import TaggerI, FeaturesetTaggerI
+
+from nltk import jsontags
+
+######################################################################
+#{ Abstract Base Classes
+######################################################################
+class SequentialBackoffTagger(TaggerI):
+ """
+ An abstract base class for taggers that tags words sequentially,
+ left to right. Tagging of individual words is performed by the
+ ``choose_tag()`` method, which should be defined by subclasses. If
+ a tagger is unable to determine a tag for the specified token,
+ then its backoff tagger is consulted.
+
+ :ivar _taggers: A list of all the taggers that should be tried to
+ tag a token (i.e., self and its backoff taggers).
+ """
+ def __init__(self, backoff=None):
+ if backoff is None:
+ self._taggers = [self]
+ else:
+ self._taggers = [self] + backoff._taggers
+
+ @property
+ def backoff(self):
+ """The backoff tagger for this tagger."""
+ return self._taggers[1] if len(self._taggers) > 1 else None
+
+ def tag(self, tokens):
+ # docs inherited from TaggerI
+ tags = []
+ for i in range(len(tokens)):
+ tags.append(self.tag_one(tokens, i, tags))
+ return list(zip(tokens, tags))
+
+ def tag_one(self, tokens, index, history):
+ """
+ Determine an appropriate tag for the specified token, and
+ return that tag. If this tagger is unable to determine a tag
+ for the specified token, then its backoff tagger is consulted.
+
+ :rtype: str
+ :type tokens: list
+ :param tokens: The list of words that are being tagged.
+ :type index: int
+ :param index: The index of the word whose tag should be
+ returned.
+ :type history: list(str)
+ :param history: A list of the tags for all words before *index*.
+ """
+ tag = None
+ for tagger in self._taggers:
+ tag = tagger.choose_tag(tokens, index, history)
+ if tag is not None: break
+ return tag
+
+ def choose_tag(self, tokens, index, history):
+ """
+ Decide which tag should be used for the specified token, and
+ return that tag. If this tagger is unable to determine a tag
+ for the specified token, return None -- do not consult
+ the backoff tagger. This method should be overridden by
+ subclasses of SequentialBackoffTagger.
+
+ :rtype: str
+ :type tokens: list
+ :param tokens: The list of words that are being tagged.
+ :type index: int
+ :param index: The index of the word whose tag should be
+ returned.
+ :type history: list(str)
+ :param history: A list of the tags for all words before *index*.
+ """
+ raise NotImplementedError()
+
+
+ at python_2_unicode_compatible
+class ContextTagger(SequentialBackoffTagger):
+ """
+ An abstract base class for sequential backoff taggers that choose
+ a tag for a token based on the value of its "context". Different
+ subclasses are used to define different contexts.
+
+ A ContextTagger chooses the tag for a token by calculating the
+ token's context, and looking up the corresponding tag in a table.
+ This table can be constructed manually; or it can be automatically
+ constructed based on a training corpus, using the ``_train()``
+ factory method.
+
+ :ivar _context_to_tag: Dictionary mapping contexts to tags.
+ """
+ def __init__(self, context_to_tag, backoff=None):
+ """
+ :param context_to_tag: A dictionary mapping contexts to tags.
+ :param backoff: The backoff tagger that should be used for this tagger.
+ """
+ SequentialBackoffTagger.__init__(self, backoff)
+ self._context_to_tag = (context_to_tag if context_to_tag else {})
+
+ def context(self, tokens, index, history):
+ """
+ :return: the context that should be used to look up the tag
+ for the specified token; or None if the specified token
+ should not be handled by this tagger.
+ :rtype: (hashable)
+ """
+ raise NotImplementedError()
+
+ def choose_tag(self, tokens, index, history):
+ context = self.context(tokens, index, history)
+ return self._context_to_tag.get(context)
+
+ def size(self):
+ """
+ :return: The number of entries in the table used by this
+ tagger to map from contexts to tags.
+ """
+ return len(self._context_to_tag)
+
+ def __repr__(self):
+ return '<%s: size=%d>' % (self.__class__.__name__, self.size())
+
+ def _train(self, tagged_corpus, cutoff=0, verbose=False):
+ """
+ Initialize this ContextTagger's ``_context_to_tag`` table
+ based on the given training data. In particular, for each
+ context ``c`` in the training data, set
+ ``_context_to_tag[c]`` to the most frequent tag for that
+ context. However, exclude any contexts that are already
+ tagged perfectly by the backoff tagger(s).
+
+ The old value of ``self._context_to_tag`` (if any) is discarded.
+
+ :param tagged_corpus: A tagged corpus. Each item should be
+ a list of (word, tag tuples.
+ :param cutoff: If the most likely tag for a context occurs
+ fewer than cutoff times, then exclude it from the
+ context-to-tag table for the new tagger.
+ """
+
+ token_count = hit_count = 0
+
+ # A context is considered 'useful' if it's not already tagged
+ # perfectly by the backoff tagger.
+ useful_contexts = set()
+
+ # Count how many times each tag occurs in each context.
+ fd = ConditionalFreqDist()
+ for sentence in tagged_corpus:
+ tokens, tags = zip(*sentence)
+ for index, (token, tag) in enumerate(sentence):
+ # Record the event.
+ token_count += 1
+ context = self.context(tokens, index, tags[:index])
+ if context is None: continue
+ fd[context][tag] += 1
+ # If the backoff got it wrong, this context is useful:
+ if (self.backoff is None or
+ tag != self.backoff.tag_one(tokens, index, tags[:index])):
+ useful_contexts.add(context)
+
+ # Build the context_to_tag table -- for each context, figure
+ # out what the most likely tag is. Only include contexts that
+ # we've seen at least `cutoff` times.
+ for context in useful_contexts:
+ best_tag = fd[context].max()
+ hits = fd[context][best_tag]
+ if hits > cutoff:
+ self._context_to_tag[context] = best_tag
+ hit_count += hits
+
+ # Display some stats, if requested.
+ if verbose:
+ size = len(self._context_to_tag)
+ backoff = 100 - (hit_count * 100.0)/ token_count
+ pruning = 100 - (size * 100.0) / len(fd.conditions())
+ print("[Trained Unigram tagger:", end=' ')
+ print("size=%d, backoff=%.2f%%, pruning=%.2f%%]" % (
+ size, backoff, pruning))
+
+######################################################################
+#{ Tagger Classes
+######################################################################
+
+ at python_2_unicode_compatible
+ at jsontags.register_tag
+class DefaultTagger(SequentialBackoffTagger):
+ """
+ A tagger that assigns the same tag to every token.
+
+ >>> from nltk.tag.sequential import DefaultTagger
+ >>> default_tagger = DefaultTagger('NN')
+ >>> list(default_tagger.tag('This is a test'.split()))
+ [('This', 'NN'), ('is', 'NN'), ('a', 'NN'), ('test', 'NN')]
+
+ This tagger is recommended as a backoff tagger, in cases where
+ a more powerful tagger is unable to assign a tag to the word
+ (e.g. because the word was not seen during training).
+
+ :param tag: The tag to assign to each token
+ :type tag: str
+ """
+
+ json_tag = 'nltk.tag.sequential.DefaultTagger'
+
+ def __init__(self, tag):
+ self._tag = tag
+ SequentialBackoffTagger.__init__(self, None)
+
+ def encode_json_obj(self):
+ return self._tag
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ tag = obj
+ return cls(tag)
+
+ def choose_tag(self, tokens, index, history):
+ return self._tag # ignore token and history
+
+ def __repr__(self):
+ return '<DefaultTagger: tag=%s>' % self._tag
+
+
+ at jsontags.register_tag
+class NgramTagger(ContextTagger):
+ """
+ A tagger that chooses a token's tag based on its word string and
+ on the preceding n word's tags. In particular, a tuple
+ (tags[i-n:i-1], words[i]) is looked up in a table, and the
+ corresponding tag is returned. N-gram taggers are typically
+ trained on a tagged corpus.
+
+ Train a new NgramTagger using the given training data or
+ the supplied model. In particular, construct a new tagger
+ whose table maps from each context (tag[i-n:i-1], word[i])
+ to the most frequent tag for that context. But exclude any
+ contexts that are already tagged perfectly by the backoff
+ tagger.
+
+ :param train: A tagged corpus consisting of a list of tagged
+ sentences, where each sentence is a list of (word, tag) tuples.
+ :param backoff: A backoff tagger, to be used by the new
+ tagger if it encounters an unknown context.
+ :param cutoff: If the most likely tag for a context occurs
+ fewer than *cutoff* times, then exclude it from the
+ context-to-tag table for the new tagger.
+ """
+ json_tag = 'nltk.tag.sequential.NgramTagger'
+
+ def __init__(self, n, train=None, model=None,
+ backoff=None, cutoff=0, verbose=False):
+ self._n = n
+ self._check_params(train, model)
+
+ ContextTagger.__init__(self, model, backoff)
+
+ if train:
+ self._train(train, cutoff, verbose)
+
+ def encode_json_obj(self):
+ return self._n, self._context_to_tag, self.backoff
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ _n, _context_to_tag, backoff = obj
+ return cls(_n, model=_context_to_tag, backoff=backoff)
+
+ def context(self, tokens, index, history):
+ tag_context = tuple(history[max(0,index-self._n+1):index])
+ return tag_context, tokens[index]
+
+
+ at jsontags.register_tag
+class UnigramTagger(NgramTagger):
+ """
+ Unigram Tagger
+
+ The UnigramTagger finds the most likely tag for each word in a training
+ corpus, and then uses that information to assign tags to new tokens.
+
+ >>> from nltk.corpus import brown
+ >>> from nltk.tag.sequential import UnigramTagger
+ >>> test_sent = brown.sents(categories='news')[0]
+ >>> unigram_tagger = UnigramTagger(brown.tagged_sents(categories='news')[:500])
+ >>> for tok, tag in unigram_tagger.tag(test_sent):
+ ... print("(%s, %s), " % (tok, tag))
+ (The, AT), (Fulton, NP-TL), (County, NN-TL), (Grand, JJ-TL),
+ (Jury, NN-TL), (said, VBD), (Friday, NR), (an, AT),
+ (investigation, NN), (of, IN), (Atlanta's, NP$), (recent, JJ),
+ (primary, NN), (election, NN), (produced, VBD), (``, ``),
+ (no, AT), (evidence, NN), ('', ''), (that, CS), (any, DTI),
+ (irregularities, NNS), (took, VBD), (place, NN), (., .),
+
+ :param train: The corpus of training data, a list of tagged sentences
+ :type train: list(list(tuple(str, str)))
+ :param model: The tagger model
+ :type model: dict
+ :param backoff: Another tagger which this tagger will consult when it is
+ unable to tag a word
+ :type backoff: TaggerI
+ :param cutoff: The number of instances of training data the tagger must see
+ in order not to use the backoff tagger
+ :type cutoff: int
+ """
+
+ json_tag = 'nltk.tag.sequential.UnigramTagger'
+
+ def __init__(self, train=None, model=None,
+ backoff=None, cutoff=0, verbose=False):
+ NgramTagger.__init__(self, 1, train, model,
+ backoff, cutoff, verbose)
+
+ def encode_json_obj(self):
+ return self._context_to_tag, self.backoff
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ _context_to_tag, backoff = obj
+ return cls(model=_context_to_tag, backoff=backoff)
+
+ def context(self, tokens, index, history):
+ return tokens[index]
+
+
+ at jsontags.register_tag
+class BigramTagger(NgramTagger):
+ """
+ A tagger that chooses a token's tag based its word string and on
+ the preceding words' tag. In particular, a tuple consisting
+ of the previous tag and the word is looked up in a table, and
+ the corresponding tag is returned.
+
+ :param train: The corpus of training data, a list of tagged sentences
+ :type train: list(list(tuple(str, str)))
+ :param model: The tagger model
+ :type model: dict
+ :param backoff: Another tagger which this tagger will consult when it is
+ unable to tag a word
+ :type backoff: TaggerI
+ :param cutoff: The number of instances of training data the tagger must see
+ in order not to use the backoff tagger
+ :type cutoff: int
+ """
+ json_tag = 'nltk.tag.sequential.BigramTagger'
+
+ def __init__(self, train=None, model=None,
+ backoff=None, cutoff=0, verbose=False):
+ NgramTagger.__init__(self, 2, train, model,
+ backoff, cutoff, verbose)
+
+ def encode_json_obj(self):
+ return self._context_to_tag, self.backoff
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ _context_to_tag, backoff = obj
+ return cls(model=_context_to_tag, backoff=backoff)
+
+
+ at jsontags.register_tag
+class TrigramTagger(NgramTagger):
+ """
+ A tagger that chooses a token's tag based its word string and on
+ the preceding two words' tags. In particular, a tuple consisting
+ of the previous two tags and the word is looked up in a table, and
+ the corresponding tag is returned.
+
+ :param train: The corpus of training data, a list of tagged sentences
+ :type train: list(list(tuple(str, str)))
+ :param model: The tagger model
+ :type model: dict
+ :param backoff: Another tagger which this tagger will consult when it is
+ unable to tag a word
+ :type backoff: TaggerI
+ :param cutoff: The number of instances of training data the tagger must see
+ in order not to use the backoff tagger
+ :type cutoff: int
+ """
+ json_tag = 'nltk.tag.sequential.TrigramTagger'
+
+ def __init__(self, train=None, model=None,
+ backoff=None, cutoff=0, verbose=False):
+ NgramTagger.__init__(self, 3, train, model,
+ backoff, cutoff, verbose)
+
+ def encode_json_obj(self):
+ return self._context_to_tag, self.backoff
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ _context_to_tag, backoff = obj
+ return cls(model=_context_to_tag, backoff=backoff)
+
+
+ at jsontags.register_tag
+class AffixTagger(ContextTagger):
+ """
+ A tagger that chooses a token's tag based on a leading or trailing
+ substring of its word string. (It is important to note that these
+ substrings are not necessarily "true" morphological affixes). In
+ particular, a fixed-length substring of the word is looked up in a
+ table, and the corresponding tag is returned. Affix taggers are
+ typically constructed by training them on a tagged corpus.
+
+ Construct a new affix tagger.
+
+ :param affix_length: The length of the affixes that should be
+ considered during training and tagging. Use negative
+ numbers for suffixes.
+ :param min_stem_length: Any words whose length is less than
+ min_stem_length+abs(affix_length) will be assigned a
+ tag of None by this tagger.
+ """
+
+ json_tag = 'nltk.tag.sequential.AffixTagger'
+
+ def __init__(self, train=None, model=None, affix_length=-3,
+ min_stem_length=2, backoff=None, cutoff=0, verbose=False):
+
+ self._check_params(train, model)
+
+ ContextTagger.__init__(self, model, backoff)
+
+ self._affix_length = affix_length
+ self._min_word_length = min_stem_length + abs(affix_length)
+
+ if train:
+ self._train(train, cutoff, verbose)
+
+ def encode_json_obj(self):
+ return self._affix_length, self._min_word_length, self._context_to_tag, self.backoff
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ _affix_length, _min_word_length, _context_to_tag, backoff = obj
+ return cls(
+ affix_length=_affix_length,
+ min_stem_length=_min_word_length - abs(_affix_length),
+ model=_context_to_tag,
+ backoff=backoff
+ )
+
+ def context(self, tokens, index, history):
+ token = tokens[index]
+ if len(token) < self._min_word_length:
+ return None
+ elif self._affix_length > 0:
+ return token[:self._affix_length]
+ else:
+ return token[self._affix_length:]
+
+
+ at python_2_unicode_compatible
+ at jsontags.register_tag
+class RegexpTagger(SequentialBackoffTagger):
+ """
+ Regular Expression Tagger
+
+ The RegexpTagger assigns tags to tokens by comparing their
+ word strings to a series of regular expressions. The following tagger
+ uses word suffixes to make guesses about the correct Brown Corpus part
+ of speech tag:
+
+ >>> from nltk.corpus import brown
+ >>> from nltk.tag.sequential import RegexpTagger
+ >>> test_sent = brown.sents(categories='news')[0]
+ >>> regexp_tagger = RegexpTagger(
+ ... [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
+ ... (r'(The|the|A|a|An|an)$', 'AT'), # articles
+ ... (r'.*able$', 'JJ'), # adjectives
+ ... (r'.*ness$', 'NN'), # nouns formed from adjectives
+ ... (r'.*ly$', 'RB'), # adverbs
+ ... (r'.*s$', 'NNS'), # plural nouns
+ ... (r'.*ing$', 'VBG'), # gerunds
+ ... (r'.*ed$', 'VBD'), # past tense verbs
+ ... (r'.*', 'NN') # nouns (default)
+ ... ])
+ >>> regexp_tagger
+ <Regexp Tagger: size=9>
+ >>> regexp_tagger.tag(test_sent)
+ [('The', 'AT'), ('Fulton', 'NN'), ('County', 'NN'), ('Grand', 'NN'), ('Jury', 'NN'),
+ ('said', 'NN'), ('Friday', 'NN'), ('an', 'AT'), ('investigation', 'NN'), ('of', 'NN'),
+ ("Atlanta's", 'NNS'), ('recent', 'NN'), ('primary', 'NN'), ('election', 'NN'),
+ ('produced', 'VBD'), ('``', 'NN'), ('no', 'NN'), ('evidence', 'NN'), ("''", 'NN'),
+ ('that', 'NN'), ('any', 'NN'), ('irregularities', 'NNS'), ('took', 'NN'),
+ ('place', 'NN'), ('.', 'NN')]
+
+ :type regexps: list(tuple(str, str))
+ :param regexps: A list of ``(regexp, tag)`` pairs, each of
+ which indicates that a word matching ``regexp`` should
+ be tagged with ``tag``. The pairs will be evalutated in
+ order. If none of the regexps match a word, then the
+ optional backoff tagger is invoked, else it is
+ assigned the tag None.
+ """
+
+ json_tag = 'nltk.tag.sequential.RegexpTagger'
+
+ def __init__(self, regexps, backoff=None):
+ """
+ """
+ SequentialBackoffTagger.__init__(self, backoff)
+ labels = ['g'+str(i) for i in range(len(regexps))]
+ tags = [tag for regex, tag in regexps]
+ self._map = dict(zip(labels, tags))
+ regexps_labels = [(regex, label) for ((regex,tag),label) in zip(regexps,labels)]
+ self._regexs = re.compile('|'.join('(?P<%s>%s)' % (label, regex) for regex,label in regexps_labels))
+ self._size=len(regexps)
+
+ def encode_json_obj(self):
+ return self._map, self._regexs.pattern, self._size, self.backoff
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ _map, _regexs, _size, backoff = obj
+ self = cls(())
+ self._map = _map
+ self._regexs = re.compile(_regexs)
+ self._size = _size
+ SequentialBackoffTagger.__init__(self, backoff)
+ return self
+
+ def choose_tag(self, tokens, index, history):
+ m = self._regexs.match(tokens[index])
+ if m:
+ return self._map[m.lastgroup]
+ return None
+
+ def __repr__(self):
+ return '<Regexp Tagger: size=%d>' % self._size
+
+
+ at python_2_unicode_compatible
+class ClassifierBasedTagger(SequentialBackoffTagger, FeaturesetTaggerI):
+ """
+ A sequential tagger that uses a classifier to choose the tag for
+ each token in a sentence. The featureset input for the classifier
+ is generated by a feature detector function::
+
+ feature_detector(tokens, index, history) -> featureset
+
+ Where tokens is the list of unlabeled tokens in the sentence;
+ index is the index of the token for which feature detection
+ should be performed; and history is list of the tags for all
+ tokens before index.
+
+ Construct a new classifier-based sequential tagger.
+
+ :param feature_detector: A function used to generate the
+ featureset input for the classifier::
+ feature_detector(tokens, index, history) -> featureset
+
+ :param train: A tagged corpus consisting of a list of tagged
+ sentences, where each sentence is a list of (word, tag) tuples.
+
+ :param backoff: A backoff tagger, to be used by the new tagger
+ if it encounters an unknown context.
+
+ :param classifier_builder: A function used to train a new
+ classifier based on the data in *train*. It should take
+ one argument, a list of labeled featuresets (i.e.,
+ (featureset, label) tuples).
+
+ :param classifier: The classifier that should be used by the
+ tagger. This is only useful if you want to manually
+ construct the classifier; normally, you would use *train*
+ instead.
+
+ :param backoff: A backoff tagger, used if this tagger is
+ unable to determine a tag for a given token.
+
+ :param cutoff_prob: If specified, then this tagger will fall
+ back on its backoff tagger if the probability of the most
+ likely tag is less than *cutoff_prob*.
+ """
+ def __init__(self, feature_detector=None, train=None,
+ classifier_builder=NaiveBayesClassifier.train,
+ classifier=None, backoff=None,
+ cutoff_prob=None, verbose=False):
+ self._check_params(train, classifier)
+
+ SequentialBackoffTagger.__init__(self, backoff)
+
+ if (train and classifier) or (not train and not classifier):
+ raise ValueError('Must specify either training data or '
+ 'trained classifier.')
+
+ if feature_detector is not None:
+ self._feature_detector = feature_detector
+ # The feature detector function, used to generate a featureset
+ # or each token: feature_detector(tokens, index, history) -> featureset
+
+ self._cutoff_prob = cutoff_prob
+ """Cutoff probability for tagging -- if the probability of the
+ most likely tag is less than this, then use backoff."""
+
+ self._classifier = classifier
+ """The classifier used to choose a tag for each token."""
+
+ if train:
+ self._train(train, classifier_builder, verbose)
+
+ def choose_tag(self, tokens, index, history):
+ # Use our feature detector to get the featureset.
+ featureset = self.feature_detector(tokens, index, history)
+
+ # Use the classifier to pick a tag. If a cutoff probability
+ # was specified, then check that the tag's probability is
+ # higher than that cutoff first; otherwise, return None.
+ if self._cutoff_prob is None:
+ return self._classifier.classify(featureset)
+
+ pdist = self._classifier.prob_classify(featureset)
+ tag = pdist.max()
+ return tag if pdist.prob(tag) >= self._cutoff_prob else None
+
+ def _train(self, tagged_corpus, classifier_builder, verbose):
+ """
+ Build a new classifier, based on the given training data
+ *tagged_corpus*.
+ """
+
+ classifier_corpus = []
+ if verbose:
+ print('Constructing training corpus for classifier.')
+
+ for sentence in tagged_corpus:
+ history = []
+ untagged_sentence, tags = zip(*sentence)
+ for index in range(len(sentence)):
+ featureset = self.feature_detector(untagged_sentence,
+ index, history)
+ classifier_corpus.append( (featureset, tags[index]) )
+ history.append(tags[index])
+
+ if verbose:
+ print('Training classifier (%d instances)' % len(classifier_corpus))
+ self._classifier = classifier_builder(classifier_corpus)
+
+ def __repr__(self):
+ return '<ClassifierBasedTagger: %r>' % self._classifier
+
+ def feature_detector(self, tokens, index, history):
+ """
+ Return the feature detector that this tagger uses to generate
+ featuresets for its classifier. The feature detector is a
+ function with the signature::
+
+ feature_detector(tokens, index, history) -> featureset
+
+ See ``classifier()``
+ """
+ return self._feature_detector(tokens, index, history)
+
+ def classifier(self):
+ """
+ Return the classifier that this tagger uses to choose a tag
+ for each word in a sentence. The input for this classifier is
+ generated using this tagger's feature detector.
+ See ``feature_detector()``
+ """
+ return self._classifier
+
+class ClassifierBasedPOSTagger(ClassifierBasedTagger):
+ """
+ A classifier based part of speech tagger.
+ """
+ def feature_detector(self, tokens, index, history):
+ word = tokens[index]
+ if index == 0:
+ prevword = prevprevword = None
+ prevtag = prevprevtag = None
+ elif index == 1:
+ prevword = tokens[index-1].lower()
+ prevprevword = None
+ prevtag = history[index-1]
+ prevprevtag = None
+ else:
+ prevword = tokens[index-1].lower()
+ prevprevword = tokens[index-2].lower()
+ prevtag = history[index-1]
+ prevprevtag = history[index-2]
+
+ if re.match('[0-9]+(\.[0-9]*)?|[0-9]*\.[0-9]+$', word):
+ shape = 'number'
+ elif re.match('\W+$', word):
+ shape = 'punct'
+ elif re.match('[A-Z][a-z]+$', word):
+ shape = 'upcase'
+ elif re.match('[a-z]+$', word):
+ shape = 'downcase'
+ elif re.match('\w+$', word):
+ shape = 'mixedcase'
+ else:
+ shape = 'other'
+
+ features = {
+ 'prevtag': prevtag,
+ 'prevprevtag': prevprevtag,
+ 'word': word,
+ 'word.lower': word.lower(),
+ 'suffix3': word.lower()[-3:],
+ 'suffix2': word.lower()[-2:],
+ 'suffix1': word.lower()[-1:],
+ 'prevprevword': prevprevword,
+ 'prevword': prevword,
+ 'prevtag+word': '%s+%s' % (prevtag, word.lower()),
+ 'prevprevtag+word': '%s+%s' % (prevprevtag, word.lower()),
+ 'prevword+word': '%s+%s' % (prevword, word.lower()),
+ 'shape': shape,
+ }
+ return features
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/stanford.py b/nltk/tag/stanford.py
new file mode 100644
index 0000000..5f732e8
--- /dev/null
+++ b/nltk/tag/stanford.py
@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the Stanford NER-tagger
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Nitin Madnani <nmadnani at ets.org>
+# Rami Al-Rfou' <ralrfou at cs.stonybrook.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A module for interfacing with the Stanford taggers.
+"""
+
+import os
+import tempfile
+from subprocess import PIPE
+import warnings
+
+from nltk.internals import find_file, find_jar, config_java, java, _java_options
+from nltk.tag.api import TaggerI
+from nltk import compat
+
+_stanford_url = 'http://nlp.stanford.edu/software'
+
+class StanfordTagger(TaggerI):
+ """
+ An interface to Stanford taggers. Subclasses must define:
+
+ - ``_cmd`` property: A property that returns the command that will be
+ executed.
+ - ``_SEPARATOR``: Class constant that represents that character that
+ is used to separate the tokens from their tags.
+ - ``_JAR`` file: Class constant that represents the jar file name.
+ """
+
+ _SEPARATOR = ''
+ _JAR = ''
+
+ def __init__(self, path_to_model, path_to_jar=None, encoding='ascii', verbose=False, java_options='-mx1000m'):
+
+ if not self._JAR:
+ warnings.warn('The StanfordTagger class is not meant to be '
+ 'instanciated directly. Did you mean POS- or NERTagger?')
+ self._stanford_jar = find_jar(
+ self._JAR, path_to_jar,
+ searchpath=(), url=_stanford_url,
+ verbose=verbose)
+
+ self._stanford_model = find_file(path_to_model,
+ env_vars=('STANFORD_MODELS',), verbose=verbose)
+ self._encoding = encoding
+ self.java_options = java_options
+
+ @property
+ def _cmd(self):
+ raise NotImplementedError
+
+ def tag(self, tokens):
+ return self.tag_sents([tokens])[0]
+
+ def tag_sents(self, sentences):
+ encoding = self._encoding
+ default_options = ' '.join(_java_options)
+ config_java(options=self.java_options, verbose=False)
+
+ # Create a temporary input file
+ _input_fh, self._input_file_path = tempfile.mkstemp(text=True)
+
+ self._cmd.extend(['-encoding', encoding])
+
+ # Write the actual sentences to the temporary input file
+ _input_fh = os.fdopen(_input_fh, 'wb')
+ _input = '\n'.join((' '.join(x) for x in sentences))
+ if isinstance(_input, compat.text_type) and encoding:
+ _input = _input.encode(encoding)
+ _input_fh.write(_input)
+ _input_fh.close()
+
+ # Run the tagger and get the output
+ stanpos_output, _stderr = java(self._cmd,classpath=self._stanford_jar,
+ stdout=PIPE, stderr=PIPE)
+ stanpos_output = stanpos_output.decode(encoding)
+
+ # Delete the temporary file
+ os.unlink(self._input_file_path)
+
+ # Return java configurations to their default values
+ config_java(options=default_options, verbose=False)
+
+ return self.parse_output(stanpos_output)
+
+ def parse_output(self, text):
+ # Output the tagged sentences
+ tagged_sentences = []
+ for tagged_sentence in text.strip().split("\n"):
+ sentence = []
+ for tagged_word in tagged_sentence.strip().split():
+ word_tags = tagged_word.strip().split(self._SEPARATOR)
+ sentence.append((''.join(word_tags[:-1]), word_tags[-1]))
+ tagged_sentences.append(sentence)
+ return tagged_sentences
+
+class POSTagger(StanfordTagger):
+ """
+ A class for pos tagging with Stanford Tagger. The input is the paths to:
+ - a model trained on training data
+ - (optionally) the path to the stanford tagger jar file. If not specified here,
+ then this jar file must be specified in the CLASSPATH envinroment variable.
+ - (optionally) the encoding of the training data (default: ASCII)
+
+ Example:
+
+ >>> from nltk.tag.stanford import POSTagger
+ >>> st = POSTagger('/usr/share/stanford-postagger/models/english-bidirectional-distsim.tagger',
+ ... '/usr/share/stanford-postagger/stanford-postagger.jar') # doctest: +SKIP
+ >>> st.tag('What is the airspeed of an unladen swallow ?'.split()) # doctest: +SKIP
+ [('What', 'WP'), ('is', 'VBZ'), ('the', 'DT'), ('airspeed', 'NN'), ('of', 'IN'), ('an', 'DT'), ('unladen', 'JJ'), ('swallow', 'VB'), ('?', '.')]
+ """
+
+ _SEPARATOR = '_'
+ _JAR = 'stanford-postagger.jar'
+
+ def __init__(self, *args, **kwargs):
+ super(POSTagger, self).__init__(*args, **kwargs)
+
+ @property
+ def _cmd(self):
+ return ['edu.stanford.nlp.tagger.maxent.MaxentTagger',
+ '-model', self._stanford_model, '-textFile',
+ self._input_file_path, '-tokenize', 'false']
+
+class NERTagger(StanfordTagger):
+ """
+ A class for ner tagging with Stanford Tagger. The input is the paths to:
+
+ - a model trained on training data
+ - (optionally) the path to the stanford tagger jar file. If not specified here,
+ then this jar file must be specified in the CLASSPATH envinroment variable.
+ - (optionally) the encoding of the training data (default: ASCII)
+
+ Example:
+
+ >>> from nltk.tag.stanford import NERTagger
+ >>> st = NERTagger('/usr/share/stanford-ner/classifiers/all.3class.distsim.crf.ser.gz',
+ ... '/usr/share/stanford-ner/stanford-ner.jar') # doctest: +SKIP
+ >>> st.tag('Rami Eid is studying at Stony Brook University in NY'.split()) # doctest: +SKIP
+ [('Rami', 'PERSON'), ('Eid', 'PERSON'), ('is', 'O'), ('studying', 'O'),
+ ('at', 'O'), ('Stony', 'ORGANIZATION'), ('Brook', 'ORGANIZATION'),
+ ('University', 'ORGANIZATION'), ('in', 'O'), ('NY', 'LOCATION')]
+ """
+
+ _SEPARATOR = '/'
+ _JAR = 'stanford-ner.jar'
+ _FORMAT = 'slashTags'
+
+ def __init__(self, *args, **kwargs):
+ super(NERTagger, self).__init__(*args, **kwargs)
+
+ @property
+ def _cmd(self):
+ return ['edu.stanford.nlp.ie.crf.CRFClassifier',
+ '-loadClassifier', self._stanford_model, '-textFile',
+ self._input_file_path, '-outputFormat', self._FORMAT]
+
+ def parse_output(self, text):
+ if self._FORMAT == 'slashTags':
+ return super(NERTagger, self).parse_output(text)
+ raise NotImplementedError
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tag/tnt.py b/nltk/tag/tnt.py
new file mode 100755
index 0000000..6faf04a
--- /dev/null
+++ b/nltk/tag/tnt.py
@@ -0,0 +1,607 @@
+# Natural Language Toolkit: TnT Tagger
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Sam Huston <sjh900 at gmail.com>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+'''
+Implementation of 'TnT - A Statisical Part of Speech Tagger'
+by Thorsten Brants
+
+http://acl.ldc.upenn.edu/A/A00/A00-1031.pdf
+'''
+from __future__ import print_function
+from math import log
+
+from operator import itemgetter
+
+from nltk.probability import FreqDist, ConditionalFreqDist
+from nltk.tag.api import TaggerI
+
+class TnT(TaggerI):
+ '''
+ TnT - Statistical POS tagger
+
+ IMPORTANT NOTES:
+
+ * DOES NOT AUTOMATICALLY DEAL WITH UNSEEN WORDS
+
+ - It is possible to provide an untrained POS tagger to
+ create tags for unknown words, see __init__ function
+
+ * SHOULD BE USED WITH SENTENCE-DELIMITED INPUT
+
+ - Due to the nature of this tagger, it works best when
+ trained over sentence delimited input.
+ - However it still produces good results if the training
+ data and testing data are separated on all punctuation eg: [,.?!]
+ - Input for training is expected to be a list of sentences
+ where each sentence is a list of (word, tag) tuples
+ - Input for tag function is a single sentence
+ Input for tagdata function is a list of sentences
+ Output is of a similar form
+
+ * Function provided to process text that is unsegmented
+
+ - Please see basic_sent_chop()
+
+
+ TnT uses a second order Markov model to produce tags for
+ a sequence of input, specifically:
+
+ argmax [Proj(P(t_i|t_i-1,t_i-2)P(w_i|t_i))] P(t_T+1 | t_T)
+
+ IE: the maximum projection of a set of probabilities
+
+ The set of possible tags for a given word is derived
+ from the training data. It is the set of all tags
+ that exact word has been assigned.
+
+ To speed up and get more precision, we can use log addition
+ to instead multiplication, specifically:
+
+ argmax [Sigma(log(P(t_i|t_i-1,t_i-2))+log(P(w_i|t_i)))] +
+ log(P(t_T+1|t_T))
+
+ The probability of a tag for a given word is the linear
+ interpolation of 3 markov models; a zero-order, first-order,
+ and a second order model.
+
+ P(t_i| t_i-1, t_i-2) = l1*P(t_i) + l2*P(t_i| t_i-1) +
+ l3*P(t_i| t_i-1, t_i-2)
+
+ A beam search is used to limit the memory usage of the algorithm.
+ The degree of the beam can be changed using N in the initialization.
+ N represents the maximum number of possible solutions to maintain
+ while tagging.
+
+ It is possible to differentiate the tags which are assigned to
+ capitalized words. However this does not result in a significant
+ gain in the accuracy of the results.
+ '''
+
+ def __init__(self, unk=None, Trained=False, N=1000, C=False):
+ '''
+ Construct a TnT statistical tagger. Tagger must be trained
+ before being used to tag input.
+
+ :param unk: instance of a POS tagger, conforms to TaggerI
+ :type unk:(TaggerI)
+ :param Trained: Indication that the POS tagger is trained or not
+ :type Trained: boolean
+ :param N: Beam search degree (see above)
+ :type N:(int)
+ :param C: Capitalization flag
+ :type C: boolean
+
+ Initializer, creates frequency distributions to be used
+ for tagging
+
+ _lx values represent the portion of the tri/bi/uni taggers
+ to be used to calculate the probability
+
+ N value is the number of possible solutions to maintain
+ while tagging. A good value for this is 1000
+
+ C is a boolean value which specifies to use or
+ not use the Capitalization of the word as additional
+ information for tagging.
+ NOTE: using capitalization may not increase the accuracy
+ of the tagger
+ '''
+
+ self._uni = FreqDist()
+ self._bi = ConditionalFreqDist()
+ self._tri = ConditionalFreqDist()
+ self._wd = ConditionalFreqDist()
+ self._eos = ConditionalFreqDist()
+ self._l1 = 0.0
+ self._l2 = 0.0
+ self._l3 = 0.0
+ self._N = N
+ self._C = C
+ self._T = Trained
+
+ self._unk = unk
+
+ # statistical tools (ignore or delete me)
+ self.unknown = 0
+ self.known = 0
+
+ def train(self, data):
+ '''
+ Uses a set of tagged data to train the tagger.
+ If an unknown word tagger is specified,
+ it is trained on the same data.
+
+ :param data: List of lists of (word, tag) tuples
+ :type data: tuple(str)
+ '''
+
+ # Ensure that local C flag is initialized before use
+ C = False
+
+ if self._unk is not None and self._T == False:
+ self._unk.train(data)
+
+ for sent in data:
+ history = [('BOS',False), ('BOS',False)]
+ for w, t in sent:
+
+ # if capitalization is requested,
+ # and the word begins with a capital
+ # set local flag C to True
+ if self._C and w[0].isupper(): C=True
+
+ self._wd[w][t] += 1
+ self._uni[(t,C)] += 1
+ self._bi[history[1]][(t,C)] += 1
+ self._tri[tuple(history)][(t,C)] += 1
+
+ history.append((t,C))
+ history.pop(0)
+
+ # set local flag C to false for the next word
+ C = False
+
+ self._eos[t]['EOS'] += 1
+
+
+ # compute lambda values from the trained frequency distributions
+ self._compute_lambda()
+
+ #(debugging -- ignore or delete me)
+ #print "lambdas"
+ #print i, self._l1, i, self._l2, i, self._l3
+
+
+ def _compute_lambda(self):
+ '''
+ creates lambda values based upon training data
+
+ NOTE: no need to explicitly reference C,
+ it is contained within the tag variable :: tag == (tag,C)
+
+ for each tag trigram (t1, t2, t3)
+ depending on the maximum value of
+ - f(t1,t2,t3)-1 / f(t1,t2)-1
+ - f(t2,t3)-1 / f(t2)-1
+ - f(t3)-1 / N-1
+
+ increment l3,l2, or l1 by f(t1,t2,t3)
+
+ ISSUES -- Resolutions:
+ if 2 values are equal, increment both lambda values
+ by (f(t1,t2,t3) / 2)
+ '''
+
+ # temporary lambda variables
+ tl1 = 0.0
+ tl2 = 0.0
+ tl3 = 0.0
+
+ # for each t1,t2 in system
+ for history in self._tri.conditions():
+ (h1, h2) = history
+
+ # for each t3 given t1,t2 in system
+ # (NOTE: tag actually represents (tag,C))
+ # However no effect within this function
+ for tag in self._tri[history].keys():
+
+ # if there has only been 1 occurrence of this tag in the data
+ # then ignore this trigram.
+ if self._uni[tag] == 1:
+ continue
+
+ # safe_div provides a safe floating point division
+ # it returns -1 if the denominator is 0
+ c3 = self._safe_div((self._tri[history][tag]-1), (self._tri[history].N()-1))
+ c2 = self._safe_div((self._bi[h2][tag]-1), (self._bi[h2].N()-1))
+ c1 = self._safe_div((self._uni[tag]-1), (self._uni.N()-1))
+
+
+ # if c1 is the maximum value:
+ if (c1 > c3) and (c1 > c2):
+ tl1 += self._tri[history][tag]
+
+ # if c2 is the maximum value
+ elif (c2 > c3) and (c2 > c1):
+ tl2 += self._tri[history][tag]
+
+ # if c3 is the maximum value
+ elif (c3 > c2) and (c3 > c1):
+ tl3 += self._tri[history][tag]
+
+ # if c3, and c2 are equal and larger than c1
+ elif (c3 == c2) and (c3 > c1):
+ tl2 += float(self._tri[history][tag]) /2.0
+ tl3 += float(self._tri[history][tag]) /2.0
+
+ # if c1, and c2 are equal and larger than c3
+ # this might be a dumb thing to do....(not sure yet)
+ elif (c2 == c1) and (c1 > c3):
+ tl1 += float(self._tri[history][tag]) /2.0
+ tl2 += float(self._tri[history][tag]) /2.0
+
+ # otherwise there might be a problem
+ # eg: all values = 0
+ else:
+ #print "Problem", c1, c2 ,c3
+ pass
+
+ # Lambda normalisation:
+ # ensures that l1+l2+l3 = 1
+ self._l1 = tl1 / (tl1+tl2+tl3)
+ self._l2 = tl2 / (tl1+tl2+tl3)
+ self._l3 = tl3 / (tl1+tl2+tl3)
+
+
+
+ def _safe_div(self, v1, v2):
+ '''
+ Safe floating point division function, does not allow division by 0
+ returns -1 if the denominator is 0
+ '''
+ if v2 == 0:
+ return -1
+ else:
+ return float(v1) / float(v2)
+
+ def tagdata(self, data):
+ '''
+ Tags each sentence in a list of sentences
+
+ :param data:list of list of words
+ :type data: [[string,],]
+ :return: list of list of (word, tag) tuples
+
+ Invokes tag(sent) function for each sentence
+ compiles the results into a list of tagged sentences
+ each tagged sentence is a list of (word, tag) tuples
+ '''
+ res = []
+ for sent in data:
+ res1 = self.tag(sent)
+ res.append(res1)
+ return res
+
+
+ def tag(self, data):
+ '''
+ Tags a single sentence
+
+ :param data: list of words
+ :type data: [string,]
+
+ :return: [(word, tag),]
+
+ Calls recursive function '_tagword'
+ to produce a list of tags
+
+ Associates the sequence of returned tags
+ with the correct words in the input sequence
+
+ returns a list of (word, tag) tuples
+ '''
+
+ current_state = [(['BOS', 'BOS'], 0.0)]
+
+ sent = list(data)
+
+ tags = self._tagword(sent, current_state)
+
+ res = []
+ for i in range(len(sent)):
+ # unpack and discard the C flags
+ (t,C) = tags[i+2]
+ res.append((sent[i], t))
+
+ return res
+
+
+ def _tagword(self, sent, current_states):
+ '''
+ :param sent : List of words remaining in the sentence
+ :type sent : [word,]
+ :param current_states : List of possible tag combinations for
+ the sentence so far, and the log probability
+ associated with each tag combination
+ :type current_states : [([tag, ], logprob), ]
+
+ Tags the first word in the sentence and
+ recursively tags the reminder of sentence
+
+ Uses formula specified above to calculate the probability
+ of a particular tag
+ '''
+
+ # if this word marks the end of the sentance,
+ # return the most probable tag
+ if sent == []:
+ (h, logp) = current_states[0]
+ return h
+
+ # otherwise there are more words to be tagged
+ word = sent[0]
+ sent = sent[1:]
+ new_states = []
+
+ # if the Capitalisation is requested,
+ # initalise the flag for this word
+ C = False
+ if self._C and word[0].isupper(): C=True
+
+ # if word is known
+ # compute the set of possible tags
+ # and their associated log probabilities
+ if word in self._wd.conditions():
+ self.known += 1
+
+ for (history, curr_sent_logprob) in current_states:
+ logprobs = []
+
+ for t in self._wd[word].keys():
+ p_uni = self._uni.freq((t,C))
+ p_bi = self._bi[history[-1]].freq((t,C))
+ p_tri = self._tri[tuple(history[-2:])].freq((t,C))
+ p_wd = float(self._wd[word][t])/float(self._uni[(t,C)])
+ p = self._l1 *p_uni + self._l2 *p_bi + self._l3 *p_tri
+ p2 = log(p, 2) + log(p_wd, 2)
+
+ logprobs.append(((t,C), p2))
+
+
+ # compute the result of appending each tag to this history
+ for (tag, logprob) in logprobs:
+ new_states.append((history + [tag],
+ curr_sent_logprob + logprob))
+
+
+
+
+ # otherwise a new word, set of possible tags is unknown
+ else:
+ self.unknown += 1
+
+ # since a set of possible tags,
+ # and the probability of each specific tag
+ # can not be returned from most classifiers:
+ # specify that any unknown words are tagged with certainty
+ p = 1
+
+ # if no unknown word tagger has been specified
+ # then use the tag 'Unk'
+ if self._unk is None:
+ tag = ('Unk',C)
+
+ # otherwise apply the unknown word tagger
+ else :
+ [(_w, t)] = list(self._unk.tag([word]))
+ tag = (t,C)
+
+ for (history, logprob) in current_states:
+ history.append(tag)
+
+ new_states = current_states
+
+
+
+ # now have computed a set of possible new_states
+
+ # sort states by log prob
+ # set is now ordered greatest to least log probability
+ new_states.sort(reverse=True, key=itemgetter(1))
+
+ # del everything after N (threshold)
+ # this is the beam search cut
+ if len(new_states) > self._N:
+ new_states = new_states[:self._N]
+
+
+ # compute the tags for the rest of the sentence
+ # return the best list of tags for the sentence
+ return self._tagword(sent, new_states)
+
+
+########################################
+# helper function -- basic sentence tokenizer
+########################################
+
+def basic_sent_chop(data, raw=True):
+ '''
+ Basic method for tokenizing input into sentences
+ for this tagger:
+
+ :param data: list of tokens (words or (word, tag) tuples)
+ :type data: str or tuple(str, str)
+ :param raw: boolean flag marking the input data
+ as a list of words or a list of tagged words
+ :type raw: bool
+ :return: list of sentences
+ sentences are a list of tokens
+ tokens are the same as the input
+
+ Function takes a list of tokens and separates the tokens into lists
+ where each list represents a sentence fragment
+ This function can separate both tagged and raw sequences into
+ basic sentences.
+
+ Sentence markers are the set of [,.!?]
+
+ This is a simple method which enhances the performance of the TnT
+ tagger. Better sentence tokenization will further enhance the results.
+ '''
+
+ new_data = []
+ curr_sent = []
+ sent_mark = [',','.','?','!']
+
+
+ if raw:
+ for word in data:
+ if word in sent_mark:
+ curr_sent.append(word)
+ new_data.append(curr_sent)
+ curr_sent = []
+ else:
+ curr_sent.append(word)
+
+ else:
+ for (word,tag) in data:
+ if word in sent_mark:
+ curr_sent.append((word,tag))
+ new_data.append(curr_sent)
+ curr_sent = []
+ else:
+ curr_sent.append((word,tag))
+ return new_data
+
+
+
+def demo():
+ from nltk.corpus import brown
+ sents = list(brown.tagged_sents())
+ test = list(brown.sents())
+
+ # create and train the tagger
+ tagger = TnT()
+ tagger.train(sents[200:1000])
+
+ # tag some data
+ tagged_data = tagger.tagdata(test[100:120])
+
+ # print results
+ for j in range(len(tagged_data)):
+ s = tagged_data[j]
+ t = sents[j+100]
+ for i in range(len(s)):
+ print(s[i],'--', t[i])
+ print()
+
+
+def demo2():
+ from nltk.corpus import treebank
+
+ d = list(treebank.tagged_sents())
+
+ t = TnT(N=1000, C=False)
+ s = TnT(N=1000, C=True)
+ t.train(d[(11)*100:])
+ s.train(d[(11)*100:])
+
+ for i in range(10):
+ tacc = t.evaluate(d[i*100:((i+1)*100)])
+ tp_un = float(t.unknown) / float(t.known +t.unknown)
+ tp_kn = float(t.known) / float(t.known + t.unknown)
+ t.unknown = 0
+ t.known = 0
+
+ print('Capitalization off:')
+ print('Accuracy:', tacc)
+ print('Percentage known:', tp_kn)
+ print('Percentage unknown:', tp_un)
+ print('Accuracy over known words:', (tacc / tp_kn))
+
+ sacc = s.evaluate(d[i*100:((i+1)*100)])
+ sp_un = float(s.unknown) / float(s.known +s.unknown)
+ sp_kn = float(s.known) / float(s.known + s.unknown)
+ s.unknown = 0
+ s.known = 0
+
+ print('Capitalization on:')
+ print('Accuracy:', sacc)
+ print('Percentage known:', sp_kn)
+ print('Percentage unknown:', sp_un)
+ print('Accuracy over known words:', (sacc / sp_kn))
+
+def demo3():
+ from nltk.corpus import treebank, brown
+
+ d = list(treebank.tagged_sents())
+ e = list(brown.tagged_sents())
+
+ d = d[:1000]
+ e = e[:1000]
+
+ d10 = int(len(d)*0.1)
+ e10 = int(len(e)*0.1)
+
+ tknacc = 0
+ sknacc = 0
+ tallacc = 0
+ sallacc = 0
+ tknown = 0
+ sknown = 0
+
+ for i in range(10):
+
+ t = TnT(N=1000, C=False)
+ s = TnT(N=1000, C=False)
+
+ dtest = d[(i*d10):((i+1)*d10)]
+ etest = e[(i*e10):((i+1)*e10)]
+
+ dtrain = d[:(i*d10)] + d[((i+1)*d10):]
+ etrain = e[:(i*e10)] + e[((i+1)*e10):]
+
+ t.train(dtrain)
+ s.train(etrain)
+
+ tacc = t.evaluate(dtest)
+ tp_un = float(t.unknown) / float(t.known +t.unknown)
+ tp_kn = float(t.known) / float(t.known + t.unknown)
+ tknown += tp_kn
+ t.unknown = 0
+ t.known = 0
+
+ sacc = s.evaluate(etest)
+ sp_un = float(s.unknown) / float(s.known + s.unknown)
+ sp_kn = float(s.known) / float(s.known + s.unknown)
+ sknown += sp_kn
+ s.unknown = 0
+ s.known = 0
+
+ tknacc += (tacc / tp_kn)
+ sknacc += (sacc / tp_kn)
+ tallacc += tacc
+ sallacc += sacc
+
+ #print i+1, (tacc / tp_kn), i+1, (sacc / tp_kn), i+1, tacc, i+1, sacc
+
+
+ print("brown: acc over words known:", 10 * tknacc)
+ print(" : overall accuracy:", 10 * tallacc)
+ print(" : words known:", 10 * tknown)
+ print("treebank: acc over words known:", 10 * sknacc)
+ print(" : overall accuracy:", 10 * sallacc)
+ print(" : words known:", 10 * sknown)
+
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
diff --git a/nltk/tag/util.py b/nltk/tag/util.py
new file mode 100644
index 0000000..1bd0ec5
--- /dev/null
+++ b/nltk/tag/util.py
@@ -0,0 +1,75 @@
+# Natural Language Toolkit: Tagger Utilities
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+def str2tuple(s, sep='/'):
+ """
+ Given the string representation of a tagged token, return the
+ corresponding tuple representation. The rightmost occurrence of
+ *sep* in *s* will be used to divide *s* into a word string and
+ a tag string. If *sep* does not occur in *s*, return (s, None).
+
+ >>> from nltk.tag.util import str2tuple
+ >>> str2tuple('fly/NN')
+ ('fly', 'NN')
+
+ :type s: str
+ :param s: The string representation of a tagged token.
+ :type sep: str
+ :param sep: The separator string used to separate word strings
+ from tags.
+ """
+ loc = s.rfind(sep)
+ if loc >= 0:
+ return (s[:loc], s[loc+len(sep):].upper())
+ else:
+ return (s, None)
+
+def tuple2str(tagged_token, sep='/'):
+ """
+ Given the tuple representation of a tagged token, return the
+ corresponding string representation. This representation is
+ formed by concatenating the token's word string, followed by the
+ separator, followed by the token's tag. (If the tag is None,
+ then just return the bare word string.)
+
+ >>> from nltk.tag.util import tuple2str
+ >>> tagged_token = ('fly', 'NN')
+ >>> tuple2str(tagged_token)
+ 'fly/NN'
+
+ :type tagged_token: tuple(str, str)
+ :param tagged_token: The tuple representation of a tagged token.
+ :type sep: str
+ :param sep: The separator string used to separate word strings
+ from tags.
+ """
+ word, tag = tagged_token
+ if tag is None:
+ return word
+ else:
+ assert sep not in tag, 'tag may not contain sep!'
+ return '%s%s%s' % (word, sep, tag)
+
+def untag(tagged_sentence):
+ """
+ Given a tagged sentence, return an untagged version of that
+ sentence. I.e., return a list containing the first element
+ of each tuple in *tagged_sentence*.
+
+ >>> from nltk.tag.util import untag
+ >>> untag([('John', 'NNP'), ('saw', 'VBD'), ('Mary', 'NNP')])
+ ['John', 'saw', 'Mary']
+
+ """
+ return [w for (w, t) in tagged_sentence]
+
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tbl/__init__.py b/nltk/tbl/__init__.py
new file mode 100644
index 0000000..ed7292b
--- /dev/null
+++ b/nltk/tbl/__init__.py
@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Transformation Based Learning
+
+A general purpose package for Transformation Based Learning,
+currently used by nltk.tag.BrillTagger.
+"""
+
+from nltk.tbl.template import Template
+#API: Template(...), Template.expand(...)
+
+from nltk.tbl.feature import Feature
+#API: Feature(...), Feature.expand(...)
+
+from nltk.tbl.rule import Rule
+#API: Rule.format(...), Rule.templatetid
+
+from nltk.tbl.erroranalysis import error_list
+
diff --git a/nltk/tbl/api.py b/nltk/tbl/api.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/nltk/tbl/api.py
@@ -0,0 +1 @@
+
diff --git a/nltk/tbl/demo.py b/nltk/tbl/demo.py
new file mode 100644
index 0000000..6df31c0
--- /dev/null
+++ b/nltk/tbl/demo.py
@@ -0,0 +1,367 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function, absolute_import, division
+import os
+import pickle
+
+import random
+import time
+
+from nltk.corpus import treebank
+
+from nltk.tbl import error_list, Template
+from nltk.tag.brill import Word, Pos
+from nltk.tag import BrillTaggerTrainer, RegexpTagger, UnigramTagger
+
+def demo():
+ """
+ Run a demo with defaults. See source comments for details,
+ or docstrings of any of the more specific demo_* functions.
+ """
+ postag()
+
+def demo_repr_rule_format():
+ """
+ Exemplify repr(Rule) (see also str(Rule) and Rule.format("verbose"))
+ """
+ postag(ruleformat="repr")
+
+def demo_str_rule_format():
+ """
+ Exemplify repr(Rule) (see also str(Rule) and Rule.format("verbose"))
+ """
+ postag(ruleformat="str")
+
+def demo_verbose_rule_format():
+ """
+ Exemplify Rule.format("verbose")
+ """
+ postag(ruleformat="verbose")
+
+def demo_multiposition_feature():
+ """
+ The feature/s of a template takes a list of positions
+ relative to the current word where the feature should be
+ looked for, conceptually joined by logical OR. For instance,
+ Pos([-1, 1]), given a value V, will hold whenever V is found
+ one step to the left and/or one step to the right.
+
+ For contiguous ranges, a 2-arg form giving inclusive end
+ points can also be used: Pos(-3, -1) is the same as the arg
+ below.
+ """
+ postag(templates=[Template(Pos([-3,-2,-1]))])
+
+def demo_multifeature_template():
+ """
+ Templates can have more than a single feature.
+ """
+ postag(templates=[Template(Word([0]), Pos([-2,-1]))])
+
+def demo_template_statistics():
+ """
+ Show aggregate statistics per template. Little used templates are
+ candidates for deletion, much used templates may possibly be refined.
+
+ Deleting unused templates is mostly about saving time and/or space:
+ training is basically O(T) in the number of templates T
+ (also in terms of memory usage, which often will be the limiting factor).
+ """
+ postag(incremental_stats=True, template_stats=True)
+
+def demo_generated_templates():
+ """
+ Template.expand and Feature.expand are class methods facilitating
+ generating large amounts of templates. See their documentation for
+ details.
+
+ Note: training with 500 templates can easily fill all available
+ even on relatively small corpora
+ """
+ wordtpls = Word.expand([-1,0,1], [1,2], excludezero=False)
+ tagtpls = Pos.expand([-2,-1,0,1], [1,2], excludezero=True)
+ templates = list(Template.expand([wordtpls, tagtpls], combinations=(1,3)))
+ print("Generated {0} templates for transformation-based learning".format(len(templates)))
+ postag(templates=templates, incremental_stats=True, template_stats=True)
+
+def demo_learning_curve():
+ """
+ Plot a learning curve -- the contribution on tagging accuracy of
+ the individual rules.
+ Note: requires matplotlib
+ """
+ postag(incremental_stats=True, separate_baseline_data=True, learning_curve_output="learningcurve.png")
+
+def demo_error_analysis():
+ """
+ Writes a file with context for each erroneous word after tagging testing data
+ """
+ postag(error_output="errors.txt")
+
+def demo_serialize_tagger():
+ """
+ Serializes the learned tagger to a file in pickle format; reloads it
+ and validates the process.
+ """
+ postag(serialize_output="tagger.pcl")
+
+def demo_high_accuracy_rules():
+ """
+ Discard rules with low accuracy. This may hurt performance a bit,
+ but will often produce rules which are more interesting read to a human.
+ """
+ postag(num_sents=3000, min_acc=0.96, min_score=10)
+
+def postag(
+ templates=None,
+ tagged_data=None,
+ num_sents=1000,
+ max_rules=300,
+ min_score=3,
+ min_acc=None,
+ train=0.8,
+ trace=3,
+ randomize=False,
+ ruleformat="str",
+ incremental_stats=False,
+ template_stats=False,
+ error_output=None,
+ serialize_output=None,
+ learning_curve_output=None,
+ learning_curve_take=300,
+ baseline_backoff_tagger=None,
+ separate_baseline_data=False,
+ cache_baseline_tagger=None):
+ """
+ Brill Tagger Demonstration
+ :param templates: how many sentences of training and testing data to use
+ :type templates: list of Template
+
+ :param tagged_data: maximum number of rule instances to create
+ :type tagged_data: C{int}
+
+ :param num_sents: how many sentences of training and testing data to use
+ :type num_sents: C{int}
+
+ :param max_rules: maximum number of rule instances to create
+ :type max_rules: C{int}
+
+ :param min_score: the minimum score for a rule in order for it to be considered
+ :type min_score: C{int}
+
+ :param min_acc: the minimum score for a rule in order for it to be considered
+ :type min_acc: C{float}
+
+ :param train: the fraction of the the corpus to be used for training (1=all)
+ :type train: C{float}
+
+ :param trace: the level of diagnostic tracing output to produce (0-4)
+ :type trace: C{int}
+
+ :param randomize: whether the training data should be a random subset of the corpus
+ :type randomize: C{bool}
+
+ :param ruleformat: rule output format, one of "str", "repr", "verbose"
+ :type ruleformat: C{str}
+
+ :param incremental_stats: if true, will tag incrementally and collect stats for each rule (rather slow)
+ :type incremental_stats: C{bool}
+
+ :param template_stats: if true, will print per-template statistics collected in training and (optionally) testing
+ :type template_stats: C{bool}
+
+ :param error_output: the file where errors will be saved
+ :type error_output: C{string}
+
+ :param serialize_output: the file where the learned tbl tagger will be saved
+ :type serialize_output: C{string}
+
+ :param learning_curve_output: filename of plot of learning curve(s) (train and also test, if available)
+ :type learning_curve_output: C{string}
+
+ :param learning_curve_take: how many rules plotted
+ :type learning_curve_take: C{int}
+
+ :param baseline_backoff_tagger: the file where rules will be saved
+ :type baseline_backoff_tagger: tagger
+
+ :param separate_baseline_data: use a fraction of the training data exclusively for training baseline
+ :type separate_baseline_data: C{bool}
+
+ :param cache_baseline_tagger: cache baseline tagger to this file (only interesting as a temporary workaround to get
+ deterministic output from the baseline unigram tagger between python versions)
+ :type cache_baseline_tagger: C{string}
+
+
+ Note on separate_baseline_data: if True, reuse training data both for baseline and rule learner. This
+ is fast and fine for a demo, but is likely to generalize worse on unseen data.
+ Also cannot be sensibly used for learning curves on training data (the baseline will be artificially high).
+ """
+
+ # defaults
+ baseline_backoff_tagger = baseline_backoff_tagger or REGEXP_TAGGER
+ if templates is None:
+ from nltk.tag.brill import describe_template_sets, brill24
+ # some pre-built template sets taken from typical systems or publications are
+ # available. Print a list with describe_template_sets()
+ # for instance:
+ templates = brill24()
+ (training_data, baseline_data, gold_data, testing_data) = \
+ _demo_prepare_data(tagged_data, train, num_sents, randomize, separate_baseline_data)
+
+ # creating (or reloading from cache) a baseline tagger (unigram tagger)
+ # this is just a mechanism for getting deterministic output from the baseline between
+ # python versions
+ if cache_baseline_tagger:
+ if not os.path.exists(cache_baseline_tagger):
+ baseline_tagger = UnigramTagger(baseline_data, backoff=baseline_backoff_tagger)
+ with open(cache_baseline_tagger, 'w') as print_rules:
+ pickle.dump(baseline_tagger, print_rules)
+ print("Trained baseline tagger, pickled it to {0}".format(cache_baseline_tagger))
+ with open(cache_baseline_tagger, "r") as print_rules:
+ baseline_tagger= pickle.load(print_rules)
+ print("Reloaded pickled tagger from {0}".format(cache_baseline_tagger))
+ else:
+ baseline_tagger = UnigramTagger(baseline_data, backoff=baseline_backoff_tagger)
+ print("Trained baseline tagger")
+ if gold_data:
+ print(" Accuracy on test set: {0:0.4f}".format(baseline_tagger.evaluate(gold_data)))
+
+ # creating a Brill tagger
+ tbrill = time.time()
+ trainer = BrillTaggerTrainer(baseline_tagger, templates, trace, ruleformat=ruleformat)
+ print("Training tbl tagger...")
+ brill_tagger = trainer.train(training_data, max_rules, min_score, min_acc)
+ print("Trained tbl tagger in {0:0.2f} seconds".format(time.time() - tbrill))
+ if gold_data:
+ print(" Accuracy on test set: %.4f" % brill_tagger.evaluate(gold_data))
+
+ # printing the learned rules, if learned silently
+ if trace == 1:
+ print("\nLearned rules: ")
+ for (ruleno, rule) in enumerate(brill_tagger.rules(),1):
+ print("{0:4d} {1:s}".format(ruleno, rule.format(ruleformat)))
+
+
+ # printing template statistics (optionally including comparison with the training data)
+ # note: if not separate_baseline_data, then baseline accuracy will be artificially high
+ if incremental_stats:
+ print("Incrementally tagging the test data, collecting individual rule statistics")
+ (taggedtest, teststats) = brill_tagger.batch_tag_incremental(testing_data, gold_data)
+ print(" Rule statistics collected")
+ if not separate_baseline_data:
+ print("WARNING: train_stats asked for separate_baseline_data=True; the baseline "
+ "will be artificially high")
+ trainstats = brill_tagger.train_stats()
+ if template_stats:
+ brill_tagger.print_template_statistics(teststats)
+ if learning_curve_output:
+ _demo_plot(learning_curve_output, teststats, trainstats, take=learning_curve_take)
+ print("Wrote plot of learning curve to {0}".format(learning_curve_output))
+ else:
+ print("Tagging the test data")
+ taggedtest = brill_tagger.batch_tag(testing_data)
+ if template_stats:
+ brill_tagger.print_template_statistics()
+
+ # writing error analysis to file
+ if error_output is not None:
+ with open(error_output, 'w') as f:
+ f.write('Errors for Brill Tagger %r\n\n' % serialize_output)
+ for e in error_list(gold_data, taggedtest):
+ f.write(e+'\n')
+ print("Wrote tagger errors including context to {0}".format(error_output))
+
+ # serializing the tagger to a pickle file and reloading (just to see it works)
+ if serialize_output is not None:
+ taggedtest = brill_tagger.batch_tag(testing_data)
+ with open(serialize_output, 'w') as print_rules:
+ pickle.dump(brill_tagger, print_rules)
+ print("Wrote pickled tagger to {0}".format(serialize_output))
+ with open(serialize_output, "r") as print_rules:
+ brill_tagger_reloaded = pickle.load(print_rules)
+ print("Reloaded pickled tagger from {0}".format(serialize_output))
+ taggedtest_reloaded = brill_tagger.batch_tag(testing_data)
+ if taggedtest == taggedtest_reloaded:
+ print("Reloaded tagger tried on test set, results identical")
+ else:
+ print("PROBLEM: Reloaded tagger gave different results on test set")
+
+def _demo_prepare_data(tagged_data, train, num_sents, randomize, separate_baseline_data):
+ # train is the proportion of data used in training; the rest is reserved
+ # for testing.
+ if tagged_data is None:
+ print("Loading tagged data from treebank... ")
+ tagged_data = treebank.tagged_sents()
+ if num_sents is None or len(tagged_data) <= num_sents:
+ num_sents = len(tagged_data)
+ if randomize:
+ random.seed(len(tagged_data))
+ random.shuffle(tagged_data)
+ cutoff = int(num_sents * train)
+ training_data = tagged_data[:cutoff]
+ gold_data = tagged_data[cutoff:num_sents]
+ testing_data = [[t[0] for t in sent] for sent in gold_data]
+ if not separate_baseline_data:
+ baseline_data = training_data
+ else:
+ bl_cutoff = len(training_data) // 3
+ (baseline_data, training_data) = (training_data[:bl_cutoff], training_data[bl_cutoff:])
+ (trainseqs, traintokens) = corpus_size(training_data)
+ (testseqs, testtokens) = corpus_size(testing_data)
+ (bltrainseqs, bltraintokens) = corpus_size(baseline_data)
+ print("Read testing data ({0:d} sents/{1:d} wds)".format(testseqs, testtokens))
+ print("Read training data ({0:d} sents/{1:d} wds)".format(trainseqs, traintokens))
+ print("Read baseline data ({0:d} sents/{1:d} wds) {2:s}".format(
+ bltrainseqs, bltraintokens, "" if separate_baseline_data else "[reused the training set]"))
+ return (training_data, baseline_data, gold_data, testing_data)
+
+
+def _demo_plot(learning_curve_output, teststats, trainstats=None, take=None):
+ testcurve = [teststats['initialerrors']]
+ for rulescore in teststats['rulescores']:
+ testcurve.append(testcurve[-1] - rulescore)
+ testcurve = [1 - x/teststats['tokencount'] for x in testcurve[:take]]
+
+ traincurve = [trainstats['initialerrors']]
+ for rulescore in trainstats['rulescores']:
+ traincurve.append(traincurve[-1] - rulescore)
+ traincurve = [1 - x/trainstats['tokencount'] for x in traincurve[:take]]
+
+ import matplotlib.pyplot as plt
+ r = list(range(len(testcurve)))
+ plt.plot(r, testcurve, r, traincurve)
+ plt.axis([None, None, None, 1.0])
+ plt.savefig(learning_curve_output)
+
+
+NN_CD_TAGGER = RegexpTagger(
+ [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'),
+ (r'.*', 'NN')])
+
+REGEXP_TAGGER = RegexpTagger(
+ [(r'^-?[0-9]+(.[0-9]+)?$', 'CD'), # cardinal numbers
+ (r'(The|the|A|a|An|an)$', 'AT'), # articles
+ (r'.*able$', 'JJ'), # adjectives
+ (r'.*ness$', 'NN'), # nouns formed from adjectives
+ (r'.*ly$', 'RB'), # adverbs
+ (r'.*s$', 'NNS'), # plural nouns
+ (r'.*ing$', 'VBG'), # gerunds
+ (r'.*ed$', 'VBD'), # past tense verbs
+ (r'.*', 'NN') # nouns (default)
+])
+
+
+def corpus_size(seqs):
+ return (len(seqs), sum(len(x) for x in seqs))
+
+if __name__ == '__main__':
+ demo_learning_curve()
diff --git a/nltk/tbl/erroranalysis.py b/nltk/tbl/erroranalysis.py
new file mode 100644
index 0000000..1cbbf14
--- /dev/null
+++ b/nltk/tbl/erroranalysis.py
@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function
+
+
+# returns a list of errors in string format
+
+def error_list(train_sents, test_sents):
+ """
+ Returns a list of human-readable strings indicating the errors in the
+ given tagging of the corpus.
+
+ :param train_sents: The correct tagging of the corpus
+ :type train_sents: list(tuple)
+ :param test_sents: The tagged corpus
+ :type test_sents: list(tuple)
+ """
+ hdr = (('%25s | %s | %s\n' + '-'*26+'+'+'-'*24+'+'+'-'*26) %
+ ('left context', 'word/test->gold'.center(22), 'right context'))
+ errors = [hdr]
+ for (train_sent, test_sent) in zip(train_sents, test_sents):
+ for wordnum, (word, train_pos) in enumerate(train_sent):
+ test_pos = test_sent[wordnum][1]
+ if train_pos != test_pos:
+ left = ' '.join('%s/%s' % w for w in train_sent[:wordnum])
+ right = ' '.join('%s/%s' % w for w in train_sent[wordnum+1:])
+ mid = '%s/%s->%s' % (word, test_pos, train_pos)
+ errors.append('%25s | %s | %s' %
+ (left[-25:], mid.center(22), right[:25]))
+
+ return errors
diff --git a/nltk/tbl/feature.py b/nltk/tbl/feature.py
new file mode 100644
index 0000000..2c0f3cc
--- /dev/null
+++ b/nltk/tbl/feature.py
@@ -0,0 +1,265 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import division, print_function, unicode_literals
+
+
+class Feature(object):
+ """
+ An abstract base class for Features. A Feature is a combination of
+ a specific property-computing method and a list of relative positions
+ to apply that method to.
+
+ The property-computing method, M{extract_property(tokens, index)},
+ must be implemented by every subclass. It extracts or computes a specific
+ property for the token at the current index. Typical extract_property()
+ methods return features such as the token text or tag; but more involved
+ methods may consider the entire sequence M{tokens} and
+ for instance compute the length of the sentence the token belongs to.
+
+ In addition, the subclass may have a PROPERTY_NAME, which is how
+ it will be printed (in Rules and Templates, etc). If not given, defaults
+ to the classname.
+
+ """
+ #!!FOR_FUTURE: when targeting python3 only, consider @abc.abstractmethod
+ # and metaclass=abc.ABCMeta rather than NotImplementedError
+ #http://julien.danjou.info/blog/2013/guide-python-static-class-abstract-methods
+
+ json_tag = 'nltk.tbl.Feature'
+ PROPERTY_NAME = None
+
+ def __init__(self, positions, end=None):
+ """
+ Construct a Feature which may apply at C{positions}.
+
+ #For instance, importing some concrete subclasses (Feature is abstract)
+ >>> from nltk.tag.brill import Word, Pos
+
+ #Feature Word, applying at one of [-2, -1]
+ >>> Word([-2,-1])
+ Word([-2, -1])
+
+ #Positions need not be contiguous
+ >>> Word([-2,-1, 1])
+ Word([-2, -1, 1])
+
+ #Contiguous ranges can alternatively be specified giving the
+ #two endpoints (inclusive)
+ >>> Pos(-3, -1)
+ Pos([-3, -2, -1])
+
+ #In two-arg form, start <= end is enforced
+ >>> Pos(2, 1)
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ File "nltk/tbl/template.py", line 306, in __init__
+ raise TypeError
+ ValueError: illegal interval specification: (start=2, end=1)
+
+ :type positions: list of int
+ :param positions: the positions at which this features should apply
+ :raises ValueError: illegal position specifications
+
+ An alternative calling convention, for contiguous positions only,
+ is Feature(start, end):
+
+ :type start: int
+ :param start: start of range where this feature should apply
+ :type end: int
+ :param end: end of range (NOTE: inclusive!) where this feature should apply
+
+ """
+ self.positions = None #to avoid warnings
+ if end is None:
+ self.positions = tuple(sorted(set([int(i) for i in positions])))
+ else: #positions was actually not a list, but only the start index
+ try:
+ if positions > end:
+ raise TypeError
+ self.positions = tuple(range(positions, end+1))
+ except TypeError:
+ #let any kind of erroneous spec raise ValueError
+ raise ValueError("illegal interval specification: (start={0}, end={1})".format(positions, end))
+
+ #set property name given in subclass, or otherwise name of subclass
+ self.PROPERTY_NAME = self.__class__.PROPERTY_NAME or self.__class__.__name__
+
+ def encode_json_obj(self):
+ return self.positions
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ positions = obj
+ return cls(positions)
+
+ def __repr__(self):
+ return "%s(%r)" % (
+ self.__class__.__name__, list(self.positions))
+
+ @classmethod
+ def expand(cls, starts, winlens, excludezero=False):
+ """
+ Return a list of features, one for each start point in starts
+ and for each window length in winlen. If excludezero is True,
+ no Features containing 0 in its positions will be generated
+ (many tbl trainers have a special representation for the
+ target feature at [0])
+
+ #For instance, importing a concrete subclass (Feature is abstract)
+ >>> from nltk.tag.brill import Word
+
+ #First argument gives the possible start positions, second the
+ #possible window lengths
+ >>> Word.expand([-3,-2,-1], [1])
+ [Word([-3]), Word([-2]), Word([-1])]
+
+ >>> Word.expand([-2,-1], [1])
+ [Word([-2]), Word([-1])]
+
+ >>> Word.expand([-3,-2,-1], [1,2])
+ [Word([-3]), Word([-2]), Word([-1]), Word([-3, -2]), Word([-2, -1])]
+
+ >>> Word.expand([-2,-1], [1])
+ [Word([-2]), Word([-1])]
+
+ #a third optional argument excludes all Features whose positions contain zero
+ >>> Word.expand([-2,-1,0], [1,2], excludezero=False)
+ [Word([-2]), Word([-1]), Word([0]), Word([-2, -1]), Word([-1, 0])]
+
+ >>> Word.expand([-2,-1,0], [1,2], excludezero=True)
+ [Word([-2]), Word([-1]), Word([-2, -1])]
+
+ #All window lengths must be positive
+ >>> Word.expand([-2,-1], [0])
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ File "nltk/tag/tbl/template.py", line 371, in expand
+ :param starts: where to start looking for Feature
+ ValueError: non-positive window length in [0]
+
+ :param starts: where to start looking for Feature
+ :type starts: list of ints
+ :param winlens: window lengths where to look for Feature
+ :type starts: list of ints
+ :param excludezero: do not output any Feature with 0 in any of its positions.
+ :type excludezero: bool
+ :returns: list of Features
+ :raises ValueError: for non-positive window lengths
+ """
+ if not all(x > 0 for x in winlens):
+ raise ValueError("non-positive window length in {0:s}".format(winlens))
+ xs = (starts[i:i+w] for w in winlens for i in range(len(starts)-w+1))
+ return [cls(x) for x in xs if not (excludezero and 0 in x)]
+
+ def issuperset(self, other):
+ """
+ Return True if this Feature always returns True when other does
+
+ More precisely, return True if this feature refers to the same property as other;
+ and this Feature looks at all positions that other does (and possibly
+ other positions in addition).
+
+ #For instance, importing a concrete subclass (Feature is abstract)
+ >>> from nltk.tag.brill import Word, Pos
+
+ >>> Word([-3,-2,-1]).issuperset(Word([-3,-2]))
+ True
+
+ >>> Word([-3,-2,-1]).issuperset(Word([-3,-2, 0]))
+ False
+
+ #Feature subclasses must agree
+ >>> Word([-3,-2,-1]).issuperset(Pos([-3,-2]))
+ False
+
+ :param other: feature with which to compare
+ :type other: (subclass of) Feature
+ :return: True if this feature is superset, otherwise False
+ :rtype: bool
+
+
+ """
+ return (self.__class__ is other.__class__ and
+ set(self.positions) >= set(other.positions))
+
+ def intersects(self, other):
+ """
+ Return True if the positions of this Feature intersects with those of other
+
+ More precisely, return True if this feature refers to the same property as other;
+ and there is some overlap in the positions they look at.
+
+ #For instance, importing a concrete subclass (Feature is abstract)
+ >>> from nltk.tag.brill import Word, Pos
+
+ >>> Word([-3,-2,-1]).intersects(Word([-3,-2]))
+ True
+
+ >>> Word([-3,-2,-1]).intersects(Word([-3,-2, 0]))
+ True
+
+ >>> Word([-3,-2,-1]).intersects(Word([0]))
+ False
+
+ #Feature subclasses must agree
+ >>> Word([-3,-2,-1]).intersects(Pos([-3,-2]))
+ False
+
+ :param other: feature with which to compare
+ :type other: (subclass of) Feature
+ :return: True if feature classes agree and there is some overlap in the positions they look at
+ :rtype: bool
+ """
+
+ return bool((self.__class__ is other.__class__ and
+ set(self.positions) & set(other.positions)))
+
+ #Rich comparisons for Features. With @functools.total_ordering (Python 2.7+),
+ # it will be enough to define __lt__ and __eq__
+ def __eq__(self, other):
+ return (self.__class__ is other.__class__ and
+ self.positions == other.positions)
+
+ def __lt__(self, other):
+ return (self.__class__.__name__ < other.__class__.__name__ or
+ #self.positions is a sorted tuple of ints
+ self.positions < other.positions)
+
+ def __ne__(self, other):
+ return not (self == other)
+
+ def __gt__(self, other):
+ return other < self
+
+ def __ge__(self, other):
+ return not self < other
+
+ def __le__(self, other):
+ return self < other or self == other
+
+ @staticmethod
+ def extract_property(tokens, index):
+ """
+ Any subclass of Feature must define static method extract_property(tokens, index)
+
+ :param tokens: the sequence of tokens
+ :type tokens: list of tokens
+ :param index: the current index
+ :type index: int
+ :return: feature value
+ :rtype: any (but usually scalar)
+ """
+ raise NotImplementedError
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tbl/rule.py b/nltk/tbl/rule.py
new file mode 100644
index 0000000..d23c780
--- /dev/null
+++ b/nltk/tbl/rule.py
@@ -0,0 +1,314 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function
+
+from nltk.compat import python_2_unicode_compatible, unicode_repr
+from nltk import jsontags
+
+######################################################################
+## Tag Rules
+######################################################################
+
+
+class TagRule(object):
+ """
+ An interface for tag transformations on a tagged corpus, as
+ performed by tbl taggers. Each transformation finds all tokens
+ in the corpus that are tagged with a specific original tag and
+ satisfy a specific condition, and replaces their tags with a
+ replacement tag. For any given transformation, the original
+ tag, replacement tag, and condition are fixed. Conditions may
+ depend on the token under consideration, as well as any other
+ tokens in the corpus.
+
+ Tag rules must be comparable and hashable.
+ """
+
+ def __init__(self, original_tag, replacement_tag):
+
+ self.original_tag = original_tag
+ """The tag which this TagRule may cause to be replaced."""
+
+ self.replacement_tag = replacement_tag
+ """The tag with which this TagRule may replace another tag."""
+
+ def apply(self, tokens, positions=None):
+ """
+ Apply this rule at every position in positions where it
+ applies to the given sentence. I.e., for each position p
+ in *positions*, if *tokens[p]* is tagged with this rule's
+ original tag, and satisfies this rule's condition, then set
+ its tag to be this rule's replacement tag.
+
+ :param tokens: The tagged sentence
+ :type tokens: list(tuple(str, str))
+ :type positions: list(int)
+ :param positions: The positions where the transformation is to
+ be tried. If not specified, try it at all positions.
+ :return: The indices of tokens whose tags were changed by this
+ rule.
+ :rtype: int
+ """
+ if positions is None:
+ positions = list(range(len(tokens)))
+
+ # Determine the indices at which this rule applies.
+ change = [i for i in positions if self.applies(tokens, i)]
+
+ # Make the changes. Note: this must be done in a separate
+ # step from finding applicable locations, since we don't want
+ # the rule to interact with itself.
+ for i in change:
+ tokens[i] = (tokens[i][0], self.replacement_tag)
+
+ return change
+
+ def applies(self, tokens, index):
+ """
+ :return: True if the rule would change the tag of
+ ``tokens[index]``, False otherwise
+ :rtype: bool
+ :param tokens: A tagged sentence
+ :type tokens: list(str)
+ :param index: The index to check
+ :type index: int
+ """
+ raise NotImplementedError
+
+ # Rules must be comparable and hashable for the algorithm to work
+ def __eq__(self, other):
+ raise TypeError("Rules must implement __eq__()")
+
+ def __ne__(self, other):
+ raise TypeError("Rules must implement __ne__()")
+
+ def __hash__(self):
+ raise TypeError("Rules must implement __hash__()")
+
+
+ at python_2_unicode_compatible
+ at jsontags.register_tag
+class Rule(TagRule):
+ """
+ A Rule checks the current corpus position for a certain set of conditions;
+ if they are all fulfilled, the Rule is triggered, meaning that it
+ will change tag A to tag B. For other tags than A, nothing happens.
+
+ The conditions are parameters to the Rule instance. Each condition is a feature-value pair,
+ with a set of positions to check for the value of the corresponding feature.
+ Conceptually, the positions are joined by logical OR, and the feature set by logical AND.
+
+ More formally, the Rule is then applicable to the M{n}th token iff:
+
+ - The M{n}th token is tagged with the Rule's original tag; and
+ - For each (Feature(positions), M{value}) tuple:
+ - The value of Feature of at least one token in {n+p for p in positions}
+ is M{value}.
+
+ """
+
+ json_tag='nltk.tbl.Rule'
+
+ def __init__(self, templateid, original_tag, replacement_tag, conditions):
+ """
+ Construct a new Rule that changes a token's tag from
+ C{original_tag} to C{replacement_tag} if all of the properties
+ specified in C{conditions} hold.
+
+ @type templateid: string
+ @param templateid: the template id (a zero-padded string, '001' etc,
+ so it will sort nicely)
+
+ @type conditions: C{iterable} of C{Feature}
+ @param conditions: A list of Feature(positions),
+ each of which specifies that the property (computed by
+ Feature.extract_property()) of at least one
+ token in M{n} + p in positions is C{value}.
+
+ """
+ TagRule.__init__(self, original_tag, replacement_tag)
+ self._conditions = conditions
+ self.templateid = templateid
+
+ def encode_json_obj(self):
+ return {
+ 'templateid': self.templateid,
+ 'original': self.original_tag,
+ 'replacement': self.replacement_tag,
+ 'conditions': self._conditions,
+ }
+
+ @classmethod
+ def decode_json_obj(cls, obj):
+ return cls(obj['templateid'], obj['original'], obj['replacement'], obj['conditions'])
+
+ def applies(self, tokens, index):
+ # Inherit docs from TagRule
+
+ # Does the given token have this Rule's "original tag"?
+ if tokens[index][1] != self.original_tag:
+ return False
+
+ # Check to make sure that every condition holds.
+ for (feature, val) in self._conditions:
+
+ # Look for *any* token that satisfies the condition.
+ for pos in feature.positions:
+ if not (0 <= index + pos < len(tokens)):
+ continue
+ if feature.extract_property(tokens, index+pos) == val:
+ break
+ else:
+ # No token satisfied the condition; return false.
+ return False
+
+ # Every condition checked out, so the Rule is applicable.
+ return True
+
+ def __eq__(self, other):
+ return (self is other or
+ (other is not None and
+ other.__class__ == self.__class__ and
+ self.original_tag == other.original_tag and
+ self.replacement_tag == other.replacement_tag and
+ self._conditions == other._conditions))
+
+ def __ne__(self, other):
+ return not (self==other)
+
+ def __hash__(self):
+
+ # Cache our hash value (justified by profiling.)
+ try:
+ return self.__hash
+ except:
+ self.__hash = hash(repr(self))
+ return self.__hash
+
+ def __repr__(self):
+ # Cache the repr (justified by profiling -- this is used as
+ # a sort key when deterministic=True.)
+ try:
+ return self.__repr
+ except:
+ self.__repr = ('%s(%r, %s, %s, [%s])' % (
+ self.__class__.__name__,
+ self.templateid,
+ unicode_repr(self.original_tag),
+ unicode_repr(self.replacement_tag),
+
+ # list(self._conditions) would be simpler but will not generate
+ # the same Rule.__repr__ in python 2 and 3 and thus break some tests
+ ", ".join("({0:s},{1:s})".format(f,unicode_repr(v)) for (f,v) in self._conditions)))
+
+ return self.__repr
+
+ def __str__(self):
+ def _condition_to_logic(feature, value):
+ """
+ Return a compact, predicate-logic styled string representation
+ of the given condition.
+ """
+ return ('%s:%s@[%s]' %
+ (feature.PROPERTY_NAME, value, ",".join(str(w) for w in feature.positions)))
+
+ conditions = ' & '.join([_condition_to_logic(f,v) for (f,v) in self._conditions])
+ s = ('%s->%s if %s' % (
+ self.original_tag,
+ self.replacement_tag,
+ conditions))
+ return s
+
+
+ def format(self, fmt):
+ """
+ Return a string representation of this rule.
+
+ >>> from nltk.tbl.rule import Rule
+ >>> from nltk.tag.brill import Pos
+
+ >>> r = Rule(23, "VB", "NN", [(Pos([-2,-1]), 'DT')])
+
+ #r.format("str") == str(r)
+ >>> r.format("str")
+ 'VB->NN if Pos:DT@[-2,-1]'
+
+ #r.format("repr") == repr(r)
+ >>> r.format("repr")
+ "Rule(23, 'VB', 'NN', [(Pos([-2, -1]),'DT')])"
+
+ >>> r.format("verbose")
+ 'VB -> NN if the Pos of words i-2...i-1 is "DT"'
+
+ >>> r.format("not_found")
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ File "nltk/tbl/rule.py", line 256, in format
+ raise ValueError("unknown rule format spec: {0}".format(fmt))
+ ValueError: unknown rule format spec: not_found
+ >>>
+
+ :param fmt: format specification
+ :type fmt: str
+ :return: string representation
+ :rtype: str
+ """
+ if fmt == "str":
+ return self.__str__()
+ elif fmt == "repr":
+ return self.__repr__()
+ elif fmt == "verbose":
+ return self._verbose_format()
+ else:
+ raise ValueError("unknown rule format spec: {0}".format(fmt))
+
+ def _verbose_format(self):
+ """
+ Return a wordy, human-readable string representation
+ of the given rule.
+
+ Not sure how useful this is.
+ """
+ def condition_to_str(feature, value):
+ return ('the %s of %s is "%s"' %
+ (feature.PROPERTY_NAME, range_to_str(feature.positions), value))
+
+ def range_to_str(positions):
+ if len(positions) == 1:
+ p = positions[0]
+ if p == 0:
+ return 'this word'
+ if p == -1:
+ return 'the preceding word'
+ elif p == 1:
+ return 'the following word'
+ elif p < 0:
+ return 'word i-%d' % -p
+ elif p > 0:
+ return 'word i+%d' % p
+ else:
+ # for complete compatibility with the wordy format of nltk2
+ mx = max(positions)
+ mn = min(positions)
+ if mx - mn == len(positions) - 1:
+ return 'words i%+d...i%+d' % (mn, mx)
+ else:
+ return 'words {%s}' % (",".join("i%+d" % d for d in positions),)
+
+ replacement = '%s -> %s' % (self.original_tag, self.replacement_tag)
+ conditions = (' if ' if self._conditions else "") + ', and '.join(
+ [condition_to_str(f,v) for (f,v) in self._conditions])
+ return replacement + conditions
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tbl/template.py b/nltk/tbl/template.py
new file mode 100644
index 0000000..648dfde
--- /dev/null
+++ b/nltk/tbl/template.py
@@ -0,0 +1,311 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Transformation-based learning
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Marcus Uneson <marcus.uneson at gmail.com>
+# based on previous (nltk2) version by
+# Christopher Maloof, Edward Loper, Steven Bird
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function
+import itertools as it
+from nltk.tbl.feature import Feature
+
+
+class BrillTemplateI(object):
+ """
+ An interface for generating lists of transformational rules that
+ apply at given sentence positions. ``BrillTemplateI`` is used by
+ ``Brill`` training algorithms to generate candidate rules.
+ """
+ #!!FOR_FUTURE: when targeting python3 only, consider @abc.abstractmethod
+ # and metaclass=abc.ABCMeta rather than NotImplementedError
+ #http://julien.danjou.info/blog/2013/guide-python-static-class-abstract-methods
+ def applicable_rules(self, tokens, i, correctTag):
+ """
+ Return a list of the transformational rules that would correct
+ the *i*th subtoken's tag in the given token. In particular,
+ return a list of zero or more rules that would change
+ *tokens*[i][1] to *correctTag*, if applied to *token*[i].
+
+ If the *i*th token already has the correct tag (i.e., if
+ tagged_tokens[i][1] == correctTag), then
+ ``applicable_rules()`` should return the empty list.
+
+ :param tokens: The tagged tokens being tagged.
+ :type tokens: list(tuple)
+ :param i: The index of the token whose tag should be corrected.
+ :type i: int
+ :param correctTag: The correct tag for the *i*th token.
+ :type correctTag: any
+ :rtype: list(BrillRule)
+ """
+ raise NotImplementedError
+
+ def get_neighborhood(self, token, index):
+ """
+ Returns the set of indices *i* such that
+ ``applicable_rules(token, i, ...)`` depends on the value of
+ the *index*th token of *token*.
+
+ This method is used by the "fast" Brill tagger trainer.
+
+ :param token: The tokens being tagged.
+ :type token: list(tuple)
+ :param index: The index whose neighborhood should be returned.
+ :type index: int
+ :rtype: set
+ """
+ raise NotImplementedError
+
+
+from nltk.tbl.rule import Rule
+
+
+class Template(BrillTemplateI):
+ """
+ A tbl Template that generates a list of L{Rule}s that apply at a given sentence
+ position. In particular, each C{Template} is parameterized by a list of
+ independent features (a combination of a specific
+ property to extract and a list C{L} of relative positions at which to extract
+ it) and generates all Rules that:
+
+ - use the given features, each at its own independent position; and
+ - are applicable to the given token.
+ """
+ ALLTEMPLATES = []
+ #record a unique id of form "001", for each template created
+# _ids = it.count(0)
+
+ def __init__(self, *features):
+
+ """
+ Construct a Template for generating Rules.
+
+ Takes a list of Features. A C{Feature} is a combination
+ of a specific property and its relative positions and should be
+ a subclass of L{nltk.tbl.feature.Feature}.
+
+ An alternative calling convention (kept for backwards compatibility,
+ but less expressive as it only permits one feature type) is
+ Template(Feature, (start1, end1), (start2, end2), ...)
+ In new code, that would be better written
+ Template(Feature(start1, end1), Feature(start2, end2), ...)
+
+ #For instance, importing some features
+ >>> from nltk.tbl.template import Template
+ >>> from nltk.tag.brill import Word, Pos
+
+ #create some features
+
+ >>> wfeat1, wfeat2, pfeat = (Word([-1]), Word([1,2]), Pos([-2,-1]))
+
+ #Create a single-feature template
+ >>> Template(wfeat1)
+ Template(Word([-1]))
+
+ #or a two-feature one
+ >>> Template(wfeat1, wfeat2)
+ Template(Word([-1]),Word([1, 2]))
+
+ #or a three-feature one with two different feature types
+ >>> Template(wfeat1, wfeat2, pfeat)
+ Template(Word([-1]),Word([1, 2]),Pos([-2, -1]))
+
+ #deprecated api: Feature subclass, followed by list of (start,end) pairs
+ #(permits only a single Feature)
+ >>> Template(Word, (-2,-1), (0,0))
+ Template(Word([-2, -1]),Word([0]))
+
+ #incorrect specification raises TypeError
+ >>> Template(Word, (-2,-1), Pos, (0,0))
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ File "nltk/tag/tbl/template.py", line 143, in __init__
+ raise TypeError(
+ TypeError: expected either Feature1(args), Feature2(args), ... or Feature, (start1, end1), (start2, end2), ...
+
+ :type features: list of Features
+ :param features: the features to build this Template on
+ """
+ #determine the calling form: either
+ #Template(Feature, args1, [args2, ...)]
+ #Template(Feature1(args), Feature2(args), ...)
+ if all(isinstance(f, Feature) for f in features):
+ self._features = features
+ elif issubclass(features[0], Feature) and all(isinstance(a, tuple) for a in features[1:]):
+ self._features = [features[0](*tp) for tp in features[1:]]
+ else:
+ raise TypeError(
+ "expected either Feature1(args), Feature2(args), ... or Feature, (start1, end1), (start2, end2), ...")
+ self.id = "{0:03d}".format(len(self.ALLTEMPLATES))
+ self.ALLTEMPLATES.append(self)
+
+ def __repr__(self):
+ return "%s(%s)" % (self.__class__.__name__, ",".join([str(f) for f in self._features]))
+
+ def applicable_rules(self, tokens, index, correct_tag):
+ if tokens[index][1] == correct_tag:
+ return []
+
+ # For each of this Template's features, find the conditions
+ # that are applicable for the given token.
+ # Then, generate one Rule for each combination of features
+ # (the crossproduct of the conditions).
+
+ applicable_conditions = self._applicable_conditions(tokens, index)
+ xs = list(it.product(*applicable_conditions))
+ return [Rule(self.id, tokens[index][1], correct_tag, tuple(x)) for x in xs]
+
+ def _applicable_conditions(self, tokens, index):
+ """
+ :returns: A set of all conditions for rules
+ that are applicable to C{tokens[index]}.
+ """
+ conditions = []
+
+ for feature in self._features:
+ conditions.append([])
+ for pos in feature.positions:
+ if not (0 <= index+pos < len(tokens)):
+ continue
+ value = feature.extract_property(tokens, index+pos)
+ conditions[-1].append( (feature, value) )
+ return conditions
+
+ def get_neighborhood(self, tokens, index):
+ # inherit docs from BrillTemplateI
+
+ # applicable_rules(tokens, index, ...) depends on index.
+ neighborhood = set([index]) #set literal for python 2.7+
+
+ # applicable_rules(tokens, i, ...) depends on index if
+ # i+start < index <= i+end.
+
+ allpositions = [0] + [p for feat in self._features for p in feat.positions]
+ start, end = min(allpositions), max(allpositions)
+ s = max(0, index+(-end))
+ e = min(index+(-start)+1, len(tokens))
+ for i in range(s, e):
+ neighborhood.add(i)
+ return neighborhood
+
+ @classmethod
+ def expand(cls, featurelists, combinations=None, skipintersecting=True):
+
+ """
+ Factory method to mass generate Templates from a list L of lists of Features.
+
+ #With combinations=(k1, k2), the function will in all possible ways choose k1 ... k2
+ #of the sublists in L; it will output all Templates formed by the Cartesian product
+ #of this selection, with duplicates and other semantically equivalent
+ #forms removed. Default for combinations is (1, len(L)).
+
+ The feature lists may have been specified
+ manually, or generated from Feature.expand(). For instance,
+
+ >>> from nltk.tbl.template import Template
+ >>> from nltk.tag.brill import Word, Pos
+
+ #creating some features
+ >>> (wd_0, wd_01) = (Word([0]), Word([0,1]))
+
+ >>> (pos_m2, pos_m33) = (Pos([-2]), Pos([3-2,-1,0,1,2,3]))
+
+ >>> list(Template.expand([[wd_0], [pos_m2]]))
+ [Template(Word([0])), Template(Pos([-2])), Template(Pos([-2]),Word([0]))]
+
+ >>> list(Template.expand([[wd_0, wd_01], [pos_m2]]))
+ [Template(Word([0])), Template(Word([0, 1])), Template(Pos([-2])), Template(Pos([-2]),Word([0])), Template(Pos([-2]),Word([0, 1]))]
+
+ #note: with Feature.expand(), it is very easy to generate more templates
+ #than your system can handle -- for instance,
+ >>> wordtpls = Word.expand([-2,-1,0,1], [1,2], excludezero=False)
+ >>> len(wordtpls)
+ 7
+
+ >>> postpls = Pos.expand([-3,-2,-1,0,1,2], [1,2,3], excludezero=True)
+ >>> len(postpls)
+ 9
+
+ #and now the Cartesian product of all non-empty combinations of two wordtpls and
+ #two postpls, with semantic equivalents removed
+ >>> templates = list(Template.expand([wordtpls, wordtpls, postpls, postpls]))
+ >>> len(templates)
+ 713
+
+
+ will return a list of eight templates
+ Template(Word([0])),
+ Template(Word([0, 1])),
+ Template(Pos([-2])),
+ Template(Pos([-1])),
+ Template(Pos([-2]),Word([0])),
+ Template(Pos([-1]),Word([0])),
+ Template(Pos([-2]),Word([0, 1])),
+ Template(Pos([-1]),Word([0, 1]))]
+
+
+ #Templates where one feature is a subset of another, such as
+ #Template(Word([0,1]), Word([1]), will not appear in the output.
+ #By default, this non-subset constraint is tightened to disjointness:
+ #Templates of type Template(Word([0,1]), Word([1,2]) will also be filtered out.
+ #With skipintersecting=False, then such Templates are allowed
+
+ WARNING: this method makes it very easy to fill all your memory when training
+ generated templates on any real-world corpus
+
+ :param featurelists: lists of Features, whose Cartesian product will return a set of Templates
+ :type featurelists: list of (list of Features)
+ :param combinations: given n featurelists: if combinations=k, all generated Templates will have
+ k features; if combinations=(k1,k2) they will have k1..k2 features; if None, defaults to 1..n
+ :type combinations: None, int, or (int, int)
+ :param skipintersecting: if True, do not output intersecting Templates (non-disjoint positions for some feature)
+ :type skipintersecting: bool
+ :returns: generator of Templates
+
+ """
+ def nonempty_powerset(xs): #xs is a list
+ #itertools docnonempty_powerset([1,2,3]) --> (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)
+
+ #find the correct tuple given combinations, one of {None, k, (k1,k2)}
+ k = combinations #for brevity
+ combrange = ((1, len(xs)+1) if k is None else #n over 1 .. n over n (all non-empty combinations)
+ (k, k+1) if isinstance(k, int) else #n over k (only
+ (k[0], k[1]+1)) #n over k1, n over k1+1... n over k2
+ return it.chain.from_iterable(it.combinations(xs, r)
+ for r in range(*combrange))
+ seentemplates = set()
+ for picks in nonempty_powerset(featurelists):
+ for pick in it.product(*picks):
+ if any(i != j and x.issuperset(y)
+ for (i, x) in enumerate(pick)
+ for (j,y) in enumerate(pick)):
+ continue
+ if skipintersecting and any(i != j and x.intersects(y)
+ for (i, x) in enumerate(pick)
+ for (j, y) in enumerate(pick)):
+ continue
+ thistemplate = cls(*sorted(pick))
+ strpick = str(thistemplate)
+ #!!FIXME --this is hackish
+ if strpick in seentemplates: #already added
+ cls._poptemplate()
+ continue
+ seentemplates.add(strpick)
+ yield thistemplate
+
+ @classmethod
+ def _cleartemplates(cls):
+ cls.ALLTEMPLATES = []
+
+ @classmethod
+ def _poptemplate(cls):
+ return cls.ALLTEMPLATES.pop() if cls.ALLTEMPLATES else None
+
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/test/FX8.xml b/nltk/test/FX8.xml
new file mode 100644
index 0000000..58631da
--- /dev/null
+++ b/nltk/test/FX8.xml
@@ -0,0 +1,17 @@
+<bncDoc xml:id="FX8"><teiHeader><fileDesc><titleStmt><title> General practitioner's surgery: medical consultation. Sample containing about 125 words speech recorded in public context </title><respStmt><resp> Data capture and transcription </resp><name> Longman ELT </name> </respStmt></titleStmt><editionStmt><edition>BNC XML Edition, December 2006</edition></editionStmt><extent> 125 tokens; 130 w-units; 15 s-units </extent><publicationStmt><distributor>Distributed under licence by Oxford [...]
+<stext type="OTHERSP"><u who="FX8PSUNK">
+<s n="1"><w c5="ITJ" hw="ah" pos="INTERJ">Ah </w><w c5="AV0" hw="there" pos="ADV">there </w><w c5="PNP" hw="we" pos="PRON">we </w><w c5="VBB" hw="be" pos="VERB">are</w><c c5="PUN">,</c><unclear/><c c5="PUN">.</c></s>
+<s n="2"><w c5="AV0" hw="right" pos="ADV">Right </w><unclear/><w c5="AJ0" hw="abdominal" pos="ADJ">abdominal </w><w c5="NN1" hw="wound" pos="SUBST">wound</w><c c5="PUN">, </c><w c5="PNP" hw="she" pos="PRON">she</w><w c5="VBZ" hw="be" pos="VERB">'s </w><w c5="AT0" hw="a" pos="ART">a </w><w c5="AJ0-NN1" hw="wee" pos="ADJ">wee </w><w c5="NN1" hw="bit" pos="SUBST">bit </w><pause/><w c5="VVN-AJ0" hw="confuse" pos="VERB">confused</w><c c5="PUN">.</c></s>
+<s n="3"><w c5="PNP" hw="she" pos="PRON">She </w><w c5="VDD" hw="do" pos="VERB">did</w><w c5="XX0" hw="not" pos="ADV">n't </w><w c5="VVI" hw="bother" pos="VERB">bother </w><w c5="TO0" hw="to" pos="PREP">to </w><w c5="VVI" hw="tell" pos="VERB">tell </w><w c5="PNP" hw="i" pos="PRON">me </w><w c5="CJT" hw="that" pos="CONJ">that </w><w c5="PNP" hw="she" pos="PRON">she</w><w c5="VHD" hw="have" pos="VERB">'d </w><w c5="AV0" hw="only" pos="ADV">only </w><w c5="VVN" hw="get" pos="VERB">got </w> [...]
+<s n="4"><w c5="UNC" hw="erm" pos="UNC">Erm </w><w c5="PNP" hw="she" pos="PRON">she </w><w c5="VBD" hw="be" pos="VERB">was</w><w c5="XX0" hw="not" pos="ADV">n't </w><w c5="PRP" hw="in" pos="PREP">in </w><w c5="DPS" hw="she" pos="PRON">her </w><w c5="NN1" hw="nightdress" pos="SUBST">nightdress </w><w c5="CJC" hw="but" pos="CONJ">but </w><w c5="PNP" hw="she" pos="PRON">she </w><w c5="AV0" hw="only" pos="ADV">only </w><w c5="VVD" hw="dress" pos="VERB">dressed </w><w c5="PNX" hw="herself" po [...]
+<s n="5"><align with="FX8LC001"/><w c5="CJC" hw="and" pos="CONJ">And </w><w c5="PNP" hw="you" pos="PRON">you </w><unclear/><align with="FX8LC002"/></s></u><u who="FX8PS000">
+<s n="6"><w c5="PNP" hw="she" pos="PRON">She </w><w c5="VVD" hw="say" pos="VERB">said </w><w c5="PNP" hw="she" pos="PRON">she </w><w c5="VVD" hw="go" pos="VERB">went </w><w c5="TO0" hw="to" pos="PREP">to </w><w c5="VVI" hw="buy" pos="VERB">buy </w><w c5="PNI" hw="something" pos="PRON">something </w><unclear/><w c5="PNX" hw="herself" pos="PRON">herself</w><c c5="PUN">, </c><w c5="PNP" hw="she" pos="PRON">she </w><w c5="VVD" hw="phone" pos="VERB">phoned </w><w c5="AT0" hw="the" pos="ART"> [...]
+<s n="7"><w c5="PNP" hw="she" pos="PRON">She</w><w c5="VBZ" hw="be" pos="VERB">'s </w><unclear/><w c5="AV0" hw="here" pos="ADV">here </w><w c5="CJC" hw="and" pos="CONJ">and </w><gap desc="name" reason="anonymization"/><w c5="VVZ" hw="say" pos="VERB">says </w><w c5="PNP" hw="she" pos="PRON">she </w><w c5="VM0" hw="should" pos="VERB">should </w><w c5="VBI" hw="be" pos="VERB">be </w><unclear/><w c5="AV0" hw="fortnightly" pos="ADV">fortnightly </w><unclear/><c c5="PUN">.</c></s>
+<s n="8"><pause/><w c5="AV0" hw="so" pos="ADV">So </w><w c5="PNP" hw="i" pos="PRON">I </w><w c5="VDB" hw="do" pos="VERB">do</w><w c5="XX0" hw="not" pos="ADV">n't </w><w c5="VVI" hw="know" pos="VERB">know </w><w c5="CJS" hw="whether" pos="CONJ">whether </w><w c5="PNP" hw="you" pos="PRON">you </w><w c5="VVB" hw="want" pos="VERB">want </w><w c5="TO0" hw="to" pos="PREP">to </w><w c5="VVI" hw="go" pos="VERB">go </w><w c5="CJC" hw="and" pos="CONJ">and </w><w c5="VVI" hw="see" pos="VERB">see </ [...]
+<s n="9"><unclear/><w c5="PNP" hw="it" pos="PRON">it</w><w c5="VBZ" hw="be" pos="VERB">'s </w><w c5="AV0" hw="just" pos="ADV">just </w><w c5="CJT" hw="that" pos="CONJ">that </w><w c5="PNP" hw="i" pos="PRON">I</w><w c5="VBB" hw="be" pos="VERB">'m </w><w c5="AV0" hw="never" pos="ADV">never </w><w c5="VVG" hw="gon" pos="VERB">gon</w><w c5="TO0" hw="na" pos="PREP">na </w><w c5="VVI" hw="get" pos="VERB">get </w><w c5="PRP" hw="to" pos="PREP">to </w><mw c5="PRP"><w c5="AVP" hw="up" pos="ADV">u [...]
+<s n="10"><unclear/><c c5="PUN">?</c></s></u><u who="PS22T">
+<s n="11"><w c5="ITJ" hw="yeah" pos="INTERJ">Yeah</w><c c5="PUN">.</c></s></u><u who="FX8PS000">
+<s n="12"><w c5="AV0" hw="okay" pos="ADV">Okay</w><c c5="PUN">.</c></s></u><u who="PS22T">
+<s n="13"><w c5="ITJ" hw="yeah" pos="INTERJ">Yeah</w><c c5="PUN">.</c></s></u><u who="FX8PS000">
+<s n="14"><w c5="UNC" hw="erm" pos="UNC">erm</w><c c5="PUN">, </c><w c5="ORD" hw="first" pos="ADJ">first </w><unclear/><w c5="CRD" hw="twelve" pos="ADJ">twelve </w><w c5="NN2" hw="week" pos="SUBST">weeks </w><w c5="AJ0" hw="pregnant" pos="ADJ">pregnant </w><w c5="AV0" hw="so" pos="ADV">so </w><w c5="VM0" hw="should" pos="VERB">should </w><w c5="PNP" hw="i" pos="PRON">I </w><w c5="VVI" hw="mark" pos="VERB">mark </w><w c5="PRP" hw="at" pos="PREP">at </w><w c5="AT0" hw="the" pos="ART">the [...]
+<s n="15"><w c5="UNC" hw="erm" pos="UNC">Erm </w><unclear/><w c5="DT0" hw="this" pos="ADJ">this </w><w c5="PNI" hw="one" pos="PRON">one</w><c c5="PUN">.</c><event desc="recording ends"/></s></u></stext></bncDoc>
diff --git a/nltk/test/Makefile b/nltk/test/Makefile
new file mode 100644
index 0000000..1abd2a5
--- /dev/null
+++ b/nltk/test/Makefile
@@ -0,0 +1,23 @@
+.SUFFIXES: .doctest .errs .html
+
+TESTS = $(wildcard *.doctest)
+
+ERRS := $(TESTS:.doctest=.errs)
+
+HTML = $(TESTS:.doctest=.html)
+
+.doctest.errs:
+ python ./doctest_driver.py $< > $@
+
+.doctest.html:
+ rst2html.py $< > $@
+
+all: $(ERRS)
+
+html: $(HTML)
+
+install_html:
+ cp $(HTML) ../../../nltk.github.com/howto
+
+clean:
+ rm -f *.errs
diff --git a/nltk/test/__init__.py b/nltk/test/__init__.py
new file mode 100644
index 0000000..98d7815
--- /dev/null
+++ b/nltk/test/__init__.py
@@ -0,0 +1,18 @@
+# Natural Language Toolkit: Unit Tests
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Unit tests for the NLTK modules. These tests are intended to ensure
+that source code changes don't accidentally introduce bugs.
+For instructions, please see:
+
+../../web/dev/local_testing.rst
+
+https://github.com/nltk/nltk/blob/develop/web/dev/local_testing.rst
+
+
+"""
diff --git a/nltk/test/align.doctest b/nltk/test/align.doctest
new file mode 100644
index 0000000..9f1db41
--- /dev/null
+++ b/nltk/test/align.doctest
@@ -0,0 +1,234 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+.. -*- coding: utf-8 -*-
+
+=========
+Alignment
+=========
+
+Corpus Reader
+-------------
+
+ >>> from nltk.corpus import comtrans
+ >>> words = comtrans.words('alignment-en-fr.txt')
+ >>> for word in words:
+ ... print(word)
+ Resumption
+ of
+ the
+ session
+ I
+ declare...
+ >>> als = comtrans.aligned_sents('alignment-en-fr.txt')[0]
+ >>> als # doctest: +NORMALIZE_WHITESPACE
+ AlignedSent(['Resumption', 'of', 'the', 'session'],
+ ['Reprise', 'de', 'la', 'session'],
+ Alignment([(0, 0), (1, 1), (2, 2), (3, 3)]))
+
+
+Alignment Objects
+-----------------
+
+Aligned sentences are simply a mapping between words in a sentence:
+
+ >>> print(" ".join(als.words))
+ Resumption of the session
+ >>> print(" ".join(als.mots))
+ Reprise de la session
+ >>> als.alignment
+ Alignment([(0, 0), (1, 1), (2, 2), (3, 3)])
+
+
+Usually we look at them from the perpective of a source to a target languge,
+but they are easilly inverted:
+
+ >>> als.invert() # doctest: +NORMALIZE_WHITESPACE
+ AlignedSent(['Reprise', 'de', 'la', 'session'],
+ ['Resumption', 'of', 'the', 'session'],
+ Alignment([(0, 0), (1, 1), (2, 2), (3, 3)]))
+
+
+We can create new alignments, but these need to be in the correct range of
+the corresponding sentences:
+
+ >>> from nltk.align import Alignment, AlignedSent
+ >>> als = AlignedSent(['Reprise', 'de', 'la', 'session'],
+ ... ['Resumption', 'of', 'the', 'session'],
+ ... Alignment([(0, 0), (1, 4), (2, 1), (3, 3)]))
+ Traceback (most recent call last):
+ ...
+ IndexError: Alignment is outside boundary of mots
+
+
+You can set alignments with any sequence of tuples, so long as the first two
+indexes of the tuple are the alignment indices:
+
+als.alignment = Alignment([(0, 0), (1, 1), (2, 2, "boat"), (3, 3, False, (1,2))])
+
+ >>> Alignment([(0, 0), (1, 1), (2, 2, "boat"), (3, 3, False, (1,2))])
+ Alignment([(0, 0), (1, 1), (2, 2, 'boat'), (3, 3, False, (1, 2))])
+
+
+Alignment Algorithms
+--------------------
+
+EM for IBM Model 1
+~~~~~~~~~~~~~~~~~~
+
+Here is an example from Koehn, 2010:
+
+ >>> from nltk.align import IBMModel1
+ >>> corpus = [AlignedSent(['the', 'house'], ['das', 'Haus']),
+ ... AlignedSent(['the', 'book'], ['das', 'Buch']),
+ ... AlignedSent(['a', 'book'], ['ein', 'Buch'])]
+ >>> em_ibm1 = IBMModel1(corpus, 20)
+ >>> print(round(em_ibm1.probabilities['the']['das'], 1))
+ 1.0
+ >>> print(round(em_ibm1.probabilities['book']['das'], 1))
+ 0.0
+ >>> print(round(em_ibm1.probabilities['house']['das'], 1))
+ 0.0
+ >>> print(round(em_ibm1.probabilities['the']['Buch'], 1))
+ 0.0
+ >>> print(round(em_ibm1.probabilities['book']['Buch'], 1))
+ 1.0
+ >>> print(round(em_ibm1.probabilities['a']['Buch'], 1))
+ 0.0
+ >>> print(round(em_ibm1.probabilities['book']['ein'], 1))
+ 0.0
+ >>> print(round(em_ibm1.probabilities['a']['ein'], 1))
+ 1.0
+ >>> print(round(em_ibm1.probabilities['the']['Haus'], 1))
+ 0.0
+ >>> print(round(em_ibm1.probabilities['house']['Haus'], 1))
+ 1.0
+ >>> print(round(em_ibm1.probabilities['book'][None], 1))
+ 0.5
+
+And using an NLTK corpus. We train on only 10 sentences, since it is so slow:
+
+ >>> from nltk.corpus import comtrans
+ >>> com_ibm1 = IBMModel1(comtrans.aligned_sents()[:10], 20)
+ >>> print(round(com_ibm1.probabilities['bitte']['Please'], 1))
+ 0.2
+ >>> print(round(com_ibm1.probabilities['Sitzungsperiode']['session'], 1))
+ 1.0
+
+
+Evaluation
+----------
+The evaluation metrics for alignments are usually not interested in the
+contents of alignments but more often the comparison to a "gold standard"
+alignment that has been been constructed by human experts. For this reason we
+often want to work just with raw set operations against the alignment points.
+This then gives us a very clean form for defining our evaluation metrics.
+
+.. Note::
+ The AlignedSent class has no distinction of "possible" or "sure"
+ alignments. Thus all alignments are treated as "sure".
+
+Consider the following aligned sentence for evaluation:
+
+ >>> my_als = AlignedSent(['Resumption', 'of', 'the', 'session'],
+ ... ['Reprise', 'de', 'la', 'session'],
+ ... [(0, 0), (3, 3), (1, 2), (1, 1), (1, 3)])
+
+Precision
+~~~~~~~~~
+``precision = |A∩P| / |A|``
+
+**Precision** is probably the most well known evaluation metric and there is
+already a set based implementation in NLTK as
+`nltk.metrics.scores.precision`_. Since precision is simply interested in the
+proportion of correct alignments, we calculate the ratio of the number of our
+test alignments (*A*) that match a possible alignment (*P*) over the number of
+test alignments provided. We compare to the possible alignment set don't
+penalise for coming up with an alignment that a humans would have possibly
+considered to be correct [OCH2000]_.
+
+Here are some examples:
+
+ >>> print(als.precision(set()))
+ 0.0
+ >>> print(als.precision([(0,0), (1,1), (2,2), (3,3)]))
+ 1.0
+ >>> print(als.precision([(0,0), (3,3)]))
+ 0.5
+ >>> print(als.precision([(0,0), (1,1), (2,2), (3,3), (1,2), (2,1)]))
+ 1.0
+ >>> print(my_als.precision(als))
+ 0.6
+
+
+.. _nltk.metrics.scores.precision:
+ http://nltk.googlecode.com/svn/trunk/doc/api/nltk.metrics.scores-module.html#precision
+
+
+Recall
+~~~~~~
+``recall = |A∩S| / |S|``
+
+**Recall** is another well known evaluation metric that has a set based
+implementation in NLTK as `nltk.metrics.scores.recall`_. Since recall is
+simply interested in the proportion of found alignments, we calculate the
+ratio of the number of our test alignments (*A*) that match a sure alignment
+(*S*) over the number of sure alignments. Since we are not sure about some of
+our possible alignments we don't penalise for not finding these [OCH2000]_.
+
+Here are some examples:
+
+ >>> print(als.recall(set()))
+ None
+ >>> print(als.recall([(0,0), (1,1), (2,2), (3,3)]))
+ 1.0
+ >>> print(als.recall([(0,0), (3,3)]))
+ 1.0
+ >>> print(als.recall([(0,0), (1,1), (2,2), (3,3), (1,2), (2,1)]))
+ 0.66666666666...
+ >>> print(my_als.recall(als))
+ 0.75
+
+
+.. _nltk.metrics.scores.recall:
+ http://nltk.googlecode.com/svn/trunk/doc/api/nltk.metrics.scores-module.html#recall
+
+
+Alignment Error Rate (AER)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+``AER = 1 - (|A∩S| + |A∩P|) / (|A| + |S|)``
+
+**Alignment Error Rate** is commonly used metric for assessing sentence
+alignments. It combines precision and recall metrics together such that a
+perfect alignment must have all of the sure alignments and may have some
+possible alignments [MIHALCEA2003]_ [KOEHN2010]_.
+
+.. Note::
+ [KOEHN2010]_ defines the AER as ``AER = (|A∩S| + |A∩P|) / (|A| + |S|)``
+ meaning that the best alignment would when the ``AER = 1.0``. Thus we
+ follow [MIHALCEA2003]_ more intuitive definition where we are minimising
+ the error rate.
+
+Here are some examples:
+
+ >>> print(als.alignment_error_rate(set()))
+ 1.0
+ >>> print(als.alignment_error_rate([(0,0), (1,1), (2,2), (3,3)]))
+ 0.0
+ >>> print(my_als.alignment_error_rate(als))
+ 0.33333333333...
+ >>> print(my_als.alignment_error_rate(als,
+ ... als.alignment | set([(1,2), (2,1)])))
+ 0.22222222222...
+
+
+.. [OCH2000] Och, F. and Ney, H. (2000)
+ *Statistical Machine Translation*, EAMT Workshop
+
+.. [MIHALCEA2003] Mihalcea, R. and Pedersen, T. (2003)
+ *An evaluation exercise for word alignment*, HLT-NAACL 2003
+
+.. [KOEHN2010] Koehn, P. (2010)
+ *Statistical Machine Translation*, Cambridge University Press
+
+
diff --git a/nltk/test/align_fixt.py b/nltk/test/align_fixt.py
new file mode 100644
index 0000000..37a1a42
--- /dev/null
+++ b/nltk/test/align_fixt.py
@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from nltk.corpus import teardown_module
\ No newline at end of file
diff --git a/nltk/test/all.py b/nltk/test/all.py
new file mode 100644
index 0000000..b54a40f
--- /dev/null
+++ b/nltk/test/all.py
@@ -0,0 +1,23 @@
+"""Test suite that runs all NLTK tests.
+
+This module, `nltk.test.all`, is named as the NLTK ``test_suite`` in the
+project's ``setup-eggs.py`` file. Here, we create a test suite that
+runs all of our doctests, and return it for processing by the setuptools
+test harness.
+
+"""
+import doctest, unittest
+from glob import glob
+import os.path
+
+def additional_tests():
+ #print "here-000000000000000"
+ #print "-----", glob(os.path.join(os.path.dirname(__file__), '*.doctest'))
+ dir = os.path.dirname(__file__)
+ paths = glob(os.path.join(dir, '*.doctest'))
+ files = [ os.path.basename(path) for path in paths ]
+ return unittest.TestSuite(
+ [ doctest.DocFileSuite(file) for file in files ]
+ )
+#if os.path.split(path)[-1] != 'index.rst'
+# skips time-dependent doctest in index.rst
diff --git a/nltk/test/bnc.doctest b/nltk/test/bnc.doctest
new file mode 100644
index 0000000..9350b48
--- /dev/null
+++ b/nltk/test/bnc.doctest
@@ -0,0 +1,55 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+ >>> from nltk.corpus.reader import BNCCorpusReader
+ >>> bnc = BNCCorpusReader(root='.', fileids=r'FX8.xml')
+
+Checking the word access.
+-------------------------
+
+ >>> len(bnc.words())
+ 151
+
+ >>> bnc.words()[:6]
+ ['Ah', 'there', 'we', 'are', ',', '.']
+ >>> bnc.words(stem=True)[:6]
+ ['ah', 'there', 'we', 'be', ',', '.']
+
+ >>> bnc.tagged_words()[:6]
+ [('Ah', 'INTERJ'), ('there', 'ADV'), ('we', 'PRON'), ('are', 'VERB'), (',', 'PUN'), ('.', 'PUN')]
+
+ >>> bnc.tagged_words(c5=True)[:6]
+ [('Ah', 'ITJ'), ('there', 'AV0'), ('we', 'PNP'), ('are', 'VBB'), (',', 'PUN'), ('.', 'PUN')]
+
+Testing access to the sentences.
+--------------------------------
+
+ >>> len(bnc.sents())
+ 15
+
+ >>> bnc.sents()[0]
+ ['Ah', 'there', 'we', 'are', ',', '.']
+ >>> bnc.sents(stem=True)[0]
+ ['ah', 'there', 'we', 'be', ',', '.']
+
+ >>> bnc.tagged_sents()[0]
+ [('Ah', 'INTERJ'), ('there', 'ADV'), ('we', 'PRON'), ('are', 'VERB'), (',', 'PUN'), ('.', 'PUN')]
+ >>> bnc.tagged_sents(c5=True)[0]
+ [('Ah', 'ITJ'), ('there', 'AV0'), ('we', 'PNP'), ('are', 'VBB'), (',', 'PUN'), ('.', 'PUN')]
+
+A not lazy loader.
+-----------------
+
+ >>> eager = BNCCorpusReader(root='.', fileids=r'FX8.xml', lazy=False)
+
+ >>> len(eager.words())
+ 151
+ >>> eager.words(stem=True)[6:17]
+ ['right', 'abdominal', 'wound', ',', 'she', 'be', 'a', 'wee', 'bit', 'confuse', '.']
+
+ >>> eager.tagged_words()[6:11]
+ [('Right', 'ADV'), ('abdominal', 'ADJ'), ('wound', 'SUBST'), (',', 'PUN'), ('she', 'PRON')]
+ >>> eager.tagged_words(c5=True)[6:17]
+ [('Right', 'AV0'), ('abdominal', 'AJ0'), ('wound', 'NN1'), (',', 'PUN'), ('she', 'PNP'), ("'s", 'VBZ'), ('a', 'AT0'), ('wee', 'AJ0-NN1'), ('bit', 'NN1'), ('confused', 'VVN-AJ0'), ('.', 'PUN')]
+ >>> len(eager.sents())
+ 15
diff --git a/nltk/test/ccg.doctest b/nltk/test/ccg.doctest
new file mode 100644
index 0000000..857c846
--- /dev/null
+++ b/nltk/test/ccg.doctest
@@ -0,0 +1,277 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==============================
+Combinatory Categorial Grammar
+==============================
+
+For more information, please see:
+http://nltk.googlecode.com/svn/trunk/doc/contrib/ccg/ccg.pdf
+
+Relative Clauses
+----------------
+
+ >>> from nltk.ccg import chart, lexicon
+
+Construct a lexicon:
+
+ >>> lex = lexicon.parseLexicon('''
+ ... :- S, NP, N, VP
+ ...
+ ... Det :: NP/N
+ ... Pro :: NP
+ ... Modal :: S\\NP/VP
+ ...
+ ... TV :: VP/NP
+ ... DTV :: TV/NP
+ ...
+ ... the => Det
+ ...
+ ... that => Det
+ ... that => NP
+ ...
+ ... I => Pro
+ ... you => Pro
+ ... we => Pro
+ ...
+ ... chef => N
+ ... cake => N
+ ... children => N
+ ... dough => N
+ ...
+ ... will => Modal
+ ... should => Modal
+ ... might => Modal
+ ... must => Modal
+ ...
+ ... and => var\\.,var/.,var
+ ...
+ ... to => VP[to]/VP
+ ...
+ ... without => (VP\\VP)/VP[ing]
+ ...
+ ... be => TV
+ ... cook => TV
+ ... eat => TV
+ ...
+ ... cooking => VP[ing]/NP
+ ...
+ ... give => DTV
+ ...
+ ... is => (S\\NP)/NP
+ ... prefer => (S\\NP)/NP
+ ...
+ ... which => (N\\N)/(S/NP)
+ ...
+ ... persuade => (VP/VP[to])/NP
+ ... ''')
+
+ >>> parser = chart.CCGChartParser(lex, chart.DefaultRuleSet)
+ >>> for parse in parser.parse("you prefer that cake".split()):
+ ... chart.printCCGDerivation(parse)
+ ... break
+ ...
+ you prefer that cake
+ NP ((S\NP)/NP) (NP/N) N
+ -------------->
+ NP
+ --------------------------->
+ (S\NP)
+ --------------------------------<
+ S
+
+ >>> for parse in parser.parse("that is the cake which you prefer".split()):
+ ... chart.printCCGDerivation(parse)
+ ... break
+ ...
+ that is the cake which you prefer
+ NP ((S\NP)/NP) (NP/N) N ((N\N)/(S/NP)) NP ((S\NP)/NP)
+ ----->T
+ (S/(S\NP))
+ ------------------>B
+ (S/NP)
+ ---------------------------------->
+ (N\N)
+ ----------------------------------------<
+ N
+ ------------------------------------------------>
+ NP
+ ------------------------------------------------------------->
+ (S\NP)
+ -------------------------------------------------------------------<
+ S
+
+
+Some other sentences to try:
+"that is the cake which we will persuade the chef to cook"
+"that is the cake which we will persuade the chef to give the children"
+
+ >>> sent = "that is the dough which you will eat without cooking".split()
+ >>> nosub_parser = chart.CCGChartParser(lex, chart.ApplicationRuleSet +
+ ... chart.CompositionRuleSet + chart.TypeRaiseRuleSet)
+
+Without Substitution (no output)
+
+ >>> for parse in nosub_parser.parse(sent):
+ ... chart.printCCGDerivation(parse)
+
+With Substitution:
+
+ >>> for parse in parser.parse(sent):
+ ... chart.printCCGDerivation(parse)
+ ... break
+ ...
+ that is the dough which you will eat without cooking
+ NP ((S\NP)/NP) (NP/N) N ((N\N)/(S/NP)) NP ((S\NP)/VP) (VP/NP) ((VP\VP)/VP['ing']) (VP['ing']/NP)
+ ----->T
+ (S/(S\NP))
+ ------------------------------------->B
+ ((VP\VP)/NP)
+ ----------------------------------------------<Sx
+ (VP/NP)
+ ----------------------------------------------------------->B
+ ((S\NP)/NP)
+ ---------------------------------------------------------------->B
+ (S/NP)
+ -------------------------------------------------------------------------------->
+ (N\N)
+ ---------------------------------------------------------------------------------------<
+ N
+ ----------------------------------------------------------------------------------------------->
+ NP
+ ------------------------------------------------------------------------------------------------------------>
+ (S\NP)
+ ------------------------------------------------------------------------------------------------------------------<
+ S
+
+
+Conjunction
+-----------
+
+ >>> from nltk.ccg.chart import CCGChartParser, ApplicationRuleSet, CompositionRuleSet
+ >>> from nltk.ccg.chart import SubstitutionRuleSet, TypeRaiseRuleSet, printCCGDerivation
+ >>> from nltk.ccg import lexicon
+
+Lexicons for the tests:
+
+ >>> test1_lex = '''
+ ... :- S,N,NP,VP
+ ... I => NP
+ ... you => NP
+ ... will => S\\NP/VP
+ ... cook => VP/NP
+ ... which => (N\\N)/(S/NP)
+ ... and => var\\.,var/.,var
+ ... might => S\\NP/VP
+ ... eat => VP/NP
+ ... the => NP/N
+ ... mushrooms => N
+ ... parsnips => N'''
+ >>> test2_lex = '''
+ ... :- N, S, NP, VP
+ ... articles => N
+ ... the => NP/N
+ ... and => var\\.,var/.,var
+ ... which => (N\\N)/(S/NP)
+ ... I => NP
+ ... anyone => NP
+ ... will => (S/VP)\\NP
+ ... file => VP/NP
+ ... without => (VP\\VP)/VP[ing]
+ ... forget => VP/NP
+ ... reading => VP[ing]/NP
+ ... '''
+
+Tests handling of conjunctions.
+Note that while the two derivations are different, they are semantically equivalent.
+
+ >>> lex = lexicon.parseLexicon(test1_lex)
+ >>> parser = CCGChartParser(lex, ApplicationRuleSet + CompositionRuleSet + SubstitutionRuleSet)
+ >>> for parse in parser.parse("I will cook and might eat the mushrooms and parsnips".split()):
+ ... printCCGDerivation(parse) # doctest: +NORMALIZE_WHITESPACE
+ I will cook and might eat the mushrooms and parsnips
+ NP ((S\NP)/VP) (VP/NP) ((_var2\.,_var2)/.,_var2) ((S\NP)/VP) (VP/NP) (NP/N) N ((_var2\.,_var2)/.,_var2) N
+ ---------------------->B
+ ((S\NP)/NP)
+ ---------------------->B
+ ((S\NP)/NP)
+ ------------------------------------------------->
+ (((S\NP)/NP)\.,((S\NP)/NP))
+ -----------------------------------------------------------------------<
+ ((S\NP)/NP)
+ ------------------------------------->
+ (N\.,N)
+ ------------------------------------------------<
+ N
+ -------------------------------------------------------->
+ NP
+ ------------------------------------------------------------------------------------------------------------------------------->
+ (S\NP)
+ -----------------------------------------------------------------------------------------------------------------------------------<
+ S
+ I will cook and might eat the mushrooms and parsnips
+ NP ((S\NP)/VP) (VP/NP) ((_var2\.,_var2)/.,_var2) ((S\NP)/VP) (VP/NP) (NP/N) N ((_var2\.,_var2)/.,_var2) N
+ ---------------------->B
+ ((S\NP)/NP)
+ ---------------------->B
+ ((S\NP)/NP)
+ ------------------------------------------------->
+ (((S\NP)/NP)\.,((S\NP)/NP))
+ -----------------------------------------------------------------------<
+ ((S\NP)/NP)
+ ------------------------------------------------------------------------------->B
+ ((S\NP)/N)
+ ------------------------------------->
+ (N\.,N)
+ ------------------------------------------------<
+ N
+ ------------------------------------------------------------------------------------------------------------------------------->
+ (S\NP)
+ -----------------------------------------------------------------------------------------------------------------------------------<
+ S
+
+
+Tests handling subject extraction.
+Interesting to point that the two parses are clearly semantically different.
+
+ >>> lex = lexicon.parseLexicon(test2_lex)
+ >>> parser = CCGChartParser(lex, ApplicationRuleSet + CompositionRuleSet + SubstitutionRuleSet)
+ >>> for parse in parser.parse("articles which I will file and forget without reading".split()):
+ ... printCCGDerivation(parse) # doctest: +NORMALIZE_WHITESPACE
+ articles which I will file and forget without reading
+ N ((N\N)/(S/NP)) NP ((S/VP)\NP) (VP/NP) ((_var3\.,_var3)/.,_var3) (VP/NP) ((VP\VP)/VP['ing']) (VP['ing']/NP)
+ -----------------<
+ (S/VP)
+ ------------------------------------->B
+ ((VP\VP)/NP)
+ ----------------------------------------------<Sx
+ (VP/NP)
+ ------------------------------------------------------------------------->
+ ((VP/NP)\.,(VP/NP))
+ ----------------------------------------------------------------------------------<
+ (VP/NP)
+ --------------------------------------------------------------------------------------------------->B
+ (S/NP)
+ ------------------------------------------------------------------------------------------------------------------->
+ (N\N)
+ -----------------------------------------------------------------------------------------------------------------------------<
+ N
+ articles which I will file and forget without reading
+ N ((N\N)/(S/NP)) NP ((S/VP)\NP) (VP/NP) ((_var3\.,_var3)/.,_var3) (VP/NP) ((VP\VP)/VP['ing']) (VP['ing']/NP)
+ -----------------<
+ (S/VP)
+ ------------------------------------>
+ ((VP/NP)\.,(VP/NP))
+ ---------------------------------------------<
+ (VP/NP)
+ ------------------------------------->B
+ ((VP\VP)/NP)
+ ----------------------------------------------------------------------------------<Sx
+ (VP/NP)
+ --------------------------------------------------------------------------------------------------->B
+ (S/NP)
+ ------------------------------------------------------------------------------------------------------------------->
+ (N\N)
+ -----------------------------------------------------------------------------------------------------------------------------<
+ N
+
diff --git a/nltk/test/chat80.doctest b/nltk/test/chat80.doctest
new file mode 100644
index 0000000..74b0ba8
--- /dev/null
+++ b/nltk/test/chat80.doctest
@@ -0,0 +1,234 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=======
+Chat-80
+=======
+
+Chat-80 was a natural language system which allowed the user to
+interrogate a Prolog knowledge base in the domain of world
+geography. It was developed in the early '80s by Warren and Pereira; see
+`<http://acl.ldc.upenn.edu/J/J82/J82-3002.pdf>`_ for a description and
+`<http://www.cis.upenn.edu/~pereira/oldies.html>`_ for the source
+files.
+
+The ``chat80`` module contains functions to extract data from the Chat-80
+relation files ('the world database'), and convert then into a format
+that can be incorporated in the FOL models of
+``nltk.sem.evaluate``. The code assumes that the Prolog
+input files are available in the NLTK corpora directory.
+
+The Chat-80 World Database consists of the following files::
+
+ world0.pl
+ rivers.pl
+ cities.pl
+ countries.pl
+ contain.pl
+ borders.pl
+
+This module uses a slightly modified version of ``world0.pl``, in which
+a set of Prolog rules have been omitted. The modified file is named
+``world1.pl``. Currently, the file ``rivers.pl`` is not read in, since
+it uses a list rather than a string in the second field.
+
+Reading Chat-80 Files
+=====================
+
+Chat-80 relations are like tables in a relational database. The
+relation acts as the name of the table; the first argument acts as the
+'primary key'; and subsequent arguments are further fields in the
+table. In general, the name of the table provides a label for a unary
+predicate whose extension is all the primary keys. For example,
+relations in ``cities.pl`` are of the following form::
+
+ 'city(athens,greece,1368).'
+
+Here, ``'athens'`` is the key, and will be mapped to a member of the
+unary predicate *city*.
+
+By analogy with NLTK corpora, ``chat80`` defines a number of 'items'
+which correspond to the relations.
+
+ >>> from nltk.sem import chat80
+ >>> print(chat80.items) # doctest: +ELLIPSIS
+ ('borders', 'circle_of_lat', 'circle_of_long', 'city', ...)
+
+The fields in the table are mapped to binary predicates. The first
+argument of the predicate is the primary key, while the second
+argument is the data in the relevant field. Thus, in the above
+example, the third field is mapped to the binary predicate
+*population_of*, whose extension is a set of pairs such as
+``'(athens, 1368)'``.
+
+An exception to this general framework is required by the relations in
+the files ``borders.pl`` and ``contains.pl``. These contain facts of the
+following form::
+
+ 'borders(albania,greece).'
+
+ 'contains0(africa,central_africa).'
+
+We do not want to form a unary concept out the element in
+the first field of these records, and we want the label of the binary
+relation just to be ``'border'``/``'contain'`` respectively.
+
+In order to drive the extraction process, we use 'relation metadata bundles'
+which are Python dictionaries such as the following::
+
+ city = {'label': 'city',
+ 'closures': [],
+ 'schema': ['city', 'country', 'population'],
+ 'filename': 'cities.pl'}
+
+According to this, the file ``city['filename']`` contains a list of
+relational tuples (or more accurately, the corresponding strings in
+Prolog form) whose predicate symbol is ``city['label']`` and whose
+relational schema is ``city['schema']``. The notion of a ``closure`` is
+discussed in the next section.
+
+Concepts
+========
+In order to encapsulate the results of the extraction, a class of
+``Concept``\ s is introduced. A ``Concept`` object has a number of
+attributes, in particular a ``prefLabel``, an arity and ``extension``.
+
+ >>> c1 = chat80.Concept('dog', arity=1, extension=set(['d1', 'd2']))
+ >>> print(c1)
+ Label = 'dog'
+ Arity = 1
+ Extension = ['d1', 'd2']
+
+
+
+The ``extension`` attribute makes it easier to inspect the output of
+the extraction.
+
+ >>> schema = ['city', 'country', 'population']
+ >>> concepts = chat80.clause2concepts('cities.pl', 'city', schema)
+ >>> concepts
+ [Concept('city'), Concept('country_of'), Concept('population_of')]
+ >>> for c in concepts: # doctest: +NORMALIZE_WHITESPACE
+ ... print("%s:\n\t%s" % (c.prefLabel, c.extension[:4]))
+ city:
+ ['athens', 'bangkok', 'barcelona', 'berlin']
+ country_of:
+ [('athens', 'greece'), ('bangkok', 'thailand'), ('barcelona', 'spain'), ('berlin', 'east_germany')]
+ population_of:
+ [('athens', '1368'), ('bangkok', '1178'), ('barcelona', '1280'), ('berlin', '3481')]
+
+In addition, the ``extension`` can be further
+processed: in the case of the ``'border'`` relation, we check that the
+relation is **symmetric**, and in the case of the ``'contain'``
+relation, we carry out the **transitive closure**. The closure
+properties associated with a concept is indicated in the relation
+metadata, as indicated earlier.
+
+ >>> borders = set([('a1', 'a2'), ('a2', 'a3')])
+ >>> c2 = chat80.Concept('borders', arity=2, extension=borders)
+ >>> print(c2)
+ Label = 'borders'
+ Arity = 2
+ Extension = [('a1', 'a2'), ('a2', 'a3')]
+ >>> c3 = chat80.Concept('borders', arity=2, closures=['symmetric'], extension=borders)
+ >>> c3.close()
+ >>> print(c3)
+ Label = 'borders'
+ Arity = 2
+ Extension = [('a1', 'a2'), ('a2', 'a1'), ('a2', 'a3'), ('a3', 'a2')]
+
+The ``extension`` of a ``Concept`` object is then incorporated into a
+``Valuation`` object.
+
+Persistence
+===========
+The functions ``val_dump`` and ``val_load`` are provided to allow a
+valuation to be stored in a persistent database and re-loaded, rather
+than having to be re-computed each time.
+
+Individuals and Lexical Items
+=============================
+As well as deriving relations from the Chat-80 data, we also create a
+set of individual constants, one for each entity in the domain. The
+individual constants are string-identical to the entities. For
+example, given a data item such as ``'zloty'``, we add to the valuation
+a pair ``('zloty', 'zloty')``. In order to parse English sentences that
+refer to these entities, we also create a lexical item such as the
+following for each individual constant::
+
+ PropN[num=sg, sem=<\P.(P zloty)>] -> 'Zloty'
+
+The set of rules is written to the file ``chat_pnames.fcfg`` in the
+current directory.
+
+SQL Query
+=========
+
+The ``city`` relation is also available in RDB form and can be queried
+using SQL statements.
+
+ >>> import nltk
+ >>> q = "SELECT City, Population FROM city_table WHERE Country = 'china' and Population > 1000"
+ >>> for answer in chat80.sql_query('corpora/city_database/city.db', q):
+ ... print("%-10s %4s" % answer)
+ canton 1496
+ chungking 1100
+ mukden 1551
+ peking 2031
+ shanghai 5407
+ tientsin 1795
+
+The (deliberately naive) grammar ``sql.fcfg`` translates from English
+to SQL:
+
+ >>> nltk.data.show_cfg('grammars/book_grammars/sql0.fcfg')
+ % start S
+ S[SEM=(?np + WHERE + ?vp)] -> NP[SEM=?np] VP[SEM=?vp]
+ VP[SEM=(?v + ?pp)] -> IV[SEM=?v] PP[SEM=?pp]
+ VP[SEM=(?v + ?ap)] -> IV[SEM=?v] AP[SEM=?ap]
+ NP[SEM=(?det + ?n)] -> Det[SEM=?det] N[SEM=?n]
+ PP[SEM=(?p + ?np)] -> P[SEM=?p] NP[SEM=?np]
+ AP[SEM=?pp] -> A[SEM=?a] PP[SEM=?pp]
+ NP[SEM='Country="greece"'] -> 'Greece'
+ NP[SEM='Country="china"'] -> 'China'
+ Det[SEM='SELECT'] -> 'Which' | 'What'
+ N[SEM='City FROM city_table'] -> 'cities'
+ IV[SEM=''] -> 'are'
+ A[SEM=''] -> 'located'
+ P[SEM=''] -> 'in'
+
+Given this grammar, we can express, and then execute, queries in English.
+
+ >>> cp = nltk.parse.load_parser('grammars/book_grammars/sql0.fcfg')
+ >>> query = 'What cities are in China'
+ >>> for tree in cp.parse(query.split()):
+ ... answer = tree.label()['SEM']
+ ... q = " ".join(answer)
+ ... print(q)
+ ...
+ SELECT City FROM city_table WHERE Country="china"
+
+ >>> rows = chat80.sql_query('corpora/city_database/city.db', q)
+ >>> for r in rows: print("%s" % r, end=' ')
+ canton chungking dairen harbin kowloon mukden peking shanghai sian tientsin
+
+
+Using Valuations
+-----------------
+
+In order to convert such an extension into a valuation, we use the
+``make_valuation()`` method; setting ``read=True`` creates and returns
+a new ``Valuation`` object which contains the results.
+
+ >>> val = chat80.make_valuation(concepts, read=True)
+ >>> 'calcutta' in val['city']
+ True
+ >>> [town for (town, country) in val['country_of'] if country == 'india']
+ ['bombay', 'calcutta', 'delhi', 'hyderabad', 'madras']
+ >>> dom = val.domain
+ >>> g = nltk.sem.Assignment(dom)
+ >>> m = nltk.sem.Model(dom, val)
+ >>> m.evaluate(r'population_of(jakarta, 533)', g)
+ True
+
+
diff --git a/nltk/test/childes.doctest b/nltk/test/childes.doctest
new file mode 100644
index 0000000..7900c54
--- /dev/null
+++ b/nltk/test/childes.doctest
@@ -0,0 +1,184 @@
+=======================
+ CHILDES Corpus Readers
+=======================
+
+Read the XML version of the CHILDES corpus.
+
+How to use CHILDESCorpusReader
+==============================
+
+Read the CHILDESCorpusReader class and read the CHILDES corpus saved in
+the nltk_data directory.
+
+ >>> import nltk
+ >>> from nltk.corpus.reader import CHILDESCorpusReader
+ >>> corpus_root = nltk.data.find('corpora/childes/data-xml/Eng-USA-MOR/')
+
+Reading files in the Valian corpus (Valian, 1991).
+
+ >>> valian = CHILDESCorpusReader(corpus_root, 'Valian/.*.xml')
+ >>> valian.fileids()
+ ['Valian/01a.xml', 'Valian/01b.xml', 'Valian/02a.xml', 'Valian/02b.xml',...
+
+Count the number of files
+
+ >>> len(valian.fileids())
+ 43
+
+Printing properties of the corpus files.
+
+ >>> corpus_data = valian.corpus(valian.fileids())
+ >>> print(corpus_data[0]['Lang'])
+ eng
+ >>> for key in sorted(corpus_data[0].keys()):
+ ... print(key, ": ", corpus_data[0][key])
+ Corpus : valian
+ Date : 1986-03-04
+ Id : 01a
+ Lang : eng
+ Version : 2.0.1
+ {http://www.w3.org/2001/XMLSchema-instance}schemaLocation : http://www.talkbank.org/ns/talkbank http://talkbank.org/software/talkbank.xsd
+
+Printing information of participants of the corpus. The most common codes for
+the participants are 'CHI' (target child), 'MOT' (mother), and 'INV' (investigator).
+
+ >>> corpus_participants = valian.participants(valian.fileids())
+ >>> for this_corpus_participants in corpus_participants[:2]:
+ ... for key in sorted(this_corpus_participants.keys()):
+ ... dct = this_corpus_participants[key]
+ ... print(key, ": ", [(k, dct[k]) for k in sorted(dct.keys())])
+ CHI : [('age', 'P2Y1M3D'), ('group', 'normal'), ('id', 'CHI'), ('language', 'eng'), ('role', 'Target_Child'), ('sex', 'female')]
+ INV : [('id', 'INV'), ('language', 'eng'), ('role', 'Investigator')]
+ MOT : [('id', 'MOT'), ('language', 'eng'), ('role', 'Mother')]
+ CHI : [('age', 'P2Y1M12D'), ('group', 'normal'), ('id', 'CHI'), ('language', 'eng'), ('role', 'Target_Child'), ('sex', 'female')]
+ INV : [('id', 'INV'), ('language', 'eng'), ('role', 'Investigator')]
+ MOT : [('id', 'MOT'), ('language', 'eng'), ('role', 'Mother')]
+
+printing words.
+
+ >>> valian.words('Valian/01a.xml')
+ ['at', 'Parent', "Lastname's", 'house', 'with', 'Child', 'Lastname', ...
+
+printing sentences.
+
+ >>> valian.sents('Valian/01a.xml')
+ [['at', 'Parent', "Lastname's", 'house', 'with', 'Child', 'Lastname',
+ 'and', 'it', 'is', 'March', 'fourth', 'I', 'believe', 'and', 'when',
+ 'was', "Parent's", 'birthday'], ["Child's"], ['oh', "I'm", 'sorry'],
+ ["that's", 'okay'], ...
+
+You can specify the participants with the argument *speaker*.
+
+ >>> valian.words('Valian/01a.xml',speaker=['INV'])
+ ['at', 'Parent', "Lastname's", 'house', 'with', 'Child', 'Lastname', ...
+ >>> valian.words('Valian/01a.xml',speaker=['MOT'])
+ ["Child's", "that's", 'okay', 'February', 'first', 'nineteen', ...
+ >>> valian.words('Valian/01a.xml',speaker=['CHI'])
+ ['tape', 'it', 'up', 'and', 'two', 'tape', 'players', 'have',...
+
+
+tagged_words() and tagged_sents() return the usual (word,pos) tuple lists.
+POS tags in the CHILDES are automatically assigned by MOR and POST programs
+(MacWhinney, 2000).
+
+ >>> valian.tagged_words('Valian/01a.xml')[:30]
+ [('at', 'prep'), ('Parent', 'n:prop'), ("Lastname's", 'n:prop'), ('house', 'n'),
+ ('with', 'prep'), ('Child', 'n:prop'), ('Lastname', 'n:prop'), ('and', 'coord'),
+ ('it', 'pro'), ('is', 'v:cop'), ('March', 'n:prop'), ('fourth', 'adj'),
+ ('I', 'pro:sub'), ('believe', 'v'), ('and', 'coord'), ('when', 'adv:wh'),
+ ('was', 'v:cop'), ("Parent's", 'n:prop'), ('birthday', 'n'), ("Child's", 'n:prop'),
+ ('oh', 'co'), ("I'm", 'pro:sub'), ('sorry', 'adj'), ("that's", 'pro:dem'),
+ ('okay', 'adj'), ('February', 'n:prop'), ('first', 'adj'),
+ ('nineteen', 'det:num'), ('eighty', 'det:num'), ('four', 'det:num')]
+
+ >>> valian.tagged_sents('Valian/01a.xml')[:10]
+ [[('at', 'prep'), ('Parent', 'n:prop'), ("Lastname's", 'n:prop'), ('house', 'n'),
+ ('with', 'prep'), ('Child', 'n:prop'), ('Lastname', 'n:prop'), ('and', 'coord'),
+ ('it', 'pro'), ('is', 'v:cop'), ('March', 'n:prop'), ('fourth', 'adj'),
+ ('I', 'pro:sub'), ('believe', 'v'), ('and', 'coord'), ('when', 'adv:wh'),
+ ('was', 'v:cop'), ("Parent's", 'n:prop'), ('birthday', 'n')],
+ [("Child's", 'n:prop')], [('oh', 'co'), ("I'm", 'pro:sub'), ('sorry', 'adj')],
+ [("that's", 'pro:dem'), ('okay', 'adj')],
+ [('February', 'n:prop'), ('first', 'adj'), ('nineteen', 'det:num'),
+ ('eighty', 'det:num'), ('four', 'det:num')],
+ [('great', 'adj')],
+ [('and', 'coord'), ("she's", 'pro:sub'), ('two', 'det:num'), ('years', 'n'), ('old', 'adj')],
+ [('correct', 'adj')],
+ [('okay', 'co')], [('she', 'pro:sub'), ('just', 'adv:int'), ('turned', 'part'), ('two', 'det:num'),
+ ('a', 'det'), ('month', 'n'), ('ago', 'adv')]]
+
+When the argument *stem* is true, the word stems (e.g., 'is' -> 'be-3PS') are
+used instread of the original words.
+
+ >>> valian.words('Valian/01a.xml')[:30]
+ ['at', 'Parent', "Lastname's", 'house', 'with', 'Child', 'Lastname', 'and', 'it', 'is', ...
+ >>> valian.words('Valian/01a.xml',stem=True)[:30]
+ ['at', 'Parent', 'Lastname', 's', 'house', 'with', 'Child', 'Lastname', 'and', 'it', 'be-3S', ...
+
+When the argument *replace* is true, the replaced words are used instread of
+the original words.
+
+ >>> valian.words('Valian/01a.xml',speaker='CHI')[247]
+ 'tikteat'
+ >>> valian.words('Valian/01a.xml',speaker='CHI',replace=True)[247]
+ 'trick'
+
+When the argument *relation* is true, the relational relationships in the
+sentence are returned. See Sagae et al. (2010) for details of the relational
+structure adopted in the CHILDES.
+
+ >>> valian.words('Valian/01a.xml',relation=True)[:10]
+ [[('at', 'prep', '1|0|ROOT'), ('Parent', 'n', '2|5|VOC'), ('Lastname', 'n', '3|5|MOD'), ('s', 'poss', '4|5|MOD'), ('house', 'n', '5|1|POBJ'), ('with', 'prep', '6|1|JCT'), ('Child', 'n', '7|8|NAME'), ('Lastname', 'n', '8|6|POBJ'), ('and', 'coord', '9|8|COORD'), ('it', 'pro', '10|11|SUBJ'), ('be-3S', 'v', '11|9|COMP'), ('March', 'n', '12|11|PRED'), ('fourth', 'adj', '13|12|MOD'), ('I', 'pro', '15|16|SUBJ'), ('believe', 'v', '16|14|ROOT'), ('and', 'coord', '18|17|ROOT'), ('when', 'adv', [...]
+
+Printing age. When the argument *month* is true, the age information in
+the CHILDES format is converted into the number of months.
+
+ >>> valian.age()
+ ['P2Y1M3D', 'P2Y1M12D', 'P1Y9M21D', 'P1Y9M28D', 'P2Y1M23D', ...
+ >>> valian.age('Valian/01a.xml')
+ ['P2Y1M3D']
+ >>> valian.age('Valian/01a.xml',month=True)
+ [25]
+
+Printing MLU. The criteria for the MLU computation is broadly based on
+Brown (1973).
+
+ >>> valian.MLU()
+ [2.3574660633484..., 2.292682926829..., 3.492857142857..., 2.961783439490...,
+ 2.0842696629213..., 3.169811320754..., 3.137404580152..., 3.0578034682080...,
+ 4.090163934426..., 3.488372093023..., 2.8773584905660..., 3.4792899408284...,
+ 4.0111940298507..., 3.456790123456..., 4.487603305785..., 4.007936507936...,
+ 5.25, 5.154696132596..., ...]
+
+ >>> valian.MLU('Valian/01a.xml')
+ [2.35746606334...]
+
+
+Basic stuff
+==============================
+
+Count the number of words and sentences of each file.
+
+ >>> valian = CHILDESCorpusReader(corpus_root, 'Valian/.*.xml')
+ >>> for this_file in valian.fileids()[:6]:
+ ... print(valian.corpus(this_file)[0]['Corpus'], valian.corpus(this_file)[0]['Id'])
+ ... print("num of words: %i" % len(valian.words(this_file)))
+ ... print("num of sents: %i" % len(valian.sents(this_file)))
+ valian 01a
+ num of words: 3606
+ num of sents: 1027
+ valian 01b
+ num of words: 4376
+ num of sents: 1274
+ valian 02a
+ num of words: 2673
+ num of sents: 801
+ valian 02b
+ num of words: 5020
+ num of sents: 1583
+ valian 03a
+ num of words: 2743
+ num of sents: 988
+ valian 03b
+ num of words: 4409
+ num of sents: 1397
diff --git a/nltk/test/childes_fixt.py b/nltk/test/childes_fixt.py
new file mode 100644
index 0000000..8904eaa
--- /dev/null
+++ b/nltk/test/childes_fixt.py
@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+def setup_module(module):
+ from nose import SkipTest
+ import nltk.data
+ try:
+ nltk.data.find('corpora/childes/data-xml/Eng-USA-MOR/')
+ except LookupError as e:
+ print(e)
+ raise SkipTest("The CHILDES corpus is not found. "
+ "It should be manually downloaded and saved/unpacked "
+ "to [NLTK_Data_Dir]/corpora/childes/")
diff --git a/nltk/test/chunk.doctest b/nltk/test/chunk.doctest
new file mode 100644
index 0000000..24a08ba
--- /dev/null
+++ b/nltk/test/chunk.doctest
@@ -0,0 +1,373 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==========
+ Chunking
+==========
+
+ >>> from nltk.chunk import *
+ >>> from nltk.chunk.util import *
+ >>> from nltk.chunk.regexp import *
+ >>> from nltk import Tree
+
+ >>> tagged_text = "[ The/DT cat/NN ] sat/VBD on/IN [ the/DT mat/NN ] [ the/DT dog/NN ] chewed/VBD ./."
+ >>> gold_chunked_text = tagstr2tree(tagged_text)
+ >>> unchunked_text = gold_chunked_text.flatten()
+
+Chunking uses a special regexp syntax for rules that delimit the chunks. These
+rules must be converted to 'regular' regular expressions before a sentence can
+be chunked.
+
+ >>> tag_pattern = "<DT>?<JJ>*<NN.*>"
+ >>> regexp_pattern = tag_pattern2re_pattern(tag_pattern)
+ >>> regexp_pattern
+ '(<(DT)>)?(<(JJ)>)*(<(NN[^\\{\\}<>]*)>)'
+
+Construct some new chunking rules.
+
+ >>> chunk_rule = ChunkRule("<.*>+", "Chunk everything")
+ >>> chink_rule = ChinkRule("<VBD|IN|\.>", "Chink on verbs/prepositions")
+ >>> split_rule = SplitRule("<DT><NN>", "<DT><NN>",
+ ... "Split successive determiner/noun pairs")
+
+
+Create and score a series of chunk parsers, successively more complex.
+
+ >>> chunk_parser = RegexpChunkParser([chunk_rule], chunk_label='NP')
+ >>> chunked_text = chunk_parser.parse(unchunked_text)
+ >>> print(chunked_text)
+ (S
+ (NP
+ The/DT
+ cat/NN
+ sat/VBD
+ on/IN
+ the/DT
+ mat/NN
+ the/DT
+ dog/NN
+ chewed/VBD
+ ./.))
+
+ >>> chunkscore = ChunkScore()
+ >>> chunkscore.score(gold_chunked_text, chunked_text)
+ >>> print(chunkscore.precision())
+ 0.0
+
+ >>> print(chunkscore.recall())
+ 0.0
+
+ >>> print(chunkscore.f_measure())
+ 0
+
+ >>> for chunk in sorted(chunkscore.missed()): print(chunk)
+ (NP The/DT cat/NN)
+ (NP the/DT dog/NN)
+ (NP the/DT mat/NN)
+
+ >>> for chunk in chunkscore.incorrect(): print(chunk)
+ (NP
+ The/DT
+ cat/NN
+ sat/VBD
+ on/IN
+ the/DT
+ mat/NN
+ the/DT
+ dog/NN
+ chewed/VBD
+ ./.)
+
+ >>> chunk_parser = RegexpChunkParser([chunk_rule, chink_rule],
+ ... chunk_label='NP')
+ >>> chunked_text = chunk_parser.parse(unchunked_text)
+ >>> print(chunked_text)
+ (S
+ (NP The/DT cat/NN)
+ sat/VBD
+ on/IN
+ (NP the/DT mat/NN the/DT dog/NN)
+ chewed/VBD
+ ./.)
+ >>> assert chunked_text == chunk_parser.parse(list(unchunked_text))
+
+ >>> chunkscore = ChunkScore()
+ >>> chunkscore.score(gold_chunked_text, chunked_text)
+ >>> chunkscore.precision()
+ 0.5
+
+ >>> print(chunkscore.recall())
+ 0.33333333...
+
+ >>> print(chunkscore.f_measure())
+ 0.4
+
+ >>> for chunk in sorted(chunkscore.missed()): print(chunk)
+ (NP the/DT dog/NN)
+ (NP the/DT mat/NN)
+
+ >>> for chunk in chunkscore.incorrect(): print(chunk)
+ (NP the/DT mat/NN the/DT dog/NN)
+
+ >>> chunk_parser = RegexpChunkParser([chunk_rule, chink_rule, split_rule],
+ ... chunk_label='NP')
+ >>> chunked_text = chunk_parser.parse(unchunked_text, trace=True)
+ # Input:
+ <DT> <NN> <VBD> <IN> <DT> <NN> <DT> <NN> <VBD> <.>
+ # Chunk everything:
+ {<DT> <NN> <VBD> <IN> <DT> <NN> <DT> <NN> <VBD> <.>}
+ # Chink on verbs/prepositions:
+ {<DT> <NN>} <VBD> <IN> {<DT> <NN> <DT> <NN>} <VBD> <.>
+ # Split successive determiner/noun pairs:
+ {<DT> <NN>} <VBD> <IN> {<DT> <NN>}{<DT> <NN>} <VBD> <.>
+ >>> print(chunked_text)
+ (S
+ (NP The/DT cat/NN)
+ sat/VBD
+ on/IN
+ (NP the/DT mat/NN)
+ (NP the/DT dog/NN)
+ chewed/VBD
+ ./.)
+
+ >>> chunkscore = ChunkScore()
+ >>> chunkscore.score(gold_chunked_text, chunked_text)
+ >>> chunkscore.precision()
+ 1.0
+
+ >>> chunkscore.recall()
+ 1.0
+
+ >>> chunkscore.f_measure()
+ 1.0
+
+ >>> chunkscore.missed()
+ []
+
+ >>> chunkscore.incorrect()
+ []
+
+ >>> chunk_parser.rules() # doctest: +NORMALIZE_WHITESPACE
+ [<ChunkRule: '<.*>+'>, <ChinkRule: '<VBD|IN|\\.>'>,
+ <SplitRule: '<DT><NN>', '<DT><NN>'>]
+
+Printing parsers:
+
+ >>> print(repr(chunk_parser))
+ <RegexpChunkParser with 3 rules>
+ >>> print(chunk_parser)
+ RegexpChunkParser with 3 rules:
+ Chunk everything
+ <ChunkRule: '<.*>+'>
+ Chink on verbs/prepositions
+ <ChinkRule: '<VBD|IN|\\.>'>
+ Split successive determiner/noun pairs
+ <SplitRule: '<DT><NN>', '<DT><NN>'>
+
+Regression Tests
+~~~~~~~~~~~~~~~~
+ChunkParserI
+------------
+`ChunkParserI` is an abstract interface -- it is not meant to be
+instantiated directly.
+
+ >>> ChunkParserI().parse([])
+ Traceback (most recent call last):
+ . . .
+ NotImplementedError
+
+
+ChunkString
+-----------
+ChunkString can be built from a tree of tagged tuples, a tree of
+trees, or a mixed list of both:
+
+ >>> t1 = Tree('S', [('w%d' % i, 't%d' % i) for i in range(10)])
+ >>> t2 = Tree('S', [Tree('t0', []), Tree('t1', ['c1'])])
+ >>> t3 = Tree('S', [('w0', 't0'), Tree('t1', ['c1'])])
+ >>> ChunkString(t1)
+ <ChunkString: '<t0><t1><t2><t3><t4><t5><t6><t7><t8><t9>'>
+ >>> ChunkString(t2)
+ <ChunkString: '<t0><t1>'>
+ >>> ChunkString(t3)
+ <ChunkString: '<t0><t1>'>
+
+Other values generate an error:
+
+ >>> ChunkString(Tree('S', ['x']))
+ Traceback (most recent call last):
+ . . .
+ ValueError: chunk structures must contain tagged tokens or trees
+
+The `str()` for a chunk string adds spaces to it, which makes it line
+up with `str()` output for other chunk strings over the same
+underlying input.
+
+ >>> cs = ChunkString(t1)
+ >>> print(cs)
+ <t0> <t1> <t2> <t3> <t4> <t5> <t6> <t7> <t8> <t9>
+ >>> cs.xform('<t3>', '{<t3>}')
+ >>> print(cs)
+ <t0> <t1> <t2> {<t3>} <t4> <t5> <t6> <t7> <t8> <t9>
+
+The `_verify()` method makes sure that our transforms don't corrupt
+the chunk string. By setting debug_level=2, `_verify()` will be
+called at the end of every call to `xform`.
+
+ >>> cs = ChunkString(t1, debug_level=3)
+
+ >>> # tag not marked with <...>:
+ >>> cs.xform('<t3>', 't3')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Transformation generated invalid chunkstring:
+ <t0><t1><t2>t3<t4><t5><t6><t7><t8><t9>
+
+ >>> # brackets not balanced:
+ >>> cs.xform('<t3>', '{<t3>')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Transformation generated invalid chunkstring:
+ <t0><t1><t2>{<t3><t4><t5><t6><t7><t8><t9>
+
+ >>> # nested brackets:
+ >>> cs.xform('<t3><t4><t5>', '{<t3>{<t4>}<t5>}')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Transformation generated invalid chunkstring:
+ <t0><t1><t2>{<t3>{<t4>}<t5>}<t6><t7><t8><t9>
+
+ >>> # modified tags:
+ >>> cs.xform('<t3>', '<t9>')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Transformation generated invalid chunkstring: tag changed
+
+ >>> # added tags:
+ >>> cs.xform('<t9>', '<t9><t10>')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Transformation generated invalid chunkstring: tag changed
+
+Chunking Rules
+--------------
+
+Test the different rule constructors & __repr__ methods:
+
+ >>> r1 = RegexpChunkRule('<a|b>'+ChunkString.IN_CHINK_PATTERN,
+ ... '{<a|b>}', 'chunk <a> and <b>')
+ >>> r2 = RegexpChunkRule(re.compile('<a|b>'+ChunkString.IN_CHINK_PATTERN),
+ ... '{<a|b>}', 'chunk <a> and <b>')
+ >>> r3 = ChunkRule('<a|b>', 'chunk <a> and <b>')
+ >>> r4 = ChinkRule('<a|b>', 'chink <a> and <b>')
+ >>> r5 = UnChunkRule('<a|b>', 'unchunk <a> and <b>')
+ >>> r6 = MergeRule('<a>', '<b>', 'merge <a> w/ <b>')
+ >>> r7 = SplitRule('<a>', '<b>', 'split <a> from <b>')
+ >>> r8 = ExpandLeftRule('<a>', '<b>', 'expand left <a> <b>')
+ >>> r9 = ExpandRightRule('<a>', '<b>', 'expand right <a> <b>')
+ >>> for rule in r1, r2, r3, r4, r5, r6, r7, r8, r9:
+ ... print(rule)
+ <RegexpChunkRule: '<a|b>(?=[^\\}]*(\\{|$))'->'{<a|b>}'>
+ <RegexpChunkRule: '<a|b>(?=[^\\}]*(\\{|$))'->'{<a|b>}'>
+ <ChunkRule: '<a|b>'>
+ <ChinkRule: '<a|b>'>
+ <UnChunkRule: '<a|b>'>
+ <MergeRule: '<a>', '<b>'>
+ <SplitRule: '<a>', '<b>'>
+ <ExpandLeftRule: '<a>', '<b>'>
+ <ExpandRightRule: '<a>', '<b>'>
+
+`tag_pattern2re_pattern()` complains if the tag pattern looks problematic:
+
+ >>> tag_pattern2re_pattern('{}')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Bad tag pattern: '{}'
+
+RegexpChunkParser
+-----------------
+
+A warning is printed when parsing an empty sentence:
+
+ >>> parser = RegexpChunkParser([ChunkRule('<a>', '')])
+ >>> parser.parse(Tree('S', []))
+ Warning: parsing empty text
+ Tree('S', [])
+
+RegexpParser
+------------
+
+ >>> parser = RegexpParser('''
+ ... NP: {<DT>? <JJ>* <NN>*} # NP
+ ... P: {<IN>} # Preposition
+ ... V: {<V.*>} # Verb
+ ... PP: {<P> <NP>} # PP -> P NP
+ ... VP: {<V> <NP|PP>*} # VP -> V (NP|PP)*
+ ... ''')
+ >>> print(repr(parser))
+ <chunk.RegexpParser with 5 stages>
+ >>> print(parser)
+ chunk.RegexpParser with 5 stages:
+ RegexpChunkParser with 1 rules:
+ NP <ChunkRule: '<DT>? <JJ>* <NN>*'>
+ RegexpChunkParser with 1 rules:
+ Preposition <ChunkRule: '<IN>'>
+ RegexpChunkParser with 1 rules:
+ Verb <ChunkRule: '<V.*>'>
+ RegexpChunkParser with 1 rules:
+ PP -> P NP <ChunkRule: '<P> <NP>'>
+ RegexpChunkParser with 1 rules:
+ VP -> V (NP|PP)* <ChunkRule: '<V> <NP|PP>*'>
+ >>> print(parser.parse(unchunked_text, trace=True))
+ # Input:
+ <DT> <NN> <VBD> <IN> <DT> <NN> <DT> <NN> <VBD> <.>
+ # NP:
+ {<DT> <NN>} <VBD> <IN> {<DT> <NN>}{<DT> <NN>} <VBD> <.>
+ # Input:
+ <NP> <VBD> <IN> <NP> <NP> <VBD> <.>
+ # Preposition:
+ <NP> <VBD> {<IN>} <NP> <NP> <VBD> <.>
+ # Input:
+ <NP> <VBD> <P> <NP> <NP> <VBD> <.>
+ # Verb:
+ <NP> {<VBD>} <P> <NP> <NP> {<VBD>} <.>
+ # Input:
+ <NP> <V> <P> <NP> <NP> <V> <.>
+ # PP -> P NP:
+ <NP> <V> {<P> <NP>} <NP> <V> <.>
+ # Input:
+ <NP> <V> <PP> <NP> <V> <.>
+ # VP -> V (NP|PP)*:
+ <NP> {<V> <PP> <NP>}{<V>} <.>
+ (S
+ (NP The/DT cat/NN)
+ (VP
+ (V sat/VBD)
+ (PP (P on/IN) (NP the/DT mat/NN))
+ (NP the/DT dog/NN))
+ (VP (V chewed/VBD))
+ ./.)
+
+Test parsing of other rule types:
+
+ >>> print(RegexpParser('''
+ ... X:
+ ... }<a><b>{ # chink rule
+ ... <a>}{<b> # split rule
+ ... <a>{}<b> # merge rule
+ ... <a>{<b>}<c> # chunk rule w/ context
+ ... '''))
+ chunk.RegexpParser with 1 stages:
+ RegexpChunkParser with 4 rules:
+ chink rule <ChinkRule: '<a><b>'>
+ split rule <SplitRule: '<a>', '<b>'>
+ merge rule <MergeRule: '<a>', '<b>'>
+ chunk rule w/ context <ChunkRuleWithContext: '<a>', '<b>', '<c>'>
+
+Illegal patterns give an error message:
+
+ >>> print(RegexpParser('X: {<foo>} {<bar>}'))
+ Traceback (most recent call last):
+ . . .
+ ValueError: Illegal chunk pattern: {<foo>} {<bar>}
+
diff --git a/nltk/test/classify.doctest b/nltk/test/classify.doctest
new file mode 100644
index 0000000..892603b
--- /dev/null
+++ b/nltk/test/classify.doctest
@@ -0,0 +1,185 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=============
+ Classifiers
+=============
+
+Classifiers label tokens with category labels (or *class labels*).
+Typically, labels are represented with strings (such as ``"health"``
+or ``"sports"``. In NLTK, classifiers are defined using classes that
+implement the `ClassifyI` interface:
+
+ >>> import nltk
+ >>> nltk.usage(nltk.classify.ClassifierI)
+ ClassifierI supports the following operations:
+ - self.classify(featureset)
+ - self.classify_many(featuresets)
+ - self.labels()
+ - self.prob_classify(featureset)
+ - self.prob_classify_many(featuresets)
+
+NLTK defines several classifier classes:
+
+- `ConditionalExponentialClassifier`
+- `DecisionTreeClassifier`
+- `MaxentClassifier`
+- `NaiveBayesClassifier`
+- `WekaClassifier`
+
+Classifiers are typically created by training them on a training
+corpus.
+
+
+Regression Tests
+~~~~~~~~~~~~~~~~
+
+We define a very simple training corpus with 3 binary features: ['a',
+'b', 'c'], and are two labels: ['x', 'y']. We use a simple feature set so
+that the correct answers can be calculated analytically (although we
+haven't done this yet for all tests).
+
+ >>> train = [
+ ... (dict(a=1,b=1,c=1), 'y'),
+ ... (dict(a=1,b=1,c=1), 'x'),
+ ... (dict(a=1,b=1,c=0), 'y'),
+ ... (dict(a=0,b=1,c=1), 'x'),
+ ... (dict(a=0,b=1,c=1), 'y'),
+ ... (dict(a=0,b=0,c=1), 'y'),
+ ... (dict(a=0,b=1,c=0), 'x'),
+ ... (dict(a=0,b=0,c=0), 'x'),
+ ... (dict(a=0,b=1,c=1), 'y'),
+ ... ]
+ >>> test = [
+ ... (dict(a=1,b=0,c=1)), # unseen
+ ... (dict(a=1,b=0,c=0)), # unseen
+ ... (dict(a=0,b=1,c=1)), # seen 3 times, labels=y,y,x
+ ... (dict(a=0,b=1,c=0)), # seen 1 time, label=x
+ ... ]
+
+Test the Naive Bayes classifier:
+
+ >>> classifier = nltk.classify.NaiveBayesClassifier.train(train)
+ >>> sorted(classifier.labels())
+ ['x', 'y']
+ >>> classifier.classify_many(test)
+ ['y', 'x', 'y', 'x']
+ >>> for pdist in classifier.prob_classify_many(test):
+ ... print('%.4f %.4f' % (pdist.prob('x'), pdist.prob('y')))
+ 0.3203 0.6797
+ 0.5857 0.4143
+ 0.3792 0.6208
+ 0.6470 0.3530
+ >>> classifier.show_most_informative_features()
+ Most Informative Features
+ c = 0 x : y = 2.0 : 1.0
+ c = 1 y : x = 1.5 : 1.0
+ a = 1 y : x = 1.4 : 1.0
+ b = 0 x : y = 1.2 : 1.0
+ a = 0 x : y = 1.2 : 1.0
+ b = 1 y : x = 1.1 : 1.0
+
+Test the Decision Tree classifier:
+
+ >>> classifier = nltk.classify.DecisionTreeClassifier.train(
+ ... train, entropy_cutoff=0,
+ ... support_cutoff=0)
+ >>> sorted(classifier.labels())
+ ['x', 'y']
+ >>> print(classifier)
+ c=0? .................................................. x
+ a=0? ................................................ x
+ a=1? ................................................ y
+ c=1? .................................................. y
+ <BLANKLINE>
+ >>> classifier.classify_many(test)
+ ['y', 'y', 'y', 'x']
+ >>> for pdist in classifier.prob_classify_many(test):
+ ... print('%.4f %.4f' % (pdist.prob('x'), pdist.prob('y')))
+ Traceback (most recent call last):
+ . . .
+ NotImplementedError
+
+Test SklearnClassifier, which requires the scikit-learn package.
+
+ >>> from nltk.classify import SklearnClassifier
+ >>> from sklearn.naive_bayes import BernoulliNB
+ >>> from sklearn.svm import SVC
+ >>> train_data = [({"a": 4, "b": 1, "c": 0}, "ham"),
+ ... ({"a": 5, "b": 2, "c": 1}, "ham"),
+ ... ({"a": 0, "b": 3, "c": 4}, "spam"),
+ ... ({"a": 5, "b": 1, "c": 1}, "ham"),
+ ... ({"a": 1, "b": 4, "c": 3}, "spam")]
+ >>> classif = SklearnClassifier(BernoulliNB()).train(train_data)
+ >>> test_data = [{"a": 3, "b": 2, "c": 1},
+ ... {"a": 0, "b": 3, "c": 7}]
+ >>> classif.classify_many(test_data)
+ ['ham', 'spam']
+ >>> classif = SklearnClassifier(SVC(), sparse=False).train(train_data)
+ >>> classif.classify_many(test_data)
+ ['ham', 'spam']
+
+Test the Maximum Entropy classifier training algorithms; they should all
+generate the same results.
+
+ >>> def print_maxent_test_header():
+ ... print(' '*11+''.join([' test[%s] ' % i
+ ... for i in range(len(test))]))
+ ... print(' '*11+' p(x) p(y)'*len(test))
+ ... print('-'*(11+15*len(test)))
+
+ >>> def test_maxent(algorithm):
+ ... print('%11s' % algorithm, end=' ')
+ ... try:
+ ... classifier = nltk.classify.MaxentClassifier.train(
+ ... train, algorithm, trace=0, max_iter=1000)
+ ... except Exception as e:
+ ... print('Error: %r' % e)
+ ... return
+ ...
+ ... for featureset in test:
+ ... pdist = classifier.prob_classify(featureset)
+ ... print('%8.2f%6.2f' % (pdist.prob('x'), pdist.prob('y')), end=' ')
+ ... print()
+
+ >>> print_maxent_test_header(); test_maxent('GIS'); test_maxent('IIS')
+ test[0] test[1] test[2] test[3]
+ p(x) p(y) p(x) p(y) p(x) p(y) p(x) p(y)
+ -----------------------------------------------------------------------
+ GIS 0.16 0.84 0.46 0.54 0.41 0.59 0.76 0.24
+ IIS 0.16 0.84 0.46 0.54 0.41 0.59 0.76 0.24
+
+ >>> test_maxent('MEGAM'); test_maxent('TADM') # doctest: +SKIP
+ MEGAM 0.16 0.84 0.46 0.54 0.41 0.59 0.76 0.24
+ TADM 0.16 0.84 0.46 0.54 0.41 0.59 0.76 0.24
+
+
+
+Regression tests for TypedMaxentFeatureEncoding
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ >>> from nltk.classify import maxent
+ >>> train = [
+ ... ({'a': 1, 'b': 1, 'c': 1}, 'y'),
+ ... ({'a': 5, 'b': 5, 'c': 5}, 'x'),
+ ... ({'a': 0.9, 'b': 0.9, 'c': 0.9}, 'y'),
+ ... ({'a': 5.5, 'b': 5.4, 'c': 5.3}, 'x'),
+ ... ({'a': 0.8, 'b': 1.2, 'c': 1}, 'y'),
+ ... ({'a': 5.1, 'b': 4.9, 'c': 5.2}, 'x')
+ ... ]
+
+ >>> test = [
+ ... {'a': 1, 'b': 0.8, 'c': 1.2},
+ ... {'a': 5.2, 'b': 5.1, 'c': 5}
+ ... ]
+
+ >>> encoding = maxent.TypedMaxentFeatureEncoding.train(
+ ... train, count_cutoff=3, alwayson_features=True)
+
+ >>> classifier = maxent.MaxentClassifier.train(
+ ... train, bernoulli=False, encoding=encoding, trace=0)
+
+ >>> classifier.classify_many(test)
+ ['y', 'x']
+
+
diff --git a/nltk/test/classify_fixt.py b/nltk/test/classify_fixt.py
new file mode 100644
index 0000000..fbf771f
--- /dev/null
+++ b/nltk/test/classify_fixt.py
@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+# most of classify.doctest requires numpy
+def setup_module(module):
+ from nose import SkipTest
+ try:
+ import numpy
+ except ImportError:
+ raise SkipTest("classify.doctest requires numpy")
\ No newline at end of file
diff --git a/nltk/test/collocations.doctest b/nltk/test/collocations.doctest
new file mode 100644
index 0000000..a390714
--- /dev/null
+++ b/nltk/test/collocations.doctest
@@ -0,0 +1,276 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==============
+ Collocations
+==============
+
+Overview
+~~~~~~~~
+
+Collocations are expressions of multiple words which commonly co-occur. For
+example, the top ten bigram collocations in Genesis are listed below, as
+measured using Pointwise Mutual Information.
+
+ >>> import nltk
+ >>> from nltk.collocations import *
+ >>> bigram_measures = nltk.collocations.BigramAssocMeasures()
+ >>> trigram_measures = nltk.collocations.TrigramAssocMeasures()
+ >>> finder = BigramCollocationFinder.from_words(
+ ... nltk.corpus.genesis.words('english-web.txt'))
+ >>> finder.nbest(bigram_measures.pmi, 10) # doctest: +NORMALIZE_WHITESPACE
+ [(u'Allon', u'Bacuth'), (u'Ashteroth', u'Karnaim'), (u'Ben', u'Ammi'),
+ (u'En', u'Mishpat'), (u'Jegar', u'Sahadutha'), (u'Salt', u'Sea'),
+ (u'Whoever', u'sheds'), (u'appoint', u'overseers'), (u'aromatic', u'resin'),
+ (u'cutting', u'instrument')]
+
+While these words are highly collocated, the expressions are also very
+infrequent. Therefore it is useful to apply filters, such as ignoring all
+bigrams which occur less than three times in the corpus:
+
+ >>> finder.apply_freq_filter(3)
+ >>> finder.nbest(bigram_measures.pmi, 10) # doctest: +NORMALIZE_WHITESPACE
+ [(u'Beer', u'Lahai'), (u'Lahai', u'Roi'), (u'gray', u'hairs'),
+ (u'Most', u'High'), (u'ewe', u'lambs'), (u'many', u'colors'),
+ (u'burnt', u'offering'), (u'Paddan', u'Aram'), (u'east', u'wind'),
+ (u'living', u'creature')]
+
+We may similarly find collocations among tagged words:
+
+ >>> finder = BigramCollocationFinder.from_words(
+ ... nltk.corpus.brown.tagged_words('ca01', tagset='universal'))
+ >>> finder.nbest(bigram_measures.pmi, 5) # doctest: +NORMALIZE_WHITESPACE
+ [(('1,119', 'NUM'), ('votes', 'NOUN')),
+ (('1962', 'NUM'), ("governor's", 'NOUN')),
+ (('637', 'NUM'), ('E.', 'NOUN')),
+ (('Alpharetta', 'NOUN'), ('prison', 'NOUN')),
+ (('Bar', 'NOUN'), ('Association', 'NOUN'))]
+
+Or tags alone:
+
+ >>> finder = BigramCollocationFinder.from_words(t for w, t in
+ ... nltk.corpus.brown.tagged_words('ca01', tagset='universal'))
+ >>> finder.nbest(bigram_measures.pmi, 10) # doctest: +NORMALIZE_WHITESPACE
+ [('PRT', 'VERB'), ('PRON', 'VERB'), ('ADP', 'DET'), ('.', 'PRON'), ('DET', 'ADJ'),
+ ('CONJ', 'PRON'), ('ADP', 'NUM'), ('NUM', '.'), ('ADV', 'ADV'), ('VERB', 'ADV')]
+
+Or spanning intervening words:
+
+ >>> finder = BigramCollocationFinder.from_words(
+ ... nltk.corpus.genesis.words('english-web.txt'),
+ ... window_size = 20)
+ >>> finder.apply_freq_filter(2)
+ >>> ignored_words = nltk.corpus.stopwords.words('english')
+ >>> finder.apply_word_filter(lambda w: len(w) < 3 or w.lower() in ignored_words)
+ >>> finder.nbest(bigram_measures.likelihood_ratio, 10) # doctest: +NORMALIZE_WHITESPACE
+ [(u'chief', u'chief'), (u'became', u'father'), (u'years', u'became'),
+ (u'hundred', u'years'), (u'lived', u'became'), (u'king', u'king'),
+ (u'lived', u'years'), (u'became', u'became'), (u'chief', u'chiefs'),
+ (u'hundred', u'became')]
+
+Finders
+~~~~~~~
+
+The collocations package provides collocation finders which by default
+consider all ngrams in a text as candidate collocations:
+
+ >>> text = "I do not like green eggs and ham, I do not like them Sam I am!"
+ >>> tokens = nltk.wordpunct_tokenize(text)
+ >>> finder = BigramCollocationFinder.from_words(tokens)
+ >>> scored = finder.score_ngrams(bigram_measures.raw_freq)
+ >>> sorted(bigram for bigram, score in scored) # doctest: +NORMALIZE_WHITESPACE
+ [(',', 'I'), ('I', 'am'), ('I', 'do'), ('Sam', 'I'), ('am', '!'),
+ ('and', 'ham'), ('do', 'not'), ('eggs', 'and'), ('green', 'eggs'),
+ ('ham', ','), ('like', 'green'), ('like', 'them'), ('not', 'like'),
+ ('them', 'Sam')]
+
+We could otherwise construct the collocation finder from manually-derived
+FreqDists:
+
+ >>> word_fd = nltk.FreqDist(tokens)
+ >>> bigram_fd = nltk.FreqDist(nltk.bigrams(tokens))
+ >>> finder = BigramCollocationFinder(word_fd, bigram_fd)
+ >>> scored == finder.score_ngrams(bigram_measures.raw_freq)
+ True
+
+A similar interface is provided for trigrams:
+
+ >>> finder = TrigramCollocationFinder.from_words(tokens)
+ >>> scored = finder.score_ngrams(trigram_measures.raw_freq)
+ >>> set(trigram for trigram, score in scored) == set(nltk.trigrams(tokens))
+ True
+
+We may want to select only the top n results:
+
+ >>> sorted(finder.nbest(trigram_measures.raw_freq, 2))
+ [('I', 'do', 'not'), ('do', 'not', 'like')]
+
+Alternatively, we can select those above a minimum score value:
+
+ >>> sorted(finder.above_score(trigram_measures.raw_freq,
+ ... 1.0 / len(tuple(nltk.trigrams(tokens)))))
+ [('I', 'do', 'not'), ('do', 'not', 'like')]
+
+Now spanning intervening words:
+
+ >>> finder = TrigramCollocationFinder.from_words(tokens)
+ >>> finder = TrigramCollocationFinder.from_words(tokens, window_size=4)
+ >>> sorted(finder.nbest(trigram_measures.raw_freq, 4))
+ [('I', 'do', 'like'), ('I', 'do', 'not'), ('I', 'not', 'like'), ('do', 'not', 'like')]
+
+A closer look at the finder's ngram frequencies:
+
+ >>> sorted(finder.ngram_fd.items(), key=lambda t: (-t[1], t[0]))[:10] # doctest: +NORMALIZE_WHITESPACE
+ [(('I', 'do', 'like'), 2), (('I', 'do', 'not'), 2), (('I', 'not', 'like'), 2),
+ (('do', 'not', 'like'), 2), ((',', 'I', 'do'), 1), ((',', 'I', 'not'), 1),
+ ((',', 'do', 'not'), 1), (('I', 'am', '!'), 1), (('Sam', 'I', '!'), 1),
+ (('Sam', 'I', 'am'), 1)]
+
+
+Filtering candidates
+~~~~~~~~~~~~~~~~~~~~
+
+All the ngrams in a text are often too many to be useful when finding
+collocations. It is generally useful to remove some words or punctuation,
+and to require a minimum frequency for candidate collocations.
+
+Given our sample text above, if we remove all trigrams containing personal
+pronouns from candidature, score_ngrams should return 6 less results, and
+'do not like' will be the only candidate which occurs more than once:
+
+ >>> finder = TrigramCollocationFinder.from_words(tokens)
+ >>> len(finder.score_ngrams(trigram_measures.raw_freq))
+ 14
+ >>> finder.apply_word_filter(lambda w: w in ('I', 'me'))
+ >>> len(finder.score_ngrams(trigram_measures.raw_freq))
+ 8
+ >>> sorted(finder.above_score(trigram_measures.raw_freq,
+ ... 1.0 / len(tuple(nltk.trigrams(tokens)))))
+ [('do', 'not', 'like')]
+
+Sometimes a filter is a function on the whole ngram, rather than each word,
+such as if we may permit 'and' to appear in the middle of a trigram, but
+not on either edge:
+
+ >>> finder.apply_ngram_filter(lambda w1, w2, w3: 'and' in (w1, w3))
+ >>> len(finder.score_ngrams(trigram_measures.raw_freq))
+ 6
+
+Finally, it is often important to remove low frequency candidates, as we
+lack sufficient evidence about their significance as collocations:
+
+ >>> finder.apply_freq_filter(2)
+ >>> len(finder.score_ngrams(trigram_measures.raw_freq))
+ 1
+
+Association measures
+~~~~~~~~~~~~~~~~~~~~
+
+A number of measures are available to score collocations or other associations.
+The arguments to measure functions are marginals of a contingency table, in the
+bigram case (n_ii, (n_ix, n_xi), n_xx)::
+
+ w1 ~w1
+ ------ ------
+ w2 | n_ii | n_oi | = n_xi
+ ------ ------
+ ~w2 | n_io | n_oo |
+ ------ ------
+ = n_ix TOTAL = n_xx
+
+We test their calculation using some known values presented in Manning and
+Schutze's text and other papers.
+
+Student's t: examples from Manning and Schutze 5.3.2
+
+ >>> print('%0.4f' % bigram_measures.student_t(8, (15828, 4675), 14307668))
+ 0.9999
+ >>> print('%0.4f' % bigram_measures.student_t(20, (42, 20), 14307668))
+ 4.4721
+
+Chi-square: examples from Manning and Schutze 5.3.3
+
+ >>> print('%0.2f' % bigram_measures.chi_sq(8, (15828, 4675), 14307668))
+ 1.55
+ >>> print('%0.0f' % bigram_measures.chi_sq(59, (67, 65), 571007))
+ 456400
+
+Likelihood ratios: examples from Dunning, CL, 1993
+
+ >>> print('%0.2f' % bigram_measures.likelihood_ratio(110, (2552, 221), 31777))
+ 270.72
+ >>> print('%0.2f' % bigram_measures.likelihood_ratio(8, (13, 32), 31777))
+ 95.29
+
+Pointwise Mutual Information: examples from Manning and Schutze 5.4
+
+ >>> print('%0.2f' % bigram_measures.pmi(20, (42, 20), 14307668))
+ 18.38
+ >>> print('%0.2f' % bigram_measures.pmi(20, (15019, 15629), 14307668))
+ 0.29
+
+TODO: Find authoritative results for trigrams.
+
+Using contingency table values
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+While frequency counts make marginals readily available for collocation
+finding, it is common to find published contingency table values. The
+collocations package therefore provides a wrapper, ContingencyMeasures, which
+wraps an association measures class, providing association measures which
+take contingency values as arguments, (n_ii, n_io, n_oi, n_oo) in the
+bigram case.
+
+ >>> from nltk.metrics import ContingencyMeasures
+ >>> cont_bigram_measures = ContingencyMeasures(bigram_measures)
+ >>> print('%0.2f' % cont_bigram_measures.likelihood_ratio(8, 5, 24, 31740))
+ 95.29
+ >>> print('%0.2f' % cont_bigram_measures.chi_sq(8, 15820, 4667, 14287173))
+ 1.55
+
+Ranking and correlation
+~~~~~~~~~~~~~~~~~~~~~~~
+
+It is useful to consider the results of finding collocations as a ranking, and
+the rankings output using different association measures can be compared using
+the Spearman correlation coefficient.
+
+Ranks can be assigned to a sorted list of results trivially by assigning
+strictly increasing ranks to each result:
+
+ >>> from nltk.metrics.spearman import *
+ >>> results_list = ['item1', 'item2', 'item3', 'item4', 'item5']
+ >>> print(list(ranks_from_sequence(results_list)))
+ [('item1', 0), ('item2', 1), ('item3', 2), ('item4', 3), ('item5', 4)]
+
+If scores are available for each result, we may allow sufficiently similar
+results (differing by no more than rank_gap) to be assigned the same rank:
+
+ >>> results_scored = [('item1', 50.0), ('item2', 40.0), ('item3', 38.0),
+ ... ('item4', 35.0), ('item5', 14.0)]
+ >>> print(list(ranks_from_scores(results_scored, rank_gap=5)))
+ [('item1', 0), ('item2', 1), ('item3', 1), ('item4', 1), ('item5', 4)]
+
+The Spearman correlation coefficient gives a number from -1.0 to 1.0 comparing
+two rankings. A coefficient of 1.0 indicates identical rankings; -1.0 indicates
+exact opposite rankings.
+
+ >>> print('%0.1f' % spearman_correlation(
+ ... ranks_from_sequence(results_list),
+ ... ranks_from_sequence(results_list)))
+ 1.0
+ >>> print('%0.1f' % spearman_correlation(
+ ... ranks_from_sequence(reversed(results_list)),
+ ... ranks_from_sequence(results_list)))
+ -1.0
+ >>> results_list2 = ['item2', 'item3', 'item1', 'item5', 'item4']
+ >>> print('%0.1f' % spearman_correlation(
+ ... ranks_from_sequence(results_list),
+ ... ranks_from_sequence(results_list2)))
+ 0.6
+ >>> print('%0.1f' % spearman_correlation(
+ ... ranks_from_sequence(reversed(results_list)),
+ ... ranks_from_sequence(results_list2)))
+ -0.6
+
+
diff --git a/nltk/test/compat.doctest b/nltk/test/compat.doctest
new file mode 100644
index 0000000..ecf5a9a
--- /dev/null
+++ b/nltk/test/compat.doctest
@@ -0,0 +1,138 @@
+
+=========================================
+NLTK Python 2.x - 3.x Compatibility Layer
+=========================================
+
+NLTK comes with a Python 2.x/3.x compatibility layer, nltk.compat
+(which is loosely based on `six <http://packages.python.org/six/>`_)::
+
+ >>> from nltk import compat
+ >>> compat.PY3
+ False
+ >>> compat.integer_types
+ (<type 'int'>, <type 'long'>)
+ >>> compat.string_types
+ (<type 'basestring'>,)
+ >>> # and so on
+
+ at python_2_unicode_compatible
+----------------------------
+
+Under Python 2.x ``__str__`` and ``__repr__`` methods must
+return bytestrings.
+
+``@python_2_unicode_compatible`` decorator allows writing these methods
+in a way compatible with Python 3.x:
+
+1) wrap a class with this decorator,
+2) define ``__str__`` and ``__repr__`` methods returning unicode text
+ (that's what they must return under Python 3.x),
+
+and they would be fixed under Python 2.x to return byte strings::
+
+ >>> from nltk.compat import python_2_unicode_compatible
+
+ >>> @python_2_unicode_compatible
+ ... class Foo(object):
+ ... def __str__(self):
+ ... return u'__str__ is called'
+ ... def __repr__(self):
+ ... return u'__repr__ is called'
+
+ >>> foo = Foo()
+ >>> foo.__str__().__class__
+ <type 'str'>
+ >>> foo.__repr__().__class__
+ <type 'str'>
+ >>> print(foo)
+ __str__ is called
+ >>> foo
+ __repr__ is called
+
+Original versions of ``__str__`` and ``__repr__`` are available as
+``__unicode__`` and ``unicode_repr``::
+
+ >>> foo.__unicode__().__class__
+ <type 'unicode'>
+ >>> foo.unicode_repr().__class__
+ <type 'unicode'>
+ >>> unicode(foo)
+ u'__str__ is called'
+ >>> foo.unicode_repr()
+ u'__repr__ is called'
+
+There is no need to wrap a subclass with ``@python_2_unicode_compatible``
+if it doesn't override ``__str__`` and ``__repr__``::
+
+ >>> class Bar(Foo):
+ ... pass
+ >>> bar = Bar()
+ >>> bar.__str__().__class__
+ <type 'str'>
+
+However, if a subclass overrides ``__str__`` or ``__repr__``,
+wrap it again::
+
+ >>> class BadBaz(Foo):
+ ... def __str__(self):
+ ... return u'Baz.__str__'
+ >>> baz = BadBaz()
+ >>> baz.__str__().__class__ # this is incorrect!
+ <type 'unicode'>
+
+ >>> @python_2_unicode_compatible
+ ... class GoodBaz(Foo):
+ ... def __str__(self):
+ ... return u'Baz.__str__'
+ >>> baz = GoodBaz()
+ >>> baz.__str__().__class__
+ <type 'str'>
+ >>> baz.__unicode__().__class__
+ <type 'unicode'>
+
+Applying ``@python_2_unicode_compatible`` to a subclass
+shouldn't break methods that was not overridden::
+
+ >>> baz.__repr__().__class__
+ <type 'str'>
+ >>> baz.unicode_repr().__class__
+ <type 'unicode'>
+
+unicode_repr
+------------
+
+Under Python 3.x ``repr(unicode_string)`` doesn't have a leading "u" letter.
+
+``nltk.compat.unicode_repr`` function may be used instead of ``repr`` and
+``"%r" % obj`` to make the output more consistent under Python 2.x and 3.x::
+
+ >>> from nltk.compat import unicode_repr
+ >>> print(repr(u"test"))
+ u'test'
+ >>> print(unicode_repr(u"test"))
+ 'test'
+
+It may be also used to get an original unescaped repr (as unicode)
+of objects which class was fixed by ``@python_2_unicode_compatible``
+decorator::
+
+ >>> @python_2_unicode_compatible
+ ... class Foo(object):
+ ... def __repr__(self):
+ ... return u'<Foo: foo>'
+
+ >>> foo = Foo()
+ >>> repr(foo)
+ '<Foo: foo>'
+ >>> unicode_repr(foo)
+ u'<Foo: foo>'
+
+For other objects it returns the same value as ``repr``::
+
+ >>> unicode_repr(5)
+ '5'
+
+It may be a good idea to use ``unicode_repr`` instead of ``%r``
+string formatting specifier inside ``__repr__`` or ``__str__``
+methods of classes fixed by ``@python_2_unicode_compatible``
+to make the output consistent between Python 2.x and 3.x.
diff --git a/nltk/test/compat_fixt.py b/nltk/test/compat_fixt.py
new file mode 100644
index 0000000..6268cf0
--- /dev/null
+++ b/nltk/test/compat_fixt.py
@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from nltk.compat import PY3
+
+def setup_module(module):
+ from nose import SkipTest
+ if PY3:
+ raise SkipTest("compat.doctest is for Python 2.x")
diff --git a/nltk/test/corpus.doctest b/nltk/test/corpus.doctest
new file mode 100644
index 0000000..23bb290
--- /dev/null
+++ b/nltk/test/corpus.doctest
@@ -0,0 +1,2048 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+================
+ Corpus Readers
+================
+
+The `nltk.corpus` package defines a collection of *corpus reader*
+classes, which can be used to access the contents of a diverse set of
+corpora. The list of available corpora is given at:
+
+http://nltk.googlecode.com/svn/trunk/nltk_data/index.xml
+
+Each corpus reader class is specialized to handle a specific
+corpus format. In addition, the `nltk.corpus` package automatically
+creates a set of corpus reader instances that can be used to access
+the corpora in the NLTK data package.
+Section `Corpus Reader Objects`_ ("Corpus Reader Objects") describes
+the corpus reader instances that can be used to read the corpora in
+the NLTK data package. Section `Corpus Reader Classes`_ ("Corpus
+Reader Classes") describes the corpus reader classes themselves, and
+discusses the issues involved in creating new corpus reader objects
+and new corpus reader classes. Section `Regression Tests`_
+("Regression Tests") contains regression tests for the corpus readers
+and associated functions and classes.
+
+.. contents:: **Table of Contents**
+ :depth: 2
+ :backlinks: none
+
+---------------------
+Corpus Reader Objects
+---------------------
+
+Overview
+========
+
+NLTK includes a diverse set of corpora which can be
+read using the ``nltk.corpus`` package. Each corpus is accessed by
+means of a "corpus reader" object from ``nltk.corpus``:
+
+ >>> import nltk.corpus
+ >>> # The Brown corpus:
+ >>> print(str(nltk.corpus.brown).replace('\\\\','/'))
+ <CategorizedTaggedCorpusReader in '.../corpora/brown'...>
+ >>> # The Penn Treebank Corpus:
+ >>> print(str(nltk.corpus.treebank).replace('\\\\','/'))
+ <BracketParseCorpusReader in '.../corpora/treebank/combined'...>
+ >>> # The Name Genders Corpus:
+ >>> print(str(nltk.corpus.names).replace('\\\\','/'))
+ <WordListCorpusReader in '.../corpora/names'...>
+ >>> # The Inaugural Address Corpus:
+ >>> print(str(nltk.corpus.inaugural).replace('\\\\','/'))
+ <PlaintextCorpusReader in '.../corpora/inaugural'...>
+
+Most corpora consist of a set of files, each containing a document (or
+other pieces of text). A list of identifiers for these files is
+accessed via the ``fileids()`` method of the corpus reader:
+
+ >>> nltk.corpus.treebank.fileids() # doctest: +ELLIPSIS
+ ['wsj_0001.mrg', 'wsj_0002.mrg', 'wsj_0003.mrg', 'wsj_0004.mrg', ...]
+ >>> nltk.corpus.inaugural.fileids() # doctest: +ELLIPSIS
+ ['1789-Washington.txt', '1793-Washington.txt', '1797-Adams.txt', ...]
+
+Each corpus reader provides a variety of methods to read data from the
+corpus, depending on the format of the corpus. For example, plaintext
+corpora support methods to read the corpus as raw text, a list of
+words, a list of sentences, or a list of paragraphs.
+
+ >>> from nltk.corpus import inaugural
+ >>> inaugural.raw('1789-Washington.txt') # doctest: +ELLIPSIS
+ 'Fellow-Citizens of the Senate ...'
+ >>> inaugural.words('1789-Washington.txt')
+ ['Fellow', '-', 'Citizens', 'of', 'the', ...]
+ >>> inaugural.sents('1789-Washington.txt') # doctest: +ELLIPSIS
+ [['Fellow', '-', 'Citizens'...], ['Among', 'the', 'vicissitudes'...]...]
+ >>> inaugural.paras('1789-Washington.txt') # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [[['Fellow', '-', 'Citizens'...]],
+ [['Among', 'the', 'vicissitudes'...],
+ ['On', 'the', 'one', 'hand', ',', 'I'...]...]...]
+
+Each of these reader methods may be given a single document's item
+name or a list of document item names. When given a list of document
+item names, the reader methods will concatenate together the contents
+of the individual documents.
+
+ >>> l1 = len(inaugural.words('1789-Washington.txt'))
+ >>> l2 = len(inaugural.words('1793-Washington.txt'))
+ >>> l3 = len(inaugural.words(['1789-Washington.txt', '1793-Washington.txt']))
+ >>> print('%s+%s == %s' % (l1, l2, l3))
+ 1538+147 == 1685
+
+If the reader methods are called without any arguments, they will
+typically load all documents in the corpus.
+
+ >>> len(inaugural.words())
+ 145735
+
+If a corpus contains a README file, it can be accessed with a ``readme()`` method:
+
+ >>> inaugural.readme()[:32]
+ 'C-Span Inaugural Address Corpus\n'
+
+Plaintext Corpora
+=================
+
+Here are the first few words from each of NLTK's plaintext corpora:
+
+ >>> nltk.corpus.abc.words()
+ ['PM', 'denies', 'knowledge', 'of', 'AWB', ...]
+ >>> nltk.corpus.genesis.words()
+ [u'In', u'the', u'beginning', u'God', u'created', ...]
+ >>> nltk.corpus.gutenberg.words(fileids='austen-emma.txt')
+ ['[', 'Emma', 'by', 'Jane', 'Austen', '1816', ...]
+ >>> nltk.corpus.inaugural.words()
+ ['Fellow', '-', 'Citizens', 'of', 'the', ...]
+ >>> nltk.corpus.state_union.words()
+ ['PRESIDENT', 'HARRY', 'S', '.', 'TRUMAN', "'", ...]
+ >>> nltk.corpus.webtext.words()
+ ['Cookie', 'Manager', ':', '"', 'Don', "'", 't', ...]
+
+Tagged Corpora
+==============
+
+In addition to the plaintext corpora, NLTK's data package also
+contains a wide variety of annotated corpora. For example, the Brown
+Corpus is annotated with part-of-speech tags, and defines additional
+methods ``tagged_*()`` which words as `(word,tag)` tuples, rather
+than just bare word strings.
+
+ >>> from nltk.corpus import brown
+ >>> print(brown.words())
+ ['The', 'Fulton', 'County', 'Grand', 'Jury', ...]
+ >>> print(brown.tagged_words())
+ [('The', 'AT'), ('Fulton', 'NP-TL'), ...]
+ >>> print(brown.sents()) # doctest: +ELLIPSIS
+ [['The', 'Fulton', 'County'...], ['The', 'jury', 'further'...], ...]
+ >>> print(brown.tagged_sents()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [[('The', 'AT'), ('Fulton', 'NP-TL')...],
+ [('The', 'AT'), ('jury', 'NN'), ('further', 'RBR')...]...]
+ >>> print(brown.paras(categories='reviews')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [[['It', 'is', 'not', 'news', 'that', 'Nathan', 'Milstein'...],
+ ['Certainly', 'not', 'in', 'Orchestra', 'Hall', 'where'...]],
+ [['There', 'was', 'about', 'that', 'song', 'something', ...],
+ ['Not', 'the', 'noblest', 'performance', 'we', 'have', ...], ...], ...]
+ >>> print(brown.tagged_paras(categories='reviews')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [[[('It', 'PPS'), ('is', 'BEZ'), ('not', '*'), ...],
+ [('Certainly', 'RB'), ('not', '*'), ('in', 'IN'), ...]],
+ [[('There', 'EX'), ('was', 'BEDZ'), ('about', 'IN'), ...],
+ [('Not', '*'), ('the', 'AT'), ('noblest', 'JJT'), ...], ...], ...]
+
+Similarly, the Indian Langauge POS-Tagged Corpus includes samples of
+Indian text annotated with part-of-speech tags:
+
+ >>> from nltk.corpus import indian
+ >>> print(indian.words()) # doctest: +SKIP
+ ['\xe0\xa6\xae\xe0\xa6\xb9\xe0\xa6\xbf\...',
+ '\xe0\xa6\xb8\xe0\xa6\xa8\xe0\xa7\x8d\xe0...', ...]
+ >>> print(indian.tagged_words()) # doctest: +SKIP
+ [('\xe0\xa6\xae\xe0\xa6\xb9\xe0\xa6\xbf...', 'NN'),
+ ('\xe0\xa6\xb8\xe0\xa6\xa8\xe0\xa7\x8d\xe0...', 'NN'), ...]
+
+Several tagged corpora support access to a simplified, universal tagset, e.g. where all nouns
+tags are collapsed to a single category ``NOUN``:
+
+ >>> print(brown.tagged_sents(tagset='universal')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [[('The', 'DET'), ('Fulton', 'NOUN'), ('County', 'NOUN'), ('Grand', 'ADJ'), ('Jury', 'NOUN'), ...],
+ [('The', 'DET'), ('jury', 'NOUN'), ('further', 'ADV'), ('said', 'VERB'), ('in', 'ADP'), ...]...]
+ >>> from nltk.corpus import conll2000, switchboard
+ >>> print(conll2000.tagged_words(tagset='universal')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [('Confidence', 'NOUN'), ('in', 'ADP'), ...]
+
+Use ``nltk.app.pos_concordance()`` to access a GUI for searching tagged corpora.
+
+Chunked Corpora
+===============
+
+The CoNLL corpora also provide chunk structures, which are encoded as
+flat trees. The CoNLL 2000 Corpus includes phrasal chunks; and the
+CoNLL 2002 Corpus includes named entity chunks.
+
+ >>> from nltk.corpus import conll2000, conll2002
+ >>> print(conll2000.sents()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [['Confidence', 'in', 'the', 'pound', 'is', 'widely', ...],
+ ['Chancellor', 'of', 'the', 'Exchequer', ...], ...]
+ >>> for tree in conll2000.chunked_sents()[:2]:
+ ... print(tree) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ (S
+ (NP Confidence/NN)
+ (PP in/IN)
+ (NP the/DT pound/NN)
+ (VP is/VBZ widely/RB expected/VBN to/TO take/VB)
+ (NP another/DT sharp/JJ dive/NN)
+ if/IN
+ ...)
+ (S
+ Chancellor/NNP
+ (PP of/IN)
+ (NP the/DT Exchequer/NNP)
+ ...)
+ >>> print(conll2002.sents()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [[u'Sao', u'Paulo', u'(', u'Brasil', u')', u',', ...], [u'-'], ...]
+ >>> for tree in conll2002.chunked_sents()[:2]:
+ ... print(tree) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ (S
+ (LOC Sao/NC Paulo/VMI)
+ (/Fpa
+ (LOC Brasil/NC)
+ )/Fpt
+ ...)
+ (S -/Fg)
+
+.. note:: Since the CONLL corpora do not contain paragraph break
+ information, these readers do not support the ``para()`` method.)
+
+.. warning:: if you call the conll corpora reader methods without any
+ arguments, they will return the contents of the entire corpus,
+ *including* the 'test' portions of the corpus.)
+
+SemCor is a subset of the Brown corpus tagged with WordNet senses and
+named entities. Both kinds of lexical items include multiword units,
+which are encoded as chunks (senses and part-of-speech tags pertain
+to the entire chunk).
+
+ >>> from nltk.corpus import semcor
+ >>> semcor.words()
+ ['The', 'Fulton', 'County', 'Grand', 'Jury', ...]
+ >>> semcor.chunks()
+ [['The'], ['Fulton', 'County', 'Grand', 'Jury'], ...]
+ >>> semcor.sents() # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [['The', 'Fulton', 'County', 'Grand', 'Jury', 'said', ...],
+ ['The', 'jury', 'further', 'said', ...], ...]
+ >>> semcor.chunk_sents() # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [[['The'], ['Fulton', 'County', 'Grand', 'Jury'], ['said'], ...
+ ['.']], [['The'], ['jury'], ['further'], ['said'], ... ['.']], ...]
+ >>> list(map(str, semcor.tagged_chunks(tag='both')[:3]))
+ ['(DT The)', "(Lemma('group.n.01.group') (NE (NNP Fulton County Grand Jury)))", "(Lemma('state.v.01.say') (VB said))"]
+ >>> [[str(c) for c in s] for s in semcor.tagged_sents(tag='both')[:2]]
+ [['(DT The)', "(Lemma('group.n.01.group') (NE (NNP Fulton County Grand Jury)))", ...
+ '(None .)'], ['(DT The)', ... '(None .)']]
+
+
+The IEER corpus is another chunked corpus. This corpus is unusual in
+that each corpus item contains multiple documents. (This reflects the
+fact that each corpus file contains multiple documents.) The IEER
+corpus defines the `parsed_docs` method, which returns the documents
+in a given item as `IEERDocument` objects:
+
+ >>> from nltk.corpus import ieer
+ >>> ieer.fileids() # doctest: +NORMALIZE_WHITESPACE
+ ['APW_19980314', 'APW_19980424', 'APW_19980429',
+ 'NYT_19980315', 'NYT_19980403', 'NYT_19980407']
+ >>> docs = ieer.parsed_docs('APW_19980314')
+ >>> print(docs[0])
+ <IEERDocument APW19980314.0391: 'Kenyans protest tax hikes'>
+ >>> print(docs[0].docno)
+ APW19980314.0391
+ >>> print(docs[0].doctype)
+ NEWS STORY
+ >>> print(docs[0].date_time)
+ 03/14/1998 10:36:00
+ >>> print(docs[0].headline)
+ (DOCUMENT Kenyans protest tax hikes)
+ >>> print(docs[0].text) # doctest: +ELLIPSIS
+ (DOCUMENT
+ (LOCATION NAIROBI)
+ ,
+ (LOCATION Kenya)
+ (
+ (ORGANIZATION AP)
+ )
+ _
+ (CARDINAL Thousands)
+ of
+ laborers,
+ ...
+ on
+ (DATE Saturday)
+ ...)
+
+Parsed Corpora
+==============
+
+The Treebank corpora provide a syntactic parse for each sentence. The
+NLTK data package includes a 10% sample of the Penn Treebank (in
+``treebank``), as well as the Sinica Treebank (in ``sinica_treebank``).
+
+Reading the Penn Treebank (Wall Street Journal sample):
+
+ >>> from nltk.corpus import treebank
+ >>> print(treebank.fileids()) # doctest: +ELLIPSIS
+ ['wsj_0001.mrg', 'wsj_0002.mrg', 'wsj_0003.mrg', 'wsj_0004.mrg', ...]
+ >>> print(treebank.words('wsj_0003.mrg'))
+ ['A', 'form', 'of', 'asbestos', 'once', 'used', ...]
+ >>> print(treebank.tagged_words('wsj_0003.mrg'))
+ [('A', 'DT'), ('form', 'NN'), ('of', 'IN'), ...]
+ >>> print(treebank.parsed_sents('wsj_0003.mrg')[0]) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ (S
+ (S-TPC-1
+ (NP-SBJ
+ (NP (NP (DT A) (NN form)) (PP (IN of) (NP (NN asbestos))))
+ (RRC ...)...)...)
+ ...
+ (VP (VBD reported) (SBAR (-NONE- 0) (S (-NONE- *T*-1))))
+ (. .))
+
+If you have access to a full installation of the Penn Treebank, NLTK
+can be configured to load it as well. Download the ``ptb`` package,
+and in the directory ``nltk_data/corpora/ptb`` place the ``BROWN``
+and ``WSJ`` directories of the Treebank installation (symlinks work
+as well). Then use the ``ptb`` module instead of ``treebank``:
+
+ >>> from nltk.corpus import ptb
+ >>> print(ptb.fileids()) # doctest: +SKIP
+ ['BROWN/CF/CF01.MRG', 'BROWN/CF/CF02.MRG', 'BROWN/CF/CF03.MRG', 'BROWN/CF/CF04.MRG', ...]
+ >>> print(ptb.words('WSJ/00/WSJ_0003.MRG')) # doctest: +SKIP
+ ['A', 'form', 'of', 'asbestos', 'once', 'used', '*', ...]
+ >>> print(ptb.tagged_words('WSJ/00/WSJ_0003.MRG')) # doctest: +SKIP
+ [('A', 'DT'), ('form', 'NN'), ('of', 'IN'), ...]
+
+...and so forth, like ``treebank`` but with extended fileids. Categories
+specified in ``allcats.txt`` can be used to filter by genre; they consist
+of ``news`` (for WSJ articles) and names of the Brown subcategories
+(``fiction``, ``humor``, ``romance``, etc.):
+
+ >>> ptb.categories() # doctest: +SKIP
+ ['adventure', 'belles_lettres', 'fiction', 'humor', 'lore', 'mystery', 'news', 'romance', 'science_fiction']
+ >>> print(ptb.fileids('news')) # doctest: +SKIP
+ ['WSJ/00/WSJ_0001.MRG', 'WSJ/00/WSJ_0002.MRG', 'WSJ/00/WSJ_0003.MRG', ...]
+ >>> print(ptb.words(categories=['humor','fiction'])) # doctest: +SKIP
+ ['Thirty-three', 'Scotty', 'did', 'not', 'go', 'back', ...]
+
+As PropBank and NomBank depend on the (WSJ portion of the) Penn Treebank,
+the modules ``propbank_ptb`` and ``nombank_ptb`` are provided for access
+to a full PTB installation.
+
+Reading the Sinica Treebank:
+
+ >>> from nltk.corpus import sinica_treebank
+ >>> print(sinica_treebank.sents()) # doctest: +SKIP
+ [['\xe4\xb8\x80'], ['\xe5\x8f\x8b\xe6\x83\x85'], ...]
+ >>> sinica_treebank.parsed_sents()[25] # doctest: +SKIP
+ Tree('S',
+ [Tree('NP',
+ [Tree('Nba', ['\xe5\x98\x89\xe7\x8f\x8d'])]),
+ Tree('V\xe2\x80\xa7\xe5\x9c\xb0',
+ [Tree('VA11', ['\xe4\xb8\x8d\xe5\x81\x9c']),
+ Tree('DE', ['\xe7\x9a\x84'])]),
+ Tree('VA4', ['\xe5\x93\xad\xe6\xb3\xa3'])])
+
+Reading the CoNLL 2007 Dependency Treebanks:
+
+ >>> from nltk.corpus import conll2007
+ >>> conll2007.sents('esp.train')[0] # doctest: +SKIP
+ ['El', 'aumento', 'del', 'índice', 'de', 'desempleo', ...]
+ >>> conll2007.parsed_sents('esp.train')[0] # doctest: +SKIP
+ <DependencyGraph with 38 nodes>
+ >>> print(conll2007.parsed_sents('esp.train')[0].tree()) # doctest: +SKIP
+ (fortaleció
+ (aumento El (del (índice (de (desempleo estadounidense)))))
+ hoy
+ considerablemente
+ (al
+ (euro
+ (cotizaba
+ ,
+ que
+ (a (15.35 las GMT))
+ se
+ (en (mercado el (de divisas) (de Fráncfort)))
+ (a 0,9452_dólares)
+ (frente_a , (0,9349_dólares los (de (mañana esta)))))))
+ .)
+
+NLTK also provides a corpus reader for the York-Toronto-Helsinki
+Parsed Corpus of Old English Prose (YCOE); but the corpus itself is
+not included in the NLTK data package. If you install it yourself,
+you can use NLTK to access it:
+
+ >>> from nltk.corpus import ycoe
+ >>> for tree in ycoe.parsed_sents('cocuraC')[:4]:
+ ... print(tree) # doctest: +SKIP
+ (CP-THT
+ (C +D+atte)
+ (IP-SUB ...)
+ ...
+ (. .))
+ (IP-MAT
+ (IP-MAT-0
+ (PP (P On) (NP (ADJ o+dre) (N wisan)))...)
+ ...
+ (. .))
+ (IP-MAT
+ (NP-NOM-x-2 *exp*)
+ (NP-DAT-1 (D^D +D+am) (ADJ^D unge+dyldegum))
+ ...
+ (. .))
+ (IP-MAT
+ (ADVP (ADV Sw+a))
+ (NP-NOM-x (PRO^N hit))
+ (ADVP-TMP (ADV^T oft))
+ ...
+ (. .))
+
+If the YCOE corpus is not available, you will get an error message
+when you try to access it:
+
+ >>> from nltk.corpus import ycoe
+ >>> print(ycoe) # doctest: +SKIP
+ Traceback (most recent call last):
+ LookupError:
+ **********************************************************************
+ Resource 'corpora/ycoe' not found. For installation
+ instructions, please see <http://nltk.org/index.php/Installation>.
+ Searched in:
+ - ...
+ **********************************************************************
+
+Word Lists and Lexicons
+=======================
+
+The NLTK data package also includes a number of lexicons and word
+lists. These are accessed just like text corpora. The following
+examples illustrate the use of the wordlist corpora:
+
+ >>> from nltk.corpus import names, stopwords, words
+ >>> words.fileids()
+ ['en', 'en-basic']
+ >>> words.words('en') # doctest: +ELLIPSIS
+ ['A', 'a', 'aa', 'aal', 'aalii', 'aam', 'Aani', 'aardvark', 'aardwolf', ...]
+
+ >>> stopwords.fileids() # doctest: +ELLIPSIS
+ ['danish', 'dutch', 'english', 'finnish', 'french', 'german', 'hungarian', ...]
+ >>> stopwords.words('portuguese') # doctest: +ELLIPSIS
+ ['de', 'a', 'o', 'que', 'e', 'do', 'da', 'em', 'um', 'para', ...]
+ >>> names.fileids()
+ ['female.txt', 'male.txt']
+ >>> names.words('male.txt') # doctest: +ELLIPSIS
+ ['Aamir', 'Aaron', 'Abbey', 'Abbie', 'Abbot', 'Abbott', ...]
+ >>> names.words('female.txt') # doctest: +ELLIPSIS
+ ['Abagael', 'Abagail', 'Abbe', 'Abbey', 'Abbi', 'Abbie', ...]
+
+The CMU Pronunciation Dictionary corpus contains pronounciation
+transcriptions for over 100,000 words. It can be accessed as a list
+of entries (where each entry consists of a word, an identifier, and a
+transcription) or as a dictionary from words to lists of
+transcriptions. Transcriptions are encoded as tuples of phoneme
+strings.
+
+ >>> from nltk.corpus import cmudict
+ >>> print(cmudict.entries()[653:659]) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [('acetate', ['AE1', 'S', 'AH0', 'T', 'EY2', 'T']),
+ ('acetic', ['AH0', 'S', 'EH1', 'T', 'IH0', 'K']),
+ ('acetic', ['AH0', 'S', 'IY1', 'T', 'IH0', 'K']),
+ ('aceto', ['AA0', 'S', 'EH1', 'T', 'OW0']),
+ ('acetochlor', ['AA0', 'S', 'EH1', 'T', 'OW0', 'K', 'L', 'AO2', 'R']),
+ ('acetone', ['AE1', 'S', 'AH0', 'T', 'OW2', 'N'])]
+ >>> # Load the entire cmudict corpus into a Python dictionary:
+ >>> transcr = cmudict.dict()
+ >>> print([transcr[w][0] for w in 'Natural Language Tool Kit'.lower().split()]) # doctest: +NORMALIZE_WHITESPACE
+ [['N', 'AE1', 'CH', 'ER0', 'AH0', 'L'],
+ ['L', 'AE1', 'NG', 'G', 'W', 'AH0', 'JH'],
+ ['T', 'UW1', 'L'],
+ ['K', 'IH1', 'T']]
+
+
+WordNet
+=======
+
+Please see the separate WordNet howto.
+
+FrameNet
+========
+
+Please see the separate FrameNet howto.
+
+PropBank
+========
+
+Please see the separate PropBank howto.
+
+SentiWordNet
+============
+
+Please see the separate SentiWordNet howto.
+
+Categorized Corpora
+===================
+
+Several corpora included with NLTK contain documents that have been categorized for
+topic, genre, polarity, etc. In addition to the standard corpus interface, these
+corpora provide access to the list of categories and the mapping between the documents
+and their categories (in both directions). Access the categories using the ``categories()``
+method, e.g.:
+
+ >>> from nltk.corpus import brown, movie_reviews, reuters
+ >>> brown.categories() # doctest: +NORMALIZE_WHITESPACE
+ ['adventure', 'belles_lettres', 'editorial', 'fiction', 'government', 'hobbies', 'humor',
+ 'learned', 'lore', 'mystery', 'news', 'religion', 'reviews', 'romance', 'science_fiction']
+ >>> movie_reviews.categories()
+ ['neg', 'pos']
+ >>> reuters.categories() # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+ ['acq', 'alum', 'barley', 'bop', 'carcass', 'castor-oil', 'cocoa',
+ 'coconut', 'coconut-oil', 'coffee', 'copper', 'copra-cake', 'corn',
+ 'cotton', 'cotton-oil', 'cpi', 'cpu', 'crude', 'dfl', 'dlr', ...]
+
+This method has an optional argument that specifies a document or a list
+of documents, allowing us to map from (one or more) documents to (one or more) categories:
+
+ >>> brown.categories('ca01')
+ ['news']
+ >>> brown.categories(['ca01','cb01'])
+ ['editorial', 'news']
+ >>> reuters.categories('training/9865')
+ ['barley', 'corn', 'grain', 'wheat']
+ >>> reuters.categories(['training/9865', 'training/9880'])
+ ['barley', 'corn', 'grain', 'money-fx', 'wheat']
+
+We can go back the other way using the optional argument of the ``fileids()`` method:
+
+ >>> reuters.fileids('barley') # doctest: +ELLIPSIS
+ ['test/15618', 'test/15649', 'test/15676', 'test/15728', 'test/15871', ...]
+
+Both the ``categories()`` and ``fileids()`` methods return a sorted list containing
+no duplicates.
+
+In addition to mapping between categories and documents, these corpora permit
+direct access to their contents via the categories. Instead of accessing a subset
+of a corpus by specifying one or more fileids, we can identify one or more categories, e.g.:
+
+ >>> brown.tagged_words(categories='news')
+ [('The', 'AT'), ('Fulton', 'NP-TL'), ...]
+ >>> brown.sents(categories=['editorial','reviews']) # doctest: +NORMALIZE_WHITESPACE
+ [['Assembly', 'session', 'brought', 'much', 'good'], ['The', 'General',
+ 'Assembly', ',', 'which', 'adjourns', 'today', ',', 'has', 'performed',
+ 'in', 'an', 'atmosphere', 'of', 'crisis', 'and', 'struggle', 'from',
+ 'the', 'day', 'it', 'convened', '.'], ...]
+
+Note that it is an error to specify both documents and categories.
+
+In the context of a text categorization system, we can easily test if the
+category assigned to a document is correct as follows:
+
+ >>> def classify(doc): return 'news' # Trivial classifier
+ >>> doc = 'ca01'
+ >>> classify(doc) in brown.categories(doc)
+ True
+
+
+Other Corpora
+=============
+
+senseval
+--------
+The Senseval 2 corpus is a word sense disambiguation corpus. Each
+item in the corpus corresponds to a single ambiguous word. For each
+of these words, the corpus contains a list of instances, corresponding
+to occurences of that word. Each instance provides the word; a list
+of word senses that apply to the word occurrence; and the word's
+context.
+
+ >>> from nltk.corpus import senseval
+ >>> senseval.fileids()
+ ['hard.pos', 'interest.pos', 'line.pos', 'serve.pos']
+ >>> senseval.instances('hard.pos')
+ ... # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [SensevalInstance(word='hard-a',
+ position=20,
+ context=[('``', '``'), ('he', 'PRP'), ...('hard', 'JJ'), ...],
+ senses=('HARD1',)),
+ SensevalInstance(word='hard-a',
+ position=10,
+ context=[('clever', 'NNP'), ...('hard', 'JJ'), ('time', 'NN'), ...],
+ senses=('HARD1',)), ...]
+
+The following code looks at instances of the word 'interest', and
+displays their local context (2 words on each side) and word sense(s):
+
+ >>> for inst in senseval.instances('interest.pos')[:10]:
+ ... p = inst.position
+ ... left = ' '.join(w for (w,t) in inst.context[p-2:p])
+ ... word = ' '.join(w for (w,t) in inst.context[p:p+1])
+ ... right = ' '.join(w for (w,t) in inst.context[p+1:p+3])
+ ... senses = ' '.join(inst.senses)
+ ... print('%20s |%10s | %-15s -> %s' % (left, word, right, senses))
+ declines in | interest | rates . -> interest_6
+ indicate declining | interest | rates because -> interest_6
+ in short-term | interest | rates . -> interest_6
+ 4 % | interest | in this -> interest_5
+ company with | interests | in the -> interest_5
+ , plus | interest | . -> interest_6
+ set the | interest | rate on -> interest_6
+ 's own | interest | , prompted -> interest_4
+ principal and | interest | is the -> interest_6
+ increase its | interest | to 70 -> interest_5
+
+ppattach
+--------
+The Prepositional Phrase Attachment corpus is a corpus of
+prepositional phrase attachment decisions. Each instance in the
+corpus is encoded as a ``PPAttachment`` object:
+
+ >>> from nltk.corpus import ppattach
+ >>> ppattach.attachments('training') # doctest: +NORMALIZE_WHITESPACE
+ [PPAttachment(sent='0', verb='join', noun1='board',
+ prep='as', noun2='director', attachment='V'),
+ PPAttachment(sent='1', verb='is', noun1='chairman',
+ prep='of', noun2='N.V.', attachment='N'),
+ ...]
+ >>> inst = ppattach.attachments('training')[0]
+ >>> (inst.sent, inst.verb, inst.noun1, inst.prep, inst.noun2)
+ ('0', 'join', 'board', 'as', 'director')
+ >>> inst.attachment
+ 'V'
+
+semcor
+------
+The Brown Corpus, annotated with WordNet senses.
+
+ >>> from nltk.corpus import semcor
+ >>> semcor.words('brown2/tagfiles/br-n12.xml') # doctest: +ELLIPSIS
+ ['When', 'several', 'minutes', 'had', 'passed', ...]
+ >>> sent = semcor.xml('brown2/tagfiles/br-n12.xml').findall('context/p/s')[0]
+ >>> for wordform in sent.getchildren():
+ ... print(wordform.text, end=' ')
+ ... for key in sorted(wordform.keys()):
+ ... print(key + '=' + wordform.get(key), end=' ')
+ ... print()
+ ...
+ When cmd=ignore pos=WRB
+ several cmd=done lemma=several lexsn=5:00:00:some(a):00 pos=JJ wnsn=1
+ minutes cmd=done lemma=minute lexsn=1:28:00:: pos=NN wnsn=1
+ had cmd=done ot=notag pos=VBD
+ passed cmd=done lemma=pass lexsn=2:38:03:: pos=VB wnsn=4
+ and cmd=ignore pos=CC
+ Curt cmd=done lemma=person lexsn=1:03:00:: pn=person pos=NNP rdf=person wnsn=1
+ had cmd=done ot=notag pos=VBD
+ n't cmd=done lemma=n't lexsn=4:02:00:: pos=RB wnsn=0
+ emerged cmd=done lemma=emerge lexsn=2:30:00:: pos=VB wnsn=1
+ from cmd=ignore pos=IN
+ the cmd=ignore pos=DT
+ livery_stable cmd=done lemma=livery_stable lexsn=1:06:00:: pos=NN wnsn=1
+ ,
+ Brenner cmd=done lemma=person lexsn=1:03:00:: pn=person pos=NNP rdf=person wnsn=1
+ re-entered cmd=done lemma=re-enter lexsn=2:38:00:: pos=VB wnsn=1
+ the cmd=ignore pos=DT
+ hotel cmd=done lemma=hotel lexsn=1:06:00:: pos=NN wnsn=1
+ and cmd=ignore pos=CC
+ faced cmd=done lemma=face lexsn=2:42:02:: pos=VB wnsn=4
+ Summers cmd=done lemma=person lexsn=1:03:00:: pn=person pos=NNP rdf=person wnsn=1
+ across cmd=ignore pos=IN
+ the cmd=ignore pos=DT
+ counter cmd=done lemma=counter lexsn=1:06:00:: pos=NN wnsn=1
+ .
+
+shakespeare
+-----------
+The Shakespeare corpus contains a set of Shakespeare plays, formatted
+as XML files. These corpora are returned as ElementTree objects:
+
+ >>> from nltk.corpus import shakespeare
+ >>> from xml.etree import ElementTree
+ >>> shakespeare.fileids() # doctest: +ELLIPSIS
+ ['a_and_c.xml', 'dream.xml', 'hamlet.xml', 'j_caesar.xml', ...]
+ >>> play = shakespeare.xml('dream.xml')
+ >>> print(play) # doctest: +ELLIPSIS
+ <Element 'PLAY' at ...>
+ >>> print('%s: %s' % (play[0].tag, play[0].text))
+ TITLE: A Midsummer Night's Dream
+ >>> personae = [persona.text for persona in
+ ... play.findall('PERSONAE/PERSONA')]
+ >>> print(personae) # doctest: +ELLIPSIS
+ ['THESEUS, Duke of Athens.', 'EGEUS, father to Hermia.', ...]
+ >>> # Find and print speakers not listed as personae
+ >>> names = [persona.split(',')[0] for persona in personae]
+ >>> speakers = set(speaker.text for speaker in
+ ... play.findall('*/*/*/SPEAKER'))
+ >>> print(sorted(speakers.difference(names))) # doctest: +NORMALIZE_WHITESPACE
+ ['ALL', 'COBWEB', 'DEMETRIUS', 'Fairy', 'HERNIA', 'LYSANDER',
+ 'Lion', 'MOTH', 'MUSTARDSEED', 'Moonshine', 'PEASEBLOSSOM',
+ 'Prologue', 'Pyramus', 'Thisbe', 'Wall']
+
+toolbox
+-------
+The Toolbox corpus distributed with NLTK contains a sample lexicon and
+several sample texts from the Rotokas language. The Toolbox corpus
+reader returns Toolbox files as XML ElementTree objects. The
+following example loads the Rotokas dictionary, and figures out the
+distribution of part-of-speech tags for reduplicated words.
+
+.. doctest: +SKIP
+
+ >>> from nltk.corpus import toolbox
+ >>> from nltk.probability import FreqDist
+ >>> from xml.etree import ElementTree
+ >>> import re
+ >>> rotokas = toolbox.xml('rotokas.dic')
+ >>> redup_pos_freqdist = FreqDist()
+ >>> # Note: we skip over the first record, which is actually
+ >>> # the header.
+ >>> for record in rotokas[1:]:
+ ... lexeme = record.find('lx').text
+ ... if re.match(r'(.*)\1$', lexeme):
+ ... redup_pos_freqdist[record.find('ps').text] += 1
+ >>> for item, count in redup_pos_freqdist.most_common():
+ ... print(item, count)
+ V 41
+ N 14
+ ??? 4
+
+This example displays some records from a Rotokas text:
+
+.. doctest: +SKIP
+
+ >>> river = toolbox.xml('rotokas/river.txt', key='ref')
+ >>> for record in river.findall('record')[:3]:
+ ... for piece in record:
+ ... if len(piece.text) > 60:
+ ... print('%-6s %s...' % (piece.tag, piece.text[:57]))
+ ... else:
+ ... print('%-6s %s' % (piece.tag, piece.text))
+ ref Paragraph 1
+ t ``Viapau oisio ra ovaupasi ...
+ m viapau oisio ra ovau -pa -si ...
+ g NEG this way/like this and forget -PROG -2/3.DL...
+ p NEG ??? CONJ V.I -SUFF.V.3 -SUFF.V...
+ f ``No ken lus tingting wanema samting papa i bin tok,'' Na...
+ fe ``Don't forget what Dad said,'' yelled Naomi.
+ ref 2
+ t Osa Ira ora Reviti viapau uvupasiva.
+ m osa Ira ora Reviti viapau uvu -pa -si ...
+ g as/like name and name NEG hear/smell -PROG -2/3...
+ p CONJ N.PN CONJ N.PN NEG V.T -SUFF.V.3 -SUF...
+ f Tasol Ila na David no bin harim toktok.
+ fe But Ila and David took no notice.
+ ref 3
+ t Ikaupaoro rokosiva ...
+ m ikau -pa -oro roko -si -va ...
+ g run/hurry -PROG -SIM go down -2/3.DL.M -RP ...
+ p V.T -SUFF.V.3 -SUFF.V.4 ADV -SUFF.V.4 -SUFF.VT....
+ f Tupela i bin hariap i go long wara .
+ fe They raced to the river.
+
+timit
+-----
+The NLTK data package includes a fragment of the TIMIT
+Acoustic-Phonetic Continuous Speech Corpus. This corpus is broken
+down into small speech samples, each of which is available as a wave
+file, a phonetic transcription, and a tokenized word list.
+
+ >>> from nltk.corpus import timit
+ >>> print(timit.utteranceids()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ ['dr1-fvmh0/sa1', 'dr1-fvmh0/sa2', 'dr1-fvmh0/si1466',
+ 'dr1-fvmh0/si2096', 'dr1-fvmh0/si836', 'dr1-fvmh0/sx116',
+ 'dr1-fvmh0/sx206', 'dr1-fvmh0/sx26', 'dr1-fvmh0/sx296', ...]
+
+ >>> item = timit.utteranceids()[5]
+ >>> print(timit.phones(item)) # doctest: +NORMALIZE_WHITESPACE
+ ['h#', 'k', 'l', 'ae', 's', 'pcl', 'p', 'dh', 'ax',
+ 's', 'kcl', 'k', 'r', 'ux', 'ix', 'nx', 'y', 'ax',
+ 'l', 'eh', 'f', 'tcl', 't', 'hh', 'ae', 'n', 'dcl',
+ 'd', 'h#']
+ >>> print(timit.words(item))
+ ['clasp', 'the', 'screw', 'in', 'your', 'left', 'hand']
+ >>> timit.play(item) # doctest: +SKIP
+
+The corpus reader can combine the word segmentation information with
+the phonemes to produce a single tree structure:
+
+ >>> for tree in timit.phone_trees(item):
+ ... print(tree)
+ (S
+ h#
+ (clasp k l ae s pcl p)
+ (the dh ax)
+ (screw s kcl k r ux)
+ (in ix nx)
+ (your y ax)
+ (left l eh f tcl t)
+ (hand hh ae n dcl d)
+ h#)
+
+The start time and stop time of each phoneme, word, and sentence are
+also available:
+
+ >>> print(timit.phone_times(item)) # doctest: +ELLIPSIS
+ [('h#', 0, 2190), ('k', 2190, 3430), ('l', 3430, 4326), ...]
+ >>> print(timit.word_times(item)) # doctest: +ELLIPSIS
+ [('clasp', 2190, 8804), ('the', 8804, 9734), ...]
+ >>> print(timit.sent_times(item))
+ [('Clasp the screw in your left hand.', 0, 32154)]
+
+We can use these times to play selected pieces of a speech sample:
+
+ >>> timit.play(item, 2190, 8804) # 'clasp' # doctest: +SKIP
+
+The corpus reader can also be queried for information about the
+speaker and sentence identifier for a given speech sample:
+
+ >>> print(timit.spkrid(item))
+ dr1-fvmh0
+ >>> print(timit.sentid(item))
+ sx116
+ >>> print(timit.spkrinfo(timit.spkrid(item))) # doctest: +NORMALIZE_WHITESPACE
+ SpeakerInfo(id='VMH0',
+ sex='F',
+ dr='1',
+ use='TRN',
+ recdate='03/11/86',
+ birthdate='01/08/60',
+ ht='5\'05"',
+ race='WHT',
+ edu='BS',
+ comments='BEST NEW ENGLAND ACCENT SO FAR')
+
+ >>> # List the speech samples from the same speaker:
+ >>> timit.utteranceids(spkrid=timit.spkrid(item)) # doctest: +ELLIPSIS
+ ['dr1-fvmh0/sa1', 'dr1-fvmh0/sa2', 'dr1-fvmh0/si1466', ...]
+
+rte
+---
+The RTE (Recognizing Textual Entailment) corpus was derived from the
+RTE1, RTE2 and RTE3 datasets (dev and test data), and consists of a
+list of XML-formatted 'text'/'hypothesis' pairs.
+
+ >>> from nltk.corpus import rte
+ >>> print(rte.fileids()) # doctest: +ELLIPSIS
+ ['rte1_dev.xml', 'rte1_test.xml', 'rte2_dev.xml', ..., 'rte3_test.xml']
+ >>> rtepairs = rte.pairs(['rte2_test.xml', 'rte3_test.xml'])
+ >>> print(rtepairs) # doctest: +ELLIPSIS
+ [<RTEPair: gid=2-8>, <RTEPair: gid=2-9>, <RTEPair: gid=2-15>, ...]
+
+In the gold standard test sets, each pair is labeled according to
+whether or not the text 'entails' the hypothesis; the
+entailment value is mapped to an integer 1 (True) or 0 (False).
+
+ >>> rtepairs[5]
+ <RTEPair: gid=2-23>
+ >>> rtepairs[5].text # doctest: +NORMALIZE_WHITESPACE
+ 'His wife Strida won a seat in parliament after forging an alliance
+ with the main anti-Syrian coalition in the recent election.'
+ >>> rtepairs[5].hyp
+ 'Strida elected to parliament.'
+ >>> rtepairs[5].value
+ 1
+
+The RTE corpus also supports an ``xml()`` method which produces ElementTrees.
+
+ >>> xmltree = rte.xml('rte3_dev.xml')
+ >>> xmltree # doctest: +SKIP
+ <Element entailment-corpus at ...>
+ >>> xmltree[7].findtext('t') # doctest: +NORMALIZE_WHITESPACE
+ "Mrs. Bush's approval ratings have remained very high, above 80%,
+ even as her husband's have recently dropped below 50%."
+
+verbnet
+-------
+The VerbNet corpus is a lexicon that divides verbs into classes, based
+on their syntax-semantics linking behavior. The basic elements in the
+lexicon are verb lemmas, such as 'abandon' and 'accept', and verb
+classes, which have identifiers such as 'remove-10.1' and
+'admire-31.2-1'. These class identifiers consist of a representitive
+verb selected from the class, followed by a numerical identifier. The
+list of verb lemmas, and the list of class identifiers, can be
+retrieved with the following methods:
+
+ >>> from nltk.corpus import verbnet
+ >>> verbnet.lemmas()[20:25]
+ ['accelerate', 'accept', 'acclaim', 'accompany', 'accrue']
+ >>> verbnet.classids()[:5]
+ ['accompany-51.7', 'admire-31.2', 'admire-31.2-1', 'admit-65', 'adopt-93']
+
+The `classids()` method may also be used to retrieve the classes that
+a given lemma belongs to:
+
+ >>> verbnet.classids('accept')
+ ['approve-77', 'characterize-29.2-1-1', 'obtain-13.5.2']
+
+The primary object in the lexicon is a class record, which is stored
+as an ElementTree xml object. The class record for a given class
+identifier is returned by the `vnclass()` method:
+
+ >>> verbnet.vnclass('remove-10.1') # doctest: +ELLIPSIS
+ <Element 'VNCLASS' at ...>
+
+The `vnclass()` method also accepts "short" identifiers, such as '10.1':
+
+ >>> verbnet.vnclass('10.1') # doctest: +ELLIPSIS
+ <Element 'VNCLASS' at ...>
+
+See the Verbnet documentation, or the Verbnet files, for information
+about the structure of this xml. As an example, we can retrieve a
+list of thematic roles for a given Verbnet class:
+
+ >>> vn_31_2 = verbnet.vnclass('admire-31.2')
+ >>> for themrole in vn_31_2.findall('THEMROLES/THEMROLE'):
+ ... print(themrole.attrib['type'], end=' ')
+ ... for selrestr in themrole.findall('SELRESTRS/SELRESTR'):
+ ... print('[%(Value)s%(type)s]' % selrestr.attrib, end=' ')
+ ... print()
+ Theme
+ Experiencer [+animate]
+ Predicate
+
+The Verbnet corpus also provides a variety of pretty printing
+functions that can be used to display the xml contents in a more
+consise form. The simplest such method is `pprint()`:
+
+ >>> print(verbnet.pprint('57'))
+ weather-57
+ Subclasses: (none)
+ Members: blow clear drizzle fog freeze gust hail howl lightning mist
+ mizzle pelt pour precipitate rain roar shower sleet snow spit spot
+ sprinkle storm swelter teem thaw thunder
+ Thematic roles:
+ * Theme[+concrete +force]
+ Frames:
+ Intransitive (Expletive Subject)
+ Syntax: LEX[it] LEX[[+be]] VERB
+ Semantics:
+ * weather(during(E), Weather_type, ?Theme)
+ NP (Expletive Subject, Theme Object)
+ Syntax: LEX[it] LEX[[+be]] VERB NP[Theme]
+ Semantics:
+ * weather(during(E), Weather_type, Theme)
+ PP (Expletive Subject, Theme-PP)
+ Syntax: LEX[it[+be]] VERB PREP[with] NP[Theme]
+ Semantics:
+ * weather(during(E), Weather_type, Theme)
+
+
+nps_chat
+--------
+
+The NPS Chat Corpus, Release 1.0 consists of over 10,000 posts in age-specific
+chat rooms, which have been anonymized, POS-tagged and dialogue-act tagged.
+
+ >>> print(nltk.corpus.nps_chat.words())
+ ['now', 'im', 'left', 'with', 'this', 'gay', ...]
+ >>> print(nltk.corpus.nps_chat.tagged_words())
+ [('now', 'RB'), ('im', 'PRP'), ('left', 'VBD'), ...]
+ >>> print(nltk.corpus.nps_chat.tagged_posts()) # doctest: +NORMALIZE_WHITESPACE
+ [[('now', 'RB'), ('im', 'PRP'), ('left', 'VBD'), ('with', 'IN'),
+ ('this', 'DT'), ('gay', 'JJ'), ('name', 'NN')], [(':P', 'UH')], ...]
+
+We can access the XML elements corresponding to individual posts. These elements
+have ``class`` and ``user`` attributes that we can access using ``p.attrib['class']``
+and ``p.attrib['user']``. They also have text content, accessed using ``p.text``.
+
+ >>> print(nltk.corpus.nps_chat.xml_posts()) # doctest: +ELLIPSIS
+ [<Element 'Post' at 0...>, <Element 'Post' at 0...>, ...]
+ >>> posts = nltk.corpus.nps_chat.xml_posts()
+ >>> sorted(nltk.FreqDist(p.attrib['class'] for p in posts).keys())
+ ['Accept', 'Bye', 'Clarify', 'Continuer', 'Emotion', 'Emphasis',
+ 'Greet', 'Other', 'Reject', 'Statement', 'System', 'nAnswer',
+ 'whQuestion', 'yAnswer', 'ynQuestion']
+ >>> posts[0].text
+ 'now im left with this gay name'
+
+In addition to the above methods for accessing tagged text, we can navigate
+the XML structure directly, as follows:
+
+ >>> tokens = posts[0].findall('terminals/t')
+ >>> [t.attrib['pos'] + "/" + t.attrib['word'] for t in tokens]
+ ['RB/now', 'PRP/im', 'VBD/left', 'IN/with', 'DT/this', 'JJ/gay', 'NN/name']
+
+
+---------------------
+Corpus Reader Classes
+---------------------
+
+NLTK's *corpus reader* classes are used to access the contents of a
+diverse set of corpora. Each corpus reader class is specialized to
+handle a specific corpus format. Examples include the
+`PlaintextCorpusReader`, which handles corpora that consist of a set
+of unannotated text files, and the `BracketParseCorpusReader`, which
+handles corpora that consist of files containing
+parenthesis-delineated parse trees.
+
+Automatically Created Corpus Reader Instances
+=============================================
+
+When then `nltk.corpus` module is imported, it automatically creates a
+set of corpus reader instances that can be used to access the corpora
+in the NLTK data distribution. Here is a small sample of those
+corpus reader instances:
+
+ >>> import nltk
+ >>> nltk.corpus.brown # doctest: +ELLIPSIS
+ <CategorizedTaggedCorpusReader ...>
+ >>> nltk.corpus.treebank # doctest: +ELLIPSIS
+ <BracketParseCorpusReader ...>
+ >>> nltk.corpus.names # doctest: +ELLIPSIS
+ <WordListCorpusReader ...>
+ >>> nltk.corpus.genesis # doctest: +ELLIPSIS
+ <PlaintextCorpusReader ...>
+ >>> nltk.corpus.inaugural # doctest: +ELLIPSIS
+ <PlaintextCorpusReader ...>
+
+This sample illustrates that different corpus reader classes are used
+to read different corpora; but that the same corpus reader class may
+be used for more than one corpus (e.g., ``genesis`` and ``inaugural``).
+
+Creating New Corpus Reader Instances
+====================================
+
+Although the `nltk.corpus` module automatically creates corpus reader
+instances for the corpora in the NLTK data distribution, you may
+sometimes need to create your own corpus reader. In particular, you
+would need to create your own corpus reader if you want...
+
+- To access a corpus that is not included in the NLTK data
+ distribution.
+
+- To access a full copy of a corpus for which the NLTK data
+ distribution only provides a sample.
+
+- To access a corpus using a customized corpus reader (e.g., with
+ a customized tokenizer).
+
+To create a new corpus reader, you will first need to look up the
+signature for that corpus reader's constructor. Different corpus
+readers have different constructor signatures, but most of the
+constructor signatures have the basic form::
+
+ SomeCorpusReader(root, files, ...options...)
+
+Where ``root`` is an absolute path to the directory containing the
+corpus data files; ``files`` is either a list of file names (relative
+to ``root``) or a regexp specifying which files should be included;
+and ``options`` are additional reader-specific options. For example,
+we can create a customized corpus reader for the genesis corpus that
+uses a different sentence tokenizer as follows:
+
+ >>> # Find the directory where the corpus lives.
+ >>> genesis_dir = nltk.data.find('corpora/genesis.zip').join('genesis/')
+ >>> # Create our custom sentence tokenizer.
+ >>> my_sent_tokenizer = nltk.RegexpTokenizer('[^.!?]+')
+ >>> # Create the new corpus reader object.
+ >>> my_genesis = nltk.corpus.PlaintextCorpusReader(
+ ... genesis_dir, '.*\.txt', sent_tokenizer=my_sent_tokenizer)
+ >>> # Use the new corpus reader object.
+ >>> print(my_genesis.sents('english-kjv.txt')[0]) # doctest: +NORMALIZE_WHITESPACE
+ ['In', 'the', 'beginning', 'God', 'created', 'the', 'heaven',
+ 'and', 'the', 'earth']
+
+If you wish to read your own plaintext corpus, which is stored in the
+directory '/usr/share/some-corpus', then you can create a corpus
+reader for it with::
+
+ >>> my_corpus = nltk.corpus.PlaintextCorpusReader(
+ ... '/usr/share/some-corpus', '.*\.txt') # doctest: +SKIP
+
+For a complete list of corpus reader subclasses, see the API
+documentation for `nltk.corpus.CorpusReader`.
+
+Corpus Types
+============
+
+Corpora vary widely in the types of content they include. This is
+reflected in the fact that the base class `CorpusReader` only defines
+a few general-purpose methods for listing and accessing the files that
+make up a corpus. It is up to the subclasses to define *data access
+methods* that provide access to the information in the corpus.
+However, corpus reader subclasses should be consistent in their
+definitions of these data access methods wherever possible.
+
+At a high level, corpora can be divided into three basic types:
+
+- A *token corpus* contains information about specific occurences of
+ language use (or linguistic tokens), such as dialogues or written
+ texts. Examples of token corpora are collections of written text
+ and collections of speech.
+
+- A *type corpus*, or *lexicon*, contains information about a coherent
+ set of lexical items (or linguistic types). Examples of lexicons
+ are dictionaries and word lists.
+
+- A *language description corpus* contains information about a set of
+ non-lexical linguistic constructs, such as grammar rules.
+
+However, many individual corpora blur the distinctions between these
+types. For example, corpora that are primarily lexicons may include
+token data in the form of example sentences; and corpora that are
+primarily token corpora may be accompanied by one or more word lists
+or other lexical data sets.
+
+Because corpora vary so widely in their information content, we have
+decided that it would not be wise to use separate corpus reader base
+classes for different corpus types. Instead, we simply try to make
+the corpus readers consistent wherever possible, but let them differ
+where the underlying data itself differs.
+
+Common Corpus Reader Methods
+============================
+
+As mentioned above, there are only a handful of methods that all
+corpus readers are guaranteed to implement. These methods provide
+access to the files that contain the corpus data. Every corpus is
+assumed to consist of one or more files, all located in a common root
+directory (or in subdirectories of that root directory). The absolute
+path to the root directory is stored in the ``root`` property:
+
+ >>> import os
+ >>> str(nltk.corpus.genesis.root).replace(os.path.sep,'/') # doctest: +ELLIPSIS
+ '.../nltk_data/corpora/genesis'
+
+Each file within the corpus is identified by a platform-independent
+identifier, which is basically a path string that uses ``/`` as the
+path seperator. I.e., this identifier can be converted to a relative
+path as follows:
+
+ >>> some_corpus_file_id = nltk.corpus.reuters.fileids()[0]
+ >>> import os.path
+ >>> os.path.normpath(some_corpus_file_id).replace(os.path.sep,'/')
+ 'test/14826'
+
+To get a list of all data files that make up a corpus, use the
+``fileids()`` method. In some corpora, these files will not all contain
+the same type of data; for example, for the ``nltk.corpus.timit``
+corpus, ``fileids()`` will return a list including text files, word
+segmentation files, phonetic transcription files, sound files, and
+metadata files. For corpora with diverse file types, the ``fileids()``
+method will often take one or more optional arguments, which can be
+used to get a list of the files with a specific file type:
+
+ >>> nltk.corpus.timit.fileids() # doctest: +ELLIPSIS
+ ['dr1-fvmh0/sa1.phn', 'dr1-fvmh0/sa1.txt', 'dr1-fvmh0/sa1.wav', ...]
+ >>> nltk.corpus.timit.fileids('phn') # doctest: +ELLIPSIS
+ ['dr1-fvmh0/sa1.phn', 'dr1-fvmh0/sa2.phn', 'dr1-fvmh0/si1466.phn', ...]
+
+In some corpora, the files are divided into distinct categories. For
+these corpora, the ``fileids()`` method takes an optional argument,
+which can be used to get a list of the files within a specific category:
+
+ >>> nltk.corpus.brown.fileids('hobbies') # doctest: +ELLIPSIS
+ ['ce01', 'ce02', 'ce03', 'ce04', 'ce05', 'ce06', 'ce07', ...]
+
+The ``abspath()`` method can be used to find the absolute path to a
+corpus file, given its file identifier:
+
+ >>> str(nltk.corpus.brown.abspath('ce06')).replace(os.path.sep,'/') # doctest: +ELLIPSIS
+ '.../corpora/brown/ce06'
+
+The ``abspaths()`` method can be used to find the absolute paths for
+one corpus file, a list of corpus files, or (if no fileids are specified),
+all corpus files.
+
+This method is mainly useful as a helper method when defining corpus
+data access methods, since data access methods can usually be called
+with a string argument (to get a view for a specific file), with a
+list argument (to get a view for a specific list of files), or with no
+argument (to get a view for the whole corpus).
+
+Data Access Methods
+===================
+
+Individual corpus reader subclasses typically extend this basic set of
+file-access methods with one or more *data access methods*, which provide
+easy access to the data contained in the corpus. The signatures for
+data access methods often have the basic form::
+
+ corpus_reader.some_data access(fileids=None, ...options...)
+
+Where ``fileids`` can be a single file identifier string (to get a view
+for a specific file); a list of file identifier strings (to get a view
+for a specific list of files); or None (to get a view for the entire
+corpus). Some of the common data access methods, and their return
+types, are:
+
+ - I{corpus}.words(): list of str
+ - I{corpus}.sents(): list of (list of str)
+ - I{corpus}.paras(): list of (list of (list of str))
+ - I{corpus}.tagged_words(): list of (str,str) tuple
+ - I{corpus}.tagged_sents(): list of (list of (str,str))
+ - I{corpus}.tagged_paras(): list of (list of (list of (str,str)))
+ - I{corpus}.chunked_sents(): list of (Tree w/ (str,str) leaves)
+ - I{corpus}.parsed_sents(): list of (Tree with str leaves)
+ - I{corpus}.parsed_paras(): list of (list of (Tree with str leaves))
+ - I{corpus}.xml(): A single xml ElementTree
+ - I{corpus}.raw(): str (unprocessed corpus contents)
+
+For example, the `words()` method is supported by many different
+corpora, and returns a flat list of word strings:
+
+ >>> nltk.corpus.brown.words()
+ ['The', 'Fulton', 'County', 'Grand', 'Jury', ...]
+ >>> nltk.corpus.treebank.words()
+ ['Pierre', 'Vinken', ',', '61', 'years', 'old', ...]
+ >>> nltk.corpus.conll2002.words()
+ [u'Sao', u'Paulo', u'(', u'Brasil', u')', u',', u'23', ...]
+ >>> nltk.corpus.genesis.words()
+ [u'In', u'the', u'beginning', u'God', u'created', ...]
+
+On the other hand, the `tagged_words()` method is only supported by
+corpora that include part-of-speech annotations:
+
+ >>> nltk.corpus.brown.tagged_words()
+ [('The', 'AT'), ('Fulton', 'NP-TL'), ...]
+ >>> nltk.corpus.treebank.tagged_words()
+ [('Pierre', 'NNP'), ('Vinken', 'NNP'), ...]
+ >>> nltk.corpus.conll2002.tagged_words()
+ [(u'Sao', u'NC'), (u'Paulo', u'VMI'), (u'(', u'Fpa'), ...]
+ >>> nltk.corpus.genesis.tagged_words()
+ Traceback (most recent call last):
+ ...
+ AttributeError: 'PlaintextCorpusReader' object has no attribute 'tagged_words'
+
+Although most corpus readers use file identifiers to index their
+content, some corpora use different identifiers instead. For example,
+the data access methods for the ``timit`` corpus uses *utterance
+identifiers* to select which corpus items should be returned:
+
+ >>> nltk.corpus.timit.utteranceids() # doctest: +ELLIPSIS
+ ['dr1-fvmh0/sa1', 'dr1-fvmh0/sa2', 'dr1-fvmh0/si1466', ...]
+ >>> nltk.corpus.timit.words('dr1-fvmh0/sa2')
+ ["don't", 'ask', 'me', 'to', 'carry', 'an', 'oily', 'rag', 'like', 'that']
+
+Attempting to call ``timit``\ 's data access methods with a file
+identifier will result in an exception:
+
+ >>> nltk.corpus.timit.fileids() # doctest: +ELLIPSIS
+ ['dr1-fvmh0/sa1.phn', 'dr1-fvmh0/sa1.txt', 'dr1-fvmh0/sa1.wav', ...]
+ >>> nltk.corpus.timit.words('dr1-fvmh0/sa1.txt') # doctest: +SKIP
+ Traceback (most recent call last):
+ ...
+ IOError: No such file or directory: '.../dr1-fvmh0/sa1.txt.wrd'
+
+As another example, the ``propbank`` corpus defines the ``roleset()``
+method, which expects a roleset identifier, not a file identifier:
+
+ >>> roleset = nltk.corpus.propbank.roleset('eat.01')
+ >>> from xml.etree import ElementTree as ET
+ >>> print(ET.tostring(roleset).decode('utf8')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ <roleset id="eat.01" name="consume" vncls="39.1">
+ <roles>
+ <role descr="consumer, eater" n="0">...</role>...
+ </roles>...
+ </roleset>...
+
+Stream Backed Corpus Views
+==========================
+An important feature of NLTK's corpus readers is that many of them
+access the underlying data files using "corpus views." A *corpus
+view* is an object that acts like a simple data structure (such as a
+list), but does not store the data elements in memory; instead, data
+elements are read from the underlying data files on an as-needed
+basis.
+
+By only loading items from the file on an as-needesd basis, corpus
+views maintain both memory efficiency and responsiveness. The memory
+efficiency of corpus readers is important because some corpora contain
+very large amounts of data, and storing the entire data set in memory
+could overwhelm many machines. The responsiveness is important when
+experimenting with corpora in interactive sessions and in in-class
+demonstrations.
+
+The most common corpus view is the `StreamBackedCorpusView`, which
+acts as a read-only list of tokens. Two additional corpus view
+classes, `ConcatenatedCorpusView` and `LazySubsequence`, make it
+possible to create concatenations and take slices of
+`StreamBackedCorpusView` objects without actually storing the
+resulting list-like object's elements in memory.
+
+In the future, we may add additional corpus views that act like other
+basic data structures, such as dictionaries.
+
+Writing New Corpus Readers
+==========================
+
+In order to add support for new corpus formats, it is necessary to
+define new corpus reader classes. For many corpus formats, writing
+new corpus readers is relatively streight-forward. In this section,
+we'll describe what's involved in creating a new corpus reader. If
+you do create a new corpus reader, we encourage you to contribute it
+back to the NLTK project.
+
+Don't Reinvent the Wheel
+------------------------
+Before you start writing a new corpus reader, you should check to be
+sure that the desired format can't be read using an existing corpus
+reader with appropriate constructor arguments. For example, although
+the `TaggedCorpusReader` assumes that words and tags are separated by
+``/`` characters by default, an alternative tag-separation character
+can be specified via the ``sep`` constructor argument. You should
+also check whether the new corpus format can be handled by subclassing
+an existing corpus reader, and tweaking a few methods or variables.
+
+Design
+------
+If you decide to write a new corpus reader from scratch, then you
+should first decide which data access methods you want the reader to
+provide, and what their signatures should be. You should look at
+existing corpus readers that process corpora with similar data
+contents, and try to be consistent with those corpus readers whenever
+possible.
+
+You should also consider what sets of identifiers are appropriate for
+the corpus format. Where it's practical, file identifiers should be
+used. However, for some corpora, it may make sense to use additional
+sets of identifiers. Each set of identifiers should have a distinct
+name (e.g., fileids, utteranceids, rolesets); and you should be consistent
+in using that name to refer to that identifier. Do not use parameter
+names like ``id``, which leave it unclear what type of identifier is
+required.
+
+Once you've decided what data access methods and identifiers are
+appropriate for your corpus, you should decide if there are any
+customizable parameters that you'd like the corpus reader to handle.
+These parameters make it possible to use a single corpus reader to
+handle a wider variety of corpora. The ``sep`` argument for
+`TaggedCorpusReader`, mentioned above, is an example of a customizable
+corpus reader parameter.
+
+Implementation
+--------------
+
+Constructor
+~~~~~~~~~~~
+If your corpus reader implements any customizable parameters, then
+you'll need to override the constructor. Typically, the new
+constructor will first call its base class's constructor, and then
+store the customizable parameters. For example, the
+`ConllChunkCorpusReader`\ 's constructor is defined as follows:
+
+ >>> def __init__(self, root, files, chunk_types):
+ ... CorpusReader.__init__(self, root, files)
+ ... self.chunk_types = tuple(chunk_types)
+
+If your corpus reader does not implement any customization parameters,
+then you can often just inherit the base class's constructor.
+
+Data Access Methods
+~~~~~~~~~~~~~~~~~~~
+The most common type of data access method takes an argument
+identifying which files to access, and returns a view covering those
+files. This argument may be a a single file identifier string (to get
+a view for a specific file); a list of file identifier strings (to get
+a view for a specific list of files); or None (to get a view for the
+entire corpus). The method's implementation converts this argument to
+a list of path names using the `abspaths()` method, which handles all
+three value types (string, list, and None):
+
+ >>> print(str(nltk.corpus.brown.abspaths()).replace('\\\\','/')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [FileSystemPathPointer('.../corpora/brown/ca01'),
+ FileSystemPathPointer('.../corpora/brown/ca02'), ...]
+ >>> print(str(nltk.corpus.brown.abspaths('ce06')).replace('\\\\','/')) # doctest: +ELLIPSIS
+ [FileSystemPathPointer('.../corpora/brown/ce06')]
+ >>> print(str(nltk.corpus.brown.abspaths(['ce06', 'ce07'])).replace('\\\\','/')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [FileSystemPathPointer('.../corpora/brown/ce06'),
+ FileSystemPathPointer('.../corpora/brown/ce07')]
+
+An example of this type of method is the `words()` method, defined by
+the `PlaintextCorpusReader` as follows:
+
+ >>> def words(self, fileids=None):
+ ... return concat([self.CorpusView(fileid, self._read_word_block)
+ ... for fileid in self.abspaths(fileids)])
+
+This method first uses `abspaths()` to convert ``fileids`` to a list of
+absolute paths. It then creates a corpus view for each file, using
+the `PlaintextCorpusReader._read_word_block()` method to read elements
+from the data file (see the discussion of corpus views below).
+Finally, it combines these corpus views using the
+`nltk.corpus.reader.util.concat()` function.
+
+When writing a corpus reader for a corpus that is never expected to be
+very large, it can sometimes be appropriate to read the files
+directly, rather than using a corpus view. For example, the
+`WordListCorpusView` class defines its `words()` method as follows:
+
+ >>> def words(self, fileids=None):
+ ... return concat([[w for w in open(fileid).read().split('\n') if w]
+ ... for fileid in self.abspaths(fileids)])
+
+(This is usually more appropriate for lexicons than for token corpora.)
+
+If the type of data returned by a data access method is one for which
+NLTK has a conventional representation (e.g., words, tagged words, and
+parse trees), then you should use that representation. Otherwise, you
+may find it necessary to define your own representation. For data
+structures that are relatively corpus-specific, it's usually best to
+define new classes for these elements. For example, the ``propbank``
+corpus defines the `PropbankInstance` class to store the semantic role
+labeling instances described by the corpus; and the ``ppattach``
+corpus defines the `PPAttachment` class to store the prepositional
+attachment instances described by the corpus.
+
+Corpus Views
+~~~~~~~~~~~~
+.. (Much of the content for this section is taken from the
+ StreamBackedCorpusView docstring.)
+
+The heart of a `StreamBackedCorpusView` is its *block reader*
+function, which reads zero or more tokens from a stream, and returns
+them as a list. A very simple example of a block reader is:
+
+ >>> def simple_block_reader(stream):
+ ... return stream.readline().split()
+
+This simple block reader reads a single line at a time, and returns a
+single token (consisting of a string) for each whitespace-separated
+substring on the line. A `StreamBackedCorpusView` built from this
+block reader will act like a read-only list of all the
+whitespace-seperated tokens in an underlying file.
+
+When deciding how to define the block reader for a given corpus,
+careful consideration should be given to the size of blocks handled by
+the block reader. Smaller block sizes will increase the memory
+requirements of the corpus view's internal data structures (by 2
+integers per block). On the other hand, larger block sizes may
+decrease performance for random access to the corpus. (But note that
+larger block sizes will *not* decrease performance for iteration.)
+
+Internally, the `StreamBackedCorpusView` class maintains a partial
+mapping from token index to file position, with one entry per block.
+When a token with a given index *i* is requested, the corpus view
+constructs it as follows:
+
+1. First, it searches the toknum/filepos mapping for the token index
+ closest to (but less than or equal to) *i*.
+
+2. Then, starting at the file position corresponding to that index, it
+ reads one block at a time using the block reader until it reaches
+ the requested token.
+
+The toknum/filepos mapping is created lazily: it is initially empty,
+but every time a new block is read, the block's initial token is added
+to the mapping. (Thus, the toknum/filepos map has one entry per
+block.)
+
+You can create your own corpus view in one of two ways:
+
+1. Call the `StreamBackedCorpusView` constructor, and provide your
+ block reader function via the ``block_reader`` argument.
+
+2. Subclass `StreamBackedCorpusView`, and override the
+ `read_block()` method.
+
+The first option is usually easier, but the second option can allow
+you to write a single `read_block` method whose behavior can be
+customized by different parameters to the subclass's constructor. For
+an example of this design pattern, see the `TaggedCorpusView` class,
+which is used by `TaggedCorpusView`.
+
+----------------
+Regression Tests
+----------------
+
+The following helper functions are used to create and then delete
+testing corpora that are stored in temporary directories. These
+testing corpora are used to make sure the readers work correctly.
+
+ >>> import tempfile, os.path, textwrap
+ >>> def make_testcorpus(ext='', **fileids):
+ ... root = tempfile.mkdtemp()
+ ... for fileid, contents in fileids.items():
+ ... fileid += ext
+ ... f = open(os.path.join(root, fileid), 'w')
+ ... f.write(textwrap.dedent(contents))
+ ... f.close()
+ ... return root
+ >>> def del_testcorpus(root):
+ ... for fileid in os.listdir(root):
+ ... os.remove(os.path.join(root, fileid))
+ ... os.rmdir(root)
+
+Plaintext Corpus Reader
+=======================
+The plaintext corpus reader is used to access corpora that consist of
+unprocessed plaintext data. It assumes that paragraph breaks are
+indicated by blank lines. Sentences and words can be tokenized using
+the default tokenizers, or by custom tokenizers specificed as
+parameters to the constructor.
+
+ >>> root = make_testcorpus(ext='.txt',
+ ... a="""\
+ ... This is the first sentence. Here is another
+ ... sentence! And here's a third sentence.
+ ...
+ ... This is the second paragraph. Tokenization is currently
+ ... fairly simple, so the period in Mr. gets tokenized.
+ ... """,
+ ... b="""This is the second file.""")
+
+ >>> from nltk.corpus.reader.plaintext import PlaintextCorpusReader
+
+The list of documents can be specified explicitly, or implicitly (using a
+regexp). The ``ext`` argument specifies a file extension.
+
+ >>> corpus = PlaintextCorpusReader(root, ['a.txt', 'b.txt'])
+ >>> corpus.fileids()
+ ['a.txt', 'b.txt']
+ >>> corpus = PlaintextCorpusReader(root, '.*\.txt')
+ >>> corpus.fileids()
+ ['a.txt', 'b.txt']
+
+The directory containing the corpus is corpus.root:
+
+ >>> str(corpus.root) == str(root)
+ True
+
+We can get a list of words, or the raw string:
+
+ >>> corpus.words()
+ ['This', 'is', 'the', 'first', 'sentence', '.', ...]
+ >>> corpus.raw()[:40]
+ 'This is the first sentence. Here is ano'
+
+Check that reading individual documents works, and reading all documents at
+once works:
+
+ >>> len(corpus.words()), [len(corpus.words(d)) for d in corpus.fileids()]
+ (46, [40, 6])
+ >>> corpus.words('a.txt')
+ ['This', 'is', 'the', 'first', 'sentence', '.', ...]
+ >>> corpus.words('b.txt')
+ ['This', 'is', 'the', 'second', 'file', '.']
+ >>> corpus.words()[:4], corpus.words()[-4:]
+ (['This', 'is', 'the', 'first'], ['the', 'second', 'file', '.'])
+
+We're done with the test corpus:
+
+ >>> del_testcorpus(root)
+
+Test the plaintext corpora that come with nltk:
+
+ >>> from nltk.corpus import abc, genesis, inaugural
+ >>> from nltk.corpus import state_union, webtext
+ >>> for corpus in (abc, genesis, inaugural, state_union,
+ ... webtext):
+ ... print(str(corpus).replace('\\\\','/'))
+ ... print(' ', repr(corpus.fileids())[:60])
+ ... print(' ', repr(corpus.words()[:10])[:60])
+ <PlaintextCorpusReader in '.../nltk_data/corpora/ab...'>
+ ['rural.txt', 'science.txt']
+ ['PM', 'denies', 'knowledge', 'of', 'AWB', ...
+ <PlaintextCorpusReader in '.../nltk_data/corpora/genesi...'>
+ ['english-kjv.txt', 'english-web.txt', 'finnish.txt', ...
+ ['In', 'the', 'beginning', 'God', 'created', 'the', ...
+ <PlaintextCorpusReader in '.../nltk_data/corpora/inaugura...'>
+ ['1789-Washington.txt', '1793-Washington.txt', ...
+ ['Fellow', '-', 'Citizens', 'of', 'the', 'Senate', ...
+ <PlaintextCorpusReader in '.../nltk_data/corpora/state_unio...'>
+ ['1945-Truman.txt', '1946-Truman.txt', ...
+ ['PRESIDENT', 'HARRY', 'S', '.', 'TRUMAN', "'", ...
+ <PlaintextCorpusReader in '.../nltk_data/corpora/webtex...'>
+ ['firefox.txt', 'grail.txt', 'overheard.txt', ...
+ ['Cookie', 'Manager', ':', '"', 'Don', "'", 't', ...
+
+
+Tagged Corpus Reader
+====================
+The Tagged Corpus reader can give us words, sentences, and paragraphs,
+each tagged or untagged. All of the read methods can take one item
+(in which case they return the contents of that file) or a list of
+documents (in which case they concatenate the contents of those files).
+By default, they apply to all documents in the corpus.
+
+ >>> root = make_testcorpus(
+ ... a="""\
+ ... This/det is/verb the/det first/adj sentence/noun ./punc
+ ... Here/det is/verb another/adj sentence/noun ./punc
+ ... Note/verb that/comp you/pron can/verb use/verb \
+ ... any/noun tag/noun set/noun
+ ...
+ ... This/det is/verb the/det second/adj paragraph/noun ./punc
+ ... word/n without/adj a/det tag/noun :/: hello ./punc
+ ... """,
+ ... b="""\
+ ... This/det is/verb the/det second/adj file/noun ./punc
+ ... """)
+
+ >>> from nltk.corpus.reader.tagged import TaggedCorpusReader
+ >>> corpus = TaggedCorpusReader(root, list('ab'))
+ >>> corpus.fileids()
+ ['a', 'b']
+ >>> str(corpus.root) == str(root)
+ True
+ >>> corpus.words()
+ ['This', 'is', 'the', 'first', 'sentence', '.', ...]
+ >>> corpus.sents() # doctest: +ELLIPSIS
+ [['This', 'is', 'the', 'first', ...], ['Here', 'is', 'another'...], ...]
+ >>> corpus.paras() # doctest: +ELLIPSIS
+ [[['This', ...], ['Here', ...], ...], [['This', ...], ...], ...]
+ >>> corpus.tagged_words() # doctest: +ELLIPSIS
+ [('This', 'DET'), ('is', 'VERB'), ('the', 'DET'), ...]
+ >>> corpus.tagged_sents() # doctest: +ELLIPSIS
+ [[('This', 'DET'), ('is', 'VERB'), ...], [('Here', 'DET'), ...], ...]
+ >>> corpus.tagged_paras() # doctest: +ELLIPSIS
+ [[[('This', 'DET'), ...], ...], [[('This', 'DET'), ...], ...], ...]
+ >>> corpus.raw()[:40]
+ 'This/det is/verb the/det first/adj sente'
+ >>> len(corpus.words()), [len(corpus.words(d)) for d in corpus.fileids()]
+ (38, [32, 6])
+ >>> len(corpus.sents()), [len(corpus.sents(d)) for d in corpus.fileids()]
+ (6, [5, 1])
+ >>> len(corpus.paras()), [len(corpus.paras(d)) for d in corpus.fileids()]
+ (3, [2, 1])
+ >>> print(corpus.words('a'))
+ ['This', 'is', 'the', 'first', 'sentence', '.', ...]
+ >>> print(corpus.words('b'))
+ ['This', 'is', 'the', 'second', 'file', '.']
+ >>> del_testcorpus(root)
+
+The Brown Corpus uses the tagged corpus reader:
+
+ >>> from nltk.corpus import brown
+ >>> brown.fileids() # doctest: +ELLIPSIS
+ ['ca01', 'ca02', 'ca03', 'ca04', 'ca05', 'ca06', 'ca07', ...]
+ >>> brown.categories() # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ ['adventure', 'belles_lettres', 'editorial', 'fiction', 'government', 'hobbies', 'humor',
+ 'learned', 'lore', 'mystery', 'news', 'religion', 'reviews', 'romance', 'science_fiction']
+ >>> print(repr(brown.root).replace('\\\\','/')) # doctest: +ELLIPSIS
+ FileSystemPathPointer('.../corpora/brown')
+ >>> brown.words()
+ ['The', 'Fulton', 'County', 'Grand', 'Jury', ...]
+ >>> brown.sents() # doctest: +ELLIPSIS
+ [['The', 'Fulton', 'County', 'Grand', ...], ...]
+ >>> brown.paras() # doctest: +ELLIPSIS
+ [[['The', 'Fulton', 'County', ...]], [['The', 'jury', ...]], ...]
+ >>> brown.tagged_words() # doctest: +ELLIPSIS
+ [('The', 'AT'), ('Fulton', 'NP-TL'), ...]
+ >>> brown.tagged_sents() # doctest: +ELLIPSIS
+ [[('The', 'AT'), ('Fulton', 'NP-TL'), ('County', 'NN-TL'), ...], ...]
+ >>> brown.tagged_paras() # doctest: +ELLIPSIS
+ [[[('The', 'AT'), ...]], [[('The', 'AT'), ...]], ...]
+
+Verbnet Corpus Reader
+=====================
+
+Make sure we're picking up the right number of elements:
+
+ >>> from nltk.corpus import verbnet
+ >>> len(verbnet.lemmas())
+ 3621
+ >>> len(verbnet.wordnetids())
+ 4953
+ >>> len(verbnet.classids())
+ 429
+
+Selecting classids based on various selectors:
+
+ >>> verbnet.classids(lemma='take') # doctest: +NORMALIZE_WHITESPACE
+ ['bring-11.3', 'characterize-29.2', 'convert-26.6.2', 'cost-54.2',
+ 'fit-54.3', 'performance-26.7-2', 'steal-10.5']
+ >>> verbnet.classids(wordnetid='lead%2:38:01')
+ ['accompany-51.7']
+ >>> verbnet.classids(fileid='approve-77.xml')
+ ['approve-77']
+ >>> verbnet.classids(classid='admire-31.2') # subclasses
+ ['admire-31.2-1']
+
+vnclass() accepts filenames, long ids, and short ids:
+
+ >>> a = ElementTree.tostring(verbnet.vnclass('admire-31.2.xml'))
+ >>> b = ElementTree.tostring(verbnet.vnclass('admire-31.2'))
+ >>> c = ElementTree.tostring(verbnet.vnclass('31.2'))
+ >>> a == b == c
+ True
+
+fileids() can be used to get files based on verbnet class ids:
+
+ >>> verbnet.fileids('admire-31.2')
+ ['admire-31.2.xml']
+ >>> verbnet.fileids(['admire-31.2', 'obtain-13.5.2'])
+ ['admire-31.2.xml', 'obtain-13.5.2.xml']
+ >>> verbnet.fileids('badidentifier')
+ Traceback (most recent call last):
+ . . .
+ ValueError: vnclass identifier 'badidentifier' not found
+
+longid() and shortid() can be used to convert identifiers:
+
+ >>> verbnet.longid('31.2')
+ 'admire-31.2'
+ >>> verbnet.longid('admire-31.2')
+ 'admire-31.2'
+ >>> verbnet.shortid('31.2')
+ '31.2'
+ >>> verbnet.shortid('admire-31.2')
+ '31.2'
+ >>> verbnet.longid('badidentifier')
+ Traceback (most recent call last):
+ . . .
+ ValueError: vnclass identifier 'badidentifier' not found
+ >>> verbnet.shortid('badidentifier')
+ Traceback (most recent call last):
+ . . .
+ ValueError: vnclass identifier 'badidentifier' not found
+
+Corpus View Regression Tests
+============================
+
+Select some corpus files to play with:
+
+ >>> import nltk.data
+ >>> # A very short file (160 chars):
+ >>> f1 = nltk.data.find('corpora/inaugural/README')
+ >>> # A relatively short file (791 chars):
+ >>> f2 = nltk.data.find('corpora/inaugural/1793-Washington.txt')
+ >>> # A longer file (32k chars):
+ >>> f3 = nltk.data.find('corpora/inaugural/1909-Taft.txt')
+ >>> fileids = [f1, f2, f3]
+
+
+Concatenation
+-------------
+Check that concatenation works as intended.
+
+ >>> from nltk.corpus.reader.util import *
+
+ >>> c1 = StreamBackedCorpusView(f1, read_whitespace_block, encoding='utf-8')
+ >>> c2 = StreamBackedCorpusView(f2, read_whitespace_block, encoding='utf-8')
+ >>> c3 = StreamBackedCorpusView(f3, read_whitespace_block, encoding='utf-8')
+ >>> c123 = c1+c2+c3
+ >>> print(c123)
+ ['C-Span', 'Inaugural', 'Address', 'Corpus', 'US', ...]
+
+ >>> l1 = f1.open(encoding='utf-8').read().split()
+ >>> l2 = f2.open(encoding='utf-8').read().split()
+ >>> l3 = f3.open(encoding='utf-8').read().split()
+ >>> l123 = l1+l2+l3
+
+ >>> list(c123) == l123
+ True
+
+ >>> (c1+c2+c3)[100] == l123[100]
+ True
+
+Slicing
+-------
+First, do some tests with fairly small slices. These will all
+generate tuple values.
+
+ >>> from nltk.util import LazySubsequence
+ >>> c1 = StreamBackedCorpusView(f1, read_whitespace_block, encoding='utf-8')
+ >>> l1 = f1.open(encoding='utf-8').read().split()
+ >>> print(len(c1))
+ 21
+ >>> len(c1) < LazySubsequence.MIN_SIZE
+ True
+
+Choose a list of indices, based on the length, that covers the
+important corner cases:
+
+ >>> indices = [-60, -30, -22, -21, -20, -1,
+ ... 0, 1, 10, 20, 21, 22, 30, 60]
+
+Test slicing with explicit start & stop value:
+
+ >>> for s in indices:
+ ... for e in indices:
+ ... assert list(c1[s:e]) == l1[s:e]
+
+Test slicing with stop=None:
+
+ >>> for s in indices:
+ ... assert list(c1[s:]) == l1[s:]
+
+Test slicing with start=None:
+
+ >>> for e in indices:
+ ... assert list(c1[:e]) == l1[:e]
+
+Test slicing with start=stop=None:
+
+ >>> list(c1[:]) == list(l1[:])
+ True
+
+Next, we'll do some tests with much longer slices. These will
+generate LazySubsequence objects.
+
+ >>> c3 = StreamBackedCorpusView(f3, read_whitespace_block, encoding='utf-8')
+ >>> l3 = f3.open(encoding='utf-8').read().split()
+ >>> print(len(c3))
+ 5430
+ >>> len(c3) > LazySubsequence.MIN_SIZE*2
+ True
+
+Choose a list of indices, based on the length, that covers the
+important corner cases:
+
+ >>> indices = [-12000, -6000, -5431, -5430, -5429, -3000, -200, -1,
+ ... 0, 1, 200, 3000, 5000, 5429, 5430, 5431, 6000, 12000]
+
+Test slicing with explicit start & stop value:
+
+ >>> for s in indices:
+ ... for e in indices:
+ ... assert list(c3[s:e]) == l3[s:e]
+
+Test slicing with stop=None:
+
+ >>> for s in indices:
+ ... assert list(c3[s:]) == l3[s:]
+
+Test slicing with start=None:
+
+ >>> for e in indices:
+ ... assert list(c3[:e]) == l3[:e]
+
+Test slicing with start=stop=None:
+
+ >>> list(c3[:]) == list(l3[:])
+ True
+
+Multiple Iterators
+------------------
+If multiple iterators are created for the same corpus view, their
+iteration can be interleaved:
+
+ >>> c3 = StreamBackedCorpusView(f3, read_whitespace_block)
+ >>> iterators = [c3.iterate_from(n) for n in [0,15,30,45]]
+ >>> for i in range(15):
+ ... for iterator in iterators:
+ ... print('%-15s' % next(iterator), end=' ')
+ ... print()
+ My a duties in
+ fellow heavy of a
+ citizens: weight the proper
+ Anyone of office sense
+ who responsibility. upon of
+ has If which the
+ taken not, he obligation
+ the he is which
+ oath has about the
+ I no to oath
+ have conception enter, imposes.
+ just of or The
+ taken the he office
+ must powers is of
+ feel and lacking an
+
+SeekableUnicodeStreamReader
+===========================
+
+The file-like objects provided by the ``codecs`` module unfortunately
+suffer from a bug that prevents them from working correctly with
+corpus view objects. In particular, although the expose ``seek()``
+and ``tell()`` methods, those methods do not exhibit the expected
+behavior, because they are not synchronized with the internal buffers
+that are kept by the file-like objects. For example, the ``tell()``
+method will return the file position at the end of the buffers (whose
+contents have not yet been returned by the stream); and therefore this
+file position can not be used to return to the 'current' location in
+the stream (since ``seek()`` has no way to reconstruct the buffers).
+
+To get around these problems, we define a new class,
+`SeekableUnicodeStreamReader`, to act as a file-like interface to
+files containing encoded unicode data. This class is loosely based on
+the ``codecs.StreamReader`` class. To construct a new reader, we call
+the constructor with an underlying stream and an encoding name:
+
+ >>> from io import StringIO, BytesIO
+ >>> from nltk.data import SeekableUnicodeStreamReader
+ >>> stream = BytesIO(b"""\
+ ... This is a test file.
+ ... It is encoded in ascii.
+ ... """.decode('ascii').encode('ascii'))
+ >>> reader = SeekableUnicodeStreamReader(stream, 'ascii')
+
+`SeekableUnicodeStreamReader`\ s support all of the normal operations
+supplied by a read-only stream. Note that all of the read operations
+return ``unicode`` objects (not ``str`` objects).
+
+ >>> reader.read() # read the entire file.
+ u'This is a test file.\nIt is encoded in ascii.\n'
+ >>> reader.seek(0) # rewind to the start.
+ >>> reader.read(5) # read at most 5 bytes.
+ u'This '
+ >>> reader.readline() # read to the end of the line.
+ u'is a test file.\n'
+ >>> reader.seek(0) # rewind to the start.
+ >>> for line in reader:
+ ... print(repr(line)) # iterate over lines
+ u'This is a test file.\n'
+ u'It is encoded in ascii.\n'
+ >>> reader.seek(0) # rewind to the start.
+ >>> reader.readlines() # read a list of line strings
+ [u'This is a test file.\n', u'It is encoded in ascii.\n']
+ >>> reader.close()
+
+Size argument to ``read()``
+---------------------------
+The ``size`` argument to ``read()`` specifies the maximum number of
+*bytes* to read, not the maximum number of *characters*. Thus, for
+encodings that use multiple bytes per character, it may return fewer
+characters than the ``size`` argument:
+
+ >>> stream = BytesIO(b"""\
+ ... This is a test file.
+ ... It is encoded in utf-16.
+ ... """.decode('ascii').encode('utf-16'))
+ >>> reader = SeekableUnicodeStreamReader(stream, 'utf-16')
+ >>> reader.read(10)
+ u'This '
+
+If a read block ends in the middle of the byte string encoding a
+single character, then that byte string is stored in an internal
+buffer, and re-used on the next call to ``read()``. However, if the
+size argument is too small to read even a single character, even
+though at least one character is available, then the ``read()`` method
+will read additional bytes until it can return a single character.
+This ensures that the ``read()`` method does not return an empty
+string, which could be mistaken for indicating the end of the file.
+
+ >>> reader.seek(0) # rewind to the start.
+ >>> reader.read(1) # we actually need to read 4 bytes
+ u'T'
+ >>> int(reader.tell())
+ 4
+
+The ``readline()`` method may read more than a single line of text, in
+which case it stores the text that it does not return in a buffer. If
+this buffer is not empty, then its contents will be included in the
+value returned by the next call to ``read()``, regardless of the
+``size`` argument, since they are available without reading any new
+bytes from the stream:
+
+ >>> reader.seek(0) # rewind to the start.
+ >>> reader.readline() # stores extra text in a buffer
+ u'This is a test file.\n'
+ >>> print(reader.linebuffer) # examine the buffer contents
+ [u'It is encoded i']
+ >>> reader.read(0) # returns the contents of the buffer
+ u'It is encoded i'
+ >>> print(reader.linebuffer) # examine the buffer contents
+ None
+
+Seek and Tell
+-------------
+In addition to these basic read operations,
+`SeekableUnicodeStreamReader` also supports the ``seek()`` and
+``tell()`` operations. However, some care must still be taken when
+using these operations. In particular, the only file offsets that
+should be passed to ``seek()`` are ``0`` and any offset that has been
+returned by ``tell``.
+
+ >>> stream = BytesIO(b"""\
+ ... This is a test file.
+ ... It is encoded in utf-16.
+ ... """.decode('ascii').encode('utf-16'))
+ >>> reader = SeekableUnicodeStreamReader(stream, 'utf-16')
+ >>> reader.read(20)
+ u'This is a '
+ >>> pos = reader.tell(); print(pos)
+ 22
+ >>> reader.read(20)
+ u'test file.'
+ >>> reader.seek(pos) # rewind to the position from tell.
+ >>> reader.read(20)
+ u'test file.'
+
+The ``seek()`` and ``tell()`` methods work property even when
+``readline()`` is used.
+
+ >>> stream = BytesIO(b"""\
+ ... This is a test file.
+ ... It is encoded in utf-16.
+ ... """.decode('ascii').encode('utf-16'))
+ >>> reader = SeekableUnicodeStreamReader(stream, 'utf-16')
+ >>> reader.readline()
+ u'This is a test file.\n'
+ >>> pos = reader.tell(); print(pos)
+ 44
+ >>> reader.readline()
+ u'It is encoded in utf-16.\n'
+ >>> reader.seek(pos) # rewind to the position from tell.
+ >>> reader.readline()
+ u'It is encoded in utf-16.\n'
+
+
+Squashed Bugs
+=============
+
+svn 5276 fixed a bug in the comment-stripping behavior of
+parse_sexpr_block.
+
+ >>> from io import StringIO
+ >>> from nltk.corpus.reader.util import read_sexpr_block
+ >>> f = StringIO(b"""
+ ... (a b c)
+ ... # This line is a comment.
+ ... (d e f\ng h)""".decode('ascii'))
+ >>> print(read_sexpr_block(f, block_size=38, comment_char='#'))
+ ['(a b c)']
+ >>> print(read_sexpr_block(f, block_size=38, comment_char='#'))
+ ['(d e f\ng h)']
+
+svn 5277 fixed a bug in parse_sexpr_block, which would cause it to
+enter an infinite loop if a file ended mid-sexpr, or ended with a
+token that was not followed by whitespace. A related bug caused
+an infinite loop if the corpus ended in an unmatched close paren --
+this was fixed in svn 5279
+
+ >>> f = StringIO(b"""
+ ... This file ends mid-sexpr
+ ... (hello (world""".decode('ascii'))
+ >>> for i in range(3): print(read_sexpr_block(f))
+ ['This', 'file', 'ends', 'mid-sexpr']
+ ['(hello (world']
+ []
+
+ >>> f = StringIO(b"This file has no trailing whitespace.".decode('ascii'))
+ >>> for i in range(3): print(read_sexpr_block(f))
+ ['This', 'file', 'has', 'no', 'trailing']
+ ['whitespace.']
+ []
+
+ >>> # Bug fixed in 5279:
+ >>> f = StringIO(b"a b c)".decode('ascii'))
+ >>> for i in range(3): print(read_sexpr_block(f))
+ ['a', 'b']
+ ['c)']
+ []
+
+
+svn 5624 & 5265 fixed a bug in ConcatenatedCorpusView, which caused it
+to return the wrong items when indexed starting at any index beyond
+the first file.
+
+ >>> import nltk
+ >>> sents = nltk.corpus.brown.sents()
+ >>> print(sents[6000])
+ ['Cholesterol', 'and', 'thyroid']
+ >>> print(sents[6000])
+ ['Cholesterol', 'and', 'thyroid']
+
+svn 5728 fixed a bug in Categorized*CorpusReader, which caused them
+to return words from *all* files when just one file was specified.
+
+ >>> from nltk.corpus import reuters
+ >>> reuters.words('training/13085')
+ ['SNYDER', '&', 'lt', ';', 'SOI', '>', 'MAKES', ...]
+ >>> reuters.words('training/5082')
+ ['SHEPPARD', 'RESOURCES', 'TO', 'MERGE', 'WITH', ...]
+
+svn 7227 fixed a bug in the qc corpus reader, which prevented
+access to its tuples() method
+
+ >>> from nltk.corpus import qc
+ >>> qc.tuples('test.txt')
+ [('NUM:dist', 'How far is it from Denver to Aspen ?'), ('LOC:city', 'What county is Modesto , California in ?'), ...]
+
+
+
+
diff --git a/nltk/test/corpus_fixt.py b/nltk/test/corpus_fixt.py
new file mode 100644
index 0000000..37a1a42
--- /dev/null
+++ b/nltk/test/corpus_fixt.py
@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+from nltk.corpus import teardown_module
\ No newline at end of file
diff --git a/nltk/test/data.doctest b/nltk/test/data.doctest
new file mode 100644
index 0000000..de09e52
--- /dev/null
+++ b/nltk/test/data.doctest
@@ -0,0 +1,374 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=========================================
+ Loading Resources From the Data Package
+=========================================
+
+ >>> import nltk.data
+
+Overview
+~~~~~~~~
+The `nltk.data` module contains functions that can be used to load
+NLTK resource files, such as corpora, grammars, and saved processing
+objects.
+
+Loading Data Files
+~~~~~~~~~~~~~~~~~~
+Resources are loaded using the function `nltk.data.load()`, which
+takes as its first argument a URL specifying what file should be
+loaded. The ``nltk:`` protocol loads files from the NLTK data
+distribution:
+
+ >>> from __future__ import print_function
+ >>> tokenizer = nltk.data.load('nltk:tokenizers/punkt/english.pickle')
+ >>> tokenizer.tokenize('Hello. This is a test. It works!')
+ ['Hello.', 'This is a test.', 'It works!']
+
+It is important to note that there should be no space following the
+colon (':') in the URL; 'nltk: tokenizers/punkt/english.pickle' will
+not work!
+
+The ``nltk:`` protocol is used by default if no protocol is specified:
+
+ >>> nltk.data.load('tokenizers/punkt/english.pickle') # doctest: +ELLIPSIS
+ <nltk.tokenize.punkt.PunktSentenceTokenizer object at ...>
+
+But it is also possible to load resources from ``http:``, ``ftp:``,
+and ``file:`` URLs, e.g. ``cfg = nltk.data.load('http://example.com/path/to/toy.cfg')``
+
+ >>> # Load a grammar using an absolute path.
+ >>> url = 'file:%s' % nltk.data.find('grammars/sample_grammars/toy.cfg')
+ >>> url.replace('\\', '/') # doctest: +ELLIPSIS
+ 'file:...toy.cfg'
+ >>> print(nltk.data.load(url)) # doctest: +ELLIPSIS
+ Grammar with 14 productions (start state = S)
+ S -> NP VP
+ PP -> P NP
+ ...
+ P -> 'on'
+ P -> 'in'
+
+The second argument to the `nltk.data.load()` function specifies the
+file format, which determines how the file's contents are processed
+before they are returned by ``load()``. The formats that are
+currently supported by the data module are described by the dictionary
+`nltk.data.FORMATS`:
+
+ >>> for format, descr in sorted(nltk.data.FORMATS.items()):
+ ... print('{0:<7} {1:}'.format(format, descr)) # doctest: +NORMALIZE_WHITESPACE
+ cfg A context free grammar.
+ fcfg A feature CFG.
+ fol A list of first order logic expressions, parsed by nltk.sem.parse_fol() using nltk.sem.logic.LogicParser.
+ json A serialized python object, stored using the json module.
+ logic A list of first order logic expressions, parsed by nltk.sem.parse_logic(). Requires an additional logic_parser parameter
+ pcfg A probabilistic CFG.
+ pickle A serialized python object, stored using the pickle module.
+ raw The raw (byte string) contents of a file.
+ text The raw (unicode string) contents of a file.
+ val A semantic valuation, parsed by nltk.sem.parse_valuation().
+ yaml A serialized python object, stored using the yaml module.
+
+`nltk.data.load()` will raise a ValueError if a bad format name is
+specified:
+
+ >>> nltk.data.load('grammars/sample_grammars/toy.cfg', 'bar')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Unknown format type!
+
+By default, the ``"auto"`` format is used, which chooses a format
+based on the filename's extension. The mapping from file extensions
+to format names is specified by `nltk.data.AUTO_FORMATS`:
+
+ >>> for ext, format in sorted(nltk.data.AUTO_FORMATS.items()):
+ ... print('.%-7s -> %s' % (ext, format))
+ .cfg -> cfg
+ .fcfg -> fcfg
+ .fol -> fol
+ .json -> json
+ .logic -> logic
+ .pcfg -> pcfg
+ .pickle -> pickle
+ .text -> text
+ .txt -> text
+ .val -> val
+ .yaml -> yaml
+
+If `nltk.data.load()` is unable to determine the format based on the
+filename's extension, it will raise a ValueError:
+
+ >>> nltk.data.load('foo.bar')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Could not determine format for foo.bar based on its file
+ extension; use the "format" argument to specify the format explicitly.
+
+Note that by explicitly specifying the ``format`` argument, you can
+override the load method's default processing behavior. For example,
+to get the raw contents of any file, simply use ``format="raw"``:
+
+ >>> s = nltk.data.load('grammars/sample_grammars/toy.cfg', 'text')
+ >>> print(s) # doctest: +ELLIPSIS
+ S -> NP VP
+ PP -> P NP
+ NP -> Det N | NP PP
+ VP -> V NP | VP PP
+ ...
+
+Making Local Copies
+~~~~~~~~~~~~~~~~~~~
+.. This will not be visible in the html output: create a tempdir to
+ play in.
+ >>> import tempfile, os
+ >>> tempdir = tempfile.mkdtemp()
+ >>> old_dir = os.path.abspath('.')
+ >>> os.chdir(tempdir)
+
+The function `nltk.data.retrieve()` copies a given resource to a local
+file. This can be useful, for example, if you want to edit one of the
+sample grammars.
+
+ >>> nltk.data.retrieve('grammars/sample_grammars/toy.cfg')
+ Retrieving 'nltk:grammars/sample_grammars/toy.cfg', saving to 'toy.cfg'
+
+ >>> # Simulate editing the grammar.
+ >>> with open('toy.cfg') as inp:
+ ... s = inp.read().replace('NP', 'DP')
+ >>> with open('toy.cfg', 'w') as out:
+ ... _bytes_written = out.write(s)
+
+ >>> # Load the edited grammar, & display it.
+ >>> cfg = nltk.data.load('file:///' + os.path.abspath('toy.cfg'))
+ >>> print(cfg) # doctest: +ELLIPSIS
+ Grammar with 14 productions (start state = S)
+ S -> DP VP
+ PP -> P DP
+ ...
+ P -> 'on'
+ P -> 'in'
+
+The second argument to `nltk.data.retrieve()` specifies the filename
+for the new copy of the file. By default, the source file's filename
+is used.
+
+ >>> nltk.data.retrieve('grammars/sample_grammars/toy.cfg', 'mytoy.cfg')
+ Retrieving 'nltk:grammars/sample_grammars/toy.cfg', saving to 'mytoy.cfg'
+ >>> os.path.isfile('./mytoy.cfg')
+ True
+ >>> nltk.data.retrieve('grammars/sample_grammars/np.fcfg')
+ Retrieving 'nltk:grammars/sample_grammars/np.fcfg', saving to 'np.fcfg'
+ >>> os.path.isfile('./np.fcfg')
+ True
+
+If a file with the specified (or default) filename already exists in
+the current directory, then `nltk.data.retrieve()` will raise a
+ValueError exception. It will *not* overwrite the file:
+
+ >>> os.path.isfile('./toy.cfg')
+ True
+ >>> nltk.data.retrieve('grammars/sample_grammars/toy.cfg') # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ . . .
+ ValueError: File '...toy.cfg' already exists!
+
+.. This will not be visible in the html output: clean up the tempdir.
+ >>> os.chdir(old_dir)
+ >>> for f in os.listdir(tempdir):
+ ... os.remove(os.path.join(tempdir, f))
+ >>> os.rmdir(tempdir)
+
+Finding Files in the NLTK Data Package
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The `nltk.data.find()` function searches the NLTK data package for a
+given file, and returns a pointer to that file. This pointer can
+either be a `FileSystemPathPointer` (whose `path` attribute gives the
+absolute path of the file); or a `ZipFilePathPointer`, specifying a
+zipfile and the name of an entry within that zipfile. Both pointer
+types define the `open()` method, which can be used to read the string
+contents of the file.
+
+ >>> path = nltk.data.find('corpora/abc/rural.txt')
+ >>> str(path) # doctest: +ELLIPSIS
+ '...rural.txt'
+ >>> print(path.open().read(60).decode())
+ PM denies knowledge of AWB kickbacks
+ The Prime Minister has
+
+Alternatively, the `nltk.data.load()` function can be used with the
+keyword argument ``format="raw"``:
+
+ >>> s = nltk.data.load('corpora/abc/rural.txt', format='raw')[:60]
+ >>> print(s.decode())
+ PM denies knowledge of AWB kickbacks
+ The Prime Minister has
+
+Alternatively, you can use the keyword argument ``format="text"``:
+
+ >>> s = nltk.data.load('corpora/abc/rural.txt', format='text')[:60]
+ >>> print(s)
+ PM denies knowledge of AWB kickbacks
+ The Prime Minister has
+
+Resource Caching
+~~~~~~~~~~~~~~~~
+
+NLTK uses a weakref dictionary to maintain a cache of resources that
+have been loaded. If you load a resource that is already stored in
+the cache, then the cached copy will be returned. This behavior can
+be seen by the trace output generated when verbose=True:
+
+ >>> feat0 = nltk.data.load('grammars/book_grammars/feat0.fcfg', verbose=True)
+ <<Loading nltk:grammars/book_grammars/feat0.fcfg>>
+ >>> feat0 = nltk.data.load('grammars/book_grammars/feat0.fcfg', verbose=True)
+ <<Using cached copy of nltk:grammars/book_grammars/feat0.fcfg>>
+
+If you wish to load a resource from its source, bypassing the cache,
+use the ``cache=False`` argument to `nltk.data.load()`. This can be
+useful, for example, if the resource is loaded from a local file, and
+you are actively editing that file:
+
+ >>> feat0 = nltk.data.load('grammars/book_grammars/feat0.fcfg',cache=False,verbose=True)
+ <<Loading nltk:grammars/book_grammars/feat0.fcfg>>
+
+The cache *no longer* uses weak references. A resource will not be
+automatically expunged from the cache when no more objects are using
+it. In the following example, when we clear the variable ``feat0``,
+the reference count for the feature grammar object drops to zero.
+However, the object remains cached:
+
+ >>> del feat0
+ >>> feat0 = nltk.data.load('grammars/book_grammars/feat0.fcfg',
+ ... verbose=True)
+ <<Using cached copy of nltk:grammars/book_grammars/feat0.fcfg>>
+
+You can clear the entire contents of the cache, using
+`nltk.data.clear_cache()`:
+
+ >>> nltk.data.clear_cache()
+
+Retrieving other Data Sources
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ >>> formulas = nltk.data.load('grammars/book_grammars/background.fol')
+ >>> for f in formulas: print(str(f))
+ all x.(boxerdog(x) -> dog(x))
+ all x.(boxer(x) -> person(x))
+ all x.-(dog(x) & person(x))
+ all x.(married(x) <-> exists y.marry(x,y))
+ all x.(bark(x) -> dog(x))
+ all x y.(marry(x,y) -> (person(x) & person(y)))
+ -(Vincent = Mia)
+ -(Vincent = Fido)
+ -(Mia = Fido)
+
+Regression Tests
+~~~~~~~~~~~~~~~~
+Create a temp dir for tests that write files:
+
+ >>> import tempfile, os
+ >>> tempdir = tempfile.mkdtemp()
+ >>> old_dir = os.path.abspath('.')
+ >>> os.chdir(tempdir)
+
+The `retrieve()` function accepts all url types:
+
+ >>> urls = ['http://nltk.googlecode.com/svn/trunk/nltk/nltk/test/toy.cfg',
+ ... 'file:%s' % nltk.data.find('grammars/sample_grammars/toy.cfg'),
+ ... 'nltk:grammars/sample_grammars/toy.cfg',
+ ... 'grammars/sample_grammars/toy.cfg']
+ >>> for i, url in enumerate(urls):
+ ... nltk.data.retrieve(url, 'toy-%d.cfg' % i) # doctest: +ELLIPSIS
+ Retrieving 'http://nltk.googlecode.com/svn/trunk/nltk/nltk/test/toy.cfg', saving to 'toy-0.cfg'
+ Retrieving 'file:...toy.cfg', saving to 'toy-1.cfg'
+ Retrieving 'nltk:grammars/sample_grammars/toy.cfg', saving to 'toy-2.cfg'
+ Retrieving 'nltk:grammars/sample_grammars/toy.cfg', saving to 'toy-3.cfg'
+
+Clean up the temp dir:
+
+ >>> os.chdir(old_dir)
+ >>> for f in os.listdir(tempdir):
+ ... os.remove(os.path.join(tempdir, f))
+ >>> os.rmdir(tempdir)
+
+Lazy Loader
+-----------
+A lazy loader is a wrapper object that defers loading a resource until
+it is accessed or used in any way. This is mainly intended for
+internal use by NLTK's corpus readers.
+
+ >>> # Create a lazy loader for toy.cfg.
+ >>> ll = nltk.data.LazyLoader('grammars/sample_grammars/toy.cfg')
+
+ >>> # Show that it's not loaded yet:
+ >>> object.__repr__(ll) # doctest: +ELLIPSIS
+ '<nltk.data.LazyLoader object at ...>'
+
+ >>> # printing it is enough to cause it to be loaded:
+ >>> print(ll)
+ <Grammar with 14 productions>
+
+ >>> # Show that it's now been loaded:
+ >>> object.__repr__(ll) # doctest: +ELLIPSIS
+ '<nltk.grammar.CFG object at ...>'
+
+
+ >>> # Test that accessing an attribute also loads it:
+ >>> ll = nltk.data.LazyLoader('grammars/sample_grammars/toy.cfg')
+ >>> ll.start()
+ S
+ >>> object.__repr__(ll) # doctest: +ELLIPSIS
+ '<nltk.grammar.CFG object at ...>'
+
+Buffered Gzip Reading and Writing
+---------------------------------
+Write performance to gzip-compressed is extremely poor when the files become large.
+File creation can become a bottleneck in those cases.
+
+Read performance from large gzipped pickle files was improved in data.py by
+buffering the reads. A similar fix can be applied to writes by buffering
+the writes to a StringIO object first.
+
+This is mainly intended for internal use. The test simply tests that reading
+and writing work as intended and does not test how much improvement buffering
+provides.
+
+ >>> from nltk.compat import StringIO
+ >>> test = nltk.data.BufferedGzipFile('testbuf.gz', 'wb', size=2**10)
+ >>> ans = []
+ >>> for i in range(10000):
+ ... ans.append(str(i).encode('ascii'))
+ ... test.write(str(i).encode('ascii'))
+ >>> test.close()
+ >>> test = nltk.data.BufferedGzipFile('testbuf.gz', 'rb')
+ >>> test.read() == b''.join(ans)
+ True
+ >>> test.close()
+ >>> import os
+ >>> os.unlink('testbuf.gz')
+
+JSON Encoding and Decoding
+--------------------------
+JSON serialization is used instead of pickle for some classes.
+
+ >>> from nltk import jsontags
+ >>> from nltk.jsontags import JSONTaggedEncoder, JSONTaggedDecoder, register_tag
+ >>> @jsontags.register_tag
+ ... class JSONSerializable:
+ ... json_tag = 'JSONSerializable'
+ ...
+ ... def __init__(self, n):
+ ... self.n = n
+ ...
+ ... def encode_json_obj(self):
+ ... return self.n
+ ...
+ ... @classmethod
+ ... def decode_json_obj(cls, obj):
+ ... n = obj
+ ... return cls(n)
+ ...
+ >>> JSONTaggedEncoder().encode(JSONSerializable(1))
+ '{"!JSONSerializable": 1}'
+ >>> JSONTaggedDecoder().decode('{"!JSONSerializable": 1}').n
+ 1
+
diff --git a/nltk/test/dependency.doctest b/nltk/test/dependency.doctest
new file mode 100644
index 0000000..3193f91
--- /dev/null
+++ b/nltk/test/dependency.doctest
@@ -0,0 +1,119 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+===================
+Dependency Grammars
+===================
+
+ >>> from nltk.grammar import DependencyGrammar
+ >>> from nltk.parse import *
+
+CoNLL Data
+----------
+
+ >>> treebank_data = """Pierre NNP 2 NMOD
+ ... Vinken NNP 8 SUB
+ ... , , 2 P
+ ... 61 CD 5 NMOD
+ ... years NNS 6 AMOD
+ ... old JJ 2 NMOD
+ ... , , 2 P
+ ... will MD 0 ROOT
+ ... join VB 8 VC
+ ... the DT 11 NMOD
+ ... board NN 9 OBJ
+ ... as IN 9 VMOD
+ ... a DT 15 NMOD
+ ... nonexecutive JJ 15 NMOD
+ ... director NN 12 PMOD
+ ... Nov. NNP 9 VMOD
+ ... 29 CD 16 NMOD
+ ... . . 9 VMOD
+ ... """
+
+
+ >>> dg = DependencyGraph(treebank_data)
+ >>> print(dg.tree().pprint())
+ (will
+ (Vinken Pierre , (old (years 61)) ,)
+ (join (board the) (as (director a nonexecutive)) (Nov. 29) .))
+
+Using the dependency-parsed version of the Penn Treebank corpus sample.
+
+ >>> from nltk.corpus import dependency_treebank
+ >>> t = dependency_treebank.parsed_sents()[0]
+ >>> print(t.to_conll(3)) # doctest: +NORMALIZE_WHITESPACE
+ Pierre NNP 2
+ Vinken NNP 8
+ , , 2
+ 61 CD 5
+ years NNS 6
+ old JJ 2
+ , , 2
+ will MD 0
+ join VB 8
+ the DT 11
+ board NN 9
+ as IN 9
+ a DT 15
+ nonexecutive JJ 15
+ director NN 12
+ Nov. NNP 9
+ 29 CD 16
+ . . 8
+
+Projective Dependency Parsing
+-----------------------------
+
+ >>> grammar = DependencyGrammar.fromstring("""
+ ... 'fell' -> 'price' | 'stock'
+ ... 'price' -> 'of' 'the'
+ ... 'of' -> 'stock'
+ ... 'stock' -> 'the'
+ ... """)
+ >>> print(grammar)
+ Dependency grammar with 5 productions
+ 'fell' -> 'price'
+ 'fell' -> 'stock'
+ 'price' -> 'of' 'the'
+ 'of' -> 'stock'
+ 'stock' -> 'the'
+
+ >>> dp = ProjectiveDependencyParser(grammar)
+ >>> for t in sorted(dp.parse(['the', 'price', 'of', 'the', 'stock', 'fell'])):
+ ... print(t)
+ (fell (price the (of (stock the))))
+ (fell (price the of) (stock the))
+ (fell (price the of the) stock)
+
+Non-Projective Dependency Parsing
+---------------------------------
+
+ >>> grammar = DependencyGrammar.fromstring("""
+ ... 'taught' -> 'play' | 'man'
+ ... 'man' -> 'the'
+ ... 'play' -> 'golf' | 'dog' | 'to'
+ ... 'dog' -> 'his'
+ ... """)
+ >>> print(grammar)
+ Dependency grammar with 7 productions
+ 'taught' -> 'play'
+ 'taught' -> 'man'
+ 'man' -> 'the'
+ 'play' -> 'golf'
+ 'play' -> 'dog'
+ 'play' -> 'to'
+ 'dog' -> 'his'
+
+ >>> dp = NonprojectiveDependencyParser(grammar)
+ >>> for g in dp.parse(['the', 'man', 'taught', 'his', 'dog', 'to', 'play', 'golf']):
+ ... print(g) # doctest: +NORMALIZE_WHITESPACE
+ [{'address': 0, 'deps': 3, 'rel': 'TOP', 'tag': 'TOP', 'word': None},
+ {'address': 1, 'deps': [], 'word': 'the'},
+ {'address': 2, 'deps': [1], 'word': 'man'},
+ {'address': 3, 'deps': [2, 7], 'word': 'taught'},
+ {'address': 4, 'deps': [], 'word': 'his'},
+ {'address': 5, 'deps': [4], 'word': 'dog'},
+ {'address': 6, 'deps': [], 'word': 'to'},
+ {'address': 7, 'deps': [5, 6, 8], 'word': 'play'},
+ {'address': 8, 'deps': [], 'word': 'golf'}]
diff --git a/nltk/test/discourse.doctest b/nltk/test/discourse.doctest
new file mode 100644
index 0000000..c324c26
--- /dev/null
+++ b/nltk/test/discourse.doctest
@@ -0,0 +1,546 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==================
+Discourse Checking
+==================
+
+ >>> from nltk import *
+ >>> from nltk.sem import logic
+ >>> logic._counter._value = 0
+
+Introduction
+============
+
+The NLTK discourse module makes it possible to test consistency and
+redundancy of simple discourses, using theorem-proving and
+model-building from `nltk.inference`.
+
+The ``DiscourseTester`` constructor takes a list of sentences as a
+parameter.
+
+ >>> dt = DiscourseTester(['a boxer walks', 'every boxer chases a girl'])
+
+The ``DiscourseTester`` parses each sentence into a list of logical
+forms. Once we have created ``DiscourseTester`` object, we can
+inspect various properties of the discourse. First off, we might want
+to double-check what sentences are currently stored as the discourse.
+
+ >>> dt.sentences()
+ s0: a boxer walks
+ s1: every boxer chases a girl
+
+As you will see, each sentence receives an identifier `s`\ :subscript:`i`.
+We might also want to check what grammar the ``DiscourseTester`` is
+using (by default, ``book_grammars/discourse.fcfg``):
+
+ >>> dt.grammar() # doctest: +ELLIPSIS
+ % start S
+ # Grammar Rules
+ S[SEM = <app(?subj,?vp)>] -> NP[NUM=?n,SEM=?subj] VP[NUM=?n,SEM=?vp]
+ NP[NUM=?n,SEM=<app(?det,?nom)> ] -> Det[NUM=?n,SEM=?det] Nom[NUM=?n,SEM=?nom]
+ NP[LOC=?l,NUM=?n,SEM=?np] -> PropN[LOC=?l,NUM=?n,SEM=?np]
+ ...
+
+A different grammar can be invoked by using the optional ``gramfile``
+parameter when a ``DiscourseTester`` object is created.
+
+Readings and Threads
+====================
+
+Depending on
+the grammar used, we may find some sentences have more than one
+logical form. To check this, use the ``readings()`` method. Given a
+sentence identifier of the form `s`\ :subscript:`i`, each reading of
+that sentence is given an identifier `s`\ :sub:`i`-`r`\ :sub:`j`.
+
+
+ >>> dt.readings()
+ <BLANKLINE>
+ s0 readings:
+ <BLANKLINE>
+ s0-r0: exists z1.(boxer(z1) & walk(z1))
+ s0-r1: exists z1.(boxerdog(z1) & walk(z1))
+ <BLANKLINE>
+ s1 readings:
+ <BLANKLINE>
+ s1-r0: all z2.(boxer(z2) -> exists z3.(girl(z3) & chase(z2,z3)))
+ s1-r1: all z1.(boxerdog(z1) -> exists z2.(girl(z2) & chase(z1,z2)))
+
+
+In this case, the only source of ambiguity lies in the word *boxer*,
+which receives two translations: ``boxer`` and ``boxerdog``. The
+intention is that one of these corresponds to the ``person`` sense and
+one to the ``dog`` sense. In principle, we would also expect to see a
+quantifier scope ambiguity in ``s1``. However, the simple grammar we
+are using, namely `sem4.fcfg <sem4.fcfg>`_, doesn't support quantifier
+scope ambiguity.
+
+We can also investigate the readings of a specific sentence:
+
+ >>> dt.readings('a boxer walks')
+ The sentence 'a boxer walks' has these readings:
+ exists x.(boxer(x) & walk(x))
+ exists x.(boxerdog(x) & walk(x))
+
+Given that each sentence is two-ways ambiguous, we potentially have
+four different discourse 'threads', taking all combinations of
+readings. To see these, specify the ``threaded=True`` parameter on
+the ``readings()`` method. Again, each thread is assigned an
+identifier of the form `d`\ :sub:`i`. Following the identifier is a
+list of the readings that constitute that thread.
+
+ >>> dt.readings(threaded=True) # doctest: +NORMALIZE_WHITESPACE
+ d0: ['s0-r0', 's1-r0']
+ d1: ['s0-r0', 's1-r1']
+ d2: ['s0-r1', 's1-r0']
+ d3: ['s0-r1', 's1-r1']
+
+Of course, this simple-minded approach doesn't scale: a discourse with, say, three
+sentences, each of which has 3 readings, will generate 27 different
+threads. It is an interesting exercise to consider how to manage
+discourse ambiguity more efficiently.
+
+Checking Consistency
+====================
+
+Now, we can check whether some or all of the discourse threads are
+consistent, using the ``models()`` method. With no parameter, this
+method will try to find a model for every discourse thread in the
+current discourse. However, we can also specify just one thread, say ``d1``.
+
+ >>> dt.models('d1')
+ --------------------------------------------------------------------------------
+ Model for Discourse Thread d1
+ --------------------------------------------------------------------------------
+ % number = 1
+ % seconds = 0
+ <BLANKLINE>
+ % Interpretation of size 2
+ <BLANKLINE>
+ c1 = 0.
+ <BLANKLINE>
+ f1(0) = 0.
+ f1(1) = 0.
+ <BLANKLINE>
+ boxer(0).
+ - boxer(1).
+ <BLANKLINE>
+ - boxerdog(0).
+ - boxerdog(1).
+ <BLANKLINE>
+ - girl(0).
+ - girl(1).
+ <BLANKLINE>
+ walk(0).
+ - walk(1).
+ <BLANKLINE>
+ - chase(0,0).
+ - chase(0,1).
+ - chase(1,0).
+ - chase(1,1).
+ <BLANKLINE>
+ Consistent discourse: d1 ['s0-r0', 's1-r1']:
+ s0-r0: exists z1.(boxer(z1) & walk(z1))
+ s1-r1: all z1.(boxerdog(z1) -> exists z2.(girl(z2) & chase(z1,z2)))
+ <BLANKLINE>
+
+There are various formats for rendering **Mace4** models --- here,
+we have used the 'cooked' format (which is intended to be
+human-readable). There are a number of points to note.
+
+#. The entities in the domain are all treated as non-negative
+ integers. In this case, there are only two entities, ``0`` and
+ ``1``.
+
+#. The ``-`` symbol indicates negation. So ``0`` is the only
+ ``boxerdog`` and the only thing that ``walk``\ s. Nothing is a
+ ``boxer``, or a ``girl`` or in the ``chase`` relation. Thus the
+ universal sentence is vacuously true.
+
+#. ``c1`` is an introduced constant that denotes ``0``.
+
+#. ``f1`` is a Skolem function, but it plays no significant role in
+ this model.
+
+
+We might want to now add another sentence to the discourse, and there
+is method ``add_sentence()`` for doing just this.
+
+ >>> dt.add_sentence('John is a boxer')
+ >>> dt.sentences()
+ s0: a boxer walks
+ s1: every boxer chases a girl
+ s2: John is a boxer
+
+We can now test all the properties as before; here, we just show a
+couple of them.
+
+ >>> dt.readings()
+ <BLANKLINE>
+ s0 readings:
+ <BLANKLINE>
+ s0-r0: exists z1.(boxer(z1) & walk(z1))
+ s0-r1: exists z1.(boxerdog(z1) & walk(z1))
+ <BLANKLINE>
+ s1 readings:
+ <BLANKLINE>
+ s1-r0: all z1.(boxer(z1) -> exists z2.(girl(z2) & chase(z1,z2)))
+ s1-r1: all z1.(boxerdog(z1) -> exists z2.(girl(z2) & chase(z1,z2)))
+ <BLANKLINE>
+ s2 readings:
+ <BLANKLINE>
+ s2-r0: boxer(John)
+ s2-r1: boxerdog(John)
+ >>> dt.readings(threaded=True) # doctest: +NORMALIZE_WHITESPACE
+ d0: ['s0-r0', 's1-r0', 's2-r0']
+ d1: ['s0-r0', 's1-r0', 's2-r1']
+ d2: ['s0-r0', 's1-r1', 's2-r0']
+ d3: ['s0-r0', 's1-r1', 's2-r1']
+ d4: ['s0-r1', 's1-r0', 's2-r0']
+ d5: ['s0-r1', 's1-r0', 's2-r1']
+ d6: ['s0-r1', 's1-r1', 's2-r0']
+ d7: ['s0-r1', 's1-r1', 's2-r1']
+
+If you are interested in a particular thread, the ``expand_threads()``
+method will remind you of what readings it consists of:
+
+ >>> thread = dt.expand_threads('d1')
+ >>> for rid, reading in thread:
+ ... print(rid, str(reading.normalize()))
+ s0-r0 exists z1.(boxer(z1) & walk(z1))
+ s1-r0 all z1.(boxer(z1) -> exists z2.(girl(z2) & chase(z1,z2)))
+ s2-r1 boxerdog(John)
+
+Suppose we have already defined a discourse, as follows:
+
+ >>> dt = DiscourseTester(['A student dances', 'Every student is a person'])
+
+Now, when we add a new sentence, is it consistent with what we already
+have? The `` consistchk=True`` parameter of ``add_sentence()`` allows
+us to check:
+
+ >>> dt.add_sentence('No person dances', consistchk=True)
+ Inconsistent discourse: d0 ['s0-r0', 's1-r0', 's2-r0']:
+ s0-r0: exists z1.(student(z1) & dance(z1))
+ s1-r0: all z1.(student(z1) -> person(z1))
+ s2-r0: -exists z1.(person(z1) & dance(z1))
+ <BLANKLINE>
+ >>> dt.readings()
+ <BLANKLINE>
+ s0 readings:
+ <BLANKLINE>
+ s0-r0: exists z1.(student(z1) & dance(z1))
+ <BLANKLINE>
+ s1 readings:
+ <BLANKLINE>
+ s1-r0: all z1.(student(z1) -> person(z1))
+ <BLANKLINE>
+ s2 readings:
+ <BLANKLINE>
+ s2-r0: -exists z1.(person(z1) & dance(z1))
+
+So let's retract the inconsistent sentence:
+
+ >>> dt.retract_sentence('No person dances', verbose=True) # doctest: +NORMALIZE_WHITESPACE
+ Current sentences are
+ s0: A student dances
+ s1: Every student is a person
+
+We can now verify that result is consistent.
+
+ >>> dt.models()
+ --------------------------------------------------------------------------------
+ Model for Discourse Thread d0
+ --------------------------------------------------------------------------------
+ % number = 1
+ % seconds = 0
+ <BLANKLINE>
+ % Interpretation of size 2
+ <BLANKLINE>
+ c1 = 0.
+ <BLANKLINE>
+ dance(0).
+ - dance(1).
+ <BLANKLINE>
+ person(0).
+ - person(1).
+ <BLANKLINE>
+ student(0).
+ - student(1).
+ <BLANKLINE>
+ Consistent discourse: d0 ['s0-r0', 's1-r0']:
+ s0-r0: exists z1.(student(z1) & dance(z1))
+ s1-r0: all z1.(student(z1) -> person(z1))
+ <BLANKLINE>
+
+Checking Informativity
+======================
+
+Let's assume that we are still trying to extend the discourse *A
+student dances.* *Every student is a person.* We add a new sentence,
+but this time, we check whether it is informative with respect to what
+has gone before.
+
+ >>> dt.add_sentence('A person dances', informchk=True)
+ Sentence 'A person dances' under reading 'exists x.(person(x) & dance(x))':
+ Not informative relative to thread 'd0'
+
+In fact, we are just checking whether the new sentence is entailed by
+the preceding discourse.
+
+ >>> dt.models()
+ --------------------------------------------------------------------------------
+ Model for Discourse Thread d0
+ --------------------------------------------------------------------------------
+ % number = 1
+ % seconds = 0
+ <BLANKLINE>
+ % Interpretation of size 2
+ <BLANKLINE>
+ c1 = 0.
+ <BLANKLINE>
+ c2 = 0.
+ <BLANKLINE>
+ dance(0).
+ - dance(1).
+ <BLANKLINE>
+ person(0).
+ - person(1).
+ <BLANKLINE>
+ student(0).
+ - student(1).
+ <BLANKLINE>
+ Consistent discourse: d0 ['s0-r0', 's1-r0', 's2-r0']:
+ s0-r0: exists z1.(student(z1) & dance(z1))
+ s1-r0: all z1.(student(z1) -> person(z1))
+ s2-r0: exists z1.(person(z1) & dance(z1))
+ <BLANKLINE>
+
+
+
+Adding Background Knowledge
+===========================
+
+Let's build a new discourse, and look at the readings of the component sentences:
+
+ >>> dt = DiscourseTester(['Vincent is a boxer', 'Fido is a boxer', 'Vincent is married', 'Fido barks'])
+ >>> dt.readings()
+ <BLANKLINE>
+ s0 readings:
+ <BLANKLINE>
+ s0-r0: boxer(Vincent)
+ s0-r1: boxerdog(Vincent)
+ <BLANKLINE>
+ s1 readings:
+ <BLANKLINE>
+ s1-r0: boxer(Fido)
+ s1-r1: boxerdog(Fido)
+ <BLANKLINE>
+ s2 readings:
+ <BLANKLINE>
+ s2-r0: married(Vincent)
+ <BLANKLINE>
+ s3 readings:
+ <BLANKLINE>
+ s3-r0: bark(Fido)
+
+This gives us a lot of threads:
+
+ >>> dt.readings(threaded=True) # doctest: +NORMALIZE_WHITESPACE
+ d0: ['s0-r0', 's1-r0', 's2-r0', 's3-r0']
+ d1: ['s0-r0', 's1-r1', 's2-r0', 's3-r0']
+ d2: ['s0-r1', 's1-r0', 's2-r0', 's3-r0']
+ d3: ['s0-r1', 's1-r1', 's2-r0', 's3-r0']
+
+
+We can eliminate some of the readings, and hence some of the threads,
+by adding background information.
+
+ >>> import nltk.data
+ >>> bg = nltk.data.load('grammars/book_grammars/background.fol')
+ >>> dt.add_background(bg)
+ >>> dt.background()
+ all x.(boxerdog(x) -> dog(x))
+ all x.(boxer(x) -> person(x))
+ all x.-(dog(x) & person(x))
+ all x.(married(x) <-> exists y.marry(x,y))
+ all x.(bark(x) -> dog(x))
+ all x y.(marry(x,y) -> (person(x) & person(y)))
+ -(Vincent = Mia)
+ -(Vincent = Fido)
+ -(Mia = Fido)
+
+The background information allows us to reject three of the threads as
+inconsistent. To see what remains, use the ``filter=True`` parameter
+on ``readings()``.
+
+ >>> dt.readings(filter=True) # doctest: +NORMALIZE_WHITESPACE
+ d1: ['s0-r0', 's1-r1', 's2-r0', 's3-r0']
+
+The ``models()`` method gives us more information about the surviving thread.
+
+ >>> dt.models()
+ --------------------------------------------------------------------------------
+ Model for Discourse Thread d0
+ --------------------------------------------------------------------------------
+ No model found!
+ <BLANKLINE>
+ --------------------------------------------------------------------------------
+ Model for Discourse Thread d1
+ --------------------------------------------------------------------------------
+ % number = 1
+ % seconds = 0
+ <BLANKLINE>
+ % Interpretation of size 3
+ <BLANKLINE>
+ Fido = 0.
+ <BLANKLINE>
+ Mia = 1.
+ <BLANKLINE>
+ Vincent = 2.
+ <BLANKLINE>
+ f1(0) = 0.
+ f1(1) = 0.
+ f1(2) = 2.
+ <BLANKLINE>
+ bark(0).
+ - bark(1).
+ - bark(2).
+ <BLANKLINE>
+ - boxer(0).
+ - boxer(1).
+ boxer(2).
+ <BLANKLINE>
+ boxerdog(0).
+ - boxerdog(1).
+ - boxerdog(2).
+ <BLANKLINE>
+ dog(0).
+ - dog(1).
+ - dog(2).
+ <BLANKLINE>
+ - married(0).
+ - married(1).
+ married(2).
+ <BLANKLINE>
+ - person(0).
+ - person(1).
+ person(2).
+ <BLANKLINE>
+ - marry(0,0).
+ - marry(0,1).
+ - marry(0,2).
+ - marry(1,0).
+ - marry(1,1).
+ - marry(1,2).
+ - marry(2,0).
+ - marry(2,1).
+ marry(2,2).
+ <BLANKLINE>
+ --------------------------------------------------------------------------------
+ Model for Discourse Thread d2
+ --------------------------------------------------------------------------------
+ No model found!
+ <BLANKLINE>
+ --------------------------------------------------------------------------------
+ Model for Discourse Thread d3
+ --------------------------------------------------------------------------------
+ No model found!
+ <BLANKLINE>
+ Inconsistent discourse: d0 ['s0-r0', 's1-r0', 's2-r0', 's3-r0']:
+ s0-r0: boxer(Vincent)
+ s1-r0: boxer(Fido)
+ s2-r0: married(Vincent)
+ s3-r0: bark(Fido)
+ <BLANKLINE>
+ Consistent discourse: d1 ['s0-r0', 's1-r1', 's2-r0', 's3-r0']:
+ s0-r0: boxer(Vincent)
+ s1-r1: boxerdog(Fido)
+ s2-r0: married(Vincent)
+ s3-r0: bark(Fido)
+ <BLANKLINE>
+ Inconsistent discourse: d2 ['s0-r1', 's1-r0', 's2-r0', 's3-r0']:
+ s0-r1: boxerdog(Vincent)
+ s1-r0: boxer(Fido)
+ s2-r0: married(Vincent)
+ s3-r0: bark(Fido)
+ <BLANKLINE>
+ Inconsistent discourse: d3 ['s0-r1', 's1-r1', 's2-r0', 's3-r0']:
+ s0-r1: boxerdog(Vincent)
+ s1-r1: boxerdog(Fido)
+ s2-r0: married(Vincent)
+ s3-r0: bark(Fido)
+ <BLANKLINE>
+
+
+.. This will not be visible in the html output: create a tempdir to
+ play in.
+ >>> import tempfile, os
+ >>> tempdir = tempfile.mkdtemp()
+ >>> old_dir = os.path.abspath('.')
+ >>> os.chdir(tempdir)
+
+In order to play around with your own version of background knowledge,
+you might want to start off with a local copy of ``background.fol``:
+
+ >>> nltk.data.retrieve('grammars/book_grammars/background.fol')
+ Retrieving 'nltk:grammars/book_grammars/background.fol', saving to 'background.fol'
+
+After you have modified the file, the ``parse_logic()`` function will parse
+the strings in the file into expressions of ``nltk.logic``.
+
+ >>> from nltk.inference.discourse import parse_fol
+ >>> mybg = parse_fol(open('background.fol').read())
+
+The result can be loaded as an argument of ``add_background()`` in the
+manner shown earlier.
+
+.. This will not be visible in the html output: clean up the tempdir.
+ >>> os.chdir(old_dir)
+ >>> for f in os.listdir(tempdir):
+ ... os.remove(os.path.join(tempdir, f))
+ >>> os.rmdir(tempdir)
+ >>> nltk.data.clear_cache()
+
+
+Regression Testing from book
+============================
+
+ >>> logic._counter._value = 0
+
+ >>> from nltk.tag import RegexpTagger
+ >>> tagger = RegexpTagger(
+ ... [('^(chases|runs)$', 'VB'),
+ ... ('^(a)$', 'ex_quant'),
+ ... ('^(every)$', 'univ_quant'),
+ ... ('^(dog|boy)$', 'NN'),
+ ... ('^(He)$', 'PRP')
+ ... ])
+ >>> rc = DrtGlueReadingCommand(depparser=MaltParser(tagger=tagger))
+ >>> dt = DiscourseTester(map(str.split, ['Every dog chases a boy', 'He runs']), rc)
+ >>> dt.readings()
+ <BLANKLINE>
+ s0 readings:
+ <BLANKLINE>
+ s0-r0: ([z2],[boy(z2), (([z5],[dog(z5)]) -> ([],[chases(z5,z2)]))])
+ s0-r1: ([],[(([z1],[dog(z1)]) -> ([z2],[boy(z2), chases(z1,z2)]))])
+ <BLANKLINE>
+ s1 readings:
+ <BLANKLINE>
+ s1-r0: ([z1],[PRO(z1), runs(z1)])
+ >>> dt.readings(show_thread_readings=True)
+ d0: ['s0-r0', 's1-r0'] : ([z1,z2],[boy(z1), (([z3],[dog(z3)]) -> ([],[chases(z3,z1)])), (z2 = z1), runs(z2)])
+ d1: ['s0-r1', 's1-r0'] : INVALID: AnaphoraResolutionException
+ >>> dt.readings(filter=True, show_thread_readings=True)
+ d0: ['s0-r0', 's1-r0'] : ([z1,z3],[boy(z1), (([z2],[dog(z2)]) -> ([],[chases(z2,z1)])), (z3 = z1), runs(z3)])
+
+ >>> logic._counter._value = 0
+
+ >>> from nltk.parse import FeatureEarleyChartParser
+ >>> from nltk.sem.drt import DrtParser
+ >>> grammar = nltk.data.load('grammars/book_grammars/drt.fcfg', logic_parser=DrtParser())
+ >>> parser = FeatureEarleyChartParser(grammar, trace=0)
+ >>> trees = parser.nbest_parse('Angus owns a dog'.split())
+ >>> print(trees[0].label()['SEM'].simplify().normalize())
+ ([z1,z2],[Angus(z1), dog(z2), own(z1,z2)])
diff --git a/nltk/test/discourse_fixt.py b/nltk/test/discourse_fixt.py
new file mode 100644
index 0000000..589801d
--- /dev/null
+++ b/nltk/test/discourse_fixt.py
@@ -0,0 +1,13 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+# FIXME: the entire discourse.doctest is skipped if Prover9/Mace4 is
+# not installed, but there are pure-python parts that don't need Prover9.
+def setup_module(module):
+ from nose import SkipTest
+ from nltk.inference.mace import Mace
+ try:
+ m = Mace()
+ m._find_binary('mace4')
+ except LookupError:
+ raise SkipTest("Mace4/Prover9 is not available so discourse.doctest is skipped")
diff --git a/nltk/test/doctest_nose_plugin.py b/nltk/test/doctest_nose_plugin.py
new file mode 100644
index 0000000..9cac82f
--- /dev/null
+++ b/nltk/test/doctest_nose_plugin.py
@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+from nose.suite import ContextList
+import re
+import sys
+import os
+import codecs
+import doctest
+from nose.plugins.base import Plugin
+from nose.util import tolist, anyp
+from nose.plugins.doctests import Doctest, log, DocFileCase
+
+ALLOW_UNICODE = doctest.register_optionflag('ALLOW_UNICODE')
+
+class _UnicodeOutputChecker(doctest.OutputChecker):
+ _literal_re = re.compile(r"(\W|^)[uU]([rR]?[\'\"])", re.UNICODE)
+
+ def _remove_u_prefixes(self, txt):
+ return re.sub(self._literal_re, r'\1\2', txt)
+
+ def check_output(self, want, got, optionflags):
+ res = doctest.OutputChecker.check_output(self, want, got, optionflags)
+ if res:
+ return True
+ if not (optionflags & ALLOW_UNICODE):
+ return False
+
+ # ALLOW_UNICODE is active and want != got
+ cleaned_want = self._remove_u_prefixes(want)
+ cleaned_got = self._remove_u_prefixes(got)
+ res = doctest.OutputChecker.check_output(self, cleaned_want, cleaned_got, optionflags)
+ return res
+
+_checker = _UnicodeOutputChecker()
+
+class DoctestPluginHelper(object):
+ """
+ This mixin adds print_function future import to all test cases.
+
+ It also adds support for:
+ '#doctest +ALLOW_UNICODE' option that
+ makes DocTestCase think u'foo' == 'foo'.
+
+ '#doctest doctestencoding=utf-8' option that
+ changes the encoding of doctest files
+ """
+ OPTION_BY_NAME = ('doctestencoding',)
+
+ def loadTestsFromFileUnicode(self, filename):
+ if self.extension and anyp(filename.endswith, self.extension):
+ name = os.path.basename(filename)
+ dh = codecs.open(filename, 'r', self.options.get('doctestencoding'))
+ try:
+ doc = dh.read()
+ finally:
+ dh.close()
+
+ fixture_context = None
+ globs = {'__file__': filename}
+ if self.fixtures:
+ base, ext = os.path.splitext(name)
+ dirname = os.path.dirname(filename)
+ sys.path.append(dirname)
+ fixt_mod = base + self.fixtures
+ try:
+ fixture_context = __import__(
+ fixt_mod, globals(), locals(), ["nop"])
+ except ImportError as e:
+ log.debug(
+ "Could not import %s: %s (%s)", fixt_mod, e, sys.path)
+ log.debug("Fixture module %s resolved to %s",
+ fixt_mod, fixture_context)
+ if hasattr(fixture_context, 'globs'):
+ globs = fixture_context.globs(globs)
+ parser = doctest.DocTestParser()
+ test = parser.get_doctest(
+ doc, globs=globs, name=name,
+ filename=filename, lineno=0)
+ if test.examples:
+ case = DocFileCase(
+ test,
+ optionflags=self.optionflags,
+ setUp=getattr(fixture_context, 'setup_test', None),
+ tearDown=getattr(fixture_context, 'teardown_test', None),
+ result_var=self.doctest_result_var)
+ if fixture_context:
+ yield ContextList((case,), context=fixture_context)
+ else:
+ yield case
+ else:
+ yield False # no tests to load
+
+ def loadTestsFromFile(self, filename):
+
+ cases = self.loadTestsFromFileUnicode(filename)
+
+ for case in cases:
+ if isinstance(case, ContextList):
+ yield ContextList([self._patchTestCase(c) for c in case], case.context)
+ else:
+ yield self._patchTestCase(case)
+
+ def loadTestsFromModule(self, module):
+ """Load doctests from the module.
+ """
+ for suite in super(DoctestPluginHelper, self).loadTestsFromModule(module):
+ cases = [self._patchTestCase(case) for case in suite._get_tests()]
+ yield self.suiteClass(cases, context=module, can_split=False)
+
+ def _patchTestCase(self, case):
+ if case:
+ case._dt_test.globs['print_function'] = print_function
+ case._dt_checker = _checker
+ return case
+
+ def configure(self, options, config):
+ # it is overriden in order to fix doctest options discovery
+
+ Plugin.configure(self, options, config)
+ self.doctest_result_var = options.doctest_result_var
+ self.doctest_tests = options.doctest_tests
+ self.extension = tolist(options.doctestExtension)
+ self.fixtures = options.doctestFixtures
+ self.finder = doctest.DocTestFinder()
+
+ #super(DoctestPluginHelper, self).configure(options, config)
+ self.optionflags = 0
+ self.options = {}
+
+ if options.doctestOptions:
+ stroptions = ",".join(options.doctestOptions).split(',')
+ for stroption in stroptions:
+ try:
+ if stroption.startswith('+'):
+ self.optionflags |= doctest.OPTIONFLAGS_BY_NAME[stroption[1:]]
+ continue
+ elif stroption.startswith('-'):
+ self.optionflags &= ~doctest.OPTIONFLAGS_BY_NAME[stroption[1:]]
+ continue
+ try:
+ key,value=stroption.split('=')
+ except ValueError:
+ pass
+ else:
+ if not key in self.OPTION_BY_NAME:
+ raise ValueError()
+ self.options[key]=value
+ continue
+ except (AttributeError, ValueError, KeyError):
+ raise ValueError("Unknown doctest option {}".format(stroption))
+ else:
+ raise ValueError("Doctest option is not a flag or a key/value pair: {} ".format(stroption))
+
+
+class DoctestFix(DoctestPluginHelper, Doctest):
+ pass
diff --git a/nltk/test/drt.doctest b/nltk/test/drt.doctest
new file mode 100644
index 0000000..0f0365f
--- /dev/null
+++ b/nltk/test/drt.doctest
@@ -0,0 +1,517 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+================================
+ Discourse Representation Theory
+================================
+
+ >>> from nltk.sem import logic
+ >>> from nltk.inference import TableauProver
+
+Overview
+========
+
+A DRS can be created with the ``DRS()`` constructor. This takes two arguments: a list of
+discourse referents and list of conditions. .
+
+ >>> from nltk.sem.drt import *
+ >>> dexpr = DrtExpression.fromstring
+ >>> man_x = dexpr('man(x)')
+ >>> walk_x = dexpr('walk(x)')
+ >>> x = dexpr('x')
+ >>> print(DRS([x], [man_x, walk_x]))
+ ([x],[man(x), walk(x)])
+
+The ``parse()`` method can also be applied directly to DRS
+expressions, which allows them to be specified more
+easily.
+
+ >>> drs1 = dexpr('([x],[man(x),walk(x)])')
+ >>> print(drs1)
+ ([x],[man(x), walk(x)])
+
+DRSs can be *merged* using the ``+`` operator.
+
+ >>> drs2 = dexpr('([y],[woman(y),stop(y)])')
+ >>> drs3 = drs1 + drs2
+ >>> print(drs3)
+ (([x],[man(x), walk(x)]) + ([y],[woman(y), stop(y)]))
+ >>> print(drs3.simplify())
+ ([x,y],[man(x), walk(x), woman(y), stop(y)])
+
+We can embed DRSs as components of an ``implies`` condition.
+
+ >>> s = '([], [(%s -> %s)])' % (drs1, drs2)
+ >>> print(dexpr(s))
+ ([],[(([x],[man(x), walk(x)]) -> ([y],[woman(y), stop(y)]))])
+
+The ``fol()`` method converts DRSs into FOL formulae.
+
+ >>> print(dexpr(r'([x],[man(x), walks(x)])').fol())
+ exists x.(man(x) & walks(x))
+ >>> print(dexpr(r'([],[(([x],[man(x)]) -> ([],[walks(x)]))])').fol())
+ all x.(man(x) -> walks(x))
+
+In order to visualize a DRS, the ``pprint()`` method can be use.
+
+ >>> drs3.pprint()
+ _________ __________
+ | x | | y |
+ (|---------| + |----------|)
+ | man(x) | | woman(y) |
+ | walk(x) | | stop(y) |
+ |_________| |__________|
+
+
+Parse to semantics
+------------------
+
+..
+ >>> logic._counter._value = 0
+
+DRSs can be used for building compositional semantics in a feature
+based grammar. To specify that we want to use DRSs, the appropriate
+logic parser needs be passed as a parameter to ``load_earley()``
+
+ >>> from nltk.parse import load_parser
+ >>> from nltk.sem.drt import _DrtParser
+ >>> parser = load_parser('grammars/book_grammars/drt.fcfg', trace=0, logic_parser=_DrtParser())
+ >>> for tree in parser.parse('a dog barks'.split()):
+ ... print(tree.label()['SEM'].simplify())
+ ...
+ ([x],[dog(x), bark(x)])
+
+Alternatively, a ``FeatStructReader`` can be passed with the ``logic_parser`` set on it
+
+ >>> from nltk.featstruct import FeatStructReader
+ >>> from nltk.grammar import FeatStructNonterminal
+ >>> parser = load_parser('grammars/book_grammars/drt.fcfg', trace=0, fstruct_reader=FeatStructReader(fdict_class=FeatStructNonterminal, logic_parser=_DrtParser()))
+ >>> for tree in parser.parse('every girl chases a dog'.split()):
+ ... print(tree.label()['SEM'].simplify().normalize())
+ ...
+ ([],[(([z1],[girl(z1)]) -> ([z2],[dog(z2), chase(z1,z2)]))])
+
+
+
+Unit Tests
+==========
+
+Parser
+------
+
+ >>> print(dexpr(r'([x,y],[sees(x,y)])'))
+ ([x,y],[sees(x,y)])
+ >>> print(dexpr(r'([x],[man(x), walks(x)])'))
+ ([x],[man(x), walks(x)])
+ >>> print(dexpr(r'\x.([],[man(x), walks(x)])'))
+ \x.([],[man(x), walks(x)])
+ >>> print(dexpr(r'\x.\y.([],[sees(x,y)])'))
+ \x y.([],[sees(x,y)])
+
+ >>> print(dexpr(r'([x,y],[(x = y)])'))
+ ([x,y],[(x = y)])
+ >>> print(dexpr(r'([x,y],[(x != y)])'))
+ ([x,y],[-(x = y)])
+
+ >>> print(dexpr(r'\x.([],[walks(x)])(john)'))
+ (\x.([],[walks(x)]))(john)
+ >>> print(dexpr(r'\R.\x.([],[big(x,R)])(\y.([],[mouse(y)]))'))
+ (\R x.([],[big(x,R)]))(\y.([],[mouse(y)]))
+
+ >>> print(dexpr(r'(([x],[walks(x)]) + ([y],[runs(y)]))'))
+ (([x],[walks(x)]) + ([y],[runs(y)]))
+ >>> print(dexpr(r'(([x,y],[walks(x), jumps(y)]) + (([z],[twos(z)]) + ([w],[runs(w)])))'))
+ (([x,y],[walks(x), jumps(y)]) + ([z],[twos(z)]) + ([w],[runs(w)]))
+ >>> print(dexpr(r'((([],[walks(x)]) + ([],[twos(x)])) + ([],[runs(x)]))'))
+ (([],[walks(x)]) + ([],[twos(x)]) + ([],[runs(x)]))
+ >>> print(dexpr(r'((([],[walks(x)]) + ([],[runs(x)])) + (([],[threes(x)]) + ([],[fours(x)])))'))
+ (([],[walks(x)]) + ([],[runs(x)]) + ([],[threes(x)]) + ([],[fours(x)]))
+
+ >>> print(dexpr(r'(([],[walks(x)]) -> ([],[runs(x)]))'))
+ (([],[walks(x)]) -> ([],[runs(x)]))
+
+ >>> print(dexpr(r'([x],[PRO(x), sees(John,x)])'))
+ ([x],[PRO(x), sees(John,x)])
+ >>> print(dexpr(r'([x],[man(x), -([],[walks(x)])])'))
+ ([x],[man(x), -([],[walks(x)])])
+ >>> print(dexpr(r'([],[(([x],[man(x)]) -> ([],[walks(x)]))])'))
+ ([],[(([x],[man(x)]) -> ([],[walks(x)]))])
+
+ >>> print(dexpr(r'DRS([x],[walk(x)])'))
+ ([x],[walk(x)])
+ >>> print(dexpr(r'DRS([x][walk(x)])'))
+ ([x],[walk(x)])
+ >>> print(dexpr(r'([x][walk(x)])'))
+ ([x],[walk(x)])
+
+``simplify()``
+--------------
+
+ >>> print(dexpr(r'\x.([],[man(x), walks(x)])(john)').simplify())
+ ([],[man(john), walks(john)])
+ >>> print(dexpr(r'\x.\y.([z],[dog(z),sees(x,y)])(john)(mary)').simplify())
+ ([z],[dog(z), sees(john,mary)])
+ >>> print(dexpr(r'\R x.([],[big(x,R)])(\y.([],[mouse(y)]))').simplify())
+ \x.([],[big(x,\y.([],[mouse(y)]))])
+
+ >>> print(dexpr(r'(([x],[walks(x)]) + ([y],[runs(y)]))').simplify())
+ ([x,y],[walks(x), runs(y)])
+ >>> print(dexpr(r'(([x,y],[walks(x), jumps(y)]) + (([z],[twos(z)]) + ([w],[runs(w)])))').simplify())
+ ([w,x,y,z],[walks(x), jumps(y), twos(z), runs(w)])
+ >>> print(dexpr(r'((([],[walks(x)]) + ([],[runs(x)]) + ([],[threes(x)]) + ([],[fours(x)])))').simplify())
+ ([],[walks(x), runs(x), threes(x), fours(x)])
+ >>> dexpr(r'([x],[man(x)])+([x],[walks(x)])').simplify() == \
+ ... dexpr(r'([x,z1],[man(x), walks(z1)])')
+ True
+ >>> dexpr(r'([y],[boy(y), (([x],[dog(x)]) -> ([],[chase(x,y)]))])+([x],[run(x)])').simplify() == \
+ ... dexpr(r'([y,z1],[boy(y), (([x],[dog(x)]) -> ([],[chase(x,y)])), run(z1)])')
+ True
+
+ >>> dexpr(r'\Q.(([x],[john(x),walks(x)]) + Q)(([x],[PRO(x),leaves(x)]))').simplify() == \
+ ... dexpr(r'([x,z1],[john(x), walks(x), PRO(z1), leaves(z1)])')
+ True
+
+ >>> logic._counter._value = 0
+ >>> print(dexpr('([],[(([x],[dog(x)]) -> ([e,y],[boy(y), chase(e), subj(e,x), obj(e,y)]))])+([e,x],[PRO(x), run(e), subj(e,x)])').simplify().normalize().normalize())
+ ([e02,z5],[(([z3],[dog(z3)]) -> ([e01,z4],[boy(z4), chase(e01), subj(e01,z3), obj(e01,z4)])), PRO(z5), run(e02), subj(e02,z5)])
+
+``fol()``
+-----------
+
+ >>> print(dexpr(r'([x,y],[sees(x,y)])').fol())
+ exists x y.sees(x,y)
+ >>> print(dexpr(r'([x],[man(x), walks(x)])').fol())
+ exists x.(man(x) & walks(x))
+ >>> print(dexpr(r'\x.([],[man(x), walks(x)])').fol())
+ \x.(man(x) & walks(x))
+ >>> print(dexpr(r'\x y.([],[sees(x,y)])').fol())
+ \x y.sees(x,y)
+
+ >>> print(dexpr(r'\x.([],[walks(x)])(john)').fol())
+ \x.walks(x)(john)
+ >>> print(dexpr(r'\R x.([],[big(x,R)])(\y.([],[mouse(y)]))').fol())
+ (\R x.big(x,R))(\y.mouse(y))
+
+ >>> print(dexpr(r'(([x],[walks(x)]) + ([y],[runs(y)]))').fol())
+ (exists x.walks(x) & exists y.runs(y))
+
+ >>> print(dexpr(r'(([],[walks(x)]) -> ([],[runs(x)]))').fol())
+ (walks(x) -> runs(x))
+
+ >>> print(dexpr(r'([x],[PRO(x), sees(John,x)])').fol())
+ exists x.(PRO(x) & sees(John,x))
+ >>> print(dexpr(r'([x],[man(x), -([],[walks(x)])])').fol())
+ exists x.(man(x) & -walks(x))
+ >>> print(dexpr(r'([],[(([x],[man(x)]) -> ([],[walks(x)]))])').fol())
+ all x.(man(x) -> walks(x))
+
+ >>> print(dexpr(r'([x],[man(x) | walks(x)])').fol())
+ exists x.(man(x) | walks(x))
+ >>> print(dexpr(r'P(x) + ([x],[walks(x)])').fol())
+ (P(x) & exists x.walks(x))
+
+``resolve_anaphora()``
+----------------------
+
+ >>> from nltk.sem.drt import AnaphoraResolutionException
+
+ >>> print(resolve_anaphora(dexpr(r'([x,y,z],[dog(x), cat(y), walks(z), PRO(z)])')))
+ ([x,y,z],[dog(x), cat(y), walks(z), (z = [x,y])])
+ >>> print(resolve_anaphora(dexpr(r'([],[(([x],[dog(x)]) -> ([y],[walks(y), PRO(y)]))])')))
+ ([],[(([x],[dog(x)]) -> ([y],[walks(y), (y = x)]))])
+ >>> print(resolve_anaphora(dexpr(r'(([x,y],[]) + ([],[PRO(x)]))')).simplify())
+ ([x,y],[(x = y)])
+ >>> try: print(resolve_anaphora(dexpr(r'([x],[walks(x), PRO(x)])')))
+ ... except AnaphoraResolutionException as e: print(e)
+ Variable 'x' does not resolve to anything.
+ >>> print(resolve_anaphora(dexpr('([e01,z6,z7],[boy(z6), PRO(z7), run(e01), subj(e01,z7)])')))
+ ([e01,z6,z7],[boy(z6), (z7 = z6), run(e01), subj(e01,z7)])
+
+``equiv()``:
+----------------
+
+ >>> a = dexpr(r'([x],[man(x), walks(x)])')
+ >>> b = dexpr(r'([x],[walks(x), man(x)])')
+ >>> print(a.equiv(b, TableauProver()))
+ True
+
+
+``replace()``:
+--------------
+
+ >>> a = dexpr(r'a')
+ >>> w = dexpr(r'w')
+ >>> x = dexpr(r'x')
+ >>> y = dexpr(r'y')
+ >>> z = dexpr(r'z')
+
+
+replace bound
+-------------
+
+ >>> print(dexpr(r'([x],[give(x,y,z)])').replace(x.variable, a, False))
+ ([x],[give(x,y,z)])
+ >>> print(dexpr(r'([x],[give(x,y,z)])').replace(x.variable, a, True))
+ ([a],[give(a,y,z)])
+
+replace unbound
+---------------
+
+ >>> print(dexpr(r'([x],[give(x,y,z)])').replace(y.variable, a, False))
+ ([x],[give(x,a,z)])
+ >>> print(dexpr(r'([x],[give(x,y,z)])').replace(y.variable, a, True))
+ ([x],[give(x,a,z)])
+
+replace unbound with bound
+--------------------------
+
+ >>> dexpr(r'([x],[give(x,y,z)])').replace(y.variable, x, False) == \
+ ... dexpr('([z1],[give(z1,x,z)])')
+ True
+ >>> dexpr(r'([x],[give(x,y,z)])').replace(y.variable, x, True) == \
+ ... dexpr('([z1],[give(z1,x,z)])')
+ True
+
+replace unbound with unbound
+----------------------------
+
+ >>> print(dexpr(r'([x],[give(x,y,z)])').replace(y.variable, z, False))
+ ([x],[give(x,z,z)])
+ >>> print(dexpr(r'([x],[give(x,y,z)])').replace(y.variable, z, True))
+ ([x],[give(x,z,z)])
+
+
+replace unbound
+---------------
+
+ >>> print(dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,z)])').replace(z.variable, a, False))
+ (([x],[P(x,y,a)]) + ([y],[Q(x,y,a)]))
+ >>> print(dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,z)])').replace(z.variable, a, True))
+ (([x],[P(x,y,a)]) + ([y],[Q(x,y,a)]))
+
+replace bound
+-------------
+
+ >>> print(dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,z)])').replace(x.variable, a, False))
+ (([x],[P(x,y,z)]) + ([y],[Q(x,y,z)]))
+ >>> print(dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,z)])').replace(x.variable, a, True))
+ (([a],[P(a,y,z)]) + ([y],[Q(a,y,z)]))
+
+replace unbound with unbound
+----------------------------
+
+ >>> print(dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,z)])').replace(z.variable, a, False))
+ (([x],[P(x,y,a)]) + ([y],[Q(x,y,a)]))
+ >>> print(dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,z)])').replace(z.variable, a, True))
+ (([x],[P(x,y,a)]) + ([y],[Q(x,y,a)]))
+
+replace unbound with bound on same side
+---------------------------------------
+
+ >>> dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,w)])').replace(z.variable, x, False) == \
+ ... dexpr(r'(([z1],[P(z1,y,x)]) + ([y],[Q(z1,y,w)]))')
+ True
+ >>> dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,w)])').replace(z.variable, x, True) == \
+ ... dexpr(r'(([z1],[P(z1,y,x)]) + ([y],[Q(z1,y,w)]))')
+ True
+
+replace unbound with bound on other side
+----------------------------------------
+
+ >>> dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,w)])').replace(w.variable, x, False) == \
+ ... dexpr(r'(([z1],[P(z1,y,z)]) + ([y],[Q(z1,y,x)]))')
+ True
+ >>> dexpr(r'([x],[P(x,y,z)])+([y],[Q(x,y,w)])').replace(w.variable, x, True) == \
+ ... dexpr(r'(([z1],[P(z1,y,z)]) + ([y],[Q(z1,y,x)]))')
+ True
+
+replace unbound with double bound
+---------------------------------
+
+ >>> dexpr(r'([x],[P(x,y,z)])+([x],[Q(x,y,w)])').replace(z.variable, x, False) == \
+ ... dexpr(r'(([z1],[P(z1,y,x)]) + ([z1],[Q(z1,y,w)]))')
+ True
+ >>> dexpr(r'([x],[P(x,y,z)])+([x],[Q(x,y,w)])').replace(z.variable, x, True) == \
+ ... dexpr(r'(([z1],[P(z1,y,x)]) + ([z1],[Q(z1,y,w)]))')
+ True
+
+
+regression tests
+----------------
+
+ >>> d = dexpr('([x],[A(c), ([y],[B(x,y,z,a)])->([z],[C(x,y,z,a)])])')
+ >>> print(d)
+ ([x],[A(c), (([y],[B(x,y,z,a)]) -> ([z],[C(x,y,z,a)]))])
+ >>> d.pprint()
+ ____________________________________
+ | x |
+ |------------------------------------|
+ | A(c) |
+ | ____________ ____________ |
+ | | y | | z | |
+ | (|------------| -> |------------|) |
+ | | B(x,y,z,a) | | C(x,y,z,a) | |
+ | |____________| |____________| |
+ |____________________________________|
+ >>> print(str(d))
+ ([x],[A(c), (([y],[B(x,y,z,a)]) -> ([z],[C(x,y,z,a)]))])
+ >>> print(d.fol())
+ exists x.(A(c) & all y.(B(x,y,z,a) -> exists z.C(x,y,z,a)))
+ >>> print(d.replace(Variable('a'), DrtVariableExpression(Variable('r'))))
+ ([x],[A(c), (([y],[B(x,y,z,r)]) -> ([z],[C(x,y,z,r)]))])
+ >>> print(d.replace(Variable('x'), DrtVariableExpression(Variable('r'))))
+ ([x],[A(c), (([y],[B(x,y,z,a)]) -> ([z],[C(x,y,z,a)]))])
+ >>> print(d.replace(Variable('y'), DrtVariableExpression(Variable('r'))))
+ ([x],[A(c), (([y],[B(x,y,z,a)]) -> ([z],[C(x,y,z,a)]))])
+ >>> print(d.replace(Variable('z'), DrtVariableExpression(Variable('r'))))
+ ([x],[A(c), (([y],[B(x,y,r,a)]) -> ([z],[C(x,y,z,a)]))])
+ >>> print(d.replace(Variable('x'), DrtVariableExpression(Variable('r')), True))
+ ([r],[A(c), (([y],[B(r,y,z,a)]) -> ([z],[C(r,y,z,a)]))])
+ >>> print(d.replace(Variable('y'), DrtVariableExpression(Variable('r')), True))
+ ([x],[A(c), (([r],[B(x,r,z,a)]) -> ([z],[C(x,r,z,a)]))])
+ >>> print(d.replace(Variable('z'), DrtVariableExpression(Variable('r')), True))
+ ([x],[A(c), (([y],[B(x,y,r,a)]) -> ([r],[C(x,y,r,a)]))])
+ >>> print(d == dexpr('([l],[A(c), ([m],[B(l,m,z,a)])->([n],[C(l,m,n,a)])])'))
+ True
+ >>> d = dexpr('([],[([x,y],[B(x,y,h), ([a,b],[dee(x,a,g)])])->([z,w],[cee(x,y,f), ([c,d],[E(x,c,d,e)])])])')
+ >>> sorted(d.free())
+ [Variable('B'), Variable('E'), Variable('e'), Variable('f'), Variable('g'), Variable('h')]
+ >>> sorted(d.variables())
+ [Variable('B'), Variable('E'), Variable('e'), Variable('f'), Variable('g'), Variable('h')]
+ >>> sorted(d.get_refs(True))
+ [Variable('a'), Variable('b'), Variable('c'), Variable('d'), Variable('w'), Variable('x'), Variable('y'), Variable('z')]
+ >>> sorted(d.conds[0].get_refs(False))
+ [Variable('x'), Variable('y')]
+ >>> print(dexpr('([x,y],[A(x,y), (x=y), ([],[B(x,y)])->([],[C(x,y)]), ([x,y],[D(x,y)])->([],[E(x,y)]), ([],[F(x,y)])->([x,y],[G(x,y)])])').eliminate_equality())
+ ([x],[A(x,x), (([],[B(x,x)]) -> ([],[C(x,x)])), (([x,y],[D(x,y)]) -> ([],[E(x,y)])), (([],[F(x,x)]) -> ([x,y],[G(x,y)]))])
+ >>> print(dexpr('([x,y],[A(x,y), (x=y)]) -> ([],[B(x,y)])').eliminate_equality())
+ (([x],[A(x,x)]) -> ([],[B(x,x)]))
+ >>> print(dexpr('([x,y],[A(x,y)]) -> ([],[B(x,y), (x=y)])').eliminate_equality())
+ (([x,y],[A(x,y)]) -> ([],[B(x,x)]))
+ >>> print(dexpr('([x,y],[A(x,y), (x=y), ([],[B(x,y)])])').eliminate_equality())
+ ([x],[A(x,x), ([],[B(x,x)])])
+ >>> print(dexpr('([x,y],[A(x,y), ([],[B(x,y), (x=y)])])').eliminate_equality())
+ ([x,y],[A(x,y), ([],[B(x,x)])])
+ >>> print(dexpr('([z8 z9 z10],[A(z8), z8=z10, z9=z10, B(z9), C(z10), D(z10)])').eliminate_equality())
+ ([z9],[A(z9), B(z9), C(z9), D(z9)])
+
+ >>> print(dexpr('([x,y],[A(x,y), (x=y), ([],[B(x,y)]), ([x,y],[C(x,y)])])').eliminate_equality())
+ ([x],[A(x,x), ([],[B(x,x)]), ([x,y],[C(x,y)])])
+ >>> print(dexpr('([x,y],[A(x,y)]) + ([],[B(x,y), (x=y)]) + ([],[C(x,y)])').eliminate_equality())
+ ([x],[A(x,x), B(x,x), C(x,x)])
+ >>> print(dexpr('([x,y],[B(x,y)])+([x,y],[C(x,y)])').replace(Variable('y'), DrtVariableExpression(Variable('x'))))
+ (([x,y],[B(x,y)]) + ([x,y],[C(x,y)]))
+ >>> print(dexpr('(([x,y],[B(x,y)])+([],[C(x,y)]))+([],[D(x,y)])').replace(Variable('y'), DrtVariableExpression(Variable('x'))))
+ (([x,y],[B(x,y)]) + ([],[C(x,y)]) + ([],[D(x,y)]))
+ >>> print(dexpr('(([],[B(x,y)])+([],[C(x,y)]))+([],[D(x,y)])').replace(Variable('y'), DrtVariableExpression(Variable('x'))))
+ (([],[B(x,x)]) + ([],[C(x,x)]) + ([],[D(x,x)]))
+ >>> print(dexpr('(([],[B(x,y), ([x,y],[A(x,y)])])+([],[C(x,y)]))+([],[D(x,y)])').replace(Variable('y'), DrtVariableExpression(Variable('x'))).normalize())
+ (([],[B(z3,z1), ([z2,z3],[A(z3,z2)])]) + ([],[C(z3,z1)]) + ([],[D(z3,z1)]))
+
+
+Parse errors
+============
+
+ >>> def parse_error(drtstring):
+ ... try: dexpr(drtstring)
+ ... except logic.LogicalExpressionException as e: print(e)
+
+ >>> parse_error(r'')
+ End of input found. Expression expected.
+ <BLANKLINE>
+ ^
+ >>> parse_error(r'(')
+ End of input found. Expression expected.
+ (
+ ^
+ >>> parse_error(r'()')
+ Unexpected token: ')'. Expression expected.
+ ()
+ ^
+ >>> parse_error(r'([')
+ End of input found. Expected token ']'.
+ ([
+ ^
+ >>> parse_error(r'([,')
+ ',' is an illegal variable name. Constants may not be quantified.
+ ([,
+ ^
+ >>> parse_error(r'([x,')
+ End of input found. Variable expected.
+ ([x,
+ ^
+ >>> parse_error(r'([]')
+ End of input found. Expected token '['.
+ ([]
+ ^
+ >>> parse_error(r'([][')
+ End of input found. Expected token ']'.
+ ([][
+ ^
+ >>> parse_error(r'([][,')
+ Unexpected token: ','. Expression expected.
+ ([][,
+ ^
+ >>> parse_error(r'([][]')
+ End of input found. Expected token ')'.
+ ([][]
+ ^
+ >>> parse_error(r'([x][man(x)]) |')
+ End of input found. Expression expected.
+ ([x][man(x)]) |
+ ^
+
+Pretty Printing
+===============
+
+ >>> dexpr(r"([],[])").pprint()
+ __
+ | |
+ |--|
+ |__|
+
+ >>> dexpr(r"([],[([x],[big(x), dog(x)]) -> ([],[bark(x)]) -([x],[walk(x)])])").pprint()
+ _____________________________
+ | |
+ |-----------------------------|
+ | ________ _________ |
+ | | x | | | |
+ | (|--------| -> |---------|) |
+ | | big(x) | | bark(x) | |
+ | | dog(x) | |_________| |
+ | |________| |
+ | _________ |
+ | | x | |
+ | __ |---------| |
+ | | | walk(x) | |
+ | |_________| |
+ |_____________________________|
+
+ >>> dexpr(r"([x,y],[x=y]) + ([z],[dog(z), walk(z)])").pprint()
+ _________ _________
+ | x y | | z |
+ (|---------| + |---------|)
+ | (x = y) | | dog(z) |
+ |_________| | walk(z) |
+ |_________|
+
+ >>> dexpr(r"([],[([x],[]) | ([y],[]) | ([z],[dog(z), walk(z)])])").pprint()
+ _______________________________
+ | |
+ |-------------------------------|
+ | ___ ___ _________ |
+ | | x | | y | | z | |
+ | (|---| | |---| | |---------|) |
+ | |___| |___| | dog(z) | |
+ | | walk(z) | |
+ | |_________| |
+ |_______________________________|
+
+ >>> dexpr(r"\P.\Q.(([x],[]) + P(x) + Q(x))(\x.([],[dog(x)]))").pprint()
+ ___ ________
+ \ | x | \ | |
+ /\ P Q.(|---| + P(x) + Q(x))( /\ x.|--------|)
+ |___| | dog(x) |
+ |________|
+
+
diff --git a/nltk/test/featgram.doctest b/nltk/test/featgram.doctest
new file mode 100644
index 0000000..d0dccb5
--- /dev/null
+++ b/nltk/test/featgram.doctest
@@ -0,0 +1,607 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=========================
+ Feature Grammar Parsing
+=========================
+
+.. include:: ../../../nltk_book/definitions.rst
+
+Grammars can be parsed from strings.
+
+ >>> from __future__ import print_function
+ >>> import nltk
+ >>> from nltk import grammar, parse
+ >>> g = """
+ ... % start DP
+ ... DP[AGR=?a] -> D[AGR=?a] N[AGR=?a]
+ ... D[AGR=[NUM='sg', PERS=3]] -> 'this' | 'that'
+ ... D[AGR=[NUM='pl', PERS=3]] -> 'these' | 'those'
+ ... D[AGR=[NUM='pl', PERS=1]] -> 'we'
+ ... D[AGR=[PERS=2]] -> 'you'
+ ... N[AGR=[NUM='sg', GND='m']] -> 'boy'
+ ... N[AGR=[NUM='pl', GND='m']] -> 'boys'
+ ... N[AGR=[NUM='sg', GND='f']] -> 'girl'
+ ... N[AGR=[NUM='pl', GND='f']] -> 'girls'
+ ... N[AGR=[NUM='sg']] -> 'student'
+ ... N[AGR=[NUM='pl']] -> 'students'
+ ... """
+ >>> grammar = grammar.FeatureGrammar.fromstring(g)
+ >>> tokens = 'these girls'.split()
+ >>> parser = parse.FeatureEarleyChartParser(grammar)
+ >>> trees = parser.parse(tokens)
+ >>> for tree in trees: print(tree)
+ (DP[AGR=[GND='f', NUM='pl', PERS=3]]
+ (D[AGR=[NUM='pl', PERS=3]] these)
+ (N[AGR=[GND='f', NUM='pl']] girls))
+
+In general, when we are trying to develop even a very small grammar,
+it is convenient to put the rules in a file where they can be edited,
+tested and revised. Let's assume that we have saved feat0cfg_ as a file named
+``'feat0.fcfg'`` and placed it in the NLTK ``data`` directory. We can
+inspect it as follows:
+
+.. _feat0cfg: http://nltk.svn.sourceforge.net/svnroot/nltk/trunk/nltk/data/grammars/feat0.fcfg
+
+ >>> nltk.data.show_cfg('grammars/book_grammars/feat0.fcfg')
+ % start S
+ # ###################
+ # Grammar Productions
+ # ###################
+ # S expansion productions
+ S -> NP[NUM=?n] VP[NUM=?n]
+ # NP expansion productions
+ NP[NUM=?n] -> N[NUM=?n]
+ NP[NUM=?n] -> PropN[NUM=?n]
+ NP[NUM=?n] -> Det[NUM=?n] N[NUM=?n]
+ NP[NUM=pl] -> N[NUM=pl]
+ # VP expansion productions
+ VP[TENSE=?t, NUM=?n] -> IV[TENSE=?t, NUM=?n]
+ VP[TENSE=?t, NUM=?n] -> TV[TENSE=?t, NUM=?n] NP
+ # ###################
+ # Lexical Productions
+ # ###################
+ Det[NUM=sg] -> 'this' | 'every'
+ Det[NUM=pl] -> 'these' | 'all'
+ Det -> 'the' | 'some' | 'several'
+ PropN[NUM=sg]-> 'Kim' | 'Jody'
+ N[NUM=sg] -> 'dog' | 'girl' | 'car' | 'child'
+ N[NUM=pl] -> 'dogs' | 'girls' | 'cars' | 'children'
+ IV[TENSE=pres, NUM=sg] -> 'disappears' | 'walks'
+ TV[TENSE=pres, NUM=sg] -> 'sees' | 'likes'
+ IV[TENSE=pres, NUM=pl] -> 'disappear' | 'walk'
+ TV[TENSE=pres, NUM=pl] -> 'see' | 'like'
+ IV[TENSE=past] -> 'disappeared' | 'walked'
+ TV[TENSE=past] -> 'saw' | 'liked'
+
+Assuming we have saved feat0cfg_ as a file named
+``'feat0.fcfg'``, the function ``parse.load_parser`` allows us to
+read the grammar into NLTK, ready for use in parsing.
+
+
+ >>> cp = parse.load_parser('grammars/book_grammars/feat0.fcfg', trace=1)
+ >>> sent = 'Kim likes children'
+ >>> tokens = sent.split()
+ >>> tokens
+ ['Kim', 'likes', 'children']
+ >>> trees = cp.parse(tokens)
+ |.Kim .like.chil.|
+ |[----] . .| [0:1] 'Kim'
+ |. [----] .| [1:2] 'likes'
+ |. . [----]| [2:3] 'children'
+ |[----] . .| [0:1] PropN[NUM='sg'] -> 'Kim' *
+ |[----] . .| [0:1] NP[NUM='sg'] -> PropN[NUM='sg'] *
+ |[----> . .| [0:1] S[] -> NP[NUM=?n] * VP[NUM=?n] {?n: 'sg'}
+ |. [----] .| [1:2] TV[NUM='sg', TENSE='pres'] -> 'likes' *
+ |. [----> .| [1:2] VP[NUM=?n, TENSE=?t] -> TV[NUM=?n, TENSE=?t] * NP[] {?n: 'sg', ?t: 'pres'}
+ |. . [----]| [2:3] N[NUM='pl'] -> 'children' *
+ |. . [----]| [2:3] NP[NUM='pl'] -> N[NUM='pl'] *
+ |. . [---->| [2:3] S[] -> NP[NUM=?n] * VP[NUM=?n] {?n: 'pl'}
+ |. [---------]| [1:3] VP[NUM='sg', TENSE='pres'] -> TV[NUM='sg', TENSE='pres'] NP[] *
+ |[==============]| [0:3] S[] -> NP[NUM='sg'] VP[NUM='sg'] *
+ >>> for tree in trees: print(tree)
+ (S[]
+ (NP[NUM='sg'] (PropN[NUM='sg'] Kim))
+ (VP[NUM='sg', TENSE='pres']
+ (TV[NUM='sg', TENSE='pres'] likes)
+ (NP[NUM='pl'] (N[NUM='pl'] children))))
+
+The parser works directly with
+the underspecified productions given by the grammar. That is, the
+Predictor rule does not attempt to compile out all admissible feature
+combinations before trying to expand the non-terminals on the left hand
+side of a production. However, when the Scanner matches an input word
+against a lexical production that has been predicted, the new edge will
+typically contain fully specified features; e.g., the edge
+[PropN[`num`:feat: = `sg`:fval:] |rarr| 'Kim', (0, 1)]. Recall from
+Chapter 8 that the Fundamental (or Completer) Rule in
+standard CFGs is used to combine an incomplete edge that's expecting a
+nonterminal *B* with a following, complete edge whose left hand side
+matches *B*. In our current setting, rather than checking for a
+complete match, we test whether the expected category *B* will
+`unify`:dt: with the left hand side *B'* of a following complete
+edge. We will explain in more detail in Section 9.2 how
+unification works; for the moment, it is enough to know that as a
+result of unification, any variable values of features in *B* will be
+instantiated by constant values in the corresponding feature structure
+in *B'*, and these instantiated values will be used in the new edge
+added by the Completer. This instantiation can be seen, for example,
+in the edge
+[NP [`num`:feat:\ =\ `sg`:fval:] |rarr| PropN[`num`:feat:\ =\ `sg`:fval:] |dot|, (0, 1)]
+in Example 9.2, where the feature `num`:feat: has been assigned the value `sg`:fval:.
+
+Feature structures in NLTK are ... Atomic feature values can be strings or
+integers.
+
+ >>> fs1 = nltk.FeatStruct(TENSE='past', NUM='sg')
+ >>> print(fs1)
+ [ NUM = 'sg' ]
+ [ TENSE = 'past' ]
+
+We can think of a feature structure as being like a Python dictionary,
+and access its values by indexing in the usual way.
+
+ >>> fs1 = nltk.FeatStruct(PER=3, NUM='pl', GND='fem')
+ >>> print(fs1['GND'])
+ fem
+
+We can also define feature structures which have complex values, as
+discussed earlier.
+
+ >>> fs2 = nltk.FeatStruct(POS='N', AGR=fs1)
+ >>> print(fs2)
+ [ [ GND = 'fem' ] ]
+ [ AGR = [ NUM = 'pl' ] ]
+ [ [ PER = 3 ] ]
+ [ ]
+ [ POS = 'N' ]
+ >>> print(fs2['AGR'])
+ [ GND = 'fem' ]
+ [ NUM = 'pl' ]
+ [ PER = 3 ]
+ >>> print(fs2['AGR']['PER'])
+ 3
+
+Feature structures can also be constructed using the ``parse()``
+method of the ``nltk.FeatStruct`` class. Note that in this case, atomic
+feature values do not need to be enclosed in quotes.
+
+ >>> f1 = nltk.FeatStruct("[NUMBER = sg]")
+ >>> f2 = nltk.FeatStruct("[PERSON = 3]")
+ >>> print(nltk.unify(f1, f2))
+ [ NUMBER = 'sg' ]
+ [ PERSON = 3 ]
+
+ >>> f1 = nltk.FeatStruct("[A = [B = b, D = d]]")
+ >>> f2 = nltk.FeatStruct("[A = [C = c, D = d]]")
+ >>> print(nltk.unify(f1, f2))
+ [ [ B = 'b' ] ]
+ [ A = [ C = 'c' ] ]
+ [ [ D = 'd' ] ]
+
+
+Feature Structures as Graphs
+----------------------------
+
+Feature structures are not inherently tied to linguistic objects; they are
+general purpose structures for representing knowledge. For example, we
+could encode information about a person in a feature structure:
+
+ >>> person01 = nltk.FeatStruct("[NAME=Lee, TELNO='01 27 86 42 96',AGE=33]")
+ >>> print(person01)
+ [ AGE = 33 ]
+ [ NAME = 'Lee' ]
+ [ TELNO = '01 27 86 42 96' ]
+
+There are a number of notations for representing reentrancy in
+matrix-style representations of feature structures. In NLTK, we adopt
+the following convention: the first occurrence of a shared feature structure
+is prefixed with an integer in parentheses, such as ``(1)``, and any
+subsequent reference to that structure uses the notation
+``->(1)``, as shown below.
+
+
+ >>> fs = nltk.FeatStruct("""[NAME=Lee, ADDRESS=(1)[NUMBER=74, STREET='rue Pascal'],
+ ... SPOUSE=[NAME=Kim, ADDRESS->(1)]]""")
+ >>> print(fs)
+ [ ADDRESS = (1) [ NUMBER = 74 ] ]
+ [ [ STREET = 'rue Pascal' ] ]
+ [ ]
+ [ NAME = 'Lee' ]
+ [ ]
+ [ SPOUSE = [ ADDRESS -> (1) ] ]
+ [ [ NAME = 'Kim' ] ]
+
+There can be any number of tags within a single feature structure.
+
+ >>> fs3 = nltk.FeatStruct("[A=(1)[B=b], C=(2)[], D->(1), E->(2)]")
+ >>> print(fs3)
+ [ A = (1) [ B = 'b' ] ]
+ [ ]
+ [ C = (2) [] ]
+ [ ]
+ [ D -> (1) ]
+ [ E -> (2) ]
+ >>> fs1 = nltk.FeatStruct(NUMBER=74, STREET='rue Pascal')
+ >>> fs2 = nltk.FeatStruct(CITY='Paris')
+ >>> print(nltk.unify(fs1, fs2))
+ [ CITY = 'Paris' ]
+ [ NUMBER = 74 ]
+ [ STREET = 'rue Pascal' ]
+
+Unification is symmetric:
+
+ >>> nltk.unify(fs1, fs2) == nltk.unify(fs2, fs1)
+ True
+
+Unification is commutative:
+
+ >>> fs3 = nltk.FeatStruct(TELNO='01 27 86 42 96')
+ >>> nltk.unify(nltk.unify(fs1, fs2), fs3) == nltk.unify(fs1, nltk.unify(fs2, fs3))
+ True
+
+Unification between `FS`:math:\ :subscript:`0` and `FS`:math:\
+:subscript:`1` will fail if the two feature structures share a path |pi|,
+but the value of |pi| in `FS`:math:\ :subscript:`0` is a distinct
+atom from the value of |pi| in `FS`:math:\ :subscript:`1`. In NLTK,
+this is implemented by setting the result of unification to be
+``None``.
+
+ >>> fs0 = nltk.FeatStruct(A='a')
+ >>> fs1 = nltk.FeatStruct(A='b')
+ >>> print(nltk.unify(fs0, fs1))
+ None
+
+Now, if we look at how unification interacts with structure-sharing,
+things become really interesting.
+
+
+
+ >>> fs0 = nltk.FeatStruct("""[NAME=Lee,
+ ... ADDRESS=[NUMBER=74,
+ ... STREET='rue Pascal'],
+ ... SPOUSE= [NAME=Kim,
+ ... ADDRESS=[NUMBER=74,
+ ... STREET='rue Pascal']]]""")
+ >>> print(fs0)
+ [ ADDRESS = [ NUMBER = 74 ] ]
+ [ [ STREET = 'rue Pascal' ] ]
+ [ ]
+ [ NAME = 'Lee' ]
+ [ ]
+ [ [ ADDRESS = [ NUMBER = 74 ] ] ]
+ [ SPOUSE = [ [ STREET = 'rue Pascal' ] ] ]
+ [ [ ] ]
+ [ [ NAME = 'Kim' ] ]
+
+
+ >>> fs1 = nltk.FeatStruct("[SPOUSE=[ADDRESS=[CITY=Paris]]]")
+ >>> print(nltk.unify(fs0, fs1))
+ [ ADDRESS = [ NUMBER = 74 ] ]
+ [ [ STREET = 'rue Pascal' ] ]
+ [ ]
+ [ NAME = 'Lee' ]
+ [ ]
+ [ [ [ CITY = 'Paris' ] ] ]
+ [ [ ADDRESS = [ NUMBER = 74 ] ] ]
+ [ SPOUSE = [ [ STREET = 'rue Pascal' ] ] ]
+ [ [ ] ]
+ [ [ NAME = 'Kim' ] ]
+
+ >>> fs2 = nltk.FeatStruct("""[NAME=Lee, ADDRESS=(1)[NUMBER=74, STREET='rue Pascal'],
+ ... SPOUSE=[NAME=Kim, ADDRESS->(1)]]""")
+
+
+ >>> print(fs2)
+ [ ADDRESS = (1) [ NUMBER = 74 ] ]
+ [ [ STREET = 'rue Pascal' ] ]
+ [ ]
+ [ NAME = 'Lee' ]
+ [ ]
+ [ SPOUSE = [ ADDRESS -> (1) ] ]
+ [ [ NAME = 'Kim' ] ]
+
+
+ >>> print(nltk.unify(fs2, fs1))
+ [ [ CITY = 'Paris' ] ]
+ [ ADDRESS = (1) [ NUMBER = 74 ] ]
+ [ [ STREET = 'rue Pascal' ] ]
+ [ ]
+ [ NAME = 'Lee' ]
+ [ ]
+ [ SPOUSE = [ ADDRESS -> (1) ] ]
+ [ [ NAME = 'Kim' ] ]
+
+
+ >>> fs1 = nltk.FeatStruct("[ADDRESS1=[NUMBER=74, STREET='rue Pascal']]")
+ >>> fs2 = nltk.FeatStruct("[ADDRESS1=?x, ADDRESS2=?x]")
+ >>> print(fs2)
+ [ ADDRESS1 = ?x ]
+ [ ADDRESS2 = ?x ]
+ >>> print(nltk.unify(fs1, fs2))
+ [ ADDRESS1 = (1) [ NUMBER = 74 ] ]
+ [ [ STREET = 'rue Pascal' ] ]
+ [ ]
+ [ ADDRESS2 -> (1) ]
+
+
+
+
+ >>> sent = 'who do you claim that you like'
+ >>> tokens = sent.split()
+ >>> cp = parse.load_parser('grammars/book_grammars/feat1.fcfg', trace=1)
+ >>> trees = cp.parse(tokens)
+ |.w.d.y.c.t.y.l.|
+ |[-] . . . . . .| [0:1] 'who'
+ |. [-] . . . . .| [1:2] 'do'
+ |. . [-] . . . .| [2:3] 'you'
+ |. . . [-] . . .| [3:4] 'claim'
+ |. . . . [-] . .| [4:5] 'that'
+ |. . . . . [-] .| [5:6] 'you'
+ |. . . . . . [-]| [6:7] 'like'
+ |# . . . . . . .| [0:0] NP[]/NP[] -> *
+ |. # . . . . . .| [1:1] NP[]/NP[] -> *
+ |. . # . . . . .| [2:2] NP[]/NP[] -> *
+ |. . . # . . . .| [3:3] NP[]/NP[] -> *
+ |. . . . # . . .| [4:4] NP[]/NP[] -> *
+ |. . . . . # . .| [5:5] NP[]/NP[] -> *
+ |. . . . . . # .| [6:6] NP[]/NP[] -> *
+ |. . . . . . . #| [7:7] NP[]/NP[] -> *
+ |[-] . . . . . .| [0:1] NP[+WH] -> 'who' *
+ |[-> . . . . . .| [0:1] S[-INV] -> NP[] * VP[] {}
+ |[-> . . . . . .| [0:1] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
+ |[-> . . . . . .| [0:1] S[-INV] -> NP[] * S[]/NP[] {}
+ |. [-] . . . . .| [1:2] V[+AUX] -> 'do' *
+ |. [-> . . . . .| [1:2] S[+INV] -> V[+AUX] * NP[] VP[] {}
+ |. [-> . . . . .| [1:2] S[+INV]/?x[] -> V[+AUX] * NP[] VP[]/?x[] {}
+ |. [-> . . . . .| [1:2] VP[] -> V[+AUX] * VP[] {}
+ |. [-> . . . . .| [1:2] VP[]/?x[] -> V[+AUX] * VP[]/?x[] {}
+ |. . [-] . . . .| [2:3] NP[-WH] -> 'you' *
+ |. . [-> . . . .| [2:3] S[-INV] -> NP[] * VP[] {}
+ |. . [-> . . . .| [2:3] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
+ |. . [-> . . . .| [2:3] S[-INV] -> NP[] * S[]/NP[] {}
+ |. [---> . . . .| [1:3] S[+INV] -> V[+AUX] NP[] * VP[] {}
+ |. [---> . . . .| [1:3] S[+INV]/?x[] -> V[+AUX] NP[] * VP[]/?x[] {}
+ |. . . [-] . . .| [3:4] V[-AUX, SUBCAT='clause'] -> 'claim' *
+ |. . . [-> . . .| [3:4] VP[] -> V[-AUX, SUBCAT='clause'] * SBar[] {}
+ |. . . [-> . . .| [3:4] VP[]/?x[] -> V[-AUX, SUBCAT='clause'] * SBar[]/?x[] {}
+ |. . . . [-] . .| [4:5] Comp[] -> 'that' *
+ |. . . . [-> . .| [4:5] SBar[] -> Comp[] * S[-INV] {}
+ |. . . . [-> . .| [4:5] SBar[]/?x[] -> Comp[] * S[-INV]/?x[] {}
+ |. . . . . [-] .| [5:6] NP[-WH] -> 'you' *
+ |. . . . . [-> .| [5:6] S[-INV] -> NP[] * VP[] {}
+ |. . . . . [-> .| [5:6] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
+ |. . . . . [-> .| [5:6] S[-INV] -> NP[] * S[]/NP[] {}
+ |. . . . . . [-]| [6:7] V[-AUX, SUBCAT='trans'] -> 'like' *
+ |. . . . . . [->| [6:7] VP[] -> V[-AUX, SUBCAT='trans'] * NP[] {}
+ |. . . . . . [->| [6:7] VP[]/?x[] -> V[-AUX, SUBCAT='trans'] * NP[]/?x[] {}
+ |. . . . . . [-]| [6:7] VP[]/NP[] -> V[-AUX, SUBCAT='trans'] NP[]/NP[] *
+ |. . . . . [---]| [5:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
+ |. . . . [-----]| [4:7] SBar[]/NP[] -> Comp[] S[-INV]/NP[] *
+ |. . . [-------]| [3:7] VP[]/NP[] -> V[-AUX, SUBCAT='clause'] SBar[]/NP[] *
+ |. . [---------]| [2:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
+ |. [-----------]| [1:7] S[+INV]/NP[] -> V[+AUX] NP[] VP[]/NP[] *
+ |[=============]| [0:7] S[-INV] -> NP[] S[]/NP[] *
+
+ >>> trees = list(trees)
+ >>> for tree in trees: print(tree)
+ (S[-INV]
+ (NP[+WH] who)
+ (S[+INV]/NP[]
+ (V[+AUX] do)
+ (NP[-WH] you)
+ (VP[]/NP[]
+ (V[-AUX, SUBCAT='clause'] claim)
+ (SBar[]/NP[]
+ (Comp[] that)
+ (S[-INV]/NP[]
+ (NP[-WH] you)
+ (VP[]/NP[] (V[-AUX, SUBCAT='trans'] like) (NP[]/NP[] )))))))
+
+A different parser should give the same parse trees, but perhaps in a different order:
+
+ >>> cp2 = parse.load_parser('grammars/book_grammars/feat1.fcfg', trace=1,
+ ... parser=parse.FeatureEarleyChartParser)
+ >>> trees2 = cp2.parse(tokens)
+ |.w.d.y.c.t.y.l.|
+ |[-] . . . . . .| [0:1] 'who'
+ |. [-] . . . . .| [1:2] 'do'
+ |. . [-] . . . .| [2:3] 'you'
+ |. . . [-] . . .| [3:4] 'claim'
+ |. . . . [-] . .| [4:5] 'that'
+ |. . . . . [-] .| [5:6] 'you'
+ |. . . . . . [-]| [6:7] 'like'
+ |> . . . . . . .| [0:0] S[-INV] -> * NP[] VP[] {}
+ |> . . . . . . .| [0:0] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
+ |> . . . . . . .| [0:0] S[-INV] -> * NP[] S[]/NP[] {}
+ |> . . . . . . .| [0:0] S[-INV] -> * Adv[+NEG] S[+INV] {}
+ |> . . . . . . .| [0:0] S[+INV] -> * V[+AUX] NP[] VP[] {}
+ |> . . . . . . .| [0:0] S[+INV]/?x[] -> * V[+AUX] NP[] VP[]/?x[] {}
+ |> . . . . . . .| [0:0] NP[+WH] -> * 'who' {}
+ |[-] . . . . . .| [0:1] NP[+WH] -> 'who' *
+ |[-> . . . . . .| [0:1] S[-INV] -> NP[] * VP[] {}
+ |[-> . . . . . .| [0:1] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
+ |[-> . . . . . .| [0:1] S[-INV] -> NP[] * S[]/NP[] {}
+ |. > . . . . . .| [1:1] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
+ |. > . . . . . .| [1:1] S[+INV]/?x[] -> * V[+AUX] NP[] VP[]/?x[] {}
+ |. > . . . . . .| [1:1] V[+AUX] -> * 'do' {}
+ |. > . . . . . .| [1:1] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
+ |. > . . . . . .| [1:1] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
+ |. > . . . . . .| [1:1] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
+ |. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='intrans'] {}
+ |. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='trans'] NP[] {}
+ |. > . . . . . .| [1:1] VP[] -> * V[-AUX, SUBCAT='clause'] SBar[] {}
+ |. > . . . . . .| [1:1] VP[] -> * V[+AUX] VP[] {}
+ |. [-] . . . . .| [1:2] V[+AUX] -> 'do' *
+ |. [-> . . . . .| [1:2] S[+INV]/?x[] -> V[+AUX] * NP[] VP[]/?x[] {}
+ |. [-> . . . . .| [1:2] VP[]/?x[] -> V[+AUX] * VP[]/?x[] {}
+ |. [-> . . . . .| [1:2] VP[] -> V[+AUX] * VP[] {}
+ |. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='intrans'] {}
+ |. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='trans'] NP[] {}
+ |. . > . . . . .| [2:2] VP[] -> * V[-AUX, SUBCAT='clause'] SBar[] {}
+ |. . > . . . . .| [2:2] VP[] -> * V[+AUX] VP[] {}
+ |. . > . . . . .| [2:2] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
+ |. . > . . . . .| [2:2] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
+ |. . > . . . . .| [2:2] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
+ |. . > . . . . .| [2:2] NP[-WH] -> * 'you' {}
+ |. . [-] . . . .| [2:3] NP[-WH] -> 'you' *
+ |. [---> . . . .| [1:3] S[+INV]/?x[] -> V[+AUX] NP[] * VP[]/?x[] {}
+ |. . . > . . . .| [3:3] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
+ |. . . > . . . .| [3:3] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
+ |. . . > . . . .| [3:3] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
+ |. . . > . . . .| [3:3] V[-AUX, SUBCAT='clause'] -> * 'claim' {}
+ |. . . [-] . . .| [3:4] V[-AUX, SUBCAT='clause'] -> 'claim' *
+ |. . . [-> . . .| [3:4] VP[]/?x[] -> V[-AUX, SUBCAT='clause'] * SBar[]/?x[] {}
+ |. . . . > . . .| [4:4] SBar[]/?x[] -> * Comp[] S[-INV]/?x[] {}
+ |. . . . > . . .| [4:4] Comp[] -> * 'that' {}
+ |. . . . [-] . .| [4:5] Comp[] -> 'that' *
+ |. . . . [-> . .| [4:5] SBar[]/?x[] -> Comp[] * S[-INV]/?x[] {}
+ |. . . . . > . .| [5:5] S[-INV]/?x[] -> * NP[] VP[]/?x[] {}
+ |. . . . . > . .| [5:5] NP[-WH] -> * 'you' {}
+ |. . . . . [-] .| [5:6] NP[-WH] -> 'you' *
+ |. . . . . [-> .| [5:6] S[-INV]/?x[] -> NP[] * VP[]/?x[] {}
+ |. . . . . . > .| [6:6] VP[]/?x[] -> * V[-AUX, SUBCAT='trans'] NP[]/?x[] {}
+ |. . . . . . > .| [6:6] VP[]/?x[] -> * V[-AUX, SUBCAT='clause'] SBar[]/?x[] {}
+ |. . . . . . > .| [6:6] VP[]/?x[] -> * V[+AUX] VP[]/?x[] {}
+ |. . . . . . > .| [6:6] V[-AUX, SUBCAT='trans'] -> * 'like' {}
+ |. . . . . . [-]| [6:7] V[-AUX, SUBCAT='trans'] -> 'like' *
+ |. . . . . . [->| [6:7] VP[]/?x[] -> V[-AUX, SUBCAT='trans'] * NP[]/?x[] {}
+ |. . . . . . . #| [7:7] NP[]/NP[] -> *
+ |. . . . . . [-]| [6:7] VP[]/NP[] -> V[-AUX, SUBCAT='trans'] NP[]/NP[] *
+ |. . . . . [---]| [5:7] S[-INV]/NP[] -> NP[] VP[]/NP[] *
+ |. . . . [-----]| [4:7] SBar[]/NP[] -> Comp[] S[-INV]/NP[] *
+ |. . . [-------]| [3:7] VP[]/NP[] -> V[-AUX, SUBCAT='clause'] SBar[]/NP[] *
+ |. [-----------]| [1:7] S[+INV]/NP[] -> V[+AUX] NP[] VP[]/NP[] *
+ |[=============]| [0:7] S[-INV] -> NP[] S[]/NP[] *
+
+ >>> sorted(trees) == sorted(trees2)
+ True
+
+
+Let's load a German grammar:
+
+ >>> cp = parse.load_parser('grammars/book_grammars/german.fcfg', trace=0)
+ >>> sent = 'die Katze sieht den Hund'
+ >>> tokens = sent.split()
+ >>> trees = cp.parse(tokens)
+ >>> for tree in trees: print(tree)
+ (S[]
+ (NP[AGR=[GND='fem', NUM='sg', PER=3], CASE='nom']
+ (Det[AGR=[GND='fem', NUM='sg', PER=3], CASE='nom'] die)
+ (N[AGR=[GND='fem', NUM='sg', PER=3]] Katze))
+ (VP[AGR=[NUM='sg', PER=3]]
+ (TV[AGR=[NUM='sg', PER=3], OBJCASE='acc'] sieht)
+ (NP[AGR=[GND='masc', NUM='sg', PER=3], CASE='acc']
+ (Det[AGR=[GND='masc', NUM='sg', PER=3], CASE='acc'] den)
+ (N[AGR=[GND='masc', NUM='sg', PER=3]] Hund))))
+
+Grammar with Binding Operators
+------------------------------
+The `bindop.fcfg`_ grammar is a semantic grammar that uses lambda
+calculus. Each element has a core semantics, which is a single lambda
+calculus expression; and a set of binding operators, which bind
+variables.
+
+.. _bindop.fcfg: http://nltk.svn.sourceforge.net/svnroot/nltk/trunk/nltk/data/grammars/bindop.fcfg
+
+In order to make the binding operators work right, they need to
+instantiate their bound variable every time they are added to the
+chart. To do this, we use a special subclass of `Chart`, called
+`InstantiateVarsChart`.
+
+ >>> from nltk.parse.featurechart import InstantiateVarsChart
+ >>> cp = parse.load_parser('grammars/sample_grammars/bindop.fcfg', trace=1,
+ ... chart_class=InstantiateVarsChart)
+ >>> print(cp.grammar())
+ Grammar with 15 productions (start state = S[])
+ S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] VP[SEM=[BO=?b2, CORE=?vp]]
+ VP[SEM=[BO={?b1+?b2}, CORE=<?v(?obj)>]] -> TV[SEM=[BO=?b1, CORE=?v]] NP[SEM=[BO=?b2, CORE=?obj]]
+ VP[SEM=?s] -> IV[SEM=?s]
+ NP[SEM=[BO={?b1+?b2+{bo(?det(?n), at x)}}, CORE=<@x>]] -> Det[SEM=[BO=?b1, CORE=?det]] N[SEM=[BO=?b2, CORE=?n]]
+ Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] -> 'a'
+ N[SEM=[BO={/}, CORE=<dog>]] -> 'dog'
+ N[SEM=[BO={/}, CORE=<dog>]] -> 'cat'
+ N[SEM=[BO={/}, CORE=<dog>]] -> 'mouse'
+ IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'barks'
+ IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'eats'
+ IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'walks'
+ TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'feeds'
+ TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'walks'
+ NP[SEM=[BO={bo(\P.P(John), at x)}, CORE=<@x>]] -> 'john'
+ NP[SEM=[BO={bo(\P.P(John), at x)}, CORE=<@x>]] -> 'alex'
+
+A simple intransitive sentence:
+
+ >>> from nltk.sem import logic
+ >>> logic._counter._value = 100
+
+ >>> trees = cp.parse('john barks'.split())
+ |. john.barks.|
+ |[-----] .| [0:1] 'john'
+ |. [-----]| [1:2] 'barks'
+ |[-----] .| [0:1] NP[SEM=[BO={bo(\P.P(John),z101)}, CORE=<z101>]] -> 'john' *
+ |[-----> .| [0:1] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.P(John),z2)}, ?subj: <IndividualVariableExpression z2>}
+ |. [-----]| [1:2] IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> 'barks' *
+ |. [-----]| [1:2] VP[SEM=[BO={/}, CORE=<\x.bark(x)>]] -> IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] *
+ |[===========]| [0:2] S[SEM=[BO={bo(\P.P(John),z2)}, CORE=<bark(z2)>]] -> NP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<z2>]] VP[SEM=[BO={/}, CORE=<\x.bark(x)>]] *
+ >>> for tree in trees: print(tree)
+ (S[SEM=[BO={bo(\P.P(John),z2)}, CORE=<bark(z2)>]]
+ (NP[SEM=[BO={bo(\P.P(John),z101)}, CORE=<z101>]] john)
+ (VP[SEM=[BO={/}, CORE=<\x.bark(x)>]]
+ (IV[SEM=[BO={/}, CORE=<\x.bark(x)>]] barks)))
+
+A transitive sentence:
+
+ >>> trees = cp.parse('john feeds a dog'.split())
+ |.joh.fee. a .dog.|
+ |[---] . . .| [0:1] 'john'
+ |. [---] . .| [1:2] 'feeds'
+ |. . [---] .| [2:3] 'a'
+ |. . . [---]| [3:4] 'dog'
+ |[---] . . .| [0:1] NP[SEM=[BO={bo(\P.P(John),z102)}, CORE=<z102>]] -> 'john' *
+ |[---> . . .| [0:1] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.P(John),z2)}, ?subj: <IndividualVariableExpression z2>}
+ |. [---] . .| [1:2] TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] -> 'feeds' *
+ |. [---> . .| [1:2] VP[SEM=[BO={?b1+?b2}, CORE=<?v(?obj)>]] -> TV[SEM=[BO=?b1, CORE=?v]] * NP[SEM=[BO=?b2, CORE=?obj]] {?b1: {/}, ?v: <LambdaExpression \x y.feed(y,x)>}
+ |. . [---] .| [2:3] Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] -> 'a' *
+ |. . [---> .| [2:3] NP[SEM=[BO={?b1+?b2+{bo(?det(?n), at x)}}, CORE=<@x>]] -> Det[SEM=[BO=?b1, CORE=?det]] * N[SEM=[BO=?b2, CORE=?n]] {?b1: {/}, ?det: <LambdaExpression \Q P.exists x.(Q(x) & P(x))>}
+ |. . . [---]| [3:4] N[SEM=[BO={/}, CORE=<dog>]] -> 'dog' *
+ |. . [-------]| [2:4] NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z103)}, CORE=<z103>]] -> Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] N[SEM=[BO={/}, CORE=<dog>]] *
+ |. . [------->| [2:4] S[SEM=[BO={?b1+?b2}, CORE=<?vp(?subj)>]] -> NP[SEM=[BO=?b1, CORE=?subj]] * VP[SEM=[BO=?b2, CORE=?vp]] {?b1: {bo(\P.exists x.(dog(x) & P(x)),z2)}, ?subj: <IndividualVariableExpression z2>}
+ |. [-----------]| [1:4] VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]] -> TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<z2>]] *
+ |[===============]| [0:4] S[SEM=[BO={bo(\P.P(John),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]] -> NP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<z2>]] VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<\y.feed(y,z3)>]] *
+
+ >>> for tree in trees: print(tree)
+ (S[SEM=[BO={bo(\P.P(John),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]]
+ (NP[SEM=[BO={bo(\P.P(John),z102)}, CORE=<z102>]] john)
+ (VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]]
+ (TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
+ (NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z103)}, CORE=<z103>]]
+ (Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
+ (N[SEM=[BO={/}, CORE=<dog>]] dog))))
+
+Turn down the verbosity:
+
+ >>> cp = parse.load_parser('grammars/sample_grammars/bindop.fcfg', trace=0,
+ ... chart_class=InstantiateVarsChart)
+
+Reuse the same lexical item twice:
+
+ >>> trees = cp.parse('john feeds john'.split())
+ >>> for tree in trees: print(tree)
+ (S[SEM=[BO={bo(\P.P(John),z2), bo(\P.P(John),z3)}, CORE=<feed(z2,z3)>]]
+ (NP[SEM=[BO={bo(\P.P(John),z104)}, CORE=<z104>]] john)
+ (VP[SEM=[BO={bo(\P.P(John),z2)}, CORE=<\y.feed(y,z2)>]]
+ (TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
+ (NP[SEM=[BO={bo(\P.P(John),z105)}, CORE=<z105>]] john)))
+
+ >>> trees = cp.parse('a dog feeds a dog'.split())
+ >>> for tree in trees: print(tree)
+ (S[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2), bo(\P.exists x.(dog(x) & P(x)),z3)}, CORE=<feed(z2,z3)>]]
+ (NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z106)}, CORE=<z106>]]
+ (Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
+ (N[SEM=[BO={/}, CORE=<dog>]] dog))
+ (VP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z2)}, CORE=<\y.feed(y,z2)>]]
+ (TV[SEM=[BO={/}, CORE=<\x y.feed(y,x)>]] feeds)
+ (NP[SEM=[BO={bo(\P.exists x.(dog(x) & P(x)),z107)}, CORE=<z107>]]
+ (Det[SEM=[BO={/}, CORE=<\Q P.exists x.(Q(x) & P(x))>]] a)
+ (N[SEM=[BO={/}, CORE=<dog>]] dog))))
diff --git a/nltk/test/featstruct.doctest b/nltk/test/featstruct.doctest
new file mode 100644
index 0000000..9ede986
--- /dev/null
+++ b/nltk/test/featstruct.doctest
@@ -0,0 +1,1230 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==================================
+ Feature Structures & Unification
+==================================
+ >>> from __future__ import print_function
+ >>> from nltk.featstruct import FeatStruct
+ >>> from nltk.sem.logic import Variable, VariableExpression, Expression
+
+.. note:: For now, featstruct uses the older lambdalogic semantics
+ module. Eventually, it should be updated to use the new first
+ order predicate logic module.
+
+Overview
+~~~~~~~~
+A feature structure is a mapping from feature identifiers to feature
+values, where feature values can be simple values (like strings or
+ints), nested feature structures, or variables:
+
+ >>> fs1 = FeatStruct(number='singular', person=3)
+ >>> print(fs1)
+ [ number = 'singular' ]
+ [ person = 3 ]
+
+Feature structure may be nested:
+
+ >>> fs2 = FeatStruct(type='NP', agr=fs1)
+ >>> print(fs2)
+ [ agr = [ number = 'singular' ] ]
+ [ [ person = 3 ] ]
+ [ ]
+ [ type = 'NP' ]
+
+Variables are used to indicate that two features should be assigned
+the same value. For example, the following feature structure requires
+that the feature fs3['agr']['number'] be bound to the same value as the
+feature fs3['subj']['number'].
+
+ >>> fs3 = FeatStruct(agr=FeatStruct(number=Variable('?n')),
+ ... subj=FeatStruct(number=Variable('?n')))
+ >>> print(fs3)
+ [ agr = [ number = ?n ] ]
+ [ ]
+ [ subj = [ number = ?n ] ]
+
+Feature structures are typically used to represent partial information
+about objects. A feature name that is not mapped to a value stands
+for a feature whose value is unknown (*not* a feature without a
+value). Two feature structures that represent (potentially
+overlapping) information about the same object can be combined by
+*unification*.
+
+ >>> print(fs2.unify(fs3))
+ [ agr = [ number = 'singular' ] ]
+ [ [ person = 3 ] ]
+ [ ]
+ [ subj = [ number = 'singular' ] ]
+ [ ]
+ [ type = 'NP' ]
+
+When two inconsistent feature structures are unified, the unification
+fails and returns ``None``.
+
+ >>> fs4 = FeatStruct(agr=FeatStruct(person=1))
+ >>> print(fs4.unify(fs2))
+ None
+ >>> print(fs2.unify(fs4))
+ None
+
+..
+ >>> del fs1, fs2, fs3, fs4 # clean-up
+
+Feature Structure Types
+-----------------------
+There are actually two types of feature structure:
+
+- *feature dictionaries*, implemented by `FeatDict`, act like
+ Python dictionaries. Feature identifiers may be strings or
+ instances of the `Feature` class.
+- *feature lists*, implemented by `FeatList`, act like Python
+ lists. Feature identifiers are integers.
+
+When you construct a feature structure using the `FeatStruct`
+constructor, it will automatically decide which type is appropriate:
+
+ >>> type(FeatStruct(number='singular'))
+ <class 'nltk.featstruct.FeatDict'>
+ >>> type(FeatStruct([1,2,3]))
+ <class 'nltk.featstruct.FeatList'>
+
+Usually, we will just use feature dictionaries; but sometimes feature
+lists can be useful too. Two feature lists will unify with each other
+only if they have equal lengths, and all of their feature values
+match. If you wish to write a feature list that contains 'unknown'
+values, you must use variables:
+
+ >>> fs1 = FeatStruct([1,2,Variable('?y')])
+ >>> fs2 = FeatStruct([1,Variable('?x'),3])
+ >>> fs1.unify(fs2)
+ [1, 2, 3]
+
+..
+ >>> del fs1, fs2 # clean-up
+
+Parsing Feature Structure Strings
+---------------------------------
+Feature structures can be constructed directly from strings. Often,
+this is more convenient than constructing them directly. NLTK can
+parse most feature strings to produce the corresponding feature
+structures. (But you must restrict your base feature values to
+strings, ints, logic expressions (`nltk.sem.logic.Expression`), and a
+few other types discussed below).
+
+Feature dictionaries are written like Python dictionaries, except that
+keys are not put in quotes; and square brackets (``[]``) are used
+instead of braces (``{}``):
+
+ >>> FeatStruct('[tense="past", agr=[number="sing", person=3]]')
+ [agr=[number='sing', person=3], tense='past']
+
+If a feature value is a single alphanumeric word, then it does not
+need to be quoted -- it will be automatically treated as a string:
+
+ >>> FeatStruct('[tense=past, agr=[number=sing, person=3]]')
+ [agr=[number='sing', person=3], tense='past']
+
+Feature lists are written like python lists:
+
+ >>> FeatStruct('[1, 2, 3]')
+ [1, 2, 3]
+
+The expression ``[]`` is treated as an empty feature dictionary, not
+an empty feature list:
+
+ >>> type(FeatStruct('[]'))
+ <class 'nltk.featstruct.FeatDict'>
+
+Feature Paths
+-------------
+Features can be specified using *feature paths*, or tuples of feature
+identifiers that specify path through the nested feature structures to
+a value.
+
+ >>> fs1 = FeatStruct('[x=1, y=[1,2,[z=3]]]')
+ >>> fs1['y']
+ [1, 2, [z=3]]
+ >>> fs1['y', 2]
+ [z=3]
+ >>> fs1['y', 2, 'z']
+ 3
+
+..
+ >>> del fs1 # clean-up
+
+Reentrance
+----------
+Feature structures may contain reentrant feature values. A *reentrant
+feature value* is a single feature structure that can be accessed via
+multiple feature paths.
+
+ >>> fs1 = FeatStruct(x='val')
+ >>> fs2 = FeatStruct(a=fs1, b=fs1)
+ >>> print(fs2)
+ [ a = (1) [ x = 'val' ] ]
+ [ ]
+ [ b -> (1) ]
+ >>> fs2
+ [a=(1)[x='val'], b->(1)]
+
+As you can see, reentrane is displayed by marking a feature structure
+with a unique identifier, in this case ``(1)``, the first time it is
+encountered; and then using the special form ``var -> id`` whenever it
+is encountered again. You can use the same notation to directly
+create reentrant feature structures from strings.
+
+ >>> FeatStruct('[a=(1)[], b->(1), c=[d->(1)]]')
+ [a=(1)[], b->(1), c=[d->(1)]]
+
+Reentrant feature structures may contain cycles:
+
+ >>> fs3 = FeatStruct('(1)[a->(1)]')
+ >>> fs3['a', 'a', 'a', 'a']
+ (1)[a->(1)]
+ >>> fs3['a', 'a', 'a', 'a'] is fs3
+ True
+
+Unification preserves the reentrance relations imposed by both of the
+unified feature structures. In the feature structure resulting from
+unification, any modifications to a reentrant feature value will be
+visible using any of its feature paths.
+
+ >>> fs3.unify(FeatStruct('[a=[b=12], c=33]'))
+ (1)[a->(1), b=12, c=33]
+
+..
+ >>> del fs1, fs2, fs3 # clean-up
+
+Feature Structure Equality
+--------------------------
+Two feature structures are considered equal if they assign the same
+values to all features, *and* they contain the same reentrances.
+
+ >>> fs1 = FeatStruct('[a=(1)[x=1], b->(1)]')
+ >>> fs2 = FeatStruct('[a=(1)[x=1], b->(1)]')
+ >>> fs3 = FeatStruct('[a=[x=1], b=[x=1]]')
+ >>> fs1 == fs1, fs1 is fs1
+ (True, True)
+ >>> fs1 == fs2, fs1 is fs2
+ (True, False)
+ >>> fs1 == fs3, fs1 is fs3
+ (False, False)
+
+Note that this differs from how Python dictionaries and lists define
+equality -- in particular, Python dictionaries and lists ignore
+reentrance relations. To test two feature structures for equality
+while ignoring reentrance relations, use the `equal_values()` method:
+
+ >>> fs1.equal_values(fs1)
+ True
+ >>> fs1.equal_values(fs2)
+ True
+ >>> fs1.equal_values(fs3)
+ True
+
+..
+ >>> del fs1, fs2, fs3 # clean-up
+
+Feature Value Sets & Feature Value Tuples
+-----------------------------------------
+`nltk.featstruct` defines two new data types that are intended to be
+used as feature values: `FeatureValueTuple` and `FeatureValueSet`.
+Both of these types are considered base values -- i.e., unification
+does *not* apply to them. However, variable binding *does* apply to
+any values that they contain.
+
+Feature value tuples are written with parentheses:
+
+ >>> fs1 = FeatStruct('[x=(?x, ?y)]')
+ >>> fs1
+ [x=(?x, ?y)]
+ >>> fs1.substitute_bindings({Variable('?x'): 1, Variable('?y'): 2})
+ [x=(1, 2)]
+
+Feature sets are written with braces:
+
+ >>> fs1 = FeatStruct('[x={?x, ?y}]')
+ >>> fs1
+ [x={?x, ?y}]
+ >>> fs1.substitute_bindings({Variable('?x'): 1, Variable('?y'): 2})
+ [x={1, 2}]
+
+In addition to the basic feature value tuple & set classes, nltk
+defines feature value unions (for sets) and feature value
+concatenations (for tuples). These are written using '+', and can be
+used to combine sets & tuples:
+
+ >>> fs1 = FeatStruct('[x=((1, 2)+?z), z=?z]')
+ >>> fs1
+ [x=((1, 2)+?z), z=?z]
+ >>> fs1.unify(FeatStruct('[z=(3, 4, 5)]'))
+ [x=(1, 2, 3, 4, 5), z=(3, 4, 5)]
+
+Thus, feature value tuples and sets can be used to build up tuples
+and sets of values over the corse of unification. For example, when
+parsing sentences using a semantic feature grammar, feature sets or
+feature tuples can be used to build a list of semantic predicates as
+the sentence is parsed.
+
+As was mentioned above, unification does not apply to feature value
+tuples and sets. One reason for this that it's impossible to define a
+single correct answer for unification when concatenation is used.
+Consider the following example:
+
+ >>> fs1 = FeatStruct('[x=(1, 2, 3, 4)]')
+ >>> fs2 = FeatStruct('[x=(?a+?b), a=?a, b=?b]')
+
+If unification applied to feature tuples, then the unification
+algorithm would have to arbitrarily choose how to divide the tuple
+(1,2,3,4) into two parts. Instead, the unification algorithm refuses
+to make this decision, and simply unifies based on value. Because
+(1,2,3,4) is not equal to (?a+?b), fs1 and fs2 will not unify:
+
+ >>> print(fs1.unify(fs2))
+ None
+
+If you need a list-like structure that unification does apply to, use
+`FeatList`.
+
+..
+ >>> del fs1, fs2 # clean-up
+
+Light-weight Feature Structures
+-------------------------------
+Many of the functions defined by `nltk.featstruct` can be applied
+directly to simple Python dictionaries and lists, rather than to
+full-fledged `FeatDict` and `FeatList` objects. In other words,
+Python ``dicts`` and ``lists`` can be used as "light-weight" feature
+structures.
+
+ >>> # Note: pprint prints dicts sorted
+ >>> from pprint import pprint
+ >>> from nltk.featstruct import unify
+ >>> pprint(unify(dict(x=1, y=dict()), dict(a='a', y=dict(b='b'))))
+ {'a': 'a', 'x': 1, 'y': {'b': 'b'}}
+
+However, you should keep in mind the following caveats:
+
+- Python dictionaries & lists ignore reentrance when checking for
+ equality between values. But two FeatStructs with different
+ reentrances are considered nonequal, even if all their base
+ values are equal.
+
+- FeatStructs can be easily frozen, allowing them to be used as
+ keys in hash tables. Python dictionaries and lists can not.
+
+- FeatStructs display reentrance in their string representations;
+ Python dictionaries and lists do not.
+
+- FeatStructs may *not* be mixed with Python dictionaries and lists
+ (e.g., when performing unification).
+
+- FeatStructs provide a number of useful methods, such as `walk()`
+ and `cyclic()`, which are not available for Python dicts & lists.
+
+In general, if your feature structures will contain any reentrances,
+or if you plan to use them as dictionary keys, it is strongly
+recommended that you use full-fledged `FeatStruct` objects.
+
+Custom Feature Values
+---------------------
+The abstract base class `CustomFeatureValue` can be used to define new
+base value types that have custom unification methods. For example,
+the following feature value type encodes a range, and defines
+unification as taking the intersection on the ranges:
+
+ >>> from nltk.compat import total_ordering
+ >>> from nltk.featstruct import CustomFeatureValue, UnificationFailure
+ >>> @total_ordering
+ ... class Range(CustomFeatureValue):
+ ... def __init__(self, low, high):
+ ... assert low <= high
+ ... self.low = low
+ ... self.high = high
+ ... def unify(self, other):
+ ... if not isinstance(other, Range):
+ ... return UnificationFailure
+ ... low = max(self.low, other.low)
+ ... high = min(self.high, other.high)
+ ... if low <= high: return Range(low, high)
+ ... else: return UnificationFailure
+ ... def __repr__(self):
+ ... return '(%s<x<%s)' % (self.low, self.high)
+ ... def __eq__(self, other):
+ ... if not isinstance(other, Range):
+ ... return False
+ ... return (self.low == other.low) and (self.high == other.high)
+ ... def __lt__(self, other):
+ ... if not isinstance(other, Range):
+ ... return True
+ ... return (self.low, self.high) < (other.low, other.high)
+
+ >>> fs1 = FeatStruct(x=Range(5,8), y=FeatStruct(z=Range(7,22)))
+ >>> print(fs1.unify(FeatStruct(x=Range(6, 22))))
+ [ x = (6<x<8) ]
+ [ ]
+ [ y = [ z = (7<x<22) ] ]
+ >>> print(fs1.unify(FeatStruct(x=Range(9, 12))))
+ None
+ >>> print(fs1.unify(FeatStruct(x=12)))
+ None
+ >>> print(fs1.unify(FeatStruct('[x=?x, y=[z=?x]]')))
+ [ x = (7<x<8) ]
+ [ ]
+ [ y = [ z = (7<x<8) ] ]
+
+Regression Tests
+~~~~~~~~~~~~~~~~
+
+Dictionary access methods (non-mutating)
+----------------------------------------
+
+ >>> fs1 = FeatStruct(a=1, b=2, c=3)
+ >>> fs2 = FeatStruct(x=fs1, y='x')
+
+Feature structures support all dictionary methods (excluding the class
+method `dict.fromkeys()`). Non-mutating methods:
+
+ >>> sorted(fs2.keys()) # keys()
+ ['x', 'y']
+ >>> sorted(fs2.values()) # values()
+ [[a=1, b=2, c=3], 'x']
+ >>> sorted(fs2.items()) # items()
+ [('x', [a=1, b=2, c=3]), ('y', 'x')]
+ >>> sorted(fs2) # __iter__()
+ ['x', 'y']
+ >>> 'a' in fs2, 'x' in fs2 # __contains__()
+ (False, True)
+ >>> fs2.has_key('a'), fs2.has_key('x') # has_key()
+ (False, True)
+ >>> fs2['x'], fs2['y'] # __getitem__()
+ ([a=1, b=2, c=3], 'x')
+ >>> fs2['a'] # __getitem__()
+ Traceback (most recent call last):
+ . . .
+ KeyError: 'a'
+ >>> fs2.get('x'), fs2.get('y'), fs2.get('a') # get()
+ ([a=1, b=2, c=3], 'x', None)
+ >>> fs2.get('x', 'hello'), fs2.get('a', 'hello') # get()
+ ([a=1, b=2, c=3], 'hello')
+ >>> len(fs1), len(fs2) # __len__
+ (3, 2)
+ >>> fs2.copy() # copy()
+ [x=[a=1, b=2, c=3], y='x']
+ >>> fs2.copy() is fs2 # copy()
+ False
+
+Note: by default, `FeatStruct.copy()` does a deep copy. Use
+`FeatStruct.copy(deep=False)` for a shallow copy.
+
+..
+ >>> del fs1, fs2 # clean-up.
+
+Dictionary access methods (mutating)
+------------------------------------
+ >>> fs1 = FeatStruct(a=1, b=2, c=3)
+ >>> fs2 = FeatStruct(x=fs1, y='x')
+
+Setting features (`__setitem__()`)
+
+ >>> fs1['c'] = 5
+ >>> fs1
+ [a=1, b=2, c=5]
+ >>> fs1['x'] = 12
+ >>> fs1
+ [a=1, b=2, c=5, x=12]
+ >>> fs2['x', 'a'] = 2
+ >>> fs2
+ [x=[a=2, b=2, c=5, x=12], y='x']
+ >>> fs1
+ [a=2, b=2, c=5, x=12]
+
+Deleting features (`__delitem__()`)
+
+ >>> del fs1['x']
+ >>> fs1
+ [a=2, b=2, c=5]
+ >>> del fs2['x', 'a']
+ >>> fs1
+ [b=2, c=5]
+
+`setdefault()`:
+
+ >>> fs1.setdefault('b', 99)
+ 2
+ >>> fs1
+ [b=2, c=5]
+ >>> fs1.setdefault('x', 99)
+ 99
+ >>> fs1
+ [b=2, c=5, x=99]
+
+`update()`:
+
+ >>> fs2.update({'a':'A', 'b':'B'}, c='C')
+ >>> fs2
+ [a='A', b='B', c='C', x=[b=2, c=5, x=99], y='x']
+
+`pop()`:
+
+ >>> fs2.pop('a')
+ 'A'
+ >>> fs2
+ [b='B', c='C', x=[b=2, c=5, x=99], y='x']
+ >>> fs2.pop('a')
+ Traceback (most recent call last):
+ . . .
+ KeyError: 'a'
+ >>> fs2.pop('a', 'foo')
+ 'foo'
+ >>> fs2
+ [b='B', c='C', x=[b=2, c=5, x=99], y='x']
+
+`clear()`:
+
+ >>> fs1.clear()
+ >>> fs1
+ []
+ >>> fs2
+ [b='B', c='C', x=[], y='x']
+
+`popitem()`:
+
+ >>> sorted([fs2.popitem() for i in range(len(fs2))])
+ [('b', 'B'), ('c', 'C'), ('x', []), ('y', 'x')]
+ >>> fs2
+ []
+
+Once a feature structure has been frozen, it may not be mutated.
+
+ >>> fs1 = FeatStruct('[x=1, y=2, z=[a=3]]')
+ >>> fs1.freeze()
+ >>> fs1.frozen()
+ True
+ >>> fs1['z'].frozen()
+ True
+
+ >>> fs1['x'] = 5
+ Traceback (most recent call last):
+ . . .
+ ValueError: Frozen FeatStructs may not be modified.
+ >>> del fs1['x']
+ Traceback (most recent call last):
+ . . .
+ ValueError: Frozen FeatStructs may not be modified.
+ >>> fs1.clear()
+ Traceback (most recent call last):
+ . . .
+ ValueError: Frozen FeatStructs may not be modified.
+ >>> fs1.pop('x')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Frozen FeatStructs may not be modified.
+ >>> fs1.popitem()
+ Traceback (most recent call last):
+ . . .
+ ValueError: Frozen FeatStructs may not be modified.
+ >>> fs1.setdefault('x')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Frozen FeatStructs may not be modified.
+ >>> fs1.update(z=22)
+ Traceback (most recent call last):
+ . . .
+ ValueError: Frozen FeatStructs may not be modified.
+
+..
+ >>> del fs1, fs2 # clean-up.
+
+Feature Paths
+-------------
+Make sure that __getitem__ with feature paths works as intended:
+
+ >>> fs1 = FeatStruct(a=1, b=2,
+ ... c=FeatStruct(
+ ... d=FeatStruct(e=12),
+ ... f=FeatStruct(g=55, h='hello')))
+ >>> fs1[()]
+ [a=1, b=2, c=[d=[e=12], f=[g=55, h='hello']]]
+ >>> fs1['a'], fs1[('a',)]
+ (1, 1)
+ >>> fs1['c','d','e']
+ 12
+ >>> fs1['c','f','g']
+ 55
+
+Feature paths that select unknown features raise KeyError:
+
+ >>> fs1['c', 'f', 'e']
+ Traceback (most recent call last):
+ . . .
+ KeyError: ('c', 'f', 'e')
+ >>> fs1['q', 'p']
+ Traceback (most recent call last):
+ . . .
+ KeyError: ('q', 'p')
+
+Feature paths that try to go 'through' a feature that's not a feature
+structure raise KeyError:
+
+ >>> fs1['a', 'b']
+ Traceback (most recent call last):
+ . . .
+ KeyError: ('a', 'b')
+
+Feature paths can go through reentrant structures:
+
+ >>> fs2 = FeatStruct('(1)[a=[b=[c->(1), d=5], e=11]]')
+ >>> fs2['a', 'b', 'c', 'a', 'e']
+ 11
+ >>> fs2['a', 'b', 'c', 'a', 'b', 'd']
+ 5
+ >>> fs2[tuple('abcabcabcabcabcabcabcabcabcabca')]
+ (1)[b=[c=[a->(1)], d=5], e=11]
+
+Indexing requires strings, `Feature`\s, or tuples; other types raise a
+TypeError:
+
+ >>> fs2[12]
+ Traceback (most recent call last):
+ . . .
+ TypeError: Expected feature name or path. Got 12.
+ >>> fs2[list('abc')]
+ Traceback (most recent call last):
+ . . .
+ TypeError: Expected feature name or path. Got ['a', 'b', 'c'].
+
+Feature paths can also be used with `get()`, `has_key()`, and
+`__contains__()`.
+
+ >>> fpath1 = tuple('abcabc')
+ >>> fpath2 = tuple('abcabz')
+ >>> fs2.get(fpath1), fs2.get(fpath2)
+ ((1)[a=[b=[c->(1), d=5], e=11]], None)
+ >>> fpath1 in fs2, fpath2 in fs2
+ (True, False)
+ >>> fs2.has_key(fpath1), fs2.has_key(fpath2)
+ (True, False)
+
+..
+ >>> del fs1, fs2 # clean-up
+
+Reading Feature Structures
+--------------------------
+
+Empty feature struct:
+
+ >>> FeatStruct('[]')
+ []
+
+Test features with integer values:
+
+ >>> FeatStruct('[a=12, b=-33, c=0]')
+ [a=12, b=-33, c=0]
+
+Test features with string values. Either single or double quotes may
+be used. Strings are evaluated just like python strings -- in
+particular, you can use escape sequences and 'u' and 'r' prefixes, and
+triple-quoted strings.
+
+ >>> FeatStruct('[a="", b="hello", c="\'", d=\'\', e=\'"\']')
+ [a='', b='hello', c="'", d='', e='"']
+ >>> FeatStruct(r'[a="\\", b="\"", c="\x6f\\y", d="12"]')
+ [a='\\', b='"', c='o\\y', d='12']
+ >>> FeatStruct(r'[b=r"a\b\c"]')
+ [b='a\\b\\c']
+ >>> FeatStruct('[x="""a"""]')
+ [x='a']
+
+Test parsing of reentrant feature structures.
+
+ >>> FeatStruct('[a=(1)[], b->(1)]')
+ [a=(1)[], b->(1)]
+ >>> FeatStruct('[a=(1)[x=1, y=2], b->(1)]')
+ [a=(1)[x=1, y=2], b->(1)]
+
+Test parsing of cyclic feature structures.
+
+ >>> FeatStruct('[a=(1)[b->(1)]]')
+ [a=(1)[b->(1)]]
+ >>> FeatStruct('(1)[a=[b=[c->(1)]]]')
+ (1)[a=[b=[c->(1)]]]
+
+Strings of the form "+name" and "-name" may be used to specify boolean
+values.
+
+ >>> FeatStruct('[-bar, +baz, +foo]')
+ [-bar, +baz, +foo]
+
+None, True, and False are recognized as values:
+
+ >>> FeatStruct('[bar=True, baz=False, foo=None]')
+ [+bar, -baz, foo=None]
+
+Special features:
+
+ >>> FeatStruct('NP/VP')
+ NP[]/VP[]
+ >>> FeatStruct('?x/?x')
+ ?x[]/?x[]
+ >>> print(FeatStruct('VP[+fin, agr=?x, tense=past]/NP[+pl, agr=?x]'))
+ [ *type* = 'VP' ]
+ [ ]
+ [ [ *type* = 'NP' ] ]
+ [ *slash* = [ agr = ?x ] ]
+ [ [ pl = True ] ]
+ [ ]
+ [ agr = ?x ]
+ [ fin = True ]
+ [ tense = 'past' ]
+
+Here the slash feature gets coerced:
+ >>> FeatStruct('[*slash*=a, x=b, *type*="NP"]')
+ NP[x='b']/a[]
+
+ >>> FeatStruct('NP[sem=<bob>]/NP')
+ NP[sem=<bob>]/NP[]
+ >>> FeatStruct('S[sem=<walk(bob)>]')
+ S[sem=<walk(bob)>]
+ >>> print(FeatStruct('NP[sem=<bob>]/NP'))
+ [ *type* = 'NP' ]
+ [ ]
+ [ *slash* = [ *type* = 'NP' ] ]
+ [ ]
+ [ sem = <bob> ]
+
+Playing with ranges:
+
+ >>> from nltk.featstruct import RangeFeature, FeatStructReader
+ >>> width = RangeFeature('width')
+ >>> reader = FeatStructReader([width])
+ >>> fs1 = reader.fromstring('[*width*=-5:12]')
+ >>> fs2 = reader.fromstring('[*width*=2:123]')
+ >>> fs3 = reader.fromstring('[*width*=-7:-2]')
+ >>> fs1.unify(fs2)
+ [*width*=(2, 12)]
+ >>> fs1.unify(fs3)
+ [*width*=(-5, -2)]
+ >>> print(fs2.unify(fs3)) # no overlap in width.
+ None
+
+The slash feature has a default value of 'False':
+
+ >>> print(FeatStruct('NP[]/VP').unify(FeatStruct('NP[]'), trace=1))
+ <BLANKLINE>
+ Unification trace:
+ / NP[]/VP[]
+ |\ NP[]
+ |
+ | Unify feature: *type*
+ | / 'NP'
+ | |\ 'NP'
+ | |
+ | +-->'NP'
+ |
+ | Unify feature: *slash*
+ | / VP[]
+ | |\ False
+ | |
+ X X <-- FAIL
+ None
+
+The demo structures from category.py. They all parse, but they don't
+do quite the right thing, -- ?x vs x.
+
+ >>> FeatStruct(pos='n', agr=FeatStruct(number='pl', gender='f'))
+ [agr=[gender='f', number='pl'], pos='n']
+ >>> FeatStruct(r'NP[sem=<bob>]/NP')
+ NP[sem=<bob>]/NP[]
+ >>> FeatStruct(r'S[sem=<app(?x, ?y)>]')
+ S[sem=<?x(?y)>]
+ >>> FeatStruct('?x/?x')
+ ?x[]/?x[]
+ >>> FeatStruct('VP[+fin, agr=?x, tense=past]/NP[+pl, agr=?x]')
+ VP[agr=?x, +fin, tense='past']/NP[agr=?x, +pl]
+ >>> FeatStruct('S[sem = <app(?subj, ?vp)>]')
+ S[sem=<?subj(?vp)>]
+
+ >>> FeatStruct('S')
+ S[]
+
+The parser also includes support for reading sets and tuples.
+
+ >>> FeatStruct('[x={1,2,2,2}, y={/}]')
+ [x={1, 2}, y={/}]
+ >>> FeatStruct('[x=(1,2,2,2), y=()]')
+ [x=(1, 2, 2, 2), y=()]
+ >>> print(FeatStruct('[x=(1,[z=(1,2,?x)],?z,{/})]'))
+ [ x = (1, [ z = (1, 2, ?x) ], ?z, {/}) ]
+
+Note that we can't put a featstruct inside a tuple, because doing so
+would hash it, and it's not frozen yet:
+
+ >>> print(FeatStruct('[x={[]}]'))
+ Traceback (most recent call last):
+ . . .
+ TypeError: FeatStructs must be frozen before they can be hashed.
+
+There's a special syntax for taking the union of sets: "{...+...}".
+The elements should only be variables or sets.
+
+ >>> FeatStruct('[x={?a+?b+{1,2,3}}]')
+ [x={?a+?b+{1, 2, 3}}]
+
+There's a special syntax for taking the concatenation of tuples:
+"(...+...)". The elements should only be variables or tuples.
+
+ >>> FeatStruct('[x=(?a+?b+(1,2,3))]')
+ [x=(?a+?b+(1, 2, 3))]
+
+Parsing gives helpful messages if your string contains an error.
+
+ >>> FeatStruct('[a=, b=5]]')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Error parsing feature structure
+ [a=, b=5]]
+ ^ Expected value
+ >>> FeatStruct('[a=12 22, b=33]')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Error parsing feature structure
+ [a=12 22, b=33]
+ ^ Expected comma
+ >>> FeatStruct('[a=5] [b=6]')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Error parsing feature structure
+ [a=5] [b=6]
+ ^ Expected end of string
+ >>> FeatStruct(' *++*')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Error parsing feature structure
+ *++*
+ ^ Expected open bracket or identifier
+ >>> FeatStruct('[x->(1)]')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Error parsing feature structure
+ [x->(1)]
+ ^ Expected bound identifier
+ >>> FeatStruct('[x->y]')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Error parsing feature structure
+ [x->y]
+ ^ Expected identifier
+ >>> FeatStruct('')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Error parsing feature structure
+ <BLANKLINE>
+ ^ Expected open bracket or identifier
+
+
+Unification
+-----------
+Very simple unifications give the expected results:
+
+ >>> FeatStruct().unify(FeatStruct())
+ []
+ >>> FeatStruct(number='singular').unify(FeatStruct())
+ [number='singular']
+ >>> FeatStruct().unify(FeatStruct(number='singular'))
+ [number='singular']
+ >>> FeatStruct(number='singular').unify(FeatStruct(person=3))
+ [number='singular', person=3]
+
+Merging nested structures:
+
+ >>> fs1 = FeatStruct('[A=[B=b]]')
+ >>> fs2 = FeatStruct('[A=[C=c]]')
+ >>> fs1.unify(fs2)
+ [A=[B='b', C='c']]
+ >>> fs2.unify(fs1)
+ [A=[B='b', C='c']]
+
+A basic case of reentrant unification
+
+ >>> fs4 = FeatStruct('[A=(1)[B=b], E=[F->(1)]]')
+ >>> fs5 = FeatStruct("[A=[C='c'], E=[F=[D='d']]]")
+ >>> fs4.unify(fs5)
+ [A=(1)[B='b', C='c', D='d'], E=[F->(1)]]
+ >>> fs5.unify(fs4)
+ [A=(1)[B='b', C='c', D='d'], E=[F->(1)]]
+
+More than 2 paths to a value
+
+ >>> fs1 = FeatStruct("[a=[],b=[],c=[],d=[]]")
+ >>> fs2 = FeatStruct('[a=(1)[], b->(1), c->(1), d->(1)]')
+ >>> fs1.unify(fs2)
+ [a=(1)[], b->(1), c->(1), d->(1)]
+
+fs1[a] gets unified with itself
+
+ >>> fs1 = FeatStruct('[x=(1)[], y->(1)]')
+ >>> fs2 = FeatStruct('[x=(1)[], y->(1)]')
+ >>> fs1.unify(fs2)
+ [x=(1)[], y->(1)]
+
+Bound variables should get forwarded appropriately
+
+ >>> fs1 = FeatStruct('[A=(1)[X=x], B->(1), C=?cvar, D=?dvar]')
+ >>> fs2 = FeatStruct('[A=(1)[Y=y], B=(2)[Z=z], C->(1), D->(2)]')
+ >>> fs1.unify(fs2)
+ [A=(1)[X='x', Y='y', Z='z'], B->(1), C->(1), D->(1)]
+ >>> fs2.unify(fs1)
+ [A=(1)[X='x', Y='y', Z='z'], B->(1), C->(1), D->(1)]
+
+Cyclic structure created by unification.
+
+ >>> fs1 = FeatStruct('[F=(1)[], G->(1)]')
+ >>> fs2 = FeatStruct('[F=[H=(2)[]], G->(2)]')
+ >>> fs3 = fs1.unify(fs2)
+ >>> fs3
+ [F=(1)[H->(1)], G->(1)]
+ >>> fs3['F'] is fs3['G']
+ True
+ >>> fs3['F'] is fs3['G']['H']
+ True
+ >>> fs3['F'] is fs3['G']['H']['H']
+ True
+ >>> fs3['F'] is fs3['F']['H']['H']['H']['H']['H']['H']['H']['H']
+ True
+
+Cyclic structure created w/ variables.
+
+ >>> fs1 = FeatStruct('[F=[H=?x]]')
+ >>> fs2 = FeatStruct('[F=?x]')
+ >>> fs3 = fs1.unify(fs2, rename_vars=False)
+ >>> fs3
+ [F=(1)[H->(1)]]
+ >>> fs3['F'] is fs3['F']['H']
+ True
+ >>> fs3['F'] is fs3['F']['H']['H']
+ True
+ >>> fs3['F'] is fs3['F']['H']['H']['H']['H']['H']['H']['H']['H']
+ True
+
+Unifying w/ a cyclic feature structure.
+
+ >>> fs4 = FeatStruct('[F=[H=[H=[H=(1)[]]]], K->(1)]')
+ >>> fs3.unify(fs4)
+ [F=(1)[H->(1)], K->(1)]
+ >>> fs4.unify(fs3)
+ [F=(1)[H->(1)], K->(1)]
+
+Variable bindings should preserve reentrance.
+
+ >>> bindings = {}
+ >>> fs1 = FeatStruct("[a=?x]")
+ >>> fs2 = fs1.unify(FeatStruct("[a=[]]"), bindings)
+ >>> fs2['a'] is bindings[Variable('?x')]
+ True
+ >>> fs2.unify(FeatStruct("[b=?x]"), bindings)
+ [a=(1)[], b->(1)]
+
+Aliased variable tests
+
+ >>> fs1 = FeatStruct("[a=?x, b=?x]")
+ >>> fs2 = FeatStruct("[b=?y, c=?y]")
+ >>> bindings = {}
+ >>> fs3 = fs1.unify(fs2, bindings)
+ >>> fs3
+ [a=?x, b=?x, c=?x]
+ >>> bindings
+ {Variable('?y'): Variable('?x')}
+ >>> fs3.unify(FeatStruct("[a=1]"))
+ [a=1, b=1, c=1]
+
+If we keep track of the bindings, then we can use the same variable
+over multiple calls to unify.
+
+ >>> bindings = {}
+ >>> fs1 = FeatStruct('[a=?x]')
+ >>> fs2 = fs1.unify(FeatStruct('[a=[]]'), bindings)
+ >>> fs2.unify(FeatStruct('[b=?x]'), bindings)
+ [a=(1)[], b->(1)]
+ >>> bindings
+ {Variable('?x'): []}
+
+..
+ >>> del fs1, fs2, fs3, fs4, fs5 # clean-up
+
+Unification Bindings
+--------------------
+
+ >>> bindings = {}
+ >>> fs1 = FeatStruct('[a=?x]')
+ >>> fs2 = FeatStruct('[a=12]')
+ >>> fs3 = FeatStruct('[b=?x]')
+ >>> fs1.unify(fs2, bindings)
+ [a=12]
+ >>> bindings
+ {Variable('?x'): 12}
+ >>> fs3.substitute_bindings(bindings)
+ [b=12]
+ >>> fs3 # substitute_bindings didn't mutate fs3.
+ [b=?x]
+ >>> fs2.unify(fs3, bindings)
+ [a=12, b=12]
+
+ >>> bindings = {}
+ >>> fs1 = FeatStruct('[a=?x, b=1]')
+ >>> fs2 = FeatStruct('[a=5, b=?x]')
+ >>> fs1.unify(fs2, bindings)
+ [a=5, b=1]
+ >>> sorted(bindings.items())
+ [(Variable('?x'), 5), (Variable('?x2'), 1)]
+
+..
+ >>> del fs1, fs2, fs3 # clean-up
+
+Expressions
+-----------
+
+ >>> e = Expression.fromstring('\\P y.P(z,y)')
+ >>> fs1 = FeatStruct(x=e, y=Variable('z'))
+ >>> fs2 = FeatStruct(y=VariableExpression(Variable('John')))
+ >>> fs1.unify(fs2)
+ [x=<\P y.P(John,y)>, y=<John>]
+
+Remove Variables
+----------------
+
+ >>> FeatStruct('[a=?x, b=12, c=[d=?y]]').remove_variables()
+ [b=12, c=[]]
+ >>> FeatStruct('(1)[a=[b=?x,c->(1)]]').remove_variables()
+ (1)[a=[c->(1)]]
+
+Equality & Hashing
+------------------
+The `equal_values` method checks whether two feature structures assign
+the same value to every feature. If the optional argument
+``check_reentrances`` is supplied, then it also returns false if there
+is any difference in the reentrances.
+
+ >>> a = FeatStruct('(1)[x->(1)]')
+ >>> b = FeatStruct('(1)[x->(1)]')
+ >>> c = FeatStruct('(1)[x=[x->(1)]]')
+ >>> d = FeatStruct('[x=(1)[x->(1)]]')
+ >>> e = FeatStruct('(1)[x=[x->(1), y=1], y=1]')
+ >>> def compare(x,y):
+ ... assert x.equal_values(y, True) == y.equal_values(x, True)
+ ... assert x.equal_values(y, False) == y.equal_values(x, False)
+ ... if x.equal_values(y, True):
+ ... assert x.equal_values(y, False)
+ ... print('equal values, same reentrance')
+ ... elif x.equal_values(y, False):
+ ... print('equal values, different reentrance')
+ ... else:
+ ... print('different values')
+
+ >>> compare(a, a)
+ equal values, same reentrance
+ >>> compare(a, b)
+ equal values, same reentrance
+ >>> compare(a, c)
+ equal values, different reentrance
+ >>> compare(a, d)
+ equal values, different reentrance
+ >>> compare(c, d)
+ equal values, different reentrance
+ >>> compare(a, e)
+ different values
+ >>> compare(c, e)
+ different values
+ >>> compare(d, e)
+ different values
+ >>> compare(e, e)
+ equal values, same reentrance
+
+Feature structures may not be hashed until they are frozen:
+
+ >>> hash(a)
+ Traceback (most recent call last):
+ . . .
+ TypeError: FeatStructs must be frozen before they can be hashed.
+ >>> a.freeze()
+ >>> v = hash(a)
+
+Feature structures define hash consistently. The following example
+looks at the hash value for each (fs1,fs2) pair; if their hash values
+are not equal, then they must not be equal. If their hash values are
+equal, then display a message, and indicate whether their values are
+indeed equal. Note that c and d currently have the same hash value,
+even though they are not equal. That is not a bug, strictly speaking,
+but it wouldn't be a bad thing if it changed.
+
+ >>> for fstruct in (a, b, c, d, e):
+ ... fstruct.freeze()
+ >>> for fs1_name in 'abcde':
+ ... for fs2_name in 'abcde':
+ ... fs1 = locals()[fs1_name]
+ ... fs2 = locals()[fs2_name]
+ ... if hash(fs1) != hash(fs2):
+ ... assert fs1 != fs2
+ ... else:
+ ... print('%s and %s have the same hash value,' %
+ ... (fs1_name, fs2_name))
+ ... if fs1 == fs2: print('and are equal')
+ ... else: print('and are not equal')
+ a and a have the same hash value, and are equal
+ a and b have the same hash value, and are equal
+ b and a have the same hash value, and are equal
+ b and b have the same hash value, and are equal
+ c and c have the same hash value, and are equal
+ c and d have the same hash value, and are not equal
+ d and c have the same hash value, and are not equal
+ d and d have the same hash value, and are equal
+ e and e have the same hash value, and are equal
+
+..
+ >>> del a, b, c, d, e, v # clean-up
+
+Tracing
+-------
+
+ >>> fs1 = FeatStruct('[a=[b=(1)[], c=?x], d->(1), e=[f=?x]]')
+ >>> fs2 = FeatStruct('[a=(1)[c="C"], e=[g->(1)]]')
+ >>> fs1.unify(fs2, trace=True)
+ <BLANKLINE>
+ Unification trace:
+ / [a=[b=(1)[], c=?x], d->(1), e=[f=?x]]
+ |\ [a=(1)[c='C'], e=[g->(1)]]
+ |
+ | Unify feature: a
+ | / [b=[], c=?x]
+ | |\ [c='C']
+ | |
+ | | Unify feature: a.c
+ | | / ?x
+ | | |\ 'C'
+ | | |
+ | | +-->Variable('?x')
+ | |
+ | +-->[b=[], c=?x]
+ | Bindings: {?x: 'C'}
+ |
+ | Unify feature: e
+ | / [f=?x]
+ | |\ [g=[c='C']]
+ | |
+ | +-->[f=?x, g=[b=[], c=?x]]
+ | Bindings: {?x: 'C'}
+ |
+ +-->[a=(1)[b=(2)[], c='C'], d->(2), e=[f='C', g->(1)]]
+ Bindings: {?x: 'C'}
+ [a=(1)[b=(2)[], c='C'], d->(2), e=[f='C', g->(1)]]
+ >>>
+ >>> fs1 = FeatStruct('[a=?x, b=?z, c=?z]')
+ >>> fs2 = FeatStruct('[a=?y, b=?y, c=?q]')
+ >>> #fs1.unify(fs2, trace=True)
+ >>>
+
+..
+ >>> del fs1, fs2 # clean-up
+
+Unification on Dicts & Lists
+----------------------------
+It's possible to do unification on dictionaries:
+
+ >>> from nltk.featstruct import unify
+ >>> pprint(unify(dict(x=1, y=dict(z=2)), dict(x=1, q=5)), width=1)
+ {'q': 5, 'x': 1, 'y': {'z': 2}}
+
+It's possible to do unification on lists as well:
+
+ >>> unify([1, 2, 3], [1, Variable('x'), 3])
+ [1, 2, 3]
+
+Mixing dicts and lists is fine:
+
+ >>> pprint(unify([dict(x=1, y=dict(z=2)),3], [dict(x=1, q=5),3]),
+ ... width=1)
+ [{'q': 5, 'x': 1, 'y': {'z': 2}}, 3]
+
+Mixing dicts and FeatStructs is discouraged:
+
+ >>> unify(dict(x=1), FeatStruct(x=1))
+ Traceback (most recent call last):
+ . . .
+ ValueError: Mixing FeatStruct objects with Python dicts and lists is not supported.
+
+But you can do it if you really want, by explicitly stating that both
+dictionaries and FeatStructs should be treated as feature structures:
+
+ >>> unify(dict(x=1), FeatStruct(x=1), fs_class=(dict, FeatStruct))
+ {'x': 1}
+
+Finding Conflicts
+-----------------
+
+ >>> from nltk.featstruct import conflicts
+ >>> fs1 = FeatStruct('[a=[b=(1)[c=2], d->(1), e=[f->(1)]]]')
+ >>> fs2 = FeatStruct('[a=[b=[c=[x=5]], d=[c=2], e=[f=[c=3]]]]')
+ >>> for path in conflicts(fs1, fs2):
+ ... print('%-8s: %r vs %r' % ('.'.join(path), fs1[path], fs2[path]))
+ a.b.c : 2 vs [x=5]
+ a.e.f.c : 2 vs 3
+
+..
+ >>> del fs1, fs2 # clean-up
+
+Retracting Bindings
+-------------------
+
+ >>> from nltk.featstruct import retract_bindings
+ >>> bindings = {}
+ >>> fs1 = FeatStruct('[a=?x, b=[c=?y]]')
+ >>> fs2 = FeatStruct('[a=(1)[c=[d=1]], b->(1)]')
+ >>> fs3 = fs1.unify(fs2, bindings)
+ >>> print(fs3)
+ [ a = (1) [ c = [ d = 1 ] ] ]
+ [ ]
+ [ b -> (1) ]
+ >>> pprint(bindings)
+ {Variable('?x'): [c=[d=1]], Variable('?y'): [d=1]}
+ >>> retract_bindings(fs3, bindings)
+ [a=?x, b=?x]
+ >>> pprint(bindings)
+ {Variable('?x'): [c=?y], Variable('?y'): [d=1]}
+
+Squashed Bugs
+~~~~~~~~~~~~~
+In svn rev 5167, unifying two feature structures that used the same
+variable would cause those variables to become aliased in the output.
+
+ >>> fs1 = FeatStruct('[a=?x]')
+ >>> fs2 = FeatStruct('[b=?x]')
+ >>> fs1.unify(fs2)
+ [a=?x, b=?x2]
+
+There was a bug in svn revision 5172 that caused `rename_variables` to
+rename variables to names that are already used.
+
+ >>> FeatStruct('[a=?x, b=?x2]').rename_variables(
+ ... vars=[Variable('?x')])
+ [a=?x3, b=?x2]
+ >>> fs1 = FeatStruct('[a=?x]')
+ >>> fs2 = FeatStruct('[a=?x, b=?x2]')
+ >>> fs1.unify(fs2)
+ [a=?x, b=?x2]
+
+There was a bug in svn rev 5167 that caused us to get the following
+example wrong. Basically the problem was that we only followed
+'forward' pointers for other, not self, when unifying two feature
+structures. (nb: this test assumes that features are unified in
+alphabetical order -- if they are not, it might pass even if the bug
+is present.)
+
+ >>> fs1 = FeatStruct('[a=[x=1], b=?x, c=?x]')
+ >>> fs2 = FeatStruct('[a=(1)[], b->(1), c=[x=2]]')
+ >>> print(fs1.unify(fs2))
+ None
+
+..
+ >>> del fs1, fs2 # clean-up
+
diff --git a/nltk/test/floresta.txt b/nltk/test/floresta.txt
new file mode 100644
index 0000000..caa23ae
--- /dev/null
+++ b/nltk/test/floresta.txt
@@ -0,0 +1,7924 @@
+O 7 e Meio � um ex-libris da noite algarvia.
+� uma das mais antigas discotecas do Algarve, situada em Albufeira, que continua a manter os tra�os decorativos e as clientelas de sempre.
+� um pouco a vers�o de uma esp�cie de �outro lado� da noite, a meio caminho entre os devaneios de uma fauna perif�rica, seja de Lisboa, Londres, Dublin ou Faro e Portim�o, e a postura circunspecta dos fi�is da casa, que dela esperam a m�sica �geracionista� dos 60 ou dos 70.
+N�o deixa de ser, nos tempos que correm, um certo �very typical� algarvio, cabe�a de cartaz para os que querem fugir a algumas movimenta��es nocturnas j� a caminho da ritualiza��o de massas, do g�nero �vamos todos ao Calypso e encontramo-nos na Locomia�.
+E assim, aos 2,5 milh�es que o Minist�rio do Planeamento e Administra��o do Territ�rio j� gasta no pagamento do pessoal afecto a estes organismos, v�m juntar-se os montantes das obras propriamente ditas, que os munic�pios, j� com projectos na m�o, v�m reivindicar junto do Executivo, como salienta aquele membro do Governo.
+E o dinheiro �n�o falta s� �s c�maras�, lembra o secret�rio de Estado, que considera que a solu��o para as autarquias � �especializarem-se em fundos comunit�rios�.
+Mas como, se muitas n�o disp�em, nos seus quadros, dos t�cnicos necess�rios?
+�Encomendem-nos a projectistas de fora� porque, se as obras vierem a ser financiadas, eles at� saem de gra�a, j� que, nesse caso, �os fundos comunit�rios pagam os projectos, o mesmo n�o acontecendo quando eles s�o feitos pelos GAT�, dado serem organismos do Estado.
+Essa poder� vir a ser uma hip�tese, at� porque, no terreno, a capacidade dos GAT est� cada vez mais enfraquecida.
+Alguns at� j� desapareceram, como o de Castro Verde, e outros t�m vindo a perder quadros.
+O primeiro fabricante mundial de �ratos� para computador, a empresa su��a Logitech, apresentou esta semana numa feira especializada que teve lugar em Basileia (Su��a) um equipamento perif�rico denominado �Audioman� que permitir� dotar os computadores de �orelhas�.
+Segundo a empresa, o aparelho permite que o aparelho grave e transcreva a voz humana.
+�Estamos a dotar os computadores de um novo sentido� disse Steve d'Averio, director de marketing para a Europa da Logitech.
+O Audioman foi desenvolvido na Su��a em apenas sete meses e comp�e-se de um microfone e de um altifalante que se podem acoplar facilmente a um computador, devendo ser comercializado ao pre�o de 290 francos su��os (28 contos).
+Junqueiro foi ainda confrontado com o facto de n�o ter falado com o ministro antes de avan�ar com a proposta.
+Jo�o Cravinho, que integra a comitiva de Jorge Sampaio na visita de Estado a Mo�ambique, ainda n�o reagiu � carta que o dirigente socialista lhe enviou.
+�N�o estou a ver que, para emitir uma opini�o, n�s tiv�ssemos de informar previamente o ministro, afirmou.
+�O senhor ministro interpretar� esta sugest�o como entender�.
+Junqueiro recordou ainda que, nas �ltimas aut�rquicas, o IGAT suspendeu as suas actividades um m�s antes das elei��es.
+Al�m do Museu do Ar, o projecto gira em torno do parque tem�tico propriamente dito.
+A zona l�dica, com os divertimentos, �reas comerciais de �souvenirs� e de restaura��o, compreende espa�os distintos para os v�rios temas, ainda em an�lise, tais como Portugal, Jap�o, Brasil, �frica e Far-West.
+Os quatro primeiros temas destinam-se a mostrar o papel de Portugal no mundo e o quinto, o �nico sem rela��o com a hist�ria nacional, � justificado pela experi�ncia de Barcelona (Port Aventura), que regista assinal�vel sucesso.
+Uma �zona petting� anexa, em coopera��o com o Jardim Zool�gico de Lisboa, destina-se a permitir o contacto das crian�as com animais.
+Tudo, claro est�, muito arborizado.
+O museu, a desenvolver sob orienta��o de uma comiss�o de not�veis, dirigida pelo Presidente da Rep�blica, est� or�ado em seis milh�es de contos, valor incomport�vel para a For�a A�rea.
+Da� que a Cameron Hall tenha ca�do do c�u.
+�A proposta � muito bem vista, porque ser� mais vantajosa do que se houver s� um p�lo de interesse no local�, afirmou o major Carlos Barbosa, das rela��es p�blicas da For�a A�rea, admitindo que, com o parque tem�tico, �se o interesse for diversificado, toda a gente fica a ganhar�.
+A pouco mais de um m�s do lan�amento nacional do Rendimento M�nimo Garantido (RMG), o n�mero de fam�lias j� abrangidas pelos projectos-piloto deste programa de apoio aos agregados mais desfavorecidos n�o p�ra de aumentar.
+7.777 fam�lias, totalizando 26.668 pessoas, est�o j� a usufruir do rendimento destinado a garantir condi��es consideradas m�nimas de sobreviv�ncia e reinser��o de cidad�os exclu�dos socialmente.
+O balan�o -- a que o P�BLICO teve acesso -- tra�ado pela comiss�o revela que o n�mero de pessoas abrangidas pelo RMG aumentou 36 por cento relativamente ao �ltimo balan�o de 30 de Mar�o.
+Este crescimento �resulta da op��o de alargar o n�mero de projectos-piloto, de modo a cobrir uma parte do territ�rio nacional at� ao dia 1 de Julho�, referiu ao P�BLICO o presidente da Comiss�o Nacional do RMG, Paulo Pedroso.
+Para tal, referiu aquele respons�vel, �foram montadas mais estruturas, mais zonas est�o abrangidas e, por isso, mais pessoas se podem candidatar�.
+E tantos foram os candidatos que o per�odo destinado a testar a aplica��o do RMG acabaria por ceder lugar a um processo efectivo de financiamento.
+A institui��o deste direito s� ser�, contudo, efectivado depois do lan�amento nacional do projecto dentro de pouco mais de um m�s.
+Mais metaf�rico foi o secret�rio de Estado do Desenvolvimento Regional, Adriano Pimp�o, que comparou o acordo a �uma embraiagem�.
+Porque � a embraiagem �que p�e o motor em contacto com as rodas que geram o movimento, que para n�s � o desenvolvimento�.
+Para que n�o surjam avarias, Pimp�o pediu aos presentes que se empenhem na execu��o dos termos do acordo, sob pena de a embraiagem se transformar em �trav�o�.
+A prop�sito, no Museu da Segunda Guerra Mundial, que a� foi aberto, a hist�ria da maior guerra no continente europeu come�a com a fotografia de Estaline a cumprimentar o ministro dos Neg�cios Estrangeiros da Alemanha nazi, ou seja, a guerra come�a com a assinatura do Pacto Molotov-Ribbentrop.
+Na cerim�nia de inaugura��o do edif�cio, Ieltsin declarou perante mais de cem mil pessoas que �a R�ssia est� perto da estabilidade pol�tica� e que �todos os problemas podem ser resolvidos � mesa das conversa��es�.
+Talvez entusiasmado pela festa da vit�ria, o Presidente russo afirmou que �chegar� o dia em que a R�ssia ajudar� o Ocidente�.
+As cerim�nias oficiais terminaram com desfiles militares e recria��es de combates a�reos entre os aviadores sovi�ticos e alem�es.
+Co-produ��o franco-eg�pcia, �O Emigrante� inspira-se na hist�ria de Jos�, filho de Jacob, contando o percurso de Ram que, h� 3000 anos, decide abandonar a sua terra �rida para se instalar no Egipto dos fara�s, centro da civiliza��o.
+Tomando a defesa do filme de Chahine, numa sala atulhada, o baston�rio dos advogados, Ahmed al-Khawaga, replicou que o realizador eg�pcio se inspirou na hist�ria de Jos�, �mas n�o se afastou das palavras do Cor�o que evoca, em termos claros, as propostas feitas ao profeta pela esposa do mestre que o comprou � chegada ao Egipto�.
+�Eu n�o sou franc�s para falar da cultura francesa, mas sei que ela deu uma contribui��o importante � cultura eg�pcia.
+E uma vez que um dos nossos artistas conclui um acordo com um artista franc�s, isso n�o nos desonra�.
+O �ltimo romance de Paul Auster -- que ainda n�o est� traduzido.
+Nashe encontra Pozzi, um jogador, com quem inicia um p�quer extravagante.
+No imenso desacerto que foi a defesa do Penafiel, o capit�o Vasco foi o homem que ainda segurou as pontas.
+Seguro, eficiente, decidido -- tivesse o Penafiel outro Vasco e provavelmente o resultado teria sido outro.
+VALTINHO -- Foi dos melhores jogadores do Penafiel, este brasileiro de pernas altas.
+Dos seus p�s ainda nasceu alguma coisa, embora o resto da equipa n�o ajudasse grande coisa.
+Lutou como lhe vemos poucas vezes e ainda teve nos p�s uma boa oportunidade de golo, mas teve que rematar em jeito e n�o em for�a, como gosta mais.
+(H� quem defenda, no entanto, que se trata de um fax ap�crifo, realmente escrito pelo deputado Jos� Magalh�es, o qual teria, ali�s, imitado a letra do ex-deputado Ant�nio Barreto.
+Fontes fidedignas -- que o mesmo � dizer, n�o jornal�sticas -- garantiram entretanto que, entre as frases atribu�das ao senhor primeiro-ministro que, de facto, lhe n�o pertencem est�o o coment�rio ao congresso �Portugal e o futuro�, a resposta ao deputado Adriano Moreira quando este o interrogou sobre a sua concep��o de federalismo e -- at�! -- a contestada frase sobre a fidedignidade dos jornalistas.
+O Benfica voltou ontem a vencer o Lokomotiv de Moscovo pela diferen�a m�nima (3-2), passando aos quartos-de-final da Ta�a das Ta�as em futebol.
+O jogo chegou a estar complicado para a equipa de Paulo Autuori, essencialmente porque o Lokomotiv marcou muito cedo (8 ') e os jogadores portugueses n�o conseguiam adaptar-se ao estado do relvado.
+Mas a expuls�o prematura e est�pida de Gurenko, quando ainda n�o estava decorrida meia hora de jogo, seguida, pouco depois, da entrada de Panduru, foi o suficiente para dar a volta ao resultado.
+O Benfica apresentou o seu esquema habitual, com Jamir e Tahar mais recuados e insistindo com a utiliza��o de Valdo na esquerda.
+Em contrapartida, o Lokomotiv actuava com tr�s centrais, com o l�bero Chugainov, como habitualmente, a comandar o jogo e o lateral-esquerdo Solomatin muito activo.
+Muito bem estavam tamb�m os tr�s m�dios -- Maminov, Drozdov e Kharlachev --, desempenhando Kosolapov as fun��es de �pivot� do ataque.
+Por isso, e tamb�m porque os russos estavam mais bem adaptados ao muito enlameado e escorregadio relvado, o Lokomotiv era mais r�pido sobre a bola, ganhando a maior parte dos duelos na zona fulcral do terreno, situada entre o meio-campo e a defesa do Benfica.
+O �cocktail� que Jupp� condena � bem real.
+Mas �-o apenas porque este Governo e esta maioria, escolheram, desde Agosto do ano passado, a imigra��o e a pol�tica da imigra��o como ponto de clivagem pol�tica para as pr�ximas legislativas, ontem mesmo marcadas para Mar�o de 1998, em simult�neo com as regionais.
+Um endurecimento n�tido existe desde ent�o neste terreno altamente perigoso.
+E raz�o desta escolha �, obviamente, a progress�o demente da Frente Nacional, que prospera sempre a apontar o imigrante como bode expiat�rio e simultaneamente como a fonte de todos males do povo franc�s.
+Foi com estupefac��o e surpresa que li, na edi��o do passado fim-de-semana, os comunicados da direc��o e da administra��o deste jornal.
+Sou um leitor ass�duo e atento do P�BLICO, desde o primeiro n�mero, e n�o poucas vezes tenho manifestado opini�es nas suas p�ginas, o que me leva agora a emitir o meu pensamento sobre o jornal, sobre quem o faz e sobre os ditos comunicados.
+1. O P�BLICO veio dar � imprensa di�ria portuguesa uma nova dimens�o e, pelo seu aparecimento, obrigou os �grandes� (�JN� e �DN�) a reformular a sua postura e tamb�m o seu grafismo.
+Tal como na �ltima final do Campeonato do Mundo, disputada em Nova Iorque e Lion, Kasparov tenta apresentar-se como o �bom reformador� contra o �mau conservador� (Karpov).
+A FIDE, que organiza a competi��o desde 1948, declarou estar pronta para defender os seus direitos em tribunal, e acusou Kasparov e Short de privilegiarem os seus interesses econ�micos ao tentarem conseguir uma verba superior � oferecida pela cidade de Manchester para organizar a final e que rondava os 250 mil contos.
+Este �golpe de Estado� deixa c�ptica a maior parte grandes mestres de xadrez (cerca de 300 em todo o mundo), que esperam ver a situa��o clarificada, independentemente da parte que acabe vencedora.
+Por outro lado, a expuls�o de Kasparov, detentor do t�tulo mundial desde 1985, criar� dificuldades � FIDE para convencer Manchester a acolher uma falsa final entre Karpov e Timman.
+Isto no caso de os dois xadrezistas aceitarem o convite.
+A bem dizer, todos t�m raz�o.
+O �rbitro porque o terreno de jogo estava quase impratic�vel.
+Henrique Calisto porque o terreno estava quase impratic�vel, o Leix�es defendera o adiamento da partida antes do seu in�cio e tinha menos um homem em campo que o advers�rio, por expuls�o de S�rgio.
+Eurico e Milton Areal porque o terreno estava quase impratic�vel, mas nem melhor nem pior do que na primeira metade do encontro e o Tirsense parecia mais fresco e estava em situa��o de vantagem num�rica.
+Quanto ao jogo, ele dividiu-se em dois per�odos distintos: antes e depois da expuls�o de S�rgio.
+Antes, o Leix�es marcou um golo, por interm�dio de Maur�cio, na sequ�ncia de um canto apontado por Barriga, e foi a equipa que melhor se adaptou �s condi��es do terreno.
+Depois, o Tirsense tomou conta da partida, criou v�rias situa��es de golo e conseguiu a igualdade atrav�s de uma grande penalidade marcada por Rui Manuel, a castigar carga de Correia sobre Batista.
+Com o dil�vio como pano de fundo, o empate traduz de forma feliz um jogo que ficou no meio.
+Durante a an�lise do relat�rio de actividades passadas, foram identificadas como principais insufici�ncias a aus�ncia de uma orienta��o nacional junto dos quadros t�cnicos, e o fraco recrutamento e pouca contribui��o na �rea da Ci�ncia e Tecnologia.
+O Partido Popular (PP), vencedor das elei��es de 3 de Mar�o, quer a plena integra��o da Espanha na Alian�a Atl�ntica, organiza��o a que Madrid aderiu em 1982, sem, no entanto, englobar as suas for�as militares nas da NATO, de acordo com os resultados do referendo de 1986.
+Um processo que ocupar� os pr�ximos ministros da Defesa e das Rela��es Exteriores e que n�o ter� a oposi��o dos socialistas de Felipe Gonz�lez, a segunda maior for�a pol�tica do pa�s.
+O russo ser� uma das seis l�nguas principais usadas por Jo�o Paulo II amanh� e depois, no Encontro Mundial da Juventude, no Santu�rio Mariano de Czestokowa, onde se prev� a presen�a de um milh�o de jovens, o dobro dos que h� dois anos se congregaram em Santiago de Compostela.
+� que, al�m de a proximidade geogr�fica da URSS e dos outros pa�ses do Leste, muita coisa aconteceu nos �ltimos 24 meses na grande Casa Comum Europeia.
+Ao chegar, �s 9 horas (TMG) de hoje, ao aeroporto de Crac�via, Jo�o Paulo II ser� recebido com um m�nimo de formalidades.
+� a quinta vez que Karol Wojtyla pisa, como Pont�fice, o solo da sua p�tria.
+Mas convencionou-se que esta sua desloca��o de tr�s dias, para presidir ao Encontro dos Jovens em Czestochawa, seria considerado um complemento ainda da sua recente viagem � Pol�nia, de 1 a 9 de Junho passado.
+Carrington fez sempre quest�o de salientar que as hip�teses de sucesso do cessar-fogo dependem sobretudo dos beligerantes.
+�Se for firmado, ningu�m ficar� mais contente do que n�s.
+Eu tentei, o senhor Vance tentou, se for respeitado, urrah!�, comentou.
+Mas se falhar?
+�Ningu�m fez mais at� agora do que o secret�rio Vance.
+Devemos tentar�.
+�A situa��o pode ser potencialmente horr�vel.
+Pela primeira vez no Haiti um padre foi assassinado por motivos pol�ticos.
+Uma mensagem dos militares no poder para mostrarem quem ainda manda no pa�s, interpretam meios eclesi�sticos, que reafirmam a disposi��o de continuar a luta �pela liberta��o do povo haitiano�.
+A Confer�ncia Haitiana de Religiosos, cuja direc��o � tida por moderada, vem respondendo ao crime com jejum, ora��es, missas ao ar livre e homilias em que o engajamento do padre Jean-Marie ao lado dos �pobres e oprimidos� � apontado como exemplo.
+Pode vir a ser o ponto de viragem na Igreja, cuja hierarquia, com a excep��o de um �nico bispo, prefere os militares golpistas a Aristide, o Presidente eleito democraticamente, ele pr�prio um padre que os salesianos expulsaram da ordem.
+H�, no ar, uma certa ideia de invas�o.
+Cavaco Silva n�o assinou o habitual despacho que d� toler�ncia de ponto na ter�a-feira de Carnaval- o que significa que, ao contr�rio do que � tradicional, este ano n�o h� o feriado do Entrudo.
+Ontem, come�ou a chegar �s direc��es-gerais (atrav�s de uma circular) a informa��o de que o dia 23 ser� um dia �normal� para os funcion�rios p�blicos.
+Registe-se que a ter�a-feira de Carnaval n�o � um feriado legal, mas t�o-s� �tradicional�: ou seja, todos os anos o primeiro-ministro tem que produzir um despacho, publicado em �Di�rio da Rep�blica�, em que decreta a toler�ncia de ponto.
+Este ano, eventualmente condicionado pela pol�mica que o op�e ao Presidente da Rep�blica em torno dos feriados (recorde-se que Soares enviou a lei para o Tribunal Constitucional) Cavaco decidiu, pura e simplesmente, �acabar� com a ter�a-feira de Carnaval.
+Ali�s, era contra as �ter�as-feiras�, propiciadoras de sugestivas �pontes�, que a lei governamental mais se batia ...
+Desde 1990 que estava na mesa a reformula��o das �secretas�.
+A primeira inten��o do Governo era ver legislado algo que, ignorando a lei, era um facto desde o in�cio: a inexist�ncia efectiva do Servi�o de Informa��es Estrat�gicas de Defesa (SIED), que nunca foi mais que uma al�nea da lei-quadro nunca levada � pr�tica.
+Com argumentos economicistas e de operacionalidade, o Executivo de Cavaco Silva sempre se escusou a concretizar o SIED, cujas compet�ncias foram, entretanto, transferidas para o SIM (Servi�os de Informa��es Militares), por via de um pol�mico acto administrativo do Governo, que assim chamava a si mat�rias da exclusiva compet�ncia da AR.
+Ou seja, em dez anos, nunca a Lei dos Servi�os de Informa��es foi integralmente cumprida, com uma estrutura que estava �no papel� sem exist�ncia pr�tica (o SIED) e outra que assegurava as fun��es da primeira (a Dinfo).
+Facto que, ao longo do tempo, foi repetidamente denunciado, tanto pelos partidos da oposi��o (onde se destacaram o PCP e o PS), como pelo Conselho de Fiscaliza��o dos Servi�os de Informa��es nomeado pela AR.
+O caso ocorreu numa noite de 1978, na ilha de Carvalo, ao largo da C�rsega.
+O pr�ncipe jantava com amigos num restaurante deste para�so para milion�rios, quando um grupo barulhento de jovens da alta sociedade italiana acostou na enseada de Palma, ao lado do seu iate, o L'Aniram.
+Os advogados da defesa sublinharam no processo que este facto perturbou altamente o �senhor de Sab�ia�.
+Naquele ano, as Brigadas Vermelhas (BR) estavam no auge da actividade terrorista, o l�der crist�o-democrata Aldo Moro acabara de ser raptado, e o pr�ncipe -- proibido de entrar em It�lia desde o ex�lio do pai em 1946 -- teria mesmo recebido amea�as das BR.
+O certo � que, pouco depois, V�tor-Emanuel apercebeu-se que um barco pneum�tico fora deslocado do seu iate e atracado ao Cocke, o navio dos jovens italianos.
+�Irritado com este acto de apropria��o�, foi buscar uma espingarda US 30 semiautom�tica, utilizada em safaris, e 31 cartuchos, e dirigiu-se para o Cocke.
+Pouco depois, o pr�ncipe aponta-lhe a arma ao ventre.
+Na confus�o que se segue, parte um primeiro tiro, depois um segundo, e os dois homens caem ao mar.
+A bordo do veleiro polaco Dar Mlodziezy, patrocinado pela Cutty Sark, os tripulantes e passageiros tiveram que esperar at� �s 13 horas de ontem para, finalmente, pisarem o cais.
+Ancorado fora do porto de C�dis durante toda a noite, o grande veleiro foi tomado de assalto pelos 18 passageiros portugueses a cantar fados at� de madrugada.
+Desde as 10 horas da manh�, as manobras de entrada do porto eram aguardadas ansiosamente e por tr�s horas toda a guarni��o permaneceu no conv�s, pronta para receber ordens do piloto a bordo do rebocador que coordenou as manobras de atraca��o ao lado do veleiro Esmeralda, da armada chilena.
+Os organizadores da regata ainda davam os �ltimos retoques nas instala��es � volta do porto quando a frota desembarcou.
+V�rias tendas ofereciam os mais variados servi�os aos tripulantes, como restaurantes, cabines telef�nicas, servi�os banc�rios e de correios enquanto na sala de imprensa os empregados da Telef�nica ainda instalavam os telefones e m�quinas de fax para o batalh�o de jornalistas que chegou � cidade.
+Mesmo com a confus�o administrativa da escala, o �show n�utico� continua a todo o pano por quatro dias.
+A Netscape Communicationns decidiu adquirir a Collabra Software por 108,7 milh�es de contos (16,3 milh�es de contos).
+A Collabra edita o �groupware� e o �software� que permitem compatibilizar diversas redes de computadores, nomeadamente as que existem no interior de uma mesma empresa.
+Os termos concretos da transac��o n�o foram tornados p�blicos mas os analistas coincidem na interpreta��o deste neg�cio como mais um passo da Netscape para transformar a Internet num meio privilegiado de comunica��o e informa��o � escala mundial.
+A Netscape � o mais importante fabricante de �software� de navega��o para a Internet.
+A LG Electronics, o terceiro maior fabricante sul-coreano de inform�tica, decidiu reduzir os pre�os dos seus computadores pessoais.
+A decis�o visa tornar mais competitivos os produtos da empresa, que est� a sofrer uma forte concorr�ncia dos seus principais concorrentes.
+As descidas oscilam, consoante os produtos, entre 9,8 e 26,9 por cento.
+Durante o primeiro semestre do corrente ano, o mercado de computadores pessoais da Coreia do Sul registou um aumento de 42 por cento relativamente a id�ntico per�odo do ano transacto, num montante de 1300 milh�es de d�lares (195 milh�es de contos).
+O an�ncio divulgado na sexta-feira pela OMS dizia que uma s�rie de testes iniciais, realizados pelo Instituto Pasteur de Paris, em amostras de sangue das primeiras nove pessoas a morrerem eram consistentes com o diagn�stico de febre de Ebola.
+Esse mesmo diagn�stico j� tinha sido feito a t�tulo provis�rio apenas com base nos sintomas dos doentes.
+Uma equipa daquela organiza��o encontra-se desde sexta-feira passada na regi�o para examinar as v�timas e colher amostras.
+O Governo gabon�s, num comunicado em que d� conta da exist�ncia de �uma epidemia� na regi�o, pede aos habitantes que n�o evacuem os doentes nem para a capital da prov�ncia, Makokou, nem para a capital nacional, Libreville, e que alertem para qualquer novo caso as autoridades sanit�rias, para que estas possam providenciar o tratamento dos doentes �in loco�.
+Recomenda-se ainda �s pessoas que n�o toquem com as m�os nuas nem nos doentes nem nos mortos e que evitem o contacto com o seu sangue, v�mitos e excrementos.
+Sempre que surge um problema, chamam-na.
+L� vai Dolores Fa�sca ver o que se passa � porta do Nacofino.
+�s vezes acontece, mas �normalmente nunca h� problemas graves�.
+C� dentro ningu�m se apercebe de que um teimoso �brio quer impor a sua presen�a.
+O homem insiste, meio zonzo, a patroa n�o cede e ele acaba por ir-se em grande dificuldade.
+Tamb�m mal atina com as manobras precisas para sair do parque de estacionamento privativo do �dancing�.
+Acaba por se lan�ar � estrada dos Quatro Caminhos rumo a Quarteira, sem temer o �bal�o� que agora transforma sopros em cadeia.
+Faz frio.
+S�o 2h25, j� s� resta um casal � mesa.
+�s 0h00 eram quatro.
+As rela��es de Hong Kong com a China est�o a condicionar a evolu��o do mercado accionista local.
+S�o as trocas comerciais, o novo aeroporto internacional, entre outros aspectos.
+Os investidores, cada vez mais sens�veis, est�o a reagir prontamente n�o tomando posi��es.
+Assim, est� a registar-se um abrandamento na procura com a consequente queda das cota��es.
+O �ndice Hang Seng caiu 2,47 por cento, fechando nos 5481,61 pontos.
+A Comiss�o Europeia considerou ontem �politicamente inoportuno� avan�ar com uma proposta de harmoniza��o dos impostos sobre os produtos energ�ticos que, a concretizar-se, poderia provocar um aumento do gas�leo em Portugal de quase 25 escudos em 2002.
+Os moradores s�o convidados a fazer desde logo a separa��o dos lixos -- condi��o necess�ria para o �xito do projecto --, que depois ser�o recolhidos por viaturas equipadas a preceito.
+Neste sistema, a recolha far-se-� �uma vez por semana para os materiais org�nicos e tr�s por semana para os restantes res�duos dom�sticos�.
+Em Gondomar, a experi�ncia come�ar� com sacos de pl�stico em vez de cestos, por vontade da pr�pria autarquia.
+Ao mesmo tempo, ir� manter-se a recolha indiferenciada tal como hoje a conhecemos.
+�Tendencialmente, o caminho ser� para aprofundar a recolha selectiva�, acentuou o mesmo respons�vel da Lipor.
+Este sistema de recolha adequa-se a edif�cios baixos, com poucos pisos.
+Em os pr�dios com muitos andares, haver� um ecoponto para todos os moradores.
+�O �nico trabalho das pessoas � separar o lixo e colocar nos dias certos os contentores e cestos para serem recolhidos.
+O resto � connosco�, garante a Lipor.
+Na Maia, cada morador receber� esses recipientes pessoalmente, da m�o de funcion�rios municipais, enquanto alunos dos cursos de Rela��es P�blicas e Psicologia do Instituto Superior da Maia explicar�o como funciona o sistema.
+Esta ac��o poder� come�ar j� em Julho, �com abordagens porta a porta�.
+Para muitos analistas o verdadeiro problema � o facto de n�o se poder falar do desenvolvimento da economia palestiniana como um facto isolado, porque ele s� faz sentido integrado no desenvolvimento de todo o M�dio Oriente.
+que tipo de rela��o econ�mica se poder� estabelecer entre Israel e os seus vizinhos �rabes?
+Ser� prematuro falar numa comunidade econ�mica entre a Jord�nia, os territ�rios palestinianos aut�nomos e Israel?
+outros defendem precisamente o contr�rio.
+Fundamental para os palestinianos � a abertura de novos mercados, tanto a Ocidente como a Oriente -- e em Israel -- para os produtos que, apesar dos �bvios problemas da sua agricultura e da ind�stria, venham a produzir.
+Outra das apostas � o turismo, esperando-se dois milh�es de visitantes por ano e a cria��o de 30 mil a 50 mil postos de emprego.
+Quanto �s anunciadas sa�das de alguns jogadores, Donner afirmou que �essas not�cias n�o foram feitas por jornalistas, mas por pataratas.
+Dizia-se que o Carlos Resende e o Filipe Cruz podiam ir para a Alemanha e o Carlos Galambas e o �lvaro Martins para o Benfica, mas � mentira.
+J� todos renovaram os seus contratos�.
+�Acho que o FC Porto s� tem de se preocupar com o Sporting, n�o pensamos em mais nada.
+A fartura de pensamento pode dar maus resultados e n�s n�o queremos ter um enfarte.
+Esta � uma prova de regularidade e s� pode beneficiar quem for mais regular�.
+R. -- N�o sou capaz.
+Sou formada em Direito, mas n�o conseguiria ensinar algu�m a ler e a escrever.
+Nunca dei uma li��o na vida, mas tenho pena, porque penso que se aprende muito a ensinar.
+P. -- Como � a sua rela��o com o piano?
+�N�o sei.
+Talvez morta�.
+Para estes haitianos, s� h� uma maneira de regressar a casa: revolu��o.
+Voltando � rua que durante d�cadas evocou o talentoso marido de D. Maria II, citemos mais um passo do j� referido relat�rio da C�mara do Porto, que nos permite avaliar at� que ponto a concretiza��o da Rua de D. Fernando ficou aqu�m das inten��es que lhe estiveram na origem.
+�A conveni�ncia desta rua � palp�vel; uma c�moda estrada desobstru�da de tortuosidades e declives, desde a Foz ao cora��o da cidade, especialmente para seges e carros, o que at� aqui mal se consegue antes de chegar ao s�tio do banco de S. Domingos [ leia-se o Banco Comercial do Porto, que fora fundado poucos anos antes e detinha autoriza��o para emitir notas ]�.
+Come�a a dar resultados a pol�tica da Uni�o Europeia de bloquear todas as tentativas su��as de gozar as vantagens da UE, sem as responsabilidades de um pa�s membro.
+Os seus governantes j� perceberam que o isolamento j� n�o d� lucro e pode levar � rejei��o.
+Quando o povo su��o recusou, em 92, a ades�o ao Espa�o Econ�mico Europeu, como j� fizera com a ONU, cometeu um grave engano.
+Foi essa volunt�ria e pretensiosa rejei��o dos vizinhos, que deixou a Su��a sem a cobertura europeia, na crise que destruiu a sua imagem.
+O fim da guerra fria, com a implos�o da URSS acabou com a import�ncia helv�tica no tabuleiro europeu, enquanto os recentes esc�ndalos de cumplicidade com os nazis, mais o roubo das economias dos judeus pelos seus bancos lhe tiraram a simpatia americana.
+No entanto, alguns dos analistas contactados pelo P�BLICO, consideram que a Sumolis tem sido �esquecida� pelo mercado e que existem boas perspectivas quanto aos resultados de 1997.
+Para al�m destes dois aspectos, surgem os habituais rumores sobre um eventual interesse comprador por parte de outros grupos empresariais do sector das bebidas, como, por exemplo, a Jer�nimo Martins.
+Uma possibilidade entretanto desmentida pelo grupo presidido por Soares dos Santos.
+�� totalmente falso que a Jer�nimo Martins esteja interessada na compra da Sumolis�, garantiu ao P�BLICO um porta-voz da empresa.
+No lado das subidas, destaca-se ainda o comportamento do Banco Totta & A�ores que, ao contr�rio dos restantes t�tulos do sector banc�rio, encerrou a ganhar 3,41 por cento.
+Movimentaram-se cerca de 285 mil t�tulos, com a cota��o de fecho a situar-se nos 4359 escudos.
+�Alguma coisa se passa � volta deste papel.
+Existe um forte interesse dos internacionais�, salientou outro respons�vel.
+Inez Teixeira � uma jovem pintora que tem exposto regularmente desde h� uns dois anos.
+Agora, num espa�o de exposi��es tamb�m recente, mostra uma s�rie de obras de pequen�ssimo formato feitas a grafite sobre tela.
+E julgamos ainda estar longe de casos como o do banco brit�nico NatWest que guardava as opini�es religiosas e pol�ticas e mesmo os h�bitos alimentares de alguns dos seus 6,5 milh�es de titulares de contas.
+Em paralelo, h� sempre o perigo de estas BD irem cair nas m�os de pessoas menos escrupulosas.
+Em Janeiro de 1994, noticiava-se a introdu��o na Alemanha de um supercomputador que regista as impress�es digitais dos candidatos a asilo pol�tico, para tentar detectar fraudes nos subs�dios da seguran�a social.
+Na mesma altura, come�ou a funcionar um sistema inform�tico que permite distribuir mais rapidamente os estrangeiros pelos campos de refugiados.
+Nada fez parar esta compila��o de dados, nem o receio de que os endere�os pudessem cair nas m�os de grupos nazis que ficariam assim a conhecer onde moram os seus �alvos�.
+Agora, a pol�cia inclina-se para que o assassinato tenha a ver com a promo��o de John Gotti Junior ao cargo de chefe da fam�lia durante o encarceramento do pai.
+Segundo fontes policiais citadas pelo �New York Times�, o atentado poderia ter partido da velha guarda do cl� Gambino.
+Desde que Gotti foi preso, em Dezembro passado, o seu filho, de 26 anos, est� a substitu�-lo, sobretudo na colecta de fundos resultantes de diversas actividades ilegais.
+De acordo com o testemunho de informadores, Junior estaria a manter uma arrog�ncia excessiva em rela��o aos velhos membros do cl�, indo ao ponto de reclamar somas superiores ao que era habitual.
+O motorista acompanhava frequentes vezes John Junior e a sua morte pode ser considerada como um aviso da velha guarda.
+Dezenas de timorenses e portugueses �ocupam� pacificamente o pavilh�o indon�sio da Expo-92, em Sevilha.
+Bush s� poder� ganhar as elei��es presidenciais se fizer cair Saddam Hussein, vaticinam os analistas pol�ticos.
+O Presidente cancela todos os compromissos e fecha-se na Casa Branca.
+Na mesma ocasi�o iniciaram-se investiga��es que incidiram sobre o �rbitro madeirense Marques da Silva, tamb�m ele suspeito de se ter deixado corromper.
+A Judici�ria aproveitou ainda o balan�o para passar buscas �s casas de Reinaldo Teles (dirigente), Jorge Gomes (funcion�rio) e Ant�nio Garrido (colaborador), todos ligados ao FC Porto, com a curiosidade de o �ltimo ser um ex-�rbitro de futebol.
+se Gu�maro era corrupto, quem eram os corruptores e porque motivo n�o foram igualmente presentes ao juiz do Tribunal de Instru��o Criminal?
+Essa � uma pergunta que ainda hoje permanece sem resposta.
+� que, apesar de todas as tentativas feitas pelos agentes da Direc��o Central de Investiga��o de Corrup��o, Fraudes e Infrac��es Econ�mico-Financeiras, o �rbitro, sujeito a diversos interrogat�rios, nunca fez qualquer revela��o que pudesse incriminar outras pessoas.
+Por outras palavras: nunca quis beneficiar do estatuto de �arrependido�.
+Alice n�o sabia o que era um fato-macaco, mas n�o teve coragem de perguntar.
+�N�o pod�amos.
+Afinal, os direitos dos trabalhadores est�o garantidos na Constitui��o.
+Os temas escolhidos s�o cinco: �Patrim�nio Virtual� (explora��o dos monumentos portugueses usando as tecnologias da realidade virtual); �Portugal Global� (Sagres como antena de expans�o e comunica��o da expans�o global e local dos portugueses); �Sacra Saturni� (Sagres como lugar de mist�rio e simbologia, tema ideal para um jogo de aventura gr�fica); �Um Milh�o de Navegadores� (Sagres como centro privilegiado de turismo cultural, com um milh�o de visitantes por ano) e �Terr�vista� (Sagres [...]
+�Numa primeira fase o trabalho ser� concretizado num ' site ' na Internet, com acesso universal e que ter� sede em Sagres.
+Essas p�ginas na Internet ter�o uma componente de refer�ncia a Sagres e � regi�o costeira do Algarve, num regime de divulga��o e promo��o do patrim�nio, com um registo diferente do cl�ssico.
+Depois, poder� haver explora��es em etapas sucessivas para outras linguagens.
+Poder-se-� tocar na realidade virtual, embora seja um objectivo a longo prazo.
+As primeiras semanas foram dedicadas ao estudo dos textos propostos pelos EUA.
+Nos �ltimos dias o ritmo tornou-se fren�tico, �manipularam-se mais de uma centena de documentos e mapas�, garante de la Pe�a.
+Segundo a �Newsweek�, as duas salas dos mapas foram o palco principal das negocia��es.
+Estavam repletas de lixo, copos de pl�stico sujos de caf�.
+Os negociadores usaram canetas de ponta de feltro para tra�ar linhas de fronteira nos cart�es plastificados com o territ�rio da ex-Jugosl�via impresso.
+Bastava um pano h�mido para fazer desaparecer as linhas, e recome�ar tudo de novo.
+�rbitros: Miguel Castro (Argentina) e Alfonso Bove (It�lia).
+Portugal -- Guilherme Silva, Paulo Almeida, V�tor Fortunato, Pedro Alves, T� Neves; Rui Lopes (2), Paulo Alves e Ant�nio Ramalho (2).
+H� muito tempo que tenho uma estranha rela��o afectuosa com esta ilha.
+No quiosque vendem-se dessas revistas de viagens que agora proliferam e que perpetuam as fantasias sobre ilhas ex�ticas.
+Sempre me pareceu estranho nunca ter lido um artigo sobre a �ilha de Santos�.
+Estava convencido que s� eu a via, s� eu a imaginava vista de cima naufragando no meio dos horr�veis autocarros lisboetas.
+Protegida pelas correntes anti-estacionamento selvagem, todos os dias suspirava de al�vio por ainda ver a ilha no seu s�tio, com as fronteiras bem definidas -- o que em si � uma das raz�es que faz das ilhas um dos nossos arqu�tipos mais resistentes.
+Outro dia reparei numa bandeira hasteada no passeio em frente.
+Era a bandeira duma organiza��o que eu desconhecia: �Amigos da Ilha de Santos�.
+Como algu�m que descobre n�o estar s� no mundo, aquela bandeira foi alimento espiritual.
+Eis que a minha ilha tinha bandeira e tudo.
+Eis que, afinal, existem mais habitantes virtuais daquele pa�s que me d� gosto imaginar como um principado independente.
+Sobretudo foi bom descobrir que h� mais gente a fazer parte do equipamento �imagin�rio� da ilha.
+Isolada dos passeios que bordejam os quarteir�es, frequentada por uma �popula��o� m�vel, provis�ria e em constante renova��o, a ilha serve de plataforma de comunica��es: apanhar um transporte, comprar um jornal com not�cias, enviar uma carta, fazer um telefonema.
+Os seus habitantes s�o do mais cosmopolita que h�.
+A cimeira sindical ib�rica j� n�o dever� realizar-se este ano, segundo apurou o P�BLICO.
+Um reuni�o ao mais alto n�vel a realizar na segunda-feira entre a CGTP e a UGT (na sede desta �ltima central) poder� desbloquear anteriores dificuldades e levar � marca��o de uma data.
+Na Europa Ocidental tem-se assistido a uma queda acentuada da fecundidade, o que fez surgir o problema da n�o substitui��o das gera��es.
+Cada vez nascem menos beb�s, com o inevit�vel envelhecimento progressivo da popula��o.
+entre 1960 e 1991, os valores da taxa de fecundidade passaram de 94,9 por cento para 47 por cento.
+Quer isto dizer que a percentagem da popula��o nacional que, em determinado per�odo, procriou baixou para metade em quatro d�cadas.
+�Os Padr�es Recentes da Fecundidade em Portugal�, estudo que ser� lan�ado na pr�xima semana, foi elaborado pelas soci�logas Ana Nunes de Almeida e Cristina Ferreira e pelas ge�grafas Filipa Ferr�o e Isabel Margarida Andr�.
+Editado pela Comiss�o para a Igualdade e para os Direitos da Mulher, este trabalho pretende contextualizar a �queda recente e vertiginosa da fecundidade em Portugal�.
+Com a acarea��o entre Paradela de Abreu e o c�nego Melo, terminaram anteontem as dilig�ncias previstas no acord�o que determinou a reabertura do processo do padre Max.
+Para a acusa��o, o balan�o � positivo, mas para a defesa um novo arquivamento do caso est� mais pr�ximo.
+At� porque o juiz acaba de indeferir uma acarea��o entre os sete suspeitos.
+0 juiz titular do processo do padre Max, Artur Oliveira, indeferiu um pedido do procurador-geral adjunto nomeado pela Procuradoria-Geral da Rep�blica para acompanhar a investiga��o deste caso, no sentido de promover uma acarea��o entre os sete indiv�duos indiciados na acusa��o provis�ria como respons�veis pelo crime-- tr�s como autores morais e os restantes como autores materiais.
+Paulo S� pedia ainda uma acarea��o entre o industrial portuense Manuel Macedo, Ramiro Moreira e o tenente da Marinha Pedro Menezes, todos testemunhas neste caso.
+Todos os nomes citados derivam desta cena, que rapidamente foi superada por outras modas.
+Qualquer deles prosseguiu na linha da pop electr�nica, todavia, s� Marc Almond e os Erasure seguem hoje uma atitude �camp�, embora ambos com algumas �nuances� que de algum modo tendem a atenuar-lhes a envolv�ncia escandalosa.
+Almond � o �nico que continua a cantar teatral e amaneirado, fazendo das suas interpreta��es casos de incandesc�ncia incontrol�vel, assente no culto do personagem instant�neo, � boa maneira tradicional do �camp� de Oscar Wilde.
+Se isso fica uma vez mais reiterado em �Memoribilia�, a compila��o dos seus �xitos a solo e nos Soft Cell, que parcialmente regravou para o efeito, tamb�m � vis�vel que nos respectivos novos v�deos colados �s antigas can��es qualquer coisa mudou em Almond.
+O exemplo paradigm�tico � �Say hello and Wave Goodbye�, cujo primeiro �clip� consistia numa verdadeira orgia de excessos e que agora � substitu�do por um teledisco de um romantismo asseado repleto de modelos em c�mara lenta, que mais se tende a ligar aos �dessexuados� Black ou Don Henley.
+Em s�ntese, aquele dinheiro seria a �ltima presta��o do pagamento do terreno.
+Em troca, assinava-se o contrato-promessa de compra e venda.
+Deolinda, com o seu advogado ao lado, deu-lhe a procura��o e Manuel foi ter com Constantino para ele lhe dar tamb�m os pap�is assinados.
+E Constantino perguntou-lhe se ele estava a brincar, pois n�o lhe ia dar procura��o nenhuma enquanto n�o recebesse a sua parte, ao que Manuel, (se ningu�m no tribunal mentiu neste aspecto), lhe replicou que j� tinha dado os 1095 contos a Deolinda e eles que dividissem a soma entre os dois.
+E viu ent�o pela cara de Constantino que tinha feito mal, mas muito mal, pois marido e mulher agora s� partilhavam uma filha, um barrac�o e um �dio horr�vel, azedo e m�tuo.
+De facto, dava a impress�o que marido e mulher, nesta �ltima fase do casamento, s� j� comunicavam atrav�s dos respectivos advogados, um luxo estranho para pessoas quase sem dinheiro.
+-- E ela disse para passar um cheque ...
+A Assembleia aprovou ainda mo��es que reclamavam a divulga��o dos resultados provis�rios da avalia��o das universidades, a fiscaliza��o da constitucionalidade da Lei de Financiamento do Ensino Superior P�blico, e insistiam nas campanhas de divulga��o das queixa dos estudantes, aproveitando, por exemplo, a presen�a da comunica��o social no pr�ximo jogo Acad�mica-Benfica.
+Entre as propostas mais ousadas, decidiu-se pedir ao Presidente da Rep�blica que proponha um referendo sobre a Lei do Financiamento, desafiar as televis�es a promoverem um debate entre o ministro da Educa��o e todos os dirigentes associativos, pintar de negro a sede da Direc��o Regional de Educa��o do Centro e remeter envelopes com folhas de papel higi�nico ao minist�rio.
+E n�o seria mais �til que o dr. Soares que, enquanto primeiro-ministro, deixou a educa��o no caos que se conhece, dirigisse mensagens a lan�ar o debate sobre o que h� a reformar no ensino, em vez de passar a vida a exigir uma televis�o t�o livre que Eduardo Moniz tenha liberdade para criticar toda a gente excepto .. o dr. Soares?
+... E, se o dr. Soares tivesse praticado desporto na escola, ser� que, hoje, pensaria da mesma maneira?
+�A VW ainda n�o tomou qualquer decis�o, porque est�o a analisar as v�rias hip�teses.
+Mira Amaral confirmou, no entanto, que Portugal � uma das possibilidades que est�o a ser estudadas pelos alem�es, tendo-lhe sido colocadas v�rias perguntas sobre as condi��es de investimento, quando da recente visita a uma das f�bricas da VW, na Alemanha.
+Para o ministro, a decis�o dever� ser tomada em meados de 1993 e, se a escolha recair sobre Portugal, essa ser� �uma boa altura� em termos de incentivos.
+�Em 1993 j� estar�o esgotadas as verbas do actual Quadro Comunit�rio de Apoio [ QCA ] e ainda n�o estar�o dispon�veis as do QCA de 1994.
+V.C. -- Mas � quase disco de prata [ faltam seis mil, segundo a editora, a BMG ]!
+Comprar um CD n�o est� ao alcance de todas as pessoas que compravam os nossos discos, mas acreditamos nos jovens, porque temos a certeza que aderem a isto.
+� l�gico que estamos a aproveitar o facto de os portugueses aderirem outra vez � m�sica popular.
+P. -- E qual � a vossa opini�o sobre a m�sica portuguesa actual?
+O que � que acham que mudou desde os anos de apogeu do conjunto?
+A R�SSIA anunciou ontem ter assinado um contrato para o fornecimento de mais dois reactores nucleares ao Ir�o, mas negou ter recebido uma encomenda para vender 4000 blindados ao Iraque.
+Quanto ao contrato com Teer�o, cujo montante n�o foi especificado, o ministro russo da Energia At�mica explicou que ele inclui a venda de dois reactores de tipo VVER-440, com uma pot�ncia de 440 megawatts, para serem instalados em Bouchehr, no sul do Ir�o.
+A R�ssia, insens�vel aos protestos dos EUA e de Israel, j� se tinha comprometido, no in�cio do ano, a instalar naquele mesmo local um reactor de 1000 megawatts, num neg�cio avaliado em mil milh�es de d�lares.
+Em rela��o ao Iraque, Valeri Progrebenkov, porta-voz da sociedade de Estado Rosvooroujenie, respons�vel pelas exporta��es militares, desmentiu a exist�ncia de uma encomenda de 4000 carros de combate russos, como afirmara o genro de Saddam Hussein que desertou para a Jord�nia.
+Segundo este, os blindados seriam entregues ao longo de v�rios anos e pagos em petr�leo, depois do levantamento das san��es impostas ao Iraque.
+Os New York Knicks venceram ter�a-feira no seu reduto os Chicago Bulls, por 96-91, passando a liderar, por 2-0, a final da Confer�ncia Leste da Liga Norte-Americana de Basquetebol Profissional (NBA), que se disputa � melhor de sete encontros.
+O Governo inaugurou pontes e estradas, mas �n�o foi capaz de inaugurar� uma Lei de Bases do Ordenamento do Territ�rio.
+A den�ncia foi feita ontem, em Faro, pelo l�der do Partido da Terra (PT).
+Ribeiro Teles defendeu, no Jardim Alameda, a necessidade de uma pol�tica que tenha �os p�s bem assentes na terra�, sem estar subordinada aos �n�meros� da macroeconomia, constru�da � custa da �degrada��o dos nossos recursos�.
+O grupo de fot�grafos participantes encontra-se para um confronto pela diversidade n�o s� de estilos, como de op��es relativamente � constru��o de uma ideia da cidade.
+Assim, Nuno F�lix da Costa, que tra�a um percurso �cronol�gico� -- com uma sequ�ncia que come�a ao amanhecer e acaba na noite �escura� lisboeta, das ruas e e dos bares -- integra constantemente o elemento ins�lito (ou tornado ins�lito pelo olhar), da exist�ncia humana e urbana, na paisagem quotidiana: a rotina dos gestos e dos comportamentos.
+�A sina do seu destino�, abre a sequ�ncia de imagens do autor, que fecha com a tr�mula, brilhante e opaca, po�tica e tensa, vida da noite.
+Ant�nio Pedro Ferreira, fot�grafo que re-afirma a sua originalidade na busca do acontecimento humano.
+N�o h� fotografias sem pessoas, ou sem os seus vest�gios, o que n�o deixa de nunca de remeter o trabalho deste fot�grafo para um dos dom�nios mais fascinantes, ou encantat�rios, m�gicos da fotografia, o �retrato�, ou a fotografia como �lente human�stica� ...
+permanece o registo dos negros, das sombras e da alus�o.
+Para 1995, a administra��o da PT calcula que os lucros atinjam valores pr�ximos dos 25 milh�es de contos e pretende manter os n�veis de investimento, verificados em 1994.
+Dentro da pol�tica de racionaliza��o de estruturas, a PT pretende ainda baixar para quatro milh�es de contos o valor das exist�ncias em armaz�m (encontra-se em sete milh�es, contra os 11 milh�es que se atingiam em Maio) e alienar 40 im�veis espalhados pelo pa�s.
+Botelho da Costa garantiu tamb�m que o or�amento para 98/99 ser� entrega ao conselho fiscal a tempo de ser analisado por este, provavelmente �no final desta semana, in�cio da pr�xima�.
+O Sporting de Braga-Guimar�es est� a suscitar grande interesse na regi�o minhota e a procura de bilhetes tem decorrido em bom ritmo.
+Embora o Guimar�es n�o tenha organizado as habituais excurs�es, � praticamente garantido que o Est�dio 1� de Maio registar� uma das melhores assist�ncias da �poca.
+Na hora de defrontar o Sporting, o Famalic�o pode finalmente respirar aliviado com a not�cia do regresso dos seus dois jogadores argelinos, Medane e Menad, que se ausentaram para disputar a Ta�a de Africa.
+Os argelinos, que desempenham um papel preponderante na equipa de Josip Skoblar, chegaram anteontem � noite a Famalic�o, quase uma semana depois de a sua equipa ter sido afastada da luta pelo ceptro m�ximo do futebol africano, tendo j� ontem treinado com o resto do plantel famalicense.
+Com o regresso de Medane e Menad, o treinador jugoslavo do Famalic�o tem � sua disposi��o todos os jogadores, com excep��o do jovem Manuel Jos�, que se encontra lesionado.
+O que � a verdade, o que � a fic��o?
+Isso n�o podemos saber.
+O que � um �escritor�?
+� um top�grafo, um investigador, um rep�rter, ou um fot�grafo, como perguntou um dia Herv� Guibert a Peter Handke, numa entrevista?
+Quando morre algu�m, e depois n�s ficamos sempre a pensar que dessa pessoa n�o sab�amos quase nada, escolhemos uma pequena parte, e, como dessa pessoa j� n�o conseguimos ver nem os olhos nem as m�os, ficamos com essa pequena parte para ocuparmos o espa�o todo que resta no s�tio da pessoa morta, ficou-nos s� essa parte, � muito pouco, por isso depois tem de ser aumentada.
+Uma parte de Herv� Guibert.
+Um livro.
+O t�tulo � �Des Aveugles�, e foi publicado na Minuit em 1985.
+Depois de �L'Image Fant�me�, de �Les Aventures Singuli�res�, de �Les Chiens�, de �Voyage Avec Deux Enfants�, de �Les Lubies d'Arthur�.
+Antes de �Mes Parents�, e muito antes dos livros-Sida.
+�Des Aveugles� � t�o nitidamente cruel e generoso como o cinismo quando � absoluto e coincide com a ingenuidade.
+O lugar do livro � uma institui��o para cegos.
+As personagens s�o cegos, atrozes e desconcertantes como nas fotografias de Sander, porque os cegos s�o pessoas que se dirigem incessantemente para um lugar intermin�vel de onde n�o podem sair e onde n�s nos sentimos gratos por n�o podermos entrar.
+Herv� Guibert na �poca era jornalista do �Le Monde�, e pediu autoriza��o ao Instituto Nacional dos Cegos para fazer uma reportagem sobre �Os Cegos e a Cultura�, h�-de ter sido o que de melhor lhe ocorreu como desculpa para passar l� uma semana.
+Diz que imaginou que �o narrador seria uma esp�cie de perverso que entra no estabelecimento gra�as a uma cumplicidade que lhe permite manter rela��es amorosas il�citas com as crian�as�.
+O livro depois mudou, e os cegos tornaram-se personagens passionais, e obscenas na sua viol�ncia, esse livro depois de escrito continha tudo, o infinito, a transpar�ncia, o medo, as imagens do escuro, continha uma crian�a cega que n�o admitia n�o ver e inventava, e Josette que torturava ratos, uma mulher casada, um marido e um amante, e depois havia uma vingan�a.
+Herv� Guibert gostava dos cegos.
+Depois dessa semana, ficou durante um ano como leitor, ia duas horas por semana.
+Apaixonou-se por um cego que n�o era aluno dele.
+Tamb�m se tornou amigo do empregado da loja onde os cegos iam buscar v�deos.
+O v�deo mais procurado pelos cegos era �Texas Chainsaw Massacre�, de Tobe Hooper.
+Foi anos mais tarde que Herv� Guibert escreveu o livro.
+� neste contexto que entre a ind�stria e a Direc��o-Geral da Qualidade do Ambiente [DGQA] nunca se interromperam os contactos t�cnicos necess�rios � atempada informa��o sobre a evolu��o verificada no cumprimento do contrato-programa e que permitir�o a continua��o de uma abordagem respons�vel da quest�o para futuro.
+Apraz-nos referir a disponibilidade da DGQA para manter com a ind�stria um di�logo s�rio e objectivo.
+N�o conhecendo outra forma de abordar um tema t�o importante como o da efectiva protec��o do ambiente de que tamb�m somos parte.
+Mas al�m das afinidades culturais, dos quais os japoneses est�o cientes, mas que muitos portugueses desconhecem, h� outros pontos de aproxima��o.
+�O tema da Expo-98, os Oceanos, ser� tamb�m abordado na Expo-2001, em Yamaguchi, cujo lema geral ser� ' O Futuro e o s�culo XXI ', ou, dito de uma forma mais po�tica e numa tradu��o mais livre do japon�s, �Rumo a um futuro onde brilhe a vida�.
+Se na primeira metada da d�cada de 80 o balan�o dos confrontos entre os dois �eternos rivais� era francamente equilibrado, na segunda metade da mesma d�cada- excep��o feita aos fabulosos 7-1 com que o Sporting venceu o Benfica na tarde de 14 de Dezembro de 1986, em Alvalade- os �encarnados� foram ganhando vantagem neste muito especial �campeonato� entre as equipas da Luz e de Alvalade.
+A maior evid�ncia para esta recente superioridade �encarnada� vai para o facto do Benfica, nas suas tr�s �ltimas desloca��es a Alvalade, ter vencido sempre os �le�es�.
+P. -- ... e mais: j� depois da derrota continuou a tentar nos jornalistas, e na comunica��o social, os bodes expiat�rios da derrota ...
+R. -- ... n�o generalizo mas houve aspectos na comunica��o social que tiveram que ver com os resultados das elei��es.
+Mas, � verdade, estava completamente convencido de que o PSD iria ganhar e, inclusive, poderia ter maioria absoluta.
+Enganei-me: n�o serie nem o primeiro nem o �ltimo, espero ..
+Observador privilegiado, o franc�s Michel Platini, ex-jogador, ex-seleccionador nacional e actual coordenador do Comit� Organizador do Campeonato do Mundo de futebol de 1998 em Fran�a, deita um olhar sobre aquilo que foi o �Mundial� dos Estados Unidos.
+Por entre cr�ticas � falta de vontade dos pol�ticos franceses, faz algumas compara��es e previs�es.
+No que se refere �s regras do jogo, considera que o futebol deve continuar a ser futebol.
+Da� que seja avesso a profundas altera��es das suas regras.
+L'�QUIPE -- De que maneira viu o recente Campeonato do Mundo de futebol?
+N�o pondo em causa o direito dos cl�nicos de exigirem melhores condi��es remunerat�rias e de trabalho, Maria Bel�m lembrou que, para se pedir mais dinheiro, �temos de mostrar que fazemos mais e melhor�.
+O recado n�o tinha apenas como destinat�rios os m�dicos algarvios, mas toda a classe.
+A este prop�sito, argumentou que as �dificuldades e os constrangimentos do pa�s� exigem um grande controlo dos gastos.
+No seu entender, o seu Minist�rio j� fez um esfor�o para que os trabalhadores da Sa�de beneficiassem de um aumento superior ao resto da fun��o p�blica.
+Por conseguinte, um novo aumento salarial, de acordo com a ministra, ter� necessariamente de ser acompanhado de um aumento de produtividade.
+Nesse sentido, adiantou que o �Minist�rio est� dispon�vel para negociar a altera��o dos regimes de trabalho, em sede dos centros de responsabilidade integrados�, tendo j� convidado os sindicatos para para iniciar esse processo.
+No caso do Hospital de Faro, onde existe um conflito entre os cardiologistas e a administra��o, renovou a confian�a no �rg�o de gest�o.
+Maria de Bel�m aproveitou esta desloca��o ao Algarve para inaugurar o centro de sa�de de Lagoa, que j� estava a funcionar, mas encontrou a maioria dos m�dicos em greve.
+Portugal encontra-se hoje mergulhado numa crise que parece ter-se instalado predominantemente na Europa, mas afecta toda a humanidade.
+N�o h� por isso, para al�m da prossecu��o das pol�ticas que privilegiam o modelo de sociedade escolhido, solu��es espec�ficas para o nosso pa�s.
+Dez can��es para lembrar a carreira de Cassandra Wilson no cat�logo JMT, a casa que lhe abriu o mundo.
+Uma das mais prometedoras cantoras dos �ltimos anos, Cassandra usa um gr�o de voz e um jeito de desenhar e se apropriar do verso que tem ra�zes em Carmen McRae.
+Nascida no seio da est�tica M-Base, de que se tornou a �nica porta-voz vocal, � medida que foi avan�ando mar adentro, Cassandra soube libertar-se do lastro que lhe ajudou a voz a crescer mas que amea�ava paralis�-la.
+Despojada da preocupa��o de fazer novo e diferente a cada passo, o canto virou �rvore, ganhou espa�o, trepou ao c�u.
+No dia em que Wilson inventar o tempo & o modo de combinar a tradi��o (de que se tem aproximado progressivamente, como o mostra, de forma exemplar, o �lbum �Blue Skies�) com os novos sons que lhe adubaram a voz (do rap e hip-hop � inquieta��o experimental vivida ao lado de Steve Coleman), uma nova porta se abrir� ao jazz vocal.
+Parte-se depois para o debate.
+Paquete de Oliveira, soci�logo, fala do drama social.
+E Carlos Narciso junta a sua voz � daqueles que se surpreendem com o facto de Domingos Pereira, condenado a 15 anos pela morte da mulher, ter cumprido apenas seis.
+Fala na �opini�o p�blica�.
+Era a tirada mais infeliz da noite.
+Com programas destes, que correm o risco de valorizar at� � exaust�o alguns dos aspectos mais s�rdidos da hist�ria de criminosos, arriscamo-nos a ter, em breve, uma opini�o p�blica a pedir agravamentos sem fim das penas, da repress�o e mesmo o ressurgimento da pena de morte.
+�Quanto mais os EUA forem capazes de deixar claro que � f�til competir com o poder americano�, menos chances haver� de que �outros alimentem ideias de perturbar a actual ordem mundial�.
+�Um outro poder, aliado mas diferente, como pode vir a ser a Uni�o Europeia, s� lhes pode ser �til�.
+E apelava ao �idealismo e ao pioneirismo� da Am�rica como o ant�doto capaz de dar sentido ao seu enorme poder.
+Mariano Gago falava na sess�o de encerramento de uma confer�ncia sobre �A cria��o da sociedade de informa��o euromediterr�nea�, um projecto que surge na sequ�ncia da confer�ncia de Barcelona (Novembro de 1995), em que foram lan�adas as bases de uma coopera��o mais estreita -- a n�vel econ�mico e pol�tico, mas tamb�m cultural -- entre os pa�ses das duas margens do Mediterr�neo.
+�As novas tecnologias da informa��o permitir�o construir rapidamente este s�mbolo moderno da nossa vontade comum de criar, entre as elites estudiosas dos nossos povos, la�os de solidariedade, compreens�o m�tua e trabalhos em comum�, declarou o ministro.
+Atentados suicidas do grupo Hamas em Israel: 25 mortos e 80 feridos.
+� a vingan�a pela morte do seu l�der Ayyash.
+Jos� Eduardo dos Santos e Jonas Savimbi re�nem-se em Libreville, concordando que um Governo de Unidade e Reconcilia��o esteja formado at� Junho ou Julho.
+Ant�nio Guterres possui uma boa rela��o com o teatro.
+Tem pena de ter pouco tempo para assistir a espect�culos, mas est� atento.
+Nos �ltimos meses ainda conseguiu escapar-se e ver �Eu, Feuerbach�, pelo Cendrev, em �vora, e �A �pera do Malandro�, pelo Seiva Troupe, no Porto.
+E gostou.
+As declara��es prestou-as ontem durante o almo�o que ofereceu, em S. Bento, �s personalidades do teatro.
+No dia em que ficou constitu�do o j�ri de apoio ao teatro n�o governamental.
+Um vizinho do P�o de A��car de Cascais j� n�o aguenta ouvir as descargas do supermercado.
+Queixas atr�s de queixas, foi conseguindo umas vit�rias.
+Agora, por�m, tudo esbarrou num muro que a empresa diz ser a solu��o mas que o vizinho n�o aceita.
+Afinal, parece que consumir fura os t�mpanos.
+A Gr�-Bretanha pediu na segunda-feira � Comunidade Europeia que fosse banido um a��car artificial usado na produ��o de refrigerantes, revelaram fontes diplom�ticas comunit�rias e industriais citadas pela Reuter.
+Os brit�nicos advogam que a subst�nca em quest�o, o ciclamato -- 30 vezes mais doce do que o a��car -- n�o � seguro para o ser humano e amea�am vetar uma lei sobre a seguran�a alimentar, que os ministros do com�rcio da CEE est�o a discutir, se o produto n�o for proibido.
+A Fran�a e a Gr�cia apoiam a Gr�-Bretanha.
+Este tipo de a��car foi banido nos Estados Unidos e na Gr�-Bretanha depois de uma s�rie de testes ter demonstrado que o produto provocava cancro nos ratos.
+A Gr�-Bretanha -- o �nico pa�s da CEE que proibiu o produto -- quer agora banir os ciclamatos da CEE baseados no argumento de que o produto leva, nos animais, � atrofia e degenera��o testicular.
+Mas a comiss�o da CEE para a seguran�a alimentar, baseada em peritos dos Doze pa�ses, aprovou os ciclamatos depois de analisar o consumo m�dio em rela��o aos humanos durante um per�odo de vida.
+Domingos, peito hirto e joelhos levantados, cerra os dentes e vence Buckner (medalha de bronze) e D�l�ze.
+Domingos n�o se tem em si de contentamento, n�o quer acreditar no que se est� a passar.
+Abra�a-se ao irm�o, que ficou em 8� lugar.
+�Ganhei a medalha de prata�, diz-lhe ofegante, mas logo se recusa a acreditar na realidade.
+S� quando Moniz Pereira lhe surge na frente, se compenetra de que era mesmo verdade.
+O treinador, de sorriso rasgado, abra�a-o.
+Lisboa, 18 de Fevereiro de 1992.
+S�o 9h 25m e Moniz Pereira arruma o autom�vel junto ao est�dio do Sporting Clube de Portugal, incrustado entre o Campo Grande e o bairro residencial lisboeta do Lumiar.
+Metade do Or�amento destina-se a despesas correntes, suscitando criticas da oposi��o, que as consideram exageradas, nomeadamente as despesas com pessoal (mais de um milh�o de contos).
+O or�amento prev� obras como a constru��o do pavilh�o gimno-desportivo de Pinhal Novo e as piscinas do Pinhal Novo, entre outras.
+Para o presidente Carlos de Sousa, trata-se de um or�amento �t�pico de um concelho rural e pobre com poucas receitas�.
+Segundo elementos recolhidos na �nica junta de freguesia do concelho, o Entroncamento tem, ap�s o recenseamento eleitoral do ano passado, 12 480 eleitores, apenas mais 372 do que no ano anterior.
+Os respons�veis desta junta, bem como os da autarquia, garantem que �este n�mero est� muito aqu�m da realidade�.
+�se Maom� n�o vai � montanha, a montanha vai a Maom�.
+Altera��es ao C�digo Civil que incidem sobre o direito da fam�lia.
+Uma proposta de lei a aprovar por a Assembleia da Rep�blica que permita a altera��o do C�digo de Procedimento Administrativo.
+O CONGRESSO Nacional Africano (ANC) reiterou ontem, em comunicado, como �insuficiente� a remodela��o ministerial realizada pelo Presidente De Klerk, na sequ�ncia do esc�ndalo sobre o financiamento secreto de Pret�ria ao partido zulu Inkhata, anunciando que vai intensificar a sua campanha para obter a mudan�a do Governo sul-africano por um Executivo de transi��o.
+UM GRUPO de alegados assaltantes, acusados do roubo de diamantes no valor de um milh�o de contos da central de escolha de uma operadora portuguesa em Angola, est� a ser submetido a interrogat�rios que forne�am pistas sobre uma presum�vel rede de tr�fico.
+Cerca de um milh�o de contos em diamantes, em gemas e para uso industrial, foi roubado na noite de 14 para 15 de Setembro da esta��o de escolha da Sociedade Portuguesa de Empreendimentos (SPE), no Ocapa, Lunda Norte.
+O que consta � que os militares t�m bastante cuidado para evitar esse tipo de situa��es.
+Sabendo antecipadamente o grau hier�rquico da personalidade que vai actuar, fazem-se representar em conformidade.
+Para um secret�rio de Estado, um vice-chefe.
+No entanto, no Minist�rio da Defesa n�o h� grandes preocupa��es quanto � funcionalidade da gest�o se houver vacatura com a prov�vel sa�da de Nogueira.
+Se os chefes militares, actualmente, despacham com o ministro de 15 em 15 dias, far�o o mesmo com o primeiro-ministro e o secret�rio de Estado governa o Minist�rio.
+� tudo uma quest�o de tempo e ainda falta conhecer os resultados do congresso do PSD e saber a altura exacta em que o Presidente da Rep�blica tomar� decis�o relativamente ao Governo.
+Entretanto, a Comiss�o Nacional de Elei��es (CNE) notificou ontem a SIC pela n�o cobertura da campanha eleitoral do candidato Jer�nimo de Sousa, na sequ�ncia de uma queixa por este apresentada.
+A esta��o de Carnaxide tem um prazo de 48 horas para responder, devendo a CNE na sua pr�xima reuni�o tomar uma decis�o, que, se for desfavor�vel � SIC, lhe poder� custar uma coima entre os mil e 10 mil escudos.
+Entre um e dez contos, exactamente.
+A Kapiti forneceu ao Banco de Com�rcio e Ind�stria (BCI) os sistemas Equation, �back-office� para opera��es internacionais, e FS-Dealer, �front-office� para opera��es cambiais.
+Mas o primeiro grande atleta de Moniz Pereira veio de Viseu, no interior nortenho, com muito frio no Inverno e muito calor no Ver�o.
+Numa freguesia rural �s portas da cidade, Vildemoinhos, nasceu e fez as primeiras corridas aquele que viria a ganhar a maratona de Los Angeles em 1984: Carlos Lopes.
+Lopes corria na inf�ncia pelas hortas, competia com amigos quando iam a festas na aldeia vizinha, atravessava velozmente vinhas e castanhais.
+E era sempre ele quem ganhava.
+Na terra natal foi torneiro-mec�nico, teve vida dura, at� que um dia deu nas vistas nos crosses nortenhos.
+O Sporting contratou-o, trouxe-o para Lisboa.
+E Moniz Pereira come�ou a trein�-lo.
+O ministro canadiano das Pescas, Brian Tobin, tinha dito, no domingo passado, estar pronto a tomar todas as medidas necess�rias para impedir 49 barcos europeus -- 38 espanh�is e 11 portugueses -- de continuarem a pescar nos grandes bancos, ao largo da Terra Nova.
+Tobin sublinhou que os pesqueiros europeus foram todos prevenidos, via r�dio, de que o Canad� proteger� os seus �stocks� de solha e palmeta, mesmo para al�m do limite das 200 milhas n�uticas.
+As informa��es dispon�veis d�o conta da presen�a de quatro fragatas, um porta-helic�pteros e avi�es canadianos a vigiar os barcos ib�ricos.
+Em causa neste diferendo est�o dois problemas: a inten��o canadiana de, pretensamente, preservar os recursos de pesca da zona e, em segundo plano, a inten��o de alargar a sua jurisdi��o a �guas internacionais para al�m das 200 milhas da zona econ�mica exclusiva.
+Os canadianos dizem querer diminuir fortemente a pesca da palmeta para evitar o seu desaparecimento, �como aconteceu com o bacalhau e o ' red fish '�, observa Ernest Loignon.
+A dupla Jorge Bica / Joaquim Capelo regressou ao comando da 42� Volta Galp a Portugal, prova pontu�vel para o �Europeu� de ralis, ap�s a disputa da segunda etapa.
+Uma vantagem de 1m34s sobre os segundos, Jos� Carlos Macedo / Miguel Borges, em Renault Clio 16V, garante ao piloto do Lancia HF Integrale uma forte dose de tranquilidade para conseguir a vit�ria na prova, o que o colocaria em excelente posi��o para a conquista do t�tulo.
+O livro foi inicialmente publicado pela Cadernos Tempo, em Mo�ambique.
+Recentemente, uma editora uma italiana descobriu-o.
+Maria Teresa Pazzolo traduziu-o, prefaciou-o e ilustrou-o, com fotografias de sua autoria.
+A AIE-Guaraldi, da Reppublica di San Marino chancelou este livro que se deixa inquietar com o desaparecimento da Ilha de Mo�ambique.
+Em Portugal, esta novela n�o foi ainda editada.
+Com a sua publica��o em It�lia, talvez �O Barco Encalhado� desenterre as amarras e aporte no Tejo.
+O que seria importante pois, de alguma forma, este livro combate a vis�o preconceituosa e lacunar que, ao longo dos tempos, � Ilha de Mo�ambique tem sido consagrada.
+Os ministros europeus do Trabalho e dos Assuntos Sociais aprovaram ontem no Luxemburgo leis comunit�rias instituindo uma licen�a m�nima de tr�s meses para assist�ncia � fam�lia e a aplica��o de regras do pa�s de acolhimento para os trabalhadores deslocados no estrangeiro, anunciou o ministro alem�o Norbert Blum.
+Estes dois projectos tinham sido objecto de um acordo pol�tico durante a �ltima reuni�o dos Quinze, a 29 de Mar�o em Bruxelas, mas faltava-lhes a adop��o formal.
+O fogo consumiu in�meros fardos de bacalhau, dois dos tr�s t�neis de secagem artificial, alguns compressores, duas carrinhas utilizadas no transporte dos trabalhadores da f�brica e um carro de marca �Triumph�, propriedade do comendador.
+Os armaz�ns e uma habita��o cont�guas � empresa foram tamb�m danificados pelo sinistro.
+Os prejuizos, disse Gon�alves Gomes, est�o cobertos por o seguro, mas ascendem a centenas de milhar de contos.
+O bacalhau queimado estava j� embalado em fardos ou disposto nos tabuleiros de secagem e pertencia a lotes de produ��o da empresa, sendo o peixe importado da Noruega e Dinamarca.
+Apesar da quantidade de bacalhau consumida pelas chamas ter sido elevada, Gon�alves Gomes afirmou que n�o ir� acontecer qualquer �desestabiliza��o do abastecimento do mercado�.
+Quanto � Empresa de Pesca de Viana estar� garantida a continuidade da labora��o, embora a n�veis de produ��o mais reduzidos, j� que s� poder� ser utilizado o �nico t�nel de secagem artificial poupado no inc�ndio.
+A compara��o � perversa e d� para os dois lados.
+O deputado social-democrata Carlos Coelho, por exemplo, at� j� se deu ao trabalho de comparar os dois documentos, apenas para provar que h� �in�meras cita��es literais� do diploma no articulado do pacto, que assim se transforma �num reposit�rio de ideias globais, que n�o acrescenta nada ao que a lei de bases e a Constitui��o portuguesa preconizam�.
+A forma como todo o processo tem vindo a ser encaminhado �, ali�s, suscept�vel de �gerar expectativas perversas�, porque � �na forma como as coisas ir�o ser levadas � pr�tica que as pessoas se dividem�.
+Escola Secund�ria de Pa�os de Ferreira -- Est� aberto pelo prazo de tr�s dias, a contar da publica��o do presente aviso, concurso para preenchimento de um hor�rio incompleto de 6 (seis) horas na disciplina de Socorrismo (11� ano).
+Na Gr�-Bretanha s�o mortas, anualmente, cerca de 70 crian�as.
+As causas s�o os maus tratos.
+Os culpados s�o quase sempre os pais.
+O n�mero de crian�as menores de cinco anos, mortas por estranhos, tem sido em m�dia de uma por ano, pelo menos desde 1982.
+Se o assass�nio de crian�as por estranhos � um tipo de crime raro, estranho � que isso aconte�a pela m�o dos pr�prios pais.
+Mais inesperado ainda � encontrar uma crian�a assassina.
+�O Dec�logo�, de Krystof Kieslowski .
+Qual foi, para si, o principal acontecimento mundial da �ltima d�cada?
+�Tr�s jovens portugueses que fazem m�sica portuguesa�.
+Assim se definem os Requiem, ex-Requiem pelos Vivos, que ap�s quase cinco anos de aus�ncia dos est�dios regressam em Outubro com um novo �lbum e uma m�o-cheia de espect�culos.
+Pelas r�dios, entretanto, roda j� �Entre o C�u e o Medo�, o single promocional.
+�Duas Viagens�, uma exposi��o de fotografias a preto-e-branco de Francisco Villa-Lobos, feitas em 1995 em T�quio, Nagasaqui e Quioto, nos intervalos de rodagem do filme �Os Olhos da �sia�, de Jo�o M�rio Grilo, pode ser vista at� 7 de Novembro na sala Laman do Centro Cultural de Bel�m.
+�Imperman�ncia -- Um Caminho para o Auto-Conhecimento� � o t�tulo de uma exposi��o / instala��o de Regina Chulam, que joga com v�rios auto-retratos pintados pela artista e que pode ser apreciada, a partir das 18h30, na Casa Fernando Pessoa (R . Coelho da Rocha, 16), em Lisboa.
+Outros dissidentes conhecidos ainda presos, envolvidos na �Primavera de Pequim� s�o Wang Juntao e Chen Zimin, jornalistas, 35 e 40 anos, respectivamente, condenados em 1990 a 13 anos de pris�o, Liu Gang, estudante, 32 anos, condenado a seis anos, Reng Wanding, oper�rio, 48 anos, condenado a sete anos, e Bao Tong, 60 anos, antigo bra�o-direito do secret�rio geral Zhao Ziyang, condenado a sete anos.
+N�o havia cruzamento, n�o se tratou de uma tentativa de ultrapassagem, nem sequer de uma travagem brusca.
+Segundo o relato da ag�ncia Lusa, o Volvo deslizou sozinho de uma garagem particular onde estava estacionado.
+E s� o inesperado encontro com o Mercedes o fez parar.
+Sabe-se l� at� onde poderia ter ido, deslizando pelo piso molhado da Rua das Flores.
+O incr�dulo automobilista, por n�o ter interlocutor, achou por bem chamar a pol�cia e relatar-lhe como tudo se passou.
+Afinal, h� que apurar responsabilidades em mat�ria de seguros, mesmo em situa��es ins�litas com esta.
+As tropas ficariam fora dessa esp�cie de estrada, que na altura n�o foi sequer definida, concentradas em determinados pontos, que tamb�m n�o foram definidos.
+v�o definir esse corredor e determinar os pontos de acantonamento dos soldados.
+Se n�o se incomodarem um ao outro � porque h� condi��es para partir para a fase seguinte.
+Que tamb�m n�o se sabe ainda qual �.
+Os �Fados d'Aqu�m e d'Al�m-Mar� que Jo�o Braga concebeu para o Centro Cultural de Bel�m traduziram-se numa exibi��o confrangedora de equ�vocos e falta de prepara��o onde couberam amigos, muito Fernando Pessoa, uma boa voz, de Rita Guerra e uma anedota brasileira de mau-gosto.
+O fado, esse, ficou aqu�m.
+O defesa portugu�s H�lder estreou-se no campeonato espanhol e encontrou logo pela frente um seu velho conhecido: o ex-sportinguista Amunike.
+Bateu-se bem, ganhou-lhes muitos lances e anulou o flanco esquerdo do ataque dos visitantes, mas acabou por ver o remate decisivo da partida tabelar no seu corpo antes de a bola chegar �s redes.
+�Fui infeliz, a bola bateu-me nas costas e entrou.
+Mas, pelo que fez na segunda parte, o Barcelona mereceu ganhar.
+H�lder cumpriu bem na sua estreia.
+Com a particularidade de ter jogado fora do seu posto habitual.
+�Penso que foi por o John Toshack saber que eu conhe�o bem o Amunike.
+Pediram-me para desenrascar, mas prefiro jogar a central�, explicou o portugu�s.
+�O H�lder tem raz�es para estar contente.
+Pelo caminho, tinha repudiado uma insinua��o de Dam�sio, que, no seu discurso, se referiu � sua comiss�o como sendo constitu�da por �cr�ticos ou opositores da actual direc��o�.
+E o mesmo fez o orador seguinte, Jos� Diogo, este representante do grupo respons�vel pela terceira proposta.
+A reuni�o prosseguia ainda � hora de fecho desta edi��o.
+Sim, porque as pessoas acabam por passar por todas, apesar das diferen�as.
+J.S.R. -- Uma das condi��es fundamentais para se criar outros gostos nas pessoas s�o os concertos.
+C� n�o h� apresenta��o ao vivo de quase nenhuma m�sica.
+E isso � muito importante.
+Mas nos demais casos em que a lei permite a cobran�a coerciva pelos tribunais fiscais, pode ser discutida a legalidade da d�vida, o que quer dizer que n�o h� t�tulo executivo.
+A n�o ser que estejamos perante um dos documentos do art.46� do C�digo do Processo Civil, aqui sim, plenamente aplic�vel, ou haja lei especial que tal disponha.
+A norte de Porto Dinheiro, na Lourinh�, elementos da Guarda Fiscal apreenderam, cerca das 4h30 de ontem, uma traineira (presumivelmente registada no porto de Peniche) que transportava 138 caixas de tabaco americano, com valor da ordem dos 22.500 contos.
+Na sequ�ncia da opera��o foram detidos quatro indiv�duos, entre os quais o mestre da embarca��o, a qual foi, tamb�m, apreendida.
+A GF confiscou ainda uma viatura ligeira de marca Bedford, envolvida na rede de contrabando, ontem descoberta.
+Quantidade id�ntica de tabaco -- seis caixas de marca Camel e 124 de Winston -- foi aprendida, na madrugada de ter�a-feira, na zona de Vila do Bispo, por elementos do comando da Guarda Fiscal do Algarve.
+A mercadoria, proveniente de um desembarque, foi encontrada numa carrinha suspeita.
+Alverca -- Sem meia d�zia de jogadores (os emprestados pelo Benfica), o rec�m-promovido Alverca ganhou o seu primeiro ponto da �poca e logo fora de casa.
+A perder, o treinador M�rio Wilson arriscou com a entrada de tr�s jogadores e saiu de Campo Maior com mais moral para enfrentar a dif�cil tarefa da manuten��o.
+Por este andar, Portugal vai ter o campeonato com o ciclo mais alargado de jogos em cada semana, com jogos de sexta a segunda-feira.
+Em It�lia os treinadores recusaram os jogos � segunda-feira, que as televis�es queriam imp�r, em Espanha tamb�m haver� jogos s� de sexta a domingo, e em lado nenhum se foi t�o longe como em Portugal dividindo as jornadas por quatro dias.
+H� raz�es t�cnicas e tamb�m de �marketing�, porque o p�blico nunca sabe a que dia joga a sua equipa e a percep��o do que se vai passando, para o p�blico em geral, tamb�m n�o � a mesma.
+A televis�o pode ter assim tanto poder?
+No que diz respeito �s 200 toneladas de farinha de carne e ossos, a respectiva incinera��o fica para mais tarde, em data a anunciar oportunamente pelo Minist�rio da Agricultura.
+Segundo este organismo, a opera��o s� ter� lugar quando for disponibilizado �o equipamento que est� a ser propositadamente constru�do para proceder � introdu��o escalonada nos fornos� da farinha obtida a partir da transforma��o das carca�as dos animais abatidos.
+Tratando-se de mat�rias muito inflam�veis -- n�o � poss�vel retirar integralmente a gordura no processo de transforma��o --, h� que evitar o risco de combust�o no momento de entrada no forno (a laborar a alta temperatura), que poderia atingir o pr�prio operador.
+Por isso, as embalagens ser�o incineradas de forma diferente do habitual, recorrendo-se a um dispositivo que permitir� �puxar� a farinha e lan��-la �por cima�, em completa seguran�a, para dentro do forno.
+Os palestinianos, que querem reivindicar a independ�ncia da Cisjord�nia e de Gaza numa fase posterior das conversa��es, insistem em ser reconhecidos como parceiros de plenos direitos, mas os israelitas, interessados apenas numa fr�gil autonomia palestiniana, querem manter-los na delega��o conjunta com a Jord�nia.
+Fontes em Washington disseram � France Presse que, para desbloquear o impasse, Israel vai propor a realiza��o simult�nea de conversa��es com uma delega��o jordano-palestiniana e com uma delega��o s� de palestinianos.
+Pedro Almodovar j� filma �Kika�, onde, com a ajuda do costureiro franc�s Jean-Paul Gaultier, criou uma esp�cie de c�mara humana.
+Nesse filme, uma criada l�sbica tem o rosto de Rossy de Palma, o perfil cubista que irrompeu em �A Lei do Desejo�.
+A actriz esteve no Festival Internacional de Teatro de Almada.
+�Chica Almodovar� o que �?
+Uma inven��o.
+Pertence �quele grupo de rostos femininos a que chamam as �chicas Almodovar�, todos eles inventados pelo cineasta de mulheres, Pedro Almodovar.
+No caso de Rossy de Palma, a primeira coisa que mostrou no cinema foi mesmo o perfil, amea�ador e desregrado como uma pintura cubista.
+Foi n' �A Lei do Desejo�, e depois aconteceu �Mulheres � Beira de um Ataque de Nervos� -- dormiu durante o filme todo, ap�s um jarro de sumo de tomate e sopor�feros --, �Ata-me!� e agora, em rodagem em Madrid, �Kika�.
+O problema do desemprego dos engenheiros e t�cnicos do �Minseredmash�, o antigo e gigantesco complexo da URSS dirigido para a constru��o do �escudo nuclear da p�tria�, � actualmente tema de acalorada discuss�o na R�ssia e constitui uma preocupa��o para os seus dirigentes.
+Os custos desta pesada estrutura tornaram-se insuport�veis para o pa�s.
+e um estabelecimento muito chique da capital oferecia aos melhores corredores um par de rel�gios em ouro.
+Era o Maxime ...
+Quando me convenci que todos os esfor�os para tomar o aquartelamento se haviam tornado perfeitamente f�teis, comecei a retirar os nossos homens em grupos de oito e dez.
+(...) As nossas baixas no combate haviam sido insignificantes, 95 por cento dos nossos mortos resultou da desumanidade do Ex�rcito ap�s a luta.
+Durante uma semana mantivemo-nos nos cumes da cordilheira da Gran Piedra, enquanto o ex�rcito ocupava as bases.
+N�o pod�amos descer, e eles n�o se atreviam a subir.
+N�o foi a for�a das armas, mas sim a fome e a sede, que acabaram por vencer a nossa resist�ncia.
+Tive de dividir os homens em grupos mais pequenos.
+outros foram escoltados por Monsenhor P�rez Serantes [ arcebispo de Santiago de Cuba ] a fim de se renderem.
+Entre os epis�dios mais importantes a transmitir, al�m da entrevista com Giacometti, destaque-se aquele em que se fala da arte de Modigliani, mais ou menos a meio da exibi��o (nem sempre a ordem � respeitada pela RTP).
+Fala-se da sua vida: do alcoolismo, do desespero; tamb�m do �charme�, da sua gentileza.
+E do drama da morte.
+Outro epis�dio importante, �Um dia na vida de Man Ray�, entrevista filmada no come�o de 1961.
+Ou uma mem�ria do surrealismo dita na primeira pessoa.
+Montparnasse Revisited � assim.
+Est� cheia destes tesouros.
+Os parlamentos da federa��o croato-mu�ulmana e dos s�rvios b�snios v�o hoje pronunciar-se sobre o novo plano de paz internacional para a rep�blica.
+Os dirigentes croatas e mu�ulmanos j� manifestaram o seu apoio ao novo projecto, mas os l�deres s�rvios da B�snia continuam a manifestar profundas reservas face ao novo mapa territorial proposto pelas �grandes pot�ncias�.
+O jogador representou na �poca passada a Associa��o Desportiva Ovarense, mas foi dispensado em virtude do seu alegado �comportamento antidesportivo�, que, na opini�o da direc��o do clube vareiro, ter� contribu�do para a derrota sofrida perante o Benfica na fase decisiva dos �play-off�.
+Sprewer ir� ter como colegas na sua nova equipa os seus compatriotas Terrence Radford- que transita da �poca passada- e Ruben Cotton, um veterano de 32 anos, naturalizado portugu�s, que na �poca passada esteve ao servi�o da Oliveirense.
+Alexandre Dias, Caetano, Moutinho, Nunes, Paulo Duarte, Rui Santos, Jos� Ruivo e Henrique Silva completam o plantel, que ser� dirigido pelo t�cnico Orlando Sim�es.
+O or�amento para a nova �poca ronda os 25 mil contos e o principal objectivo � garantir a perman�ncia do Esgueira no escal�o principal.
+O Beira Mar, a outra equipa da cidade de Aveiro, procedeu tamb�m no in�cio desta semana � substitui��o do norte-americano Mike Smith pelo seu compatriota Deshon Washington.
+Segundo respons�veis do clube aveirense, Mike Smith apresentou-se em deficientes condi��es f�sicas e sem ritmo competitivo.
+Deshon Washington, que joga na posi��o de extremo-poste, tem 23 anos de idade, mede 2,02 metros, e vai fazer dupla com o seu compatriota Kip Jones.
+O t�cnico do Beira Mar, Aniceto Carmo, tem ainda ao seu dispor os seguintes atletas: Catarino (ex-Esgueira), Paulo Sousa (ex-Salesianos), Rebelo, Traylor, Moreira, Mourinho, Alex Pires, Renato, Jo�o Miguel e Pinto.
+Tendo como objectivo intrometer-se na luta pelo t�tulo, o Beira Mar dispor�, para a nova temporada, de um or�amento a rondar os 40 mil contos, verba substancialmente inferior � da �poca passada.
+Na sua globalidade, as respostas alimentam, apesar de tudo, um certo optimismo entre os que se interessam pela sorte dos bichos.
+Para Jorge Rocha, o inqu�rito demonstrou �que h� muitas c�maras do pa�s que t�m uma atitude de grande dignidade pelos animais�, apesar de serem letra morta as parcas disposi��es da legisla��o que regulamenta esta mat�ria e do imenso atraso no nosso pa�s em rela��o a outros pa�ses europeus, onde h� inclusivamente seguros e assist�ncia � doen�a.
+De um modo geral, os animais dom�sticos que vivem no campo t�m uma exist�ncia mais feliz do que os citadinos.
+�Nas grandes cidades, os animais dom�sticos s�o normalmente mais maltratados, pois, em certa medida, entram em concorr�ncia com as pessoas, ocupando as ruas, fazendo ru�do nos pr�dios�, observa o veterin�rio municipal da C�mara do Porto, V�tor Aires.
+�Nos concelhos rurais, essa concorr�ncia n�o existe e os bichos mant�m, at�, determinadas fun��es �teis, como a guarda ou a ca�a�.
+Atr�s de Kerrigan na lista das grandes favoritas est� a japonesa Yuka Stao, quarta nos �Mundiais� e que nos �ltimos tempos bateu patinadoras de grande nome como a alem� Tanja Szewczenko e a canadiana Jos�e Chouinard.
+A China aposta tudo em Chen Lu, terceira classificada nos �ltimos dois �Mundiais�, apontada com a atleta que melhor alia a t�cnica e a beleza.
+S� depois surge Tonya Harding, campe� dos Estados Unidos.
+E a grande d�vida � se as suspeitas da agress�o � sua rival ser�o um �handicap� ou uma motiva��o.
+Homens ensopados da cabe�a aos p�s por uma chuva de petr�leo, com as chamas e o fumo deixados pelas tropas iraquianas como cen�rio de fundo.
+As imagens que Sebasti�o Salgado registou da guerra que op�s os Estados Unidos ao Iraque e que come�ou pela invas�o do Kuwait n�o mostram m�sseis Scud, nem a ferocidade dos ex�rcitos dos dois inimigos.
+Revelam a miss�o perigosa dos homens das 27 equipas de bombeiros enviadas para o �inferno� que os iraquianos atearam em 788 po�os de petr�leo quando se retiraram do Kuwait e que permitiram ao fot�grafo brasileiro concluir o seu �portfolio� sobre o homem e o trabalho.
+A fotografia foi publicada na imprensa de todo o mundo h� cerca de um ano, quando de o massacre de Bentalha, na Arg�lia, onde morreram cerca de duas centenas de pessoas, na noite de 22 para 23 de Setembro.
+Cinco meses mais tarde, a imagem de uma mulher chorando a perda dos seus oito filhos, baptizada como �piet� argelina�, voltava a ser impressa, desta feita porque o seu autor, escondido atr�s do pseud�nimo Hocine por raz�es de seguran�a, recebia o Pr�mio World Press Photo 1997.
+Tem um projecto pessoal: encenar e interpretar, com a actriz Margarida Tavares, �O Amante�, de Harold Pinter, em regime de co-produ��o no Teatro da Trindade, com estreia prevista para Julho.
+�Tive um professor, Jo�o Brites, que nos dizia sempre que t�nhamos que ter os nossos pr�prios projectos.
+Achei que era importante assumir uma postura pessoal.
+Ida e volta do domic�lio ao trabalho.
+Parti de Cabo Verde h� oito anos.
+Estava sem trabalho, fazia uns biscates de vez em quando, mas nada de s�rio.
+Se parti, foi porque para mim, como para todos os outros cabo-verdianos, s� havia uma escolha: ficar e morrer � fome ou partir, n�o importa para onde, para onde pudesse encontrar trabalho e boas condi��es para sustentar a minha fam�lia.
+Futebol -- Transmiss�o directa da cerim�nia de abertura do �Mundial� de Juniores, que se realiza no Est�dio das Antas, no Porto.
+Futebol -- Transmiss�o em directo do jogo inaugural do Campeonato do Mundo de Juniores, Portugal-Rep�blica da Irlanda, a contar para o Grupo A, a partir de as 21h00, no Est�dio das Antas.
+Porque a imprensa est� feita com a direita, como responderia o dr. Cunhal?
+Explica��o demasiado f�cil e desculpabilizadora.
+Se assim �, porque morreu a imprensa comunista, como o �Di�rio�, porque se afastaram do Partido todos os jornalistas comunistas que trabalhavam em outros �rg�os de informa��o?
+A resposta � outra.
+A resposta � que o PP, com raz�o ou sem ela, demagogicamente ou n�o, para o caso tanto faz, tem tido a capacidade de impor os debates que suscitam a separa��o de �guas, as fracturas pol�ticas de que nascem as op��es do eleitorado.
+Enquanto que o PCP � um deserto de ideias, de discuss�o e at� de coragem ideol�gica.
+Nada � mais previs�vel, nada � mais desesperadamente igual e repetitivo do que o discurso do PCP.
+Existe apenas como uma esp�cie de museu de ideias.
+O PP s� tem sete por cento dos votos, mas n�o � dif�cil imaginar que possa crescer.
+O PCP tem os mesmos sete por cento, mas quem imagina que possa subir?
+Al�m disso, a nova lei org�nica do LNIV veio atribuir-lhe mais responsabilidades (ver �Controlo da sa�de animal ganha terreno�, P�BLICO de 9-6-97) no �mbito das provas laboratoriais necess�rias ao controlo sanit�rio dos animais e seus produtos -- a par das normais compet�ncias no �mbito da Investiga��o e Desenvolvimento (I&D) e das que decorrem do seu estatuto de autoridade nacional de refer�ncia para todas as quest�es referentes � sa�de animal.
+Por tudo isso, o ministro da Agricultura entendeu que n�o era poss�vel adiar por mais tempo a decis�o relativamente ao futuro do laborat�rio.
+O despacho de Gomes da Silva aprova uma metodologia de realiza��o faseada no tempo.
+A primeira iniciativa consiste na abertura de um concurso limitado para a elabora��o, por um gabinete de projectistas, do programa preliminar e do caderno de encargos que ter� de ser apresentado no concurso p�blico (segunda fase) para a concep��o e execu��o das novas instala��es.
+Este ciclo, a iniciar muito brevemente, dever� ser cumprido at� ao final do pr�ximo ano.
+A terceira fase do processo consistir� na constru��o das instala��es propriamente ditas, cujo arranque ocorrer� em 1999 e n�o dever� estar conclu�da antes de passados tr�s anos.
+Mais de mil casais tinham-se oferecido, at� quinta-feira � noite, para recolher provisoriamente refugiados, sobretudo crian�as.
+�Mas hoje [ ontem ], quando perceberam que n�o era para adoptar, come�aram tamb�m a aparecer pessoas interessadas em receber tamb�m as m�es�, explicou uma fonte do F�rum Estudante, que est� a organizar a Miss�o Crescer em Esperan�a.
+Nas instala��es do Instituto da Juventude, �o telefone n�o p�ra de tocar�.
+Tamb�m na Madeira, a delega��o da C�ritas no Funchal tem recebido in�meras solicita��es de fam�lias do arquip�lago dispostas a receber crian�as da B�snia.
+Apesar dos muitos pedidos de adop��o, as pessoas est�o a ser informadas que as crian�as apenas ser�o recohidas temporariamente, �entre nove a doze meses�, segundo o �Jornal da Madeira�.
+Os lobos n�o t�m fronteiras, para al�m das que eles pr�prios delimitam.
+Mas, um simples salto sobre a linha que divide o Parque Natural de Montesinho da Reserva Regional de Ca�a da Serra da Culebra, na regi�o espanhola de Castilla y L�on, pode significar um passo da vida para a morte.
+Protegido em Portugal, o lobo � considerado uma esp�cie cineg�tica no lado de l� da fronteira, onde, desde h� dois anos, tem vindo a ser leiloado o seu abate e a prosperar um mercado negro em torno dos seus restos.
+Curiosamente, na confer�ncia de imprensa em que anunciou o �adeus�, o primeiro-ministro n�o apresentou justifica��es de peso para a decis�o.
+�Decidi demitir-me hoje.
+Pensei ser chegado o tempo de renovar as pessoas do Governo e come�ar de fresco no ano novo� -- afirmou, revelando que tomara a decis�o dia 1.
+A justifica��o, para muitos analistas, pode procurar-se na frase de al�vio de Murayama ap�s o seu copo de �sak�.
+A impopularidade pessoal e do seu executivo tornara-se demasiada.
+�Eles fizeram da nossa aldeia um cemit�rio�, contou uma velha.
+at� agora, as pessoas morriam de doen�a ou de velhice.
+Homens e p�s cavavam ontem freneticamente, que j� tinha passado um dia, e cami�es descarregavam cimento em quantidades que a aldeia, de casas de argila e telhados de chapa, nunca tinha visto.
+A sportinguista Teresa Machado estabeleceu no domingo, numa pequena reuni�o realizada em S�o Jacinto, perto de Aveiro, um novo recorde nacional do disco, alcan�ando a marca de 65,40m.
+ O anterior m�ximo fora conseguido no mesmo local a 8 de Agosto de 1993, com 63,70m.
+A sportinguista abriu a sua s�rie com 62,16m, subiu a seguir para 63 metros exactos e o novo m�ximo chegou na terceira ronda.
+A acabar confirmou a sua regularidade com 61,87m, 62,84m e 63,09m.
+O Sporting j� ganhou (a Ta�a de Portugal) e j� voltou a perder (com o Benfica no jogo de repeti��o, na quarta-feira, no Restelo, com meio plantel ainda sob os festejos do fim do jejum de 13 anos).
+Mas pelas bandas de Alvalade continua ainda por deslindar o mist�rio do irland�s Niall Quinn, contratado num dia pelo novo Sporting de Santana Lopes e descontratado no dia seguinte com o mesmo espalhafato medi�tico.
+S� porque, de facto, o actual clube do irland�s, o Manchester City, tratou o Sporting como �um clube terceiro-mundista�, como alegou Santana Lopes?
+Ou a hist�ria tem mais que ver com os n�meros do neg�cio (mais de 400 mil contos por um jogador � beira dos 30 anos, indesejado pelos t�cnicos de Alvalade e �chumbado� pelo respectivo departamento m�dico)?
+A ser verdade, o Sporting continua a fazer neg�cios � Cintra.
+E a portar-se � Cintra, n�o dando cavaco, sequer, ao jogador ,que at� j� vendera a casa l� na Old England.
+Ser� que est� a nascer o Sporting Cavaco Lopes?
+De um modo aparentemente displicente, mas evidentemente estudado nos seus efeitos, surgem refer�ncias a Nitsch Hermann e a Rodolf Schwarzkogler, por exemplo, personagens que, com Arnulf Rainer e G�nter Brus, compuseram um grupo de artistas vienenses capaz de, nos anos 60, provocar esc�ndalo n�o s� na cena pantanosa e hip�crita da arte austr�aca do p�s-guerra, como no conjunto da arte ocidental.
+Radicalizando a ideia das vanguardas em reduzir cada linguagem art�stica � sua especificidade, eles acabaram por inverter a l�gica de purifica��o de meios do modernismo.
+O corpo humano (na sua realidade f�sica) tornou-se suporte -- ou, se quisermos, a pr�pria mat�ria-prima da obra de arte.
+Chamaram-se a si mesmos �actionistas�, embora o movimento, que se espalhou largamente at� aos anos 70, seja designado globalmente por �body art�, ou surja nos textos de Bowie como �arte ritual�.
+Por mim, achei que foi infeliz semelhante compara��o (...) e, muito menos, em rela��o �quele grande estadista, que penso ser respeitado em todo o mundo, sempre a rigor fardado, de franciscanas e respeit�veis barbas e de linguagem t�o simples que todo o explorado da terra entende e o povo cubano venera e ama.
+Debate-se agora se ter� sido oportuno ou nefasto o reconhecimento internacional da Cro�cia e houve quem recordasse que a guerra servo-croata j� estava em curso em 1991, antes desse reconhecimento.
+Mas este n�o � o cerne da quest�o, o cerne da quest�o � a B�snia.
+a sua dignidade enquanto na��es, e logo enquanto Estados, estava firmada.
+O mesmo n�o se passa na B�snia, onde as etnias convivem ao simples n�vel da coabita��o.
+Por isso, constituiu um erro reconhecer a Cro�cia sem antes ter preparado uma solu��o vi�vel para a B�snia, e n�o h� paix�o ideol�gica capaz de o apagar, se bem que seja in�til dramatizar um erro de natureza diplom�tica.
+A urbaniza��o de Arcena passou por um complicado processo, com origem numa venda entre s�cios da empresa propriet�ria, a Eurocapital.
+Perante a falta de pagamento dos compradores, os s�cios vendedores reclamaram em tribunal a restitui��o dos seus direitos sobre a urbaniza��o, o que viriam a conseguir no in�cio deste ano.
+Durante o per�odo em que estava indefinida a propriedade dos edif�cios, grande parte deles foram ocupados por imigrantes africanos.
+Em 9 de Abril �ltimo, o tribunal de Vila Franca iniciava ac��es de despejo, para a restitui��o da totalidade da urbaniza��o � Eurocapital.
+A maior parte das cerca de 260 pessoas desalojadas instalou ent�o abrigos improvisados numa das ruas da urbaniza��o.
+A pol�cia alem� localizou, ontem de manh�, os dois criminosos que fizeram cinco ref�ns durante uma fuga de mais de 27 horas pela Alemanha, ap�s terem assaltado um banco, na segunda-feira, em Stuttgart.
+Quando tentavam controlar a identidade de um homem perto de uma cabina telef�nica em Driedorf, em Hessen, os pol�cias foram alvo de tiros, tendo respondido.
+O homem acabou por fugir a p�.
+Importantes refor�os foram j� enviados para a zona.
+Os dois criminosos tinham libertado, ontem, por volta das 4h50 locais, os tr�s ref�ns que detinham, um dos quais ferido, segundo a pol�cia de Wiesbaden.
+Os ref�ns, um homem e um casal, encontravam-se a cerca de 60 quil�metros a norte de Frankfurt.
+Ap�s uma persegui��o de mil quil�metros atrav�s do pa�s, a pol�cia a distribu�u-se pela autoestrada entre Hofheim e Wiesbade.
+Os agentes decidiram dar vantagem aos raptores por estes terem amea�ado suicidar-se com os ref�ns fazendo explodir uma granada, caso a pol�cia os seguisse a curta dist�ncia.
+Os delinquentes come�aram por sequestrar dois pol�cias no seguimento do assalto ao banco que lhes rendeu 250 mil marcos, tendo feito depois tr�s novos ref�ns.
+Os dois criminosos, um antigo soldado de elite do ex�rcito da Alemanha de Leste de 32 anos, e um su��o de 35, evadiram-se, h� tr�s semanas, de uma pris�o de Hamburgo, acusados de homic�dio e assalto � m�o armada.
+Segundo as autoridades, �n�o est� exclu�da a possibilidade de que os raptores se dirijam para Fran�a�.
+O pr�mio liter�rio Donna -Cidade de Roma, 1992, foi atribu�do a Raisa Gorbatchov, pela sua biografia-testemunho, �Io Spero�, editada no Ver�o passado, anunciou ontem, na capital italiana, a presidente do j�ri, a escritora Gabriella Sobrino.
+O galard�o ser� entregue numa cerim�nia a decorrer a 28 de Mar�o, mas ainda n�o est� confirmada a desloca��o de Raisa a It�lia.
+Telas atribu�das a Picasso, Mir� e Martinez foram descobertas ontem, pela pol�cia Judici�ria de Paris, na casa de um galerista de origem americana, Theodor Robertson.
+A descoberta aconteceu na sequ�ncia de uma busca efectuada � sua resid�ncia, em Fran�a.
+Robertson era alvo de um mandato internacional de captura, por acusa��es de fraude e falsifica��o.
+Nove empr�stimos por obriga��es e as ac��es da Luzostela foram ontem retirados da cota��o na Bolsa de Lisboa.
+Dificuldades financeiras s�o o motivo invocado para a medida.
+Os respons�veis do mercado lisboeta decidiram tamb�m extinguir a direc��o de opera��es.
+A reduzida actividade do �floor� j� n�o justificava a sua exist�ncia.
+O esc�ndalo Augusta explodiu no seguimento das investiga��es sobre o assassinato, em 1991, de um antigo ministro socialista franc�fono, Andr� Cools.
+De acordo com Philippe Moureaux, ex-vice-primeiro-ministro do mesmo partido, a sua morte ocorreu no momento em que Cools estaria prestes a revelar o pagamento das �luvas� pela firma italiana.
+em Janeiro de 1994, os �tr�s Guys� -- Guy Spitaels, ministro-presidente da Val�nia, Guy Coeme, ministro da Defesa, e Guy Mathot, ministro regional -- foram for�ados a demitir-se para assegurar a sua defesa depois de a sua imunidade parlamentar ter sido suspensa.
+Neste caso, h� duas coisas que custam sobremaneira a Fernando Neves.
+Primeiro: o processo coloca-o na ingrata posi��o de delator dos seus camaradas.
+Segundo: a queixa foi apresentada pelo governador civil, um homem que Neves chegou a louvar pela sua disponibilidade para o di�logo, que contrastou com o sil�ncio comprometido do presidente da C�mara de Castelo de Paiva, Antero Gaspar.
+�N�o estava nada � espera disto�, diz o mineiro.
+Entretanto, por causa deste processo, a agita��o amea�a voltar ao Pej�o.
+Os mineiros est�o dispostos a manifestar-se na sede do concelho em solidariedade com o seu camarada, ao mesmo tempo que est� a ser preparado um abaixo-assinado.
+�Se querem prender algu�m, t�m que nos prender a todos, porque fomos n�s todos juntos que cort�mos a estrada e barric�mos a empresa.
+Sempre quero ver se t�m pris�es que cheguem para l� meter dentro 500 mineiros!�, diz Manuel Vasconcelos.
+O m�dico Manuel Almeida Marta, principal suspeito do crime de Armamar, continua a ser procurado pelas autoridades.
+As buscas duram h� 72 horas, sem que tenha sido descoberto o paradeiro do m�dico que por pouco escapou a um linchamento popular.
+Muito mais n�o diz, nem vale a pena, apenas um sussurro curto de remorso.
+�Estou arrependido, mas n�o tinha comido nada.
+O Tribunal de Pol�cia entende as palavras de Agostinho como pesar sincero, e esta � uma pequena condi��o muito prezada pela letra e forma da lei.
+Mas uma atenuante n�o � um perd�o, e os c�digos esticam as penas em caso de reincid�ncia.
+�Ainda se considera, pela �ltima vez, ser desnecess�ria a pris�o�, dita o juiz para o escriv�o, convertendo os cinco meses na penitenci�ria em 45 mil escudos de multa, ou, em alternativa, em cem dias de deten��o.
+Atende-se, tamb�m, � �situa��o econ�mica do arguido� e autoriza-se o pagamento da multa em dez presta��es.
+Quanto ao carv�o, que j� regressara ao restaurante quando Agostinho saiu do tribunal, foi grelhar carne para quem a come.
+Esta foi a argumenta��o apresentada na reuni�o de C�mara, que mereceu, invariavelmente, alguma diplomacia na resposta dada pelo presidente.
+Disse vezes sem conta Manuel Machado que, �no que for v�lido e leg�timo�, a edilidade est� disposta a encontrar �uma alternativa poss�vel�.
+E �n�o ajuda muito t�cnicas de bloqueios a m�quinas.
+Se for a bem, tudo bem; se for a mal, arranja-se a� um '31' que s� o tribunal resolve.
+� prefer�vel concerta��o de solu��es�, advertiu.
+�Mas se a alternativa for a estrada, n�o � poss�vel�.
+por falar em alcatr�o, o presidente relembrou aos moradores que, por altura das campanha para as elei��es aut�quicas de 1993, lhe tinha sido pedido alcatr�o para o local.
+�Constata-se que nem sempre, na altura pr�pria, os mun�cipes d�o aten��o aos editais.
+Porque nenhum loteamento avan�a em Coimbra sem publica��o de editais�.
+�N�o falem em falta de di�logo.
+Nenhuma audi�ncia dos moradores foi pedida � C�mara.
+Foi apenas recebido um fax da CHEM a mandar parar as obras, mas n�o se pode retomar o PREC numa situa��o desta natureza�.
+Os dois principais partidos austr�acos chegaram ontem a um acordo para fazer redu��es or�amentais no valor de 100 mil milh�es de xelins (perto de 1500 milh�es de contos) em 1996 e 1997.
+Dois ter�os dessas economias ser�o obtidas atrav�s de cortes nas despesas do Estado Federal, prov�ncias e autoridades comunais, estando o resto dependente de um aumento de receitas.
+As medidas, anunciadas pelos ministros da Economia, Johannes Ditz (conservador), e das Finan�as, Viktor Klima (social-democrata), numa confer�ncia de imprensa comum, destinam-se a assegurar o cumprimento dos crit�rios de converg�ncia, assegurando que a A�stria possa entrar no primeiro comboio da Uni�o Econ�mica e Monet�ria.
+Sem estas redu��es, o d�fice p�blico representaria 6,1 por cento do Produto Interno Bruto (PIB) este ano e 6,5 em 1997, valores muito acima do limite de tr�s por cento imposto por Maastricht.
+PIB desce em Fran�a ...
+O PP decidiu ontem em reuni�o do grupo parlamentar adoptar a liberdade de voto e Helena Santo ficou �-vontade perante o debate parlamentar da pr�xima quinta-feira em que ser�o votados os projectos de altera��o e liberaliza��o do aborto.
+A deputada n�o decidiu ainda, contudo, se se vai abster ou votar a favor do texto de Strecht Monteiro, mas assume claramente que considera o projecto �pr�-natalista� e �vem corrigir a lei em vigor�, pelo que �merece simpatia�.
+Este afirmou que o seu objectivo � devolver � CIA a credibilidade perdida.
+Precisou que ia �realizar mudan�as profundas na gest�o da CIA�, designadamente em o que respeita aos crit�rios aplicados no recrutamento de agentes.
+�Os EUA t�m de ter a melhor capacidade do mundo para recolher informa��es�, concluiu.
+Aldricht Ames, ap�s ter sido descoberto, confessou que espiava a favor de Moscovo havia nove anos, desde Abril de 1985.
+Devido � ac��o de Ames, explicou o actual director da CIA, foi muito mais dif�cil para os EUA compreender o que se passava na URSS durante aquele per�odo cr�tico, porque ele denunciou aos sovi�ticos muitos agentes que trabalhavam para os servi�os americanos.
+Depois de Cantanhede, a forma��o apresenta-se hoje em Vila Real, no s�bado no Porto e na pr�xima quinta-feira em Tomar (ver p�gs. 10/11 deste Fim de Semana).
+At� 24 de Outubro, a Nova Filarmonia -- que actualmente � composta por 35 m�sicos -- estar� em Covilh�, Leiria, Paio Pires, Lisboa, Coimbra, Valen�a, volta a Vila Real e, finalmente, toca em Matosinhos.
+Entre os variad�ssimos patrocinadores destes concertos, encontram-se, entre outras empresas e institui��es financeiras, a Rodovi�ria Nacional, Portucel, Shell Portuguesa, Sonae, Banco Totta & A�ores e Montepio Geral.
+Entre os concertos previstos para Novembro, destaca-se o de dia 3, no Pal�cio Nacional de Queluz, quando a orquestra acompanhar o pianista Sequeira Costa, que interpretar� o concerto n� 1, opus 11, de Chopin, com o patroc�nio da GALP.
+O rio Grande nasce no concelho de Torres Vedras e atravessa todo o concelho da Lourinh� sendo as suas �guas consideradas excessivamente polu�das e impr�prias para qualquer uso, segundo dados de 1987 da Direc��o Geral da Qualidade do Ambiente.
+Neste estudo s�o apontados como principais fontes poluidores as pecu�rias, matadouros e esgotos dom�sticos sendo observados v�rios casos de contamina��o de �guas de fontan�rios e po�os.
+� sua polui��o deve-se ainda o desaparecimento dos famosos bivalves negros da �Mexelhoeira�, uma zona de rochas entre as praias da Areia Branca e do Peralta.
+As hist�rias da polui��o do rio Grande correm toda a regi�o, desde o aparecimento de cad�veres de animais na sua foz at� ao boato de um surto de hepatite B que no ano passado afastou centenas de veraneantes.
+Na perspectiva da autarquia este quadro est� em vias de melhorar pois existem j� projectos para cinco esta��es de tratamento dom�sticos das principais localidades do concelho e duas centrais de tratamento de dejectos das suiniculturas para posterior transforma��o em fertilizante, j� candidatas a fundos comunit�rios, isto para al�m das v�rias pequenas esta��es de tratamento que est�o previstas em todo o concelho.
+A C�mara Municipal aponta os complicados processos burocr�ticos como os grandes entraves para que tudo se concretize uma vez que a maioria dos apoios financeiros j� estar� garantida.
+A ajuda humanit�ria, que finalmente recome�ou a mover-se em direc��o � B�snia central, pode revelar-se uma arma de dois gumes.
+Adia o sofrimento de uma popula��o esgotada e esfomeada, mas n�o o resolve de vez.
+Crescem os receios de que antes contribua para prolongar uma guerra devastadora e alimentar os florescentes circuitos do mercado negro.
+Quanto ao projecto paisag�stico, as d�vidas persistem.
+Armindo Cordeiro, da C�mara Municipal de Lisboa, afirma ainda n�o ter sido aprovado nenhum, mas que tal vir� a acontecer a breve trecho, cabendo a responsabilidade ao Departamento de Espa�os Verves, j� que o projecto apresentado pela Junta de Freguesia foi recusado poe n�o reunir os requisitos de qualidade necess�rios.
+Tamb�m Armindo Cordeiro � perempt�rio ao afirmar que o logradouro ajardinado n�o ser� destru�do, mas pelo contr�rio ampliado, assim como melhorar� a situa��o dos moradores e lojistas da zona.
+Acabar� com o desleixo e a degrada��o do parque actualmente existente, que n�o tem cuidados de jardinagem h� muito tempo e que poder� transformar-se em poiso de marginais e toxicodependentes.
+Ab�lio Louren�o, da Comiss�o de Luta dos Professores de Educa��o Tecnol�gica (Colpet), lembra que a anterior tutela se limitou a reconverter os professores de Trabalhos Oficinais em professores de Educa��o Tecnol�gica, sem lhes dar qualquer tipo de forma��o.
+as professoras de T�xteis s� ensinam T�xteis, os professores de Electrotecnia s� d�o a �sua� mat�ria e por a� adiante.
+Nas ac��es de forma��o que a Colpet tem vindo a organizar, ficou provado que os docentes n�o cumprem os novos programas porque n�o lhes foi dada a necess�ria reciclagem.
+Como s� dominam uma das mat�rias do programa, quando s�o confrontados com o enunciado de uma Prova Global de Educa��o Tecnol�gica, os professores, na generalidade, n�o sabem resolv�-la.
+A rapariga franzina faz o que pode, mas a voz � t�o titubeante que o apresentador tem logo que ir buscar outra cantora � multid�o.
+Percebe-se que a grande massa de gente est� ali mais ou menos como o Padr�o dos Descobrimentos: para proporcionar � SIC o seu �show� de ilus�o, a ideia de que uma multid�o entusi�stica rodeia o palco, o que serve �s mil-maravilhas os primeiros planos das c�maras.
+As coisas animam-se decididamente com �Crocodile rock� de Elton John.
+Os cantores sucedem-se, Miguel �ngelo j� pula, por momentos dan�a � Elvis e p�e o concorrente a dan�ar.
+�Quantas Laura Pausini est�o aqui esta noite?�, pergunta.
+As duas primeiras jovens fraquejam e Miguel vai logo buscar outra.
+O novo parque urbano de Ermesinde, que come�a a ganhar forma junto �s ru�nas da antiga F�brica da Cer�mica (vulgo F�brica da Telha), ficar� pronto em finais deste m�s.
+� esta a expectativa assumida pelo presidente da C�mara de Valongo, Fernando Melo, empenhado em abrir aos ermesindenses a primeira fase deste novo equipamento de passeio e lazer antes que se escoe o presente mandato aut�rquico.
+Como o P�BLICO p�de constatar no local, a ideia � transformar, nesse curto espa�o de tempo, um terreno que ainda se encontra revolvido e enlameado numa �rea relvada cruzada por passeios e estadias.
+J� podem ser vistas algumas das estruturas que v�o integrar o parque, despontando no meio de um piso em rebuli�o.
+S�o os casos de alguns dos caminhos e do pequeno anfiteatro ao ar livre destinado a espect�culos estivais.
+Hoje e amanh�, decorrem na Sala Polivalente do Acarte as �ltimas de �Josa com o Violino M�gico� dos London Puppet Players.
+Porto: no Carlos Alberto, continua �Luzes de Palco�, a �ltima produ��o da Seiva Trupe .
+�Que futuro para o livro na Europa� � o mote que preside a um f�rum internacional que a Associa��o Portuguesa de Editores e Livreiros (APEL) organiza, em 16 e 17 de Fevereiro, no Centro Cultural de Bel�m, em Lisboa.
+Para al�m de representa��es de v�rias estruturas associativas europeias de editores e livreiros, est� prevista a presen�a do comiss�rio europeu com a pasta da Cultura, Martin Bangemann, bem como de v�rios membros do Parlamento Europeu.
+A discuss�o centrar-se-� em tr�s grandes pain�is: a edi��o na sociedade europeia de informa��o, a dimens�o cultural no Tratado da Uni�o Europeia e o mercado do livro na Europa: problemas de comercializa��o (pre�o fixo).
+O ministro da Cultura portugu�s, Manuel Maria Carrilho, encerra o debate.
+P. -- Qual � a diferen�a entre trabalhar em teatro e em cinema?
+R. -- No teatro tenta-se encontrar a unidade do espect�culo, cria-se um tempo, e n�o um momento -- cria-se um tempo fora do tempo.
+Em o cinema, essencialmente o que se apanha � o momento de qualquer coisa.
+� um trabalho sobre o detalhe.
+Foi em 1913 que a aventura teve in�cio.
+Jesse Lasky, a conselho de Sam Goldfish (depois Goldowyn) d� a primeira oportunidade a DeMille para fazer um filme: �The Squaw Man�.
+O palco das filmagens seria New Jersey.
+Os sic�rios do fara� Edison, cujo �trust� controlava as patentes das m�quinas de filmar for�aram o grande �xodo.
+Ei-lo de abalada para Flagstad, Arizona, com mais quatro c�mplices, entre eles o actor Dustin Farnum.
+Desiludido com o local, DeMille segue mais para Oeste (seguindo, como os pioneiros, as palavras de Horace Greeley: �Go west young man, and grew up with the country� ) e assenta arraiais num velho celeiro comprado numa zona quase deserta: Hollywood.
+Se outros j� por l� andavam, fugidos � ira fara�nica, � a partir de ent�o que a zona come�a o seu �boom�, tornando-se, em poucos anos, a capital dos sonhos.
+Nascia a nova �Terra Prometida�.
+O filme foi um �xito e DeMille at� ao fim da primeira grande guerra dirigiu e supervisou mais de metade dos filmes da sua carreira.
+Desde logo se manifestou a habilidade do �regisseur� controlando mais de um filme em simult�neo.
+� paradigm�tico o caso de �A Marca de Fogo� (The Cheat), esse filme que marcou a cr�tica francesa e o cinema de vanguarda dos anos 20, que fez dele a sua bandeira e deu ao cinema a categoria de arte (a s�tima).
+�A Marca de Fogo� foi filmado em 1914 em simult�neo com �The Golden Chance�, o primeiro de dia e o segundo de noite.
+Estreado um ano antes de �Nascimento de Uma Na��o�, �A Marca de Fogo� � uma obra surpreendente por tudo o que trouxe de novo em termos de linguagem cinematogr�fica, nos enquadramentos e na ilumina��o de interiores.
+Mas o programa cultural abre logo �s 22h00 do dia 3 de Julho, uma sexta-feira, com a actua��o da Orquestra Ligeira do Conservat�rio de Coimbra, na Pra�a do Com�rcio, e encerra no mesmo local e � mesma hora no domingo, dia 12, com os Cool Hipnoise.
+Este mesmo espa�o, no cora��o da cidade, acolhe ao longo da semana e sempre � mesma hora um espect�culo de m�sica e dan�a sevilhana (dia 6), o grupo Negros de Luz (dia 7), Fausto (dia 8), Ces�ria �vora (dia 10) e Carlos do Carmo (dia 11).
+Muito perto, nas principais ruas da Baixa de Coimbra, a anima��o ser� constante, garante a organiza��o.
+Aos saltimbancos, bombos e gigantones juntar-se-�o, ao longo dos v�rios dias de festa, a m�sica, o malabarismo, anima��es de fogo, teatro e fantoches.
+P. -- H� algo de verdadeiramente diferente em �Where In The World�, o seu �ltimo trabalho?
+R. -- Penso que o fundamental � que �Where In The World� � o primeiro �lbum mesmo �da banda�, � s� isso.
+O �lbum funciona mais como uma afirma��o, � feito como um todo, n�o soa como tr�s ou quatro discos diferentes.
+N�o se construiu um mundo diferente para cada um dos seus temas, em est�dio.
+quando, finalmente, conseguimos um disco realmente da banda, ficamos sem um dos m�sicos.
+A directoria de Faro da Pol�cia Judici�ria encerrou na passada quarta-feira, ap�s den�ncia, e na sequ�ncia de investiga��es que estavam a ser feitas no Algarve sobre o aliciamento de mulheres para a prostitui��o, um bordel na zona de Ferreiras.
+Na opera��o foram detectadas, entre outras, quatro jovens com idades compreendidas entre os 16 e os 18 anos, que eram for�adas, segundo a PJ, a exercer a prostitui��o.
+O PSD decidiu deixar hoje pronta a Lei da Greve.
+Para isso, depois do debate na generalidade agendado para hoje, pedir� a avoca��o dos artigos do seu projecto-lei, de maneira a deixar feita a discuss�o na especialidade, �despachando� assim esta quest�o antes de f�rias.
+Em protesto contra esta �pressa� social-democrata, que deitar� por terra o projecto do CDS, a oposi��o decidiu n�o dar acordo para que a Lei Org�nica do Minist�rio P�blico, que regressou ao Parlamento ferida de inconstitucionalidades, seja expurgada antes do prazo regimental.
+Assim, a aprova��o desta lei, prevista para amanh�, obrigar� � realiza��o de uma sess�o extraordin�ria, j� marcada para a pr�xima quarta-feira.
+Um l�der sempre tem os seus fi�is.
+Mas, com a normal tend�ncia para a redu��o da realidade, passou-se a falar n�o apenas na �Escola do Porto�, conceito com origem numa fal�cia regionalista j� muito afastada, mas numa j� mesquinha e inexistente �Escola do Siza�.
+A arquitectura de Siza Vieira n�o o permite, talvez porque em todo o lado seja pr�prio da lideran�a cultural, o n�o se submeter �s regras que ela pr�pria cria.
+Como estilo que nunca se chega a definir, como caminho que est� sempre interminado, a �obra do mestre� � mais a permanente inflex�o que a coer�ncia inabal�vel.
+�Apaixonei-me imediatamente pela �Casa dos Esp�ritos�.
+Telefonei mil vezes para a agente de Isabel Allende, e finalmente consegui, isso foi nos Estados Unidos, onde forcei Isabel a ver �Pelle, o Conquistador� antes da estreia na Dinamarca.
+Isabel respondeu-me que ia pensar na proposta.
+No dia seguinte ligou para mim, e disse que estava interessada.
+este filme � mesmo internacional.
+Mas a adapta��o ao cinema foi f�cil.
+A minha ambi��o era a de contar exactamente a mesma hist�ria da de Isabel.
+Depois tive que encontrar os melhores actores que h� neste mundo, o que tamb�m foi muito complicado e fascinante, e eu acho, sinceramente, que conseguimos criar uma fam�lia cred�vel para a �Casa dos Esp�ritos�.
+Opt�vamos por filmar em Portugal porque era demasiado dif�cil no Chile, onde as autoridades n�o v�em com bons olhos, ainda hoje, uma recria��o cinematogr�fica do golpe de estado contra Salvador Allende.
+mas sairia muito caro.
+E devo dizer que a parte portuguesa da nossa grande equipa tem sido eficaz e prest�vel.
+As autoridades portuguesas, entre elas os militares, t�m sido muito abertas�, concluiu Bille August que ainda revelou que �A Casa dos Esp�ritos� ter� a sua primeira estreia na Alemanha em Outubro deste ano.
+Calmamente, sem grandes foguetes de �marketing�, acaba de acontecer um daqueles pequenos passos na inform�tica que podem significar uma grande revolu��o para o mundo da comunica��o tal como o conhecemos.
+Trata-se da alian�a entre a Adobe, a mais importante empresa no ramo da concep��o de produtos para edi��o electr�nica e a Netscape, que em seis meses capturou 75 por cento do mercado de programas de navega��o na Internet com o produto que lhe deu o nome.
+M�sica irlandesa em �vora: os Wingers tocam na Feira de S�o Jo�o, naquela cidade.
+�s 21 h.
+No �mbito do Ciclo Jovens Compositores, organizado pelo Acarte da Funda��o Gulbenkian, pode escutar-se a obra �...H� Dois Ou ...�, de Ant�nio de Sousa Dias.
+Os int�rpretes s�o Jo�o Natividade (movimento), Lu�s Madureira (voz), Olga Pratts (piano), Pedro Wallenstein (contrabaixo), Ant�nio de Sousa Dias (percuss�o) e Clemente Cuba (desenho de luzes).
+�s 21h30.
+A atleta e o seu t�cnico, Jo�o Campos, elegeram a corrida de 10.000m como um meio de ela se sagrar campe� ol�mpica.
+�Apesar de ser a recordista mundial dos 5000m, � nos 10.000m que a Fernanda se exprime melhor�, considera Campos.
+Fernanda Ribeiro n�o se aterroriza sequer com as condi��es climat�ricas que, decerto, ir� encontrar -- muito calor e humidade.
+�O clima � igual para todos e n�o me preocupa muito.
+Iremos com anteced�ncia para nos adaptarmos.
+�As mais tem�veis dever�o ser as do costume, em especial a et�ope Tulu.
+No Word, o salto da vers�o 2.0c para a 6.0 significou uma altera��o profunda na orienta��o do programa.
+Em vez de acrescentar uma infinidade de caracter�sticas novas, o novo Word privilegiou a consist�ncia de funcionamento e a facilidade de acesso e de aprendizagem.
+Um dos objectivos do novo Word foi tamb�m assegurar a m�xima coer�ncia entre as vers�es para Windows e para Macintosh.
+Desde o manual (que � o mesmo para os dois produtos) ao formato de ficheiros, tudo dever� contribuir para que a mudan�a entre estas duas plataformas n�o ofere�a qualquer problema.
+por exemplo, o excelente �search and replace� tem caracter�sticas que s� existiam na vers�o 5.1 para Mac.
+O WordPerfect, por seu lado, ao passar de 5.2 para 6 tornou-se praticamente num novo produto.
+Foi, dos tr�s, o que teve a altera��o mais radical e, por isso, � o que apresenta a maior quota-parte de problemas.
+Quando as imagens do Cobe foram publicadas, elas emocionaram toda a gente.
+Afinal elas representavam o embri�o do nosso Universo.
+Um embri�o t�o primitivo que quase parecia imposs�vel recuar mais no tempo.
+No entanto, Smoot pensa que vai ser poss�vel ir ainda mais longe.
+�Penso que � poss�vel recuar ainda mais no tempo�, diz o astrof�sico.
+�Acho que vai ser poss�vel aprender algo mais sobre a infla��o do Universo e sobre o seu princ�pio.
+Acho que � mesmo poss�vel chegar at� ao Big Bang.
+Em Coimbra, os estudantes marcaram um golo � defesa menos batida da prova, marca suficiente para alcan�ar o triunfo e justificar a recupera��o das �ltimas jornadas, que j� levou a equipa a ascender ao quarto lugar na competi��o ap�s um mau in�cio de �poca.
+A lideran�a � agora repartida entre Tirsense e Rio Ave, ambos com onze pontos, contra dez do Campomaiorense, que foi empatar (2-2), no s�bado, � Madeira, frente ao Nacional, e da Acad�mica.
+Logo atr�s, surge o surpreendente Desportivo das Aves, que foi golear fora o Sporting de Espinho, por 3-0, e a Ovarense -- empate a duas bolas no Algarve com o Louletano --, ambos com nove pontos.
+A diferen�a m�nima entre os primeiros reflecte uma vez mais o equil�brio da prova, tanto mais que o primeiro e o d�cimo quarto da classifica��o geral est�o separados somente por quatro pontos.
+Refer�ncia maior para o Espinho, que, depois de, na temporada passada, ter disputado a I Divis�o, est� a ter um comportamento desastroso, pois � o �lanterna vermelha�.
+Cinco pontos em oito jogos � muito pouco para um clube que procura o regresso � prova maior do futebol portugu�s.
+Entretanto, os espinhenses disfrutam da companhia do Penafiel, que foi perder a Viseu por 3-1 e est� a repetir a m� campanha da temporada anterior.
+Nos restantes encontros, o Torrense foi garantir o empate (0-0) em Matosinhos perante o Leix�es, enquanto as vit�rias tangenciais (1-0) do Uni�o de Leiria sobre o Le�a e do Chaves contra o Felgueiras, vieram elevar para 22 os golos marcados nesta 8� jornada, que forneceu quatro vit�rias dos visitados, quatro empates e apenas uma vit�ria fora.
+O escudo esteve bastante pressionado, devido sobretudo a vendas especulativas por parte de bancos estrangeiros, e as frequentes interven��es do Banco de Portugal apresentaram-se eficazes, quebrando a tend�ncia de queda, por vezes acentuada, da divisa nacional.
+ R. -- Era o meu ano de agrega��o em Filosofia; conclui-a nesse m�s na Sorbonne comentando uma frase de Einstein sobre a compreensibilidade do mundo.
+Mas nunca fui um dirigente do movimento, apenas um simples pe�o.
+Tenho 46 anos, ou seja, perten�o exactamente a essa gera��o, facto de que me orgulho muito.
+Penso que em rela��o �s minhas filhas, que t�m hoje 23 e 20 anos, tive muito mais sorte, apesar de ter crescido numa sociedade que era certamente mais autorit�ria que a delas.
+As rela��es entre pais e filhos s�o hoje de melhor qualidade.
+P. -- As suas filhas compreendem o que se passou em Maio 68?
+Compreendem o que � que voc�s queriam?
+No momento em que a Uni�o Europeia decidiu abandonar a explora��o do carv�o de pedra, existem cinco mil mineiros portugueses no Norte de Espanha, nas prov�ncias de Le�n e das Ast�rias, condenados a assistir ao encerramento das minas onde trabalham.
+Partiram para um El Dorado, deparam-se agora com a perspectiva de uma reforma antecipada ou do desemprego for�ado.
+A UE � o principal parceiro comercial da R�ssia, representando 37 por cento do total das trocas comerciais contra 24 por cento no caso das restantes ex-rep�blicas sovi�ticas, ou quatro por cento com os Estados Unidos.
+Com exporta��es para a UE de 15,5 mil milh�es de ecus e importando o equivalente a 11,5 mil milh�es, a R�ssia mantinha em 1993 um excedente comercial, face aos europeus, de quatro mil milh�es de ecus.
+N�o � de estranhar, porque entre os comensais reunidos no dia 29 de Agosto em casa do romancista norte-americano William Styron, pr�mio Pulitzer, em Martha's Vineyard, uma ilha ao sul de Boston, Massachusetts, n�o se encontrava apenas o Presidente dos Estados Unidos, mas sobretudo esses dois grandes obcecados dos livros que s�o o Nobel colombiano Gabriel Garc�a M�rquez e o mexicano Carlos Fuentes, �ltimo pr�mio Pr�ncipe das Ast�rias de Literatura.
+Toda a gente acreditou que essa reuni�o, realizada em plena crise dos �balseros� cubanos, tinha sido programada para falar da nova crise aberta entre Cuba e os Estados Unidos e, dada a personalidade dos convivas -- Garc�a M�rquez tem uma rela��o estreita com Castro e Fuentes defende o fim do embargo norte-americano para que se inicie uma nova etapa no longo contencioso da ilha cara�ba com o seu poderoso vizinho --, deu-se como ponto assente que Cuba tinha de ter sido �o� assunto.
+�O carro est� bem equilibrado e estamos esperan�ados para a corrida�.
+�Confesso que entrei depressa demais.
+O carro atravessou-se, primeiro para a direita e depois para a esquerda, o que me levou a tirar o p� do acelerador.
+Quanto a Pedro Matos Chaves, o piloto portugu�s esteve uns furos abaixo do habitual, n�o conseguindo com o seu BMW S�rie 3 melhor que o oitavo tempo, com 1m14,667s, o que o deixou fora da �super pole�.
+O carro alem�o teimou em n�o se adaptar ao tra�ado do circuito, o que deixou Chaves algo desalentado, at� porque experimentava em Barcelona algumas novidades aerodin�micas que deveriam melhorar as �performances� do seu carro.
+S�o jovens, de facto.
+Tamb�m por isso, na FPJ pensa-se j� em Sydney.
+E espera-se o mesmo de sempre.
+Que para o pr�ximo ciclo ol�mpico o projecto n�o seja para dois, mas para quatro anos.
+P. -- Qual foi a opini�o que j� exprimiu aos parceiros (e ao Governo) sobre a concerta��o estrat�gica proposta pelo �Plano Cavaco�?
+R. -- H� muito que defendo que os acordos de concerta��o social devem ultrapassar as dimens�es temporal e de conte�do que t�m tido.
+Os acordos de concerta��o social t�m vigorado sempre por um ano.
+Agora temos uma proposta, inovadora, de m�dio prazo, ajustada a todo este ciclo que o pa�s vai atravessar de 1995 a 1999.
+Parece-me l�gico ligar um acordo de concerta��o social ao Quadro Comunit�rio de Apoio (QCA) e ao PDR, na vertente espec�fica de um acordo de concerta��o social e, mais do que isso, porque o primeiro-ministro afirmou aqui, na reuni�o, que est� disposto a que o pr�prio acordo contemple medidas de acompanhamento e de avalia��o do QCA e do PDR, visto que pode acontecer que os cen�rios macro-econ�micos e macro-sociais se alterem face ao previsto no PDR e no QCA.
+E essas altera��es dever�o ser discutidas com os parceiros sociais e n�o apenas decididas pelo Governo.
+Dois empres�rios, um chin�s de Hong Kong e outro portugu�s, encontram-se para falar de neg�cios.
+O primeiro come�a por explicar que est� ligado � constru��o de um pr�dio para habita��o e que tem tido dificuldades em encontrar uma fornecedora de material sanit�rio nas medidas exigidas.
+Porqu�?
+Porque a exiguidade de espa�o em Hong Kong � tal que at� um metro quadrado de um apartamento para habita��o social custa ouro, como quem diz, entre 600 e 800 contos.
+Para que se aproveite o espa�o ao m�ximo, h� zonas que t�m de ser �encolhidas�, o que acontece com a casa de banho.
+Por isso, em Hong Kong as medidas-padr�o ficam muitos cent�metros abaixo.
+�Ao princ�pio, quando vi ali o carro, sempre pensei que fosse o engenheiro que vinha vistoriar a obra.
+Depois, como ele nunca mais sa�a, comecei a espreitar.
+Foi ent�o que o vi tirar a boina e umas barbas pretas�, contou Jos� Lopes.
+O pedreiro adiantou que ainda esteve tentado a dirigir-se ao carro �e dizer ao fulano que o Carnaval j� acabou�.
+S� n�o o fez porque o empreiteiro da obra o demoveu.
+Mais tarde, o Ford Fiesta voltou a parar no mesmo local -- � esquina da rua da Pens�o Beira Baixa, onde Carlos Miguel almo�ou, e tamb�m muito pr�ximo de um eucaliptal onde o rapto se consumou.
+S� que, desta feita, o homem, em vez de a barba, tinha bigode, a boina desaparecera e trazia �culos escuros.
+Jos� Lopes garante ser o mesmo que vira durante a manh� e que estava a fazer uma chamada atrav�s de um telem�vel.
+O estudante nunca falsifica uma assinatura, deixa descansar o seu encarregado de educa��o, que v� em ele um grande futuro.
+O estudante n�o bebe, saboreia.
+Brennan e Laurent Filipe revelam uma maior seguran�a instrumental, a bateria de Ac�cio Salero (visiense apostado em desmentir a macrocefalia nacional que no jazz � pouco menos do que ditatorial) mant�m um grau de aten��o ao que se passa � sua volta absolutamente indispens�vel � afirma��o de qualquer bom jazzman, e o contrabaixo de David Gausden, embora fraquejando quando lhe compete marcar o sentido da marcha, cumpre razoavelmente a sua fun��o colectiva.
+Uma palavra final para sublinhar tr�s elementos essenciais, clara e inteligente -mente valorizados por Patrick Brennan: o prazer e a alegria de tocar (sem os quais, ali�s, a m�sica de Monk � inacess�vel), relembrando que, mais do que um emprego, o palco � um local de paix�o; o peso do humor no desenvolvimento das notas e, acima de tudo, a sua presen�a peripat�tica, mesmo quando instrumentalmente afastado do discurso, dan�ando os sil�ncios e coreografando os tempos.
+� esse um dos grandes segredos da m�sica do pianista (cujos bailados foram, quase sempre, interpretados como uma mera excentricidade exibicionista).
+� que � a dan�ar que Monk se entende.
+O croata Goran Ivanisevic, o norte-americano John McEnroe, o checoslovaco Petr Korda, o franc�s Henri Leconte e o holand�s Richard Krajicek foram os primeiros a assegurar a passagem � segunda ronda da Ta�a do Grand Slam em t�nis, competi��o que est� a ser disputada em Munique (Alemanha) e que est� dotada com seis milh�es de d�lares (cerca de 840 mil contos) em pr�mios.
+Ivanisevic, n� 4 do �ranking� mundial, encontrou algumas dificuldades para bater o franc�s Guy Forget (7-5 e 6-4), acabando por se imp�r na batalha do servi�o.
+O franc�s conseguiu dez ases no encontro, contra 17 do croata, que est� perto de atingir a incr�vel marca de mil ases este ano.
+Ivanisevic vai defrontar na segunda ronda o norte-americano John McEnroe, que eliminou o sueco Niklas Kulti (6-1 e 6-4).
+McEnroe, que nas �ltimas semanas tem evitado os jornalistas, depois de ter admitido dificuldades no seu casamento com a actriz Tatum O'Neal, voltou a jogar o seu melhor t�nis, empolgando a assist�ncia com todo o seu repert�rio de pancadas espectaculares.
+Henri Leconte, que venceu o sul-africano Wayne Ferreira (3-6, 6-3 e 6-0), vai defrontar na pr�xima ronda o vencedor do encontro entre Pete Sampras (EUA) e Alexander Volkov (R�ssia).
+Tamb�m Petr Korda n�o encontrou facilidades para derrotar o australiano Wally Mansur (2-6, 7-5 e 6-4), esperando agora pelo norte-americano Michael Chang, que bateu o seu compatriota Andr� Agassi (6-4 e 6-2).
+Por fim, o jovem holand�s Richard Krajicek teve o encontro mais f�cil desta primeira ronda, batendo o espanhol Em�lio Sanchez (especialista em pisos mais lentos) por 6-3 e 6-2.
+Na pr�xima jornada, Krajicek defrontar� o vencedor do encontro entre Stefan Edberg (Su�cia) e Michael Stich (Alemanha).
+Eles representam mais um aborrecimento do que uma amea�a para o Governo, comentou Sergio Ramirez, o sandinista que preside ao Parlamento.
+�O perigo � a internacionaliza��o do conflito, por causa da instabilidade que os grupos armados causam nas zonas onde operam�.
+Tony Bryant, l�der dos �Comandos L�, uma organiza��o hostil ao regime de Fidel Castro e sediada em Miami, admitiu ao �Washington Post� que est� a ajudar os �Recontras� na luta contra Violeta Chamorro e os sandinistas.
+O corpo do escritor franc�s Alain Fournier, autor de �O Grande Meaulnes�, morto no come�o da I Guerra Mundial, foi formalmente identificado, anunciou ontem a Direc��o Regional de Assuntos Culturais Franceses, na cidade de Metz.
+Os despojos do escritor franc�s era um dos 19 cad�veres descobertos em Novembro, na regi�o de Verdun, local da batalha contra o ex�rcito alem�o onde se sabia que Alain Fournier tinha sucumbido.
+O autor de �O Grande Meaulnes� (edi��o portuguesa na Rel�gio d'�gua) pertencia ao 288� Regimento de Infantaria que a� combateu.
+O seu corpo foi identificado por antrometria, sem qualquer margem para d�vidas, pela equipa chefiada pelo antrop�logo Fred�ric Adam, que comprovou a estatura do esqueleto, a sua idade � data da morte e a complei��o f�sica, por compara��o com documentos da �poca.
+Contra a lenda que falava da morte por tortura do escritor, �s m�os de soldados alem�es, os exames efectuados por Adam demonstram que �a morte de Henri Alban Fournier, dito Alain Fournier, se deveu a impactos de balas�.
+Foi em 22 de Setembro de 1914.
+R. -- N�o, de todo.
+O que acontece � que, na altura em que foram feitas as estimativas das receitas no or�amento, era j� sabido que o cen�rio internacional adiantado pelo Fundo Monet�rio Internacional poderia ser optimista.
+Isso implicou uma maior incerteza e estimativas acrescidamente prudentes.
+A confirma��o de que as perspectivas de crescimento n�o s�o, afinal, t�o favor�veis s� veio validar essa abordagem.
+P. -- Quer dizer que nunca se acreditou num crescimento de tr�s por cento e que a estimativa da receita foi feita, desde o in�cio, a pensar em dois por cento ou menos?
+Os outros operadores s�o os membros negociadores (interv�m na negocia��o em bolsa, introduzindo directamente no sistema ofertas de compra ou de venda) e os membros compensadores, que al�m de as fun��es dos anteriores, participam, designadamente, nos procedimentos de liquida��o de contratos nas datas de vencimento e no processo de constitui��o, substitui��o, redu��o e liberta��o de garantias.
+O futuro sobre OT-10 (uma taxa de juro de longo prazo) conta, � partida, com nove �market makers�, significativamente mais do que os cinco que cumprir�o ind�ntica fun��o para o futuro sobre o PSI-20 (um �ndice bolsista calculado para uma carteira de 20 t�tulos).
+O contrato sobre a Lisbor a tr�s meses (taxa de juro de curto prazo, correspondente � m�dia das taxas praticadas no mercado monet�rio interbanc�rio) s� come�ar� a ser negociado no in�cio de Julho mas j� tem garantidos seis �market makers�.
+Segundo o Instituto Nacional de Estat�stica (INE), Portugal exportou 265,2 milh�es de contos e importou 521 milh�es de contos, durante os primeiros cinco meses de 1995, o que resultou num d�fice comercial de 255,8 milh�es de contos.
+A taxa de cobertura das importa��es pelas exporta��es melhorou, situando-se em 50,9 por cento, contra 45,1 por cento nos cinco primeiros meses de 1994.
+Em Maio verificou-se um aumento de 19,3 por cento nas exporta��es e uma queda de 4,3 por cento nas importa��es, o que permitiu uma diminui��o de 19,2 por cento no d�fice comercial mensal, em rela��o ao mesmo m�s de 1994.
+As compras aos PALOP (Pa�ses Africanos de L�ngua Oficial Portuguesa) aumentaram 45,5 por cento de Janeiro a Maio deste ano, face a id�ntico per�odo de 1994, enquanto as vendas ao Jap�o subiram 34,6 por cento no mesmo per�odo.
+�O caso tem sido absolutamente devastador para a ag�ncia .. tanto em termos de p�blico como internamente�, disse um funcion�rio da Administra��o.
+�Temos de o ultrapassar ...a fim de restaurar a confian�a do Congresso e do p�blico�.
+�Creio que era tempo de Woolsey sair�.
+E deu como motivos ele n�o haver despedido quadros que n�o conseguiram detectar a tempo o trabalho de sapa que Aldrich Ames andava a realizar.
+A banda sonora de �Thirtysomething� � a de uma s�rie televisiva, dom�nio onde a m�sica vem sendo tratada pior que no cinema.
+Claro que tamb�m faz os usuais recursos a fundo de cat�logo, mas logo a esse n�vel h� uma selectividade fora do comum, e os temas repescados de Ray Charles como de Rickie Lee Jones s�o n�o apenas can��es cl�ssicas, como concordam absolutamente com o esp�rito da s�rie.
+Mais importante que isso, a m�sica composta por Snuffy Walden n�o � um mero adere�o, mas surge como outro personagem de �Thirtysomething�, de algum modo participando do seu enredo e da sua l�gica, mas com a diferen�a de ser de car�cter musical.
+Predominantes no trabalho de Walden s�o guitarra e piano, mas os instrumentos que usa s�o rel�quias do in�cio do s�culo passado, porque estes t�m em seu entender mais car�cter, ou, se se preferir, um som mais distintivo.
+Empregou, para al�m disso, tons ressonantes e profundos, de forma a aumentar a densidade emotiva.
+Da� resulta uma m�sica com uma individualidade peculiar, um misto de pureza e de permeabilidade, mas ao mesmo tempo com um sentido de passado e de amadurecimento.
+O ditador ter� sabido evitar, assim, as tr�s caracter�sticas que marcaram os regimes totalit�rios de Hitler e Estaline: moderniza��o, mobiliza��o e expans�o.
+�Salazar quis reinserir Portugal na sua tradi��o, n�o queria a moderniza��o mas a seculariza��o�, disse Braga de a Cruz.
+Tamb�m n�o tinha veleidades de mobiliza��o, pois �o seu grande intuito foi despolitizar, baixar a febre pol�tica�.
+�Ele herdou um imp�rio colonial e o seu objectivo foi nacionaliz�-lo, conserv�-lo�.
+�N�o h� nada nela que obrigue o regime a ser ditatorial.
+Nem sequer havia proibi��o de partidos�.
+Como � que, ent�o, aconteceu a ditadura?
+�Pela via administrativa�.
+Atrav�s da �policializa��o do Estado� e do esvaziamento do sistema representativo.
+�Com o tempo, o poder legislativo foi transferido para o Governo, esvaziando os poderes da Assembleia Nacional, que funcionou apenas tr�s vezes por ano�, disse Braga da Cruz, que, apesar disso, encontra algum �pluralismo� no salazarismo.
+�A Uni�o Nacional era uma grande frente onde Salazar tolerava algum pluralismo org�nico, nomeadamente a Causa Mon�rquica, que, em 1961, se n�o fosse o in�cio da guerra em �frica, teria mesmo aparecido como alternativa � Uni�o Nacional�.
+Kadhafi, da L�bia, far� o mesmo.
+A Bolsa cai e o pre�o do petr�leo sobe.
+�As pessoas que assistem a tais sess�es saiem loucas, hist�ricas, falam alto, as crian�as choram�, observa a administra��o daquele condom�nio.
+�Todas estas pr�ticas deveriam ter locais espec�ficos.
+Segundo se julga, o cinema foi licenciado para exibi��o de filmes e n�o para culto�, defende.
+A finalizar, o protesto solicita uma interven��o do presidente da Junta de Freguesia, �a fim de esclarecer esta situa��o ou, se poss�vel, acabar com as sess�es�.
+O autarca remeteu c�pias do of�cio � delega��o de sa�de respons�vel pela �rea de Alverca e � administra��o da Lusomundo, empresa propriet�ria da sala de cinema em causa.
+Afirma��es extraordin�rias exigem provas extraordin�rias.
+Drosnin e muitos dos seus seguidores aceitam acriticamente o epis�dio Rabin como �prova extraordin�ria� do c�digo da B�blia.
+Mais grave, a afirma��o dos �media� � de que foi �demonstrada cientificamente� a realidade dos c�digos.
+n�o s� as t�cnicas utilizadas n�o demonstram nada, como as conclus�es se baseiam em fal�cias, como todo o processo nada tem a ver com Ci�ncia, ao contr�rio do que as m�quinas de propaganda pretendem fazer crer.
+A t�cnica utilizada por Drosnin � muito simples.
+Na descri��o que se segue realizo algumas simplifica��es, bem como uma adapta��o do hebraico para o portugu�s.
+Passo 1: pegue num texto qualquer, de prefer�ncia grande.
+Passo 2: elimine todas as vogais, espa�os e pontua��o, ficando apenas com uma longa cadeia de consoantes.
+Passo 3: pense num n�mero inteiro, digamos 7.
+Passo 4: fa�a uma bola em torno de uma consoante qualquer da sua cadeia, e a partir dessa em torno de todas as consoantes contadas de 7 em 7 (7�, 14�, etc.), construindo uma nova cadeia de consoantes.
+Passo 5: pegue na nova cadeia e tente construir uma ou v�rias palavras com sentido introduzindo vogais onde quiser.
+Se sim, BINGO!
+Acertou.
+Se n�o, repita os passos 3 a 5 variando o que quiser at� conseguir alguma coisa interessante.
+Pode, por exemplo, come�ar a nova cadeia em qualquer das milhares de consoantes � sua escolha.
+Ou, em vez de contar de 7 em 7, pode contar de 33 em 33.
+Ou acrescentar vogais diferentes.
+Tem milhares de milh�es de varia��es ao seu dispor.
+o nome de Rabin, na �previs�o� de Drosnin, aparece contando cada 4772 letras a partir da 4333�.
+A abertura da nova Ponte Vasco da Gama parece estar a influenciar de uma forma positiva o tr�nsito na cidade de Vila Franca de Xira.
+A Pol�cia de Seguran�a P�blica vila-franquense efectuou, durante o m�s de Abril, um conjunto de recolhas de dados sobre a circula��o na cidade e detectou uma redu��o de tr�fego da ordem dos 20 por cento comparativamente com as m�dias registadas em 97.
+A mo��o apresentada pela bancada socialista defendeu com rigor o projecto do Governo para a cria��o de regi�es administrativas, considerando-o uma boa base de trabalho, e acrescentou-lhe argumentos para impor a necessidade da sua concretiza��o.
+Orlando Magalh�es recorreu a dados estat�sticos para provar que o Norte do pa�s, apesar �do seu peso econ�mico�, tem sido a �regi�o menos beneficiada pelos fundos comunit�rios�, em rela��o �s zonas do Sul e, por isso, menos desenvolvida.
+As assimetrias regionais foram, de resto, os argumentos base para defender uma mo��o que considera a regionaliza��o �uma reforma essencial ao Estado�.
+A bancada da CDU concordou e refor�ou politicamente o teor do documento socialista.
+�A regionaliza��o, al�m de promover o desenvolvimento, favorece a democracia participativa atrav�s do voto popular�, disse o comunista Ant�nio Gra�a.
+�Eu nasci e vivi em Vila Real at� aos 17 anos e por isso senti na carne o isolamento do interior�.
+No entanto, o deputado popular acusou �receios de bairrismos exacerbados� e acabou por seguir a linha do l�der do partido, ao rejeitar a cria��o de regi�es em favor de mais poder para as autarquias.
+O presidente da C�mara de Esposende, Alberto Figueiredo, anunciou ontem, que vai pedir ao ministro das Finan�as um inqu�rito a todos os fundos comunit�rios que as suas empresas receberam nos �ltimos anos.
+O pedido, feito durante uma confer�ncia de imprensa, surge na sequ�ncia das insinua��es que o candidato do PP, Franklim Torres, lhe fez de ter utilizado o seu cargo de presidente da autarquia para obter este tipo de apoios do Governo.
+Figueiredo, que se recandidata a um terceiro mandato, considera que o seu advers�rio �est� a insinuar corrup��o� e que a sua dignidade foi posta em causa, por isso n�o tem outro caminho sen�o �esclarecer estes processos para que n�o fiquem d�vidas�.
+Alguns estudos em que foram usadas defini��es de depress�o mais restritas do que as internacionais apuraram taxas mais baixas -- de tr�s a cinco por cento.
+No entanto, este especialista, psiquiatra no Hospital de Santa Maria, considera que �n�o h� raz�es m�dicas, culturais ou sociais que nos permitam pensar que a frequ�ncia da depress�o seja diferente no nosso pa�s, pelo que os dados internacionas podem ser extrapolados com alguma seguran�a�.
+As principais preocupa��es m�dicas da actualidade s�o, segundo Filipe Arriaga, �a elevada morbilidade associada � depress�o e ao suic�dio�.
+�A depress�o � o principal factor de risco do suic�dio, est� em primeiro lugar nas causas que levam ao acto suicida�, acrescenta.
+O PRIMEIRO-MINISTRO israelita, Yitzhak Rabin, e o chefe da Autoridade palestiniana, Yasser Arafat, conferenciam amanh� em Erez, ponto de passagem entre a Faixa de Gaza aut�noma e o Estado de Israel, afirmou ontem � ag�ncia noticiosa France Presse um conselheiro do chefe da OLP.
+A pe�a do jovem dramaturgo Sergi Belbel (Pr�mio Nacional da Literatura Catal� em 91, 92 e 93) tem muito de futurista.
+Mas o futuro de que fala � mais ou menos pr�ximo.
+Repress�o, discrimina��o, transgress�o s�o as molas da com�dia posta em cena por In�s C�mara Pestana.
+O repressor apela para o transgressor e vice-versa.
+A repress�o do tabagismo, por exemplo, atrai a transgress�o.
+E � dessa transgress�o que nasce o espect�culo cruel do Teatro do S�culo.
+Uma verdadeira dan�a da morte.
+Ou da vida tomada como um desporto de alt�ssima competi��o.
+� proibido fumar.
+Mas toda a gente fuma (�s escondidas) incluindo os proibicionistas.
+Se, espectador distra�do, os nomes de actores como Teresa Roby, In�s C�mara Pestana, Ant�nio Filipe, Elisabete Piecho, Manuela Pedroso, Marina Albuquerque, Orlando S�gio e Rui David n�o te dizem nada, aproveita a noite de hoje (�s 22h) para ir � Rua do S�culo ver o prod�gio de uma companhia em que os actores parecem ligados � corrente.
+Porque corrente magn�tica n�o falta � encenadora, que, em espect�culos como �Crimes e Interl�dios�, �Car�cias�, �Kvetch� e �Zucco�, fez do Teatro do S�culo uma refer�ncia incontorn�vel no panorama do teatro portugu�s nos anos 90.
+O SECRET�RIO-GERAL da Organiza��o das Na��es Unidas (ONU), Butros Butros-Ghali, deixou ontem ao princ�pio da tarde Luanda com a promessa de que at� ao fim de Agosto dever�o chegar a Angola as unidades de infantaria que faltam para se completar a Miss�o de Verifica��o (Unavem III).
+Ele pintor, 26 anos, chamado Johnson, ela professora, 28 anos, Edel de seu nome, entraram no carro de um taxista de apelido S�o Pedro, 40 anos.
+A troca de palavras menos agrad�veis come�ou logo, antes de arrancarem, por causa da curta dist�ncia da viagem.
+No sinal vermelho seguinte, a coisa ficou t�o feia que o casal abriu as portas e saiu do carro.
+O taxista n�o se ficou e, com a seguran�a que a dimens�o do seu porte lhe d�, foi atr�s deles, exigindo o seu dinheiro.
+A actual administra��o da TVI, presidida por Miguel Paes do Amaral, l�der do grupo SOCI, e a Antena 3 espanhola assinaram ontem um acordo de parceria estrat�gica, em que se comprometem a duplicar o actual capital social do quarto canal portugu�s.
+A Media Capital (do grupo SOCI), que gere a TVI, apresenta assim mais um argumento para a assembleia de credores que se realiza no pr�ximo dia 14.
+Nessa assembleia devem ser apresentados, pelo menos, dois projectos de recupera��o da TVI: o da Media Capital e da Antena 3 (a principal televis�o privada espanhola) e o do grupo Lusomundo, associado ao empres�rio macaense Stanley Ho.
+De momento, ambos os grupos est�o em contactos com os v�rios credores para assegurarem o respectivo apoio, pois � necess�rio que os votos favor�veis a uma das propostas venham de um conjunto de credores que detenham pelo menos 75 por cento da d�vida da TVI, que ascende a 17 milh�es de contos.
+Entre as mais belas �fotografias de cinema� do mundo as da ag�ncia Magnum ocupam o primeiro plano.
+s�o as que procuras os actores, mas tamb�m os maridos e mulheres e amigos; as cenas de repouso, o cinema no trabalho.
+Estas imagensque publicamos de Cartier-Bresson, Robert Capa, Eve Arnold ou Dennis Stock, entre outros, demonstra, uma vez mais, a voca��o totalizadora da Magnum, a capacidade em acompanhar de perto fen�menos mais importantes da hist�ria da humanidade.
+Uma exposi��o de fotografias, concebida em conjunto com o livro �Magnum-Cinema�, poder� ser vista em Fevereiro, em Lisboa, na Culturgest.
+Ter como amigos pessoas s�bias, bem sucedidas e eventualmente belas � um privil�gio.
+Digamos que a qualidade dos amigos, para al�m da qualidade da sua amizade, constitui para mim um factor importante de qualidade de vida.
+Que o meu querido amigo Jo�o Carlos Espada tenha acesso aos sabores duma das tradicionais bolsas da erudi��o ocidental constitui um contributo, n�o despiciendo, para a melhoria da minha qualidade de vida.
+Mais tarde ou mais cedo, quando nos encontrarmos, usufruirei, eu tamb�m, das experi�ncias de que ele agora desfruta.
+Entretanto, eu -- e os milhares de pessoas que o leram no P�BLICO (da passada segunda-feira) -- vou-me contentando com as suas Cartas de Inglaterra.
+Um quarteto formado por Bob Mover (sax alto e voz), Carlo Morena (piano), Pedro Gon�alves (contrabaixo) e Jo�o Silvestre (bateria) actua a partir das 23h, na catedral lisboeta do jazz: no Hot Clube de Portugal.
+Uma �aut�ntica� big band, constitu�da por 16 m�sicos, com alguns dos temas mais conhecidos do report�rio �standard� liderada pelo trombonista Claus Nymark e com a voz de Ana Paula Oliveira: no Speakeasy (Cais da Rocha Conde d'�bidos -- Armaz�m 115), �s 23h e 01h.
+ Alguns milhares de trabalhadores afectos � CGTP desfilaram ontem pela baixa de Lisboa em protesto contra a pol�tica econ�mica e social do Governo.
+Um boneco cabe�udo baptizado de �Santo Cavaco� foi a estrela da manifesta��o, que partiu dos Restauradores e foi at� � Pra�a da Ribeira, depois de interromper o tr�nsito das Ruas do Ouro e da Prata.
+�N�o podemos permitir que a contrata��o colectiva continue bloqueada, que o desemprego continue a aumentar, que a seguran�a social, a sa�de e a educa��o continuem a degradar-se�.
+�Somos obrigados a tornar os nossos programas o mais interessantes poss�vel, num ambiente onde existem m�ltiplas escolhas�, refere Victor Neufeld, o produtor executivo do programa 20/20, da ABC, citado pela Associated Press.
+Conseguir um bom exclusivo pode significar a entrada de milh�es de d�lares em publicidade.
+Da� que Shapiro tenha ficado t�o orgulhoso com o exclusivo dos pais dos sete irm�os g�meos.
+E, para que fique registado, o produtor afirma que a NBC n�o pagou ao casal McCaughey.
+Shapiro acredita que eles concordaram em aparecer no programa Dateline, da NBC, pura e simplesmente porque gostam de o ver.
+Quanto � presen�a dos seguran�as � porta de casa deles, foi a NBC que os contratou para manterem os �paparazzi� � dist�ncia durante a entrevista.
+Seriam 9h30 quando os tr�s homens, de quem n�o se conhecem dados identificativos, surgiram de armas na m�o e rosto coberto � porta do BNU de Massam�, uma depend�ncia recentemente inaugurada e localizada na denominada Sexta Fase da Urbaniza��o Pimenta e Rendeiro (�rea que na sua maioria ainda est� em constru��o).
+Os assaltantes atra�ram a aten��o de uma das funcion�rias, que deu de imediato o alarme, pelo que o gerente conseguiu bloquear a porta de entrada ainda antes desta poder ser aberta pelos suspeitos.
+Ap�s verificarem que a entrada havia sido bloqueada, os tr�s homens fugiram de imediato para um autom�vel Citr�en AX, de cor branca, cujas letras da matr�cula s�o AJ, e tomaram a estrada que conduz a Queluz.
+Segundo o comandante da GNR do Cac�m, entidade a quem primeiro foi comunicada a ocorr�ncia, n�o foi consumado qualquer acto violento.
+Quanto a saber donde � que vem a nossa prefer�ncia quase un�nime pela m�o direita, a quest�o permanece em aberto.
+Tanto mais quanto, hoje em dia, o hemisf�rio esquerdo (dos dextr�manos) j� n�o � considerado como o hemisf�rio cerebral �dominante�, nem como o �nico capaz de desempenhar altas fun��es mentais e de controlar movimentos precisos e subtis.
+Sabe-se que o hemisf�rio direito tamb�m � essencial para fun��es mentais de alt�ssimo n�vel, tais como o racioc�nio, a mem�ria, o reconhecimento dos rostos, das melodias musicais, etc.
+Uma explica��o poss�vel da hegemonia dextr�mana poder� ser o facto de, nos dextr�manos, o hemisf�rio cerebral esquerdo possuir liga��es nervosas sensoriais e motoras para os dois lados do corpo, enquanto o hemisf�rio direito est� quase s� ligado a um �nico lado do corpo.
+Isto poder� significar, simplesmente, que o controlo volunt�rio da m�o direita � mais f�cil do que o da m�o esquerda.
+As delega��es do Governo de Luanda e da Unita rubricam o protocolo das conversa��es de Lusaca para que haja paz em Angola.
+O acordo definitivo dever� ser assinado dentro de quinze dias.
+Marques Mendes, ministro-adjunto da presid�ncia de Conselho de Ministros com a tutela da comunica��o social, amea�a proceder a altera��es legislativas caso as televis�es n�o cheguem a acordo sobre a passagem de imagens de alegada viol�ncia.
+O provedor recomenda, assim, a �imediata transfer�ncia� do agente da PSP para um estabelecimento prisional �tutelado pelo Minist�rio da Justi�a� e �que permita compatibilizar as exig�ncias de seguran�a com o exerc�cio dos direitos conferidos a qualquer recluso em ambiente prisional normal�.
+A falta de consci�ncia do sentido de rid�culo ainda recentemente foi dada, em declara��es ao P�BLICO, por um douto professor universit�rio que classificou liminarmente de �mau plano� um PDM elaborado dentro de uma c�mara municipal com uma equipa t�cnica de qualidade e fortemente assessoriada por urbanistas experientes.
+Fazer uma gest�o urban�stica inteligente, dialogante, eficaz, informada e com bons resultados pr�ticos, exige uma tenacidade e um talento que n�o est�o ao alcance de t�cnicos cinzentos e submissos que o sistema inevitavelmente prefere, produz e atrai.
+O inc�ndio que deflagrou no final da semana passada nos concelhos de Ma��o, Abrantes e Sardoal, foi dado como extinto pelas 12h00 de ontem.
+O jogo dos campe�es escoceses, apesar das m�ltiplas tentativas dos futebolistas do Rangers de maior nome, como Hateley e Durrant, n�o s� perdia �mpeto mas morria em qualidade, quando enfrentava o meio-campo do Celtic, onde McStay se exibia a grande altura.
+Por outro lado, os defesas-centrais do Celtic, Mowbray e Grant, sempre coadjuvados por McGinlay, chegaram para reduzir a pouco o ataque do Rangers, onde Hateley n�o dispunha do apoio habitual.
+O primeiro tempo terminou sem golos.
+Na segunda parte, apesar de o Rangers ter tentado o tudo por tudo para rapidamente resolver a partida, o Celtic depressa recuperou o comando das opera��es.
+Mas o jogador russo do Rangers Mikhailichenko come�ou a evidenciar-se devido � frescura e � rapidez do seu jogo.
+Era de facto por ali que come�ava a surgir perigo para o Celtic, e n�o surpreendeu que, com pouco mais de uma hora de jogo, fosse esse mesmo Mikhailichenko a colocar a bola � frente de McCoist, de modo a permitir-lhe a marca��o do primeiro golo da partida.
+Cerca de tr�s centenas de trabalhadores municipais manifestaram-se ontem diante da C�mara Municipal de Sintra, exigindo da presidente socialista Edite Estrela o pagamento das horas extraordin�rias e das ajudas de custo e a reposi��o do subs�dio de insalubridade, penosidade e risco.
+No fim, sa�ram com algumas promessas e um �nim� para o subs�dio.
+Face a o marco, a nota verde n�o apresentou uma tend�ncia definida, oscilando entre os 1,79 e os 1,80 marcos por d�lar, enquanto no �mbito do SME se mantinha a tradicional apatia.
+Em o fecho dos mercados europeus,o marco/escudo transaccionava-se a 102,35/ 102,36 escudos por marco,tal como tinha j� acontecido ter�a e quarta-feira.
+Em o que respeita �s taxas de juro,o Bundesbank confirmou a expectativa do mercado ao anunciar uma manuten��o de todas as suas taxas directoras.
+Os bancos centrais franc�s,holand�s,austr�aco e su��o tomaram id�nticas decis�es.
+ALVO: Com este afluxo -- recorde de vota��o, a A.B.P. saiu vencedora?
+E.G.: A A.B.P.ganhou bastante e acrescento que nunca houve em Delega��o nenhuma,uma t�o grande aflu�ncia de s�cios para a vota��o,como se verificou nestas Elei��es.
+Isto significa que os Barmens n�o est�o esquecidos.
+Continuo a dizer que acho bem que os Barmens saibam que a Delega��o est� aberta para todos eles,e gostaria imenso que eles aparecessem em vez de uma vez por m�s, dez vezes porque temos todas as condi��es criadas para os receber.
+Podemos beber um copo,jogar gam�o,damas,snooker,ver televis�o,etc.
+Judo (52.500 contos)Nelson Br�s,S�lvia Henriques,Pedro Caravana,Michel Almeida,Guilherme Bentes,Pedro Soares e Filipa Cavalleri.
+Lutas Amadoras (7 mil contos)-- David Maia.
+um documento de meados do s�culo XVII refere j� uma Rua de o Reim�o.
+da� que surja muitas vezes referida como Campo do Cirne.
+Rua onde ainda hoje se concentram algumas das mais antigas casas de m�rmores e cantaria da cidade,a Avenida de Rodrigues de Freitas conta ainda com duas institui��es de especial relevo para a hist�ria da cidade,descontado o j� referido recolhimento para �rf�s: a Biblioteca P�blica Municipal do Porto -- cuja fachada lateral est� voltada para a Rua de D.Jo�o IV e para o Jardim de Marques de Oliveira (vulgo de S.L�zaro)-- e a Escola Superior de Belas-artes do Porto.
+Passados seis meses sobre o an�ncio oficial da exist�ncia da arte paleol�tica do C�a,continuamos pois � espera de iniciativas v�lidas e de boa f� para que,independentemente da decis�o final que venha a ser tomada sobre a barragem,o monumento seja estudado com os meios que a sua grandeza requer.
+At� quando permitir�o os cidad�os portugueses,cujos impostos sustentam a hierarquia do IPPAR,e cujas contas de electricidade alimentam os or�amentos sumptu�rios da EDP,que esta vergonha continue?
+O segundo dia ter� a economia como tema comum �s tr�s sess�es.
+Incluir�,naturalmente,o debate sobre a UEM,numa perspectiva europeia e portuguesa,a posi��o europeia na economia mundial e,ainda,o papel da UE no processo de reintegra��o da R�ssia na economia internacional ou a dimens�o econ�mica e pol�tica das rela��es transatl�nticas.
+Entre os participantes,estar�o Alan Milward,Jacques Attali,V�tor Const�ncio,Leonardo Ferraz de Carvalho,V�tor Martins,Artur Santos Silva,Jorge Jardim Gon�alves e Pedro Ferraz da Costa.
+Finalmente,o terceiro dia ser� dedicado a uma reflex�o mais profunda sobre essa entidade a que se chama Europa -- do ponto de vista da sua cultura e da sua hist�ria.
+Para responder � quest�o primordial -- o que � a Europa?-- interv�m na confer�ncia personalidades como Alain Finkielkraut,Edgar Morin,Susan Sontag ou Marcelo Rebelo de Sousa.
+Andr� Glucksman,Peter Schneider e Victor Cunha Rego falar�o de cultura europeia,e o americano Simon Shama abordar� a procura de uma hist�ria europeia.
+Em os mintos iniciais da segunda parte os vitorianos conseguiram de novo adormecer os seus advers�rios,nomeadamente gra�as a o trabalho laborioso de Paneira e Toni�o e a excelente marca��o que Marco exerceu sobre Zola.
+O portugu�s �secou� de tal forma o g�nio parmes�o que Ancelotti acabou por substitui-lo por Melli.
+O Parma voltou a acordar aos 66',quando Melli rematou � barra uma bola passada por Dino Baggio.
+Por�m,quando parecia que os italianos iam embalar para uma exibi��o arrasadora,aconteceu o golpe de teatro.
+Acabadinho de entrar a substituir Riva,o brasileiro Gilmar surgiu isolado na �rea italiana e deu o melhor seguimento a um centro de Quim Berto.
+Os adeptos italianos gelaram,enquanto a pequena comitiva vitoriana festejava ruidosamente na bancada central.
+de novo a pol�mica � ribalta.
+Um investimento de 27 milh�es de contos (mais seis milh�es para aquisi��o de uma colec��o pr�pria)suportado por as arcas bascas apesar de a exist�ncia de 47 �partenaires� comerciais,� sin�nimo do interesse que o governo de Euskadi,do Partido Nacionalista Basco (PNV),atribui ao evento.
+Implantado em terrenos antes ocupados por ind�strias desarticuladas por a reconvers�o industrial dos anos 80 e cujo �ferro velho� permanecia como legado da crise econ�mica da outrora laboriosa cidade de Bilbau,o Museu pretende ser um s�mbolo da regenera��o basca: de uma sociedade que quer vencer as dificuldades presentes e,em v�speras do s�culo XXI,dar novos argumentos -- os da esperan�a e da paz -- aos seus cidad�os.
+Por isso,atentos ao perder da sua influ�ncia,os etarras,atrav�s de o at� agora desconhecido �comando Katu�,tentaram destruir o s�mbolo de um novo tempo,menos rural -- � nessas zonas e entre os deserdados das cidades que a Eta faz o seu recrutamento --,mais din�mico e,sobretudo,aberto � influ�ncia exterior,algo que o nacionalismo admite com dificuldade.
+Desvalorizando,por outro lado,a posi��o assumida por o deputado independente Jos� Magalh�es,que considera �errada� a proposta de Manuel Carrilho -- �Se ele disse isso,o problema � de ele,foram as suas palavras --,Jorge Coelho reafirmou a sua confian�a no ministro da Cultura.
+�Se ele apresentou esta solu��o para resolver aquelas d�vidas,� porque deve ser uma boa solu��o�,rematou.
+�poca de Ouro abre, com chave d'ouro, o ciclo.
+Fundado por o mestre Jacob do Bandolim em 1964,o grupo tem mantido viva,ao longo de os anos,a tradi��o no dom�nio do chorinho,valorizando-a com sabedoria.
+Comp�em o grupo seis m�sicos de reconhecida craveira: Toni (viol�o),C�sar Faria (violino),Jorge Filho (cavaquinho),Ronaldo do Bandolim (bandolim),Crist�v�o Bastos (piano)e Jorginho do Pandeiro (flauta).
+Hoje e s�bado,�s 21h30,uma oportunidade �nica para confirmar,em palco,toda a alma e virtuosismo de um g�nero musical tamb�m �nico.
+Com o apoio da Funda��o luso-brasileira para o Desenvolvimento da L�ngua Portuguesa.
+�s 17 horas,pode ser visto �Adamubies� (t�tulo de um poema de Guimar�es Rosa),projecto de m�sica c�nica onde textos de Miguel Torga,Clarice Lispector,Jo�o Guimar�es Rosa e Agustina Bessa-Lu�s surgir�o �vestidos� com m�sicas reinventadas a partir de matrizes populares do Brasil,�frica e Portugal.
+A direc��o musical � de Tilike Coelho e a encena��o e cenografia de Jos� Caldas Neto.
+A estes juntar�o-se-,em palco,os cantores/m�sicos Marta Silva,Marcelo Lafontana e Pedro Ribeiro.
+O espect�culo repete domingo � mesma hora e no fim-de-semana seguinte,11 e 12,tamb�m �s 17h.
+O inter-Fiorentina era o jogo grande da terceira jornada do campeonato italiano.
+A partida at� come�ou bem para os homens da casa.
+Ronaldo,� beira de o intervalo,inaugurou o marcador para o Inter,justificando o forte investimento que a equipa milanesa fez na sua aquisi��o.
+Mas os homens de Gigi Simone tiveram que sofrer muito para levar de vencida a Fiorentina que,com golos de Serena e Batistuta,deu a volta ao resultado.
+Batistuta viu ainda dois remates seus baterem nos postes da baliza contr�ria.
+Um golo de Moriero recolocou as duas forma��es em p� de igualdade e s� a nove minutos do fim o Inter chegou ao golo da vit�ria,por interm�dio do franc�s Djorkaeff.
+Em os outros jogos do �calcio�,destaque para mais uma goleada da Juventus de Dimas (4-0 frente a o Brescia),enquanto a Lazio de Eriksson parece ter perdido a veia goleadora demonstrada em Guimar�es e foi derrotada por o rec�m-promovido Empoli,por 1-0.
+Realce tamb�m para mais uma derrota do Milan,desta feita por 2-1,frente a a Udinese.
+Agora, perante os olhos da velha Elvira, sentada no centro da sala com os xailes e o casaco que a protegem do frio, encontra-se o vestido verde com os bot�es com pequenos sabres dourados idealizado pela costureira Danilina.
+Rev�-se no Mercedes de Goebbels, apreendido na Alemanha e entregue ao marechal Oslikovsky, sentado com a filha pequena, Lora, no assento posterior.
+Uma corrida para Moscovo a chupar as bolinhas de �Kliukva v sacarnoi pudre�, feitas de a��car em p�, com uma baga siberiana de sabor agridoce no meio.
+Nessa manh�, usava pela primeira vez os sapatos de pele macia criados por Goldin, o surdo sapateiro judeu.
+Tinha cal�ado um par de pantufas inchadas de p�lo velho.
+De s�bito, ergue os olhos para voltar a mergulhar no passado que ressumava daqueles trajes velhos.
+Eis o casaco de linho claro e a saia que lhe havia dado a filha casada em It�lia.
+Uma medida grande para o seu corpo pesado, agora que tinha setenta e dois anos.
+Usou-o no navio de cruzeiro Taras Scevcenko, que partia de Odessa.
+Estava calor e ela enxugava o suor com um len�o bordado apertado entre os dedos brancos e resplandecentes dos an�is, ainda que agora de pouco valor.
+Quando o navio estava a passar no estreito dos Dardanelos e as c�pulas das mesquitas de Constantinopla pareciam pombas brancas numa atmosfera de palhetas douradas, apresenta-se-lhe uma velha senhora magra, elegante e �pripudrennaia� (empoada) que a submerge numa admira��o comovida, quase como se se tratasse de uma irm� ou de um familiar regressado do t�mulo.
+Esta senhora senta-se junto dela e, enquanto lhe acaricia as m�os e procura lev�-las at� �s faces magras, come�a a recordar-lhe o tempo passado, a sua beleza extraordin�ria, que se tornara uma lenda em toda a R�ssia e nas rep�blicas mais quentes.
+Recordou-lhe os invernos em Bakuriani, quando percorria os caminhos da aldeia a comer tangerinas e a macular, com as cascas, os montes de neve branca ao longo das estradas, por entre as casas de madeira.
+E os admiradores seguiam-na de longe, recolhendo aquelas cascas para ficarem qualquer coisa dela.
+Recordou-lhe quando, no fim da guerra, passava devagar na Avenida Gorki com o carro que pertencera a Goebbels.
+Falou da praia de Soci e do Castelo d'Aria, no cimo da colina de Dorzacos, quando o chefe da orquestra bebia copos de �gua gaseificada, apontou o pequeno restaurante do Ermitage em Moscovo, onde se comiam trutas.
+Os beijos que todos procuravam dar-lhe na Pra�a Vermelha no dia da vit�ria.
+O seu vestido escuro durante o funeral de Estaline, quando, apesar da multid�o que se apinhava ao longo dos muros, para ela havia espa�o, para que ningu�m sequer lhe tocasse.
+Depois, os longos anos de aus�ncia dos teatros, dos restaurantes e das praias da moda.
+Um aviso fugaz no grande mercado das aves, quando a indiferen�a da multid�o lhe permitia observar com tranquilidade um peixinho circunscrito � �gua de uma garrafa e j� ningu�m era atra�do pelo seu perfume, que ainda ent�o se evaporava da carne rosada e abundante.
+ Finalmente, eis que podia rev�-la e abra��-la.
+Subitamente, a mulher empoada recolhe todo o seu afecto nos olhos claros e avermelhados pela como��o.
+�Mas a senhora lembra-se de quando era nova?� -- pergunta-lhe de s�bito.
+Nesta onda de apertada concorr�ncia em que os bancos t�m vivido nos �ltimos anos, parece que n�o s�o apenas os clientes que por vezes ficam baralhados com tantos produtos novos, taxas de juro irrecus�veis e remunera��es estonteantes.
+O turbilh�o � de tal forma irresist�vel que at� as pr�prias institui��es ficam abananadas e entram, olimpicamente, pela via do disparate a toda a prova.
+Vem isto a prop�sito da recente experi�ncia do depositante de �um conceituado banco�.
+Segundo o relato do pr�prio, houve um dia em que foi � caixa do correio e l� encontrou o familiar envelope que regularmente o informa acerca do movimento da sua conta banc�ria.
+Parecia um pouco volumoso para t�o pouca coisa, mas logo se colocou a hip�tese de o extracto ir acompanhado de algum folheto de publicidade a um novo produto verdadeiramente arrasador.
+Aberto o envelope, a surpresa n�o podia ser maior.
+L� dentro, al�m do que era suposto l� estar, havia mais treze papelinhos.
+Nada menos que os extractos de conta de outros tantos clientes que a esta hora dever�o estranhar o facto de o seu banco n�o lhes passar cart�o acerca dos seus saldos.
+se tinha sido nomeado gestor de conta de toda aquela gente ou se era o banco que estava a desenvolver alguma ac��o de luta contra o sigilo banc�rio ...
+A eurodeputada portuguesa Maria Santos � uma das personalidades convidadas pela Confedera��o dos Sindicatos Agr�colas Bascos para as jornadas �Agricultura Viva para o Meio Rural�, que decorrem em Bilbau.
+Em debate est�o a rela��o entre os problemas dos agricultores europeus e as quest�es relacionadas com a defesa do meio ambiente, o desenvolvimento regional e os direitos dos consumidores.
+em Barcelos, o �quei recebe o Benfica, partida que poder� servir aos minhotos para recuperarem de uma campanha fraca (tr�s derrotas e um empate), enquanto os benfiquistas tentar�o manter o percurso vitorioso.
+Na outra partida, a Oliveirense joga em casa com o sensacional Pa�o d'Arcos, actualmente em quarto lugar, a tr�s pontos do l�der.
+O FC Porto n�o ter� especiais dificuldades, pois joga nas Antas com o HC Sintra.
+� primeira vista, Mousa, Mohammed, Abu Wahed e Khalid, o cabo, tinham a express�o deprimida e encurralada dos recrutas derrotados.
+Os uniformes estavam amarrotados e sujos, as botas imundas e rotas.
+Tinham a barba por fazer, estavam exaustos e famintos.
+Em o saco havia nacos de p�o, uma lata pequena de sumo e uma garrafa de �gua.
+Em Fran�a, a jornada deste fim-de-semana provocou grandes altera��es no posicionamento dos primeiros classificados, com vantagem para o Auxerre, que chegou ao primeiro lugar, e para o Nantes, que, apesar de derrotado, beneficiou dos maus resultados do Paris Saint-Germain e do M�naco e n�o abandonou o comando.
+Auxerre e Nantes dividem agora o primeiro lugar, ambos com 26 pontos, 35 golos marcados e 18 sofridos, seguidos pelo M�naco, ainda com 26 pontos, pelo Marselha, com 25 (menos um jogo), e pelo Paris Saint-Germain, com 24.
+A 19� jornada come�ou na sexta-feira, com a vit�ria do Marselha no Parque dos Princ�pes (0-1), perante a equipa treinada por Artur Jorge.
+No s�bado, o Nantes foi surpreendido no terreno do Toulouse (2-0), cedendo a terceira derrota da �poca, enquanto o Auxerre recebeu e venceu o Le Havre (4-1).
+No campo do Lyon, o M�naco n�o foi al�m de um empate sem golos, o que impediu a equipa de Rui Barros de se isolar no primeiro lugar.
+A prova de que a exibi��o de �O Convento� deixou muita gente baralhada podemos encontr�-la, por exemplo, naquele jornalista do �New York Post� que fala sempre em �scoops� -- do estilo �quem est� a filmar com quem� -- e que na confer�ncia de imprensa de apresenta��o do filme declarou: �Sr. Oliveira, o seu filme � o meu favorito de todo o festival�.
+Diga-se, tamb�m, que h� quem pense o contr�rio.
+Mas ontem, Catherine Deneuve e John Malkovich foram, de facto, as atrac��es da imprensa.
+Ao lado de Lu�s Miguel Cintra e de Leonor Silveira, apresentados como os guardi�es da casa de Oliveira, Deneuve e Malkovich eram os estranhos seduzidos pela �obscuridade� -- palavra de Deneuve, lembrando-se de Bu�uel.
+E como disse a actriz, �h� certas coisas que n�o se devem desvendar�.
+Malkovich, por seu lado falou em �primitive dream paintings�.
+Silenciosos, os guardi�es observavam.
+Foi com eles que o P�BLICO falou.
+basta-nos procurar compreender o conte�do do acordo agora assinado.
+E tendo integrado o n�cleo fundador da Plataforma -- sem estarmos sequer ligados � cis�o que, circunstancialmente, no PCP lhe deu origem --, � naturalmente sem satisfa��o que vemos a Plataforma dar raz�o �a posteriori� a �lvaro Cunhal.
+Acreditando na possibilidade e na necessidade da constru��o de uma terceira for�a na esquerda portuguesa, com uma l�gica nova de interven��o e funcionamento, dar raz�o a Cunhal � tamb�m dar raz�o a todos os que ainda n�o tiveram coragem de arriscar este projecto.
+O texto do �Protocolo de Coopera��o� � suficientemente vago e impreciso para ser in�cuo e, ao mesmo tempo, ter um significado pol�tico claro.
+Embora reafirmem a cada dois passos o car�cter estritamente aut�rquico do acordo, os dirigentes da PE subscrevem um texto que vai desde a reforma do sistema pol�tico em geral at� ao modelo de desenvolvimento para o pa�s.
+No �ltimo par�grafo das seis longas p�ginas, conclui-se por �uma vis�o convergente sobre a necessidade de reformar positivamente o sistema pol�tico no sentido de assegurar melhor democracia e melhor desenvolvimento, na concretiza��o de uma alternativa pol�tica ao PSD e ao Governo�.
+A 45 anos de governa��o do Partido Nacional, quatro dos quais sob a batuta de Frederik Willem de Klerk, dever� seguir-se agora um per�odo indetermin�vel de preponder�ncia pol�tica do Congresso Nacional Africano (ANC), criado em 1912 para conseguir o fim do racismo.
+As sondagens � opini�o p�blica indicam que o ANC, sob a lideran�a de Nelson Mandela, poder� conseguir de 58 a 60 por cento dos votos nas elei��es de Abril para a Constituinte.
+E alguns observadores v�em em semelhante vantagem o perigo de uma tenta��o hegem�nica, se bem que a Constitui��o interina a aprovar durante as pr�ximas semanas preveja a representa��o no governo de todos os partidos que consigam pelo menos cinco por cento dos votos.
+O lan�amento de est�gios para jovens desempregados ou � procura do primeiro emprego, o refor�o de incentivos j� existentes ao emprego por conta de outr�m ou � cria��o do pr�prio posto de trabalho bem como a sistematiza��o de informa��o sobre alternativas de forma��o e de profiss�es s�o as tr�s principais medidas do Programa para a Integra��o dos Jovens na Vida Activa.
+menos de um m�s ap�s ter vencido o Campeonato Nacional de Clubes em golfe, disputado no Oporto Golf Club, coloca na final do �Individual� dois elementos que contribuiram para a conquista desse t�tulo, Stephane Castro Ferreira e Jos� Sousa e Melo.
+O FC Porto conquista a sua oitava Ta�a de Portugal ao vencer o Sporting, por 2-1, no jogo da final�ssima disputado no Est�dio Nacional.
+No final do jogo, adeptos do Sporting lan�am garras e pedras para a tribuna de honra, onde estavam Manuela Ferreira Leite, ministra da Educa��o, e V�tor Vasques, presidente da FPF.
+Palma In�cio, ex-comandante operacional da LUAR, numa entrevista ao �Expresso�, afirma que n�o reconhece, aos que contra ele se colocam, envergadura moral para o ofender e lembra que o ELP de Sp�nola foi a organiza��o mais terrorista de Portugal.
+O templo, de configura��o quadrangular, com cerca de 15 metros de largo, foi descoberto quando S�rgio Coutinho, propriet�rio de um terreno, ali quis fazer um estabelecimento de turismo rural.
+Os arque�logos chamados ao local avaliaram o achado como sendo da �poca de J�lio C�sar, mas s� agora, ap�s diversas investiga��es das arque�logas Ana Arruda e Catarina Viegas, acompanhadas por t�cnicos do Instituto Portugu�s do Patrim�nio Arquitect�nico e Arqueol�gico, se chegou a um �consenso pleno�.
+Na mesma zona em que foi encontrado o templo, a Alc��ova, a caminho das Portas do Sol, foram ainda descobertas cisternas romanas que est�o tamb�m a ser objecto de escava��es e estudos arqueol�gicos.
+Para o autarca Jos� Noras, em declara��es � ag�ncia Lusa, o achado arqueol�gico �permite descodificar a presen�a romana na velha �Scalabis� e explicar a import�ncia estrat�gica de Santar�m no contexto da Pen�nsula dessa �poca�.
+O munic�pio vai agora preservar o monumento e promover o seu conhecimento por arque�logos e estudantes.
+Ontem, at� os futebolistas brasileiros Ronaldo, Roberto Carlos e Denilson ajudaram � festa (�O nosso favorito era Kuerten; agora � Rios�), aplaudindo o vencedor de p�.
+O pr�ximo advers�rio ser� o �artista� Hicham Arazi que garantiu nova presen�a nos quartos-de-final do Grand Slam franc�s.
+Vindo do anonimato em 1997 gra�as a quatro vit�rias consecutivas (a �ltima das quais sobre Marcelo Rios) em Roland Garros, Arazi (47� ATP) parece disposto a fazer melhor este ano depois de ter eliminado Alberto Berasategui (cabe�a de s�rie n�16), um dos tenistas em melhor forma esta �poca sobre a terra batida (vit�rias-derrotas: 18-4).
+�N�o tenho feito grandes resultados ultimamente, mas quando come�o a bater umas bolas aqui sinto-me logo melhor�, reconheceu o marroquino ap�s o triunfo por 6-2, 6-4, 3-6 e 6-3.
+A economia � muito, mas n�o � tudo para aferir da for�a vital que sai das entranhas de um povo.
+O sistema encontra-se dispon�vel no mercado portugu�s e faz parte do pacote proposto aos franchisados.
+Migu�ns Cardoso, da Triunfo / Il Fornaio, considera que, em termos de �software�, uma solu��o integrada e o apoio do consultor inform�tico � a solu��o mais adequada � gest�o de uma rede comercial deste g�nero.
+O Grupo apostou nesta solu��o desde 1990 e as perspectivas s�o para a abertura de uma nova loja por m�s.
+Uma associa��o de consumidores de Coimbra exigiu de novo a extin��o do Instituto do Consumidor, que acusa de ter gasto em 1994 dinheiro sem proveito vis�vel para os consumidores.
+A Deco tem uma posi��o oposta e o organismo visado limita-se a propor uma leitura atenta do relat�rio de actividades desenvolvidas.
+O Instituto do Consumidor (IC) gastou em 1994 mais de meio milh�o de contos sem resultados vis�veis, acusa a Associa��o Portuguesa de Direito do Consumo (APDC), que endere�ou uma carta ao primeiro-ministro propondo a �extin��o�, ainda antes das elei��es, daquela estrutura do Minist�rio do Ambiente.
+Cada dia que passa torna mais evidente que o voto dos eleitores do Entre Douro e Minho vai ser decisivo para o resultado final do referendo.
+� para esta regi�o que se viram os olhos dos partid�rios do �sim�, ansiosos por uma vota��o que compense a quebra anunciada pelas sondagens noutras zonas do pa�s.
+Mas � tamb�m neste espa�o que cabem as esperan�as dos que querem travar o modelo das oito regi�es.
+se o �n�o� na primeira pergunta do referendo for maiorit�rio na mais populosa das oito regi�es propostas, isso significar�, quase de certeza, a certid�o de �bito do modelo nascido do acordo entre o PS e o PCP.
+�Se isso vier a acontecer, a regionaliza��o ficar� adiada por 10 ou 15 anos�, reconhece Narciso Miranda, l�der da Federa��o do Porto do PS, que acredita numa �vit�ria esmagadora� do �sim� no Entre Douro e Minho.
+Pense-se em Kingsley Amis, Malcolm Bradbury e Albert Finney.
+Dois escritores, um actor.
+Pense-se no romance que o primeiro escreveu -- �The Green Man� --, que o segundo adaptou e o terceiro interpretou.
+Pense-se num enredo mirabolante, centrado num her�i desfeito, o anti-her�i Maurice Allington, e na maneira como o impens�vel -- o fantasma de Thomas Underhill -- o cerca.
+No fim, obt�m-se uma s�rie de televis�o.
+Uma boa s�rie de televis�o: a que a RTP estreou ontem � noite, na TV2, e � qual nada ligou -- pouco mais do que �primeiro epis�dio� escreveu na apresenta��o.
+Motor: Econ�mico, com grande elasticidade e bom n�vel de pot�ncia.
+Caixa: Mais um Toledo �salvo� por uma caixa bem escalonada, com destaque para as tr�s primeiras velocidades.
+-- � o �nico candidato que me parece capaz de promover uma mudan�a de paradigma na vida pol�tica americana.
+Clinton � mais novo na idade e nas ideias.
+Bush endureceu na atitude de conserva��o a todo o pre�o de uma ordem hist�rica condenada.
+A Am�rica tem de preparar-se para o futuro.
+Com Bush, n�o conseguir� faz�-lo.
+-- George Bush.
+-- Em primeiro lugar porque conhece os problemas europeus; em segundo lugar pelo papel que desempenhou na pol�tica mundial nos �ltimos quatro anos e, em terceiro lugar, porque o CDS � sempre mais ligado aos republicanos do que aos democratas.
+Para Eurico de Melo s� faz sentido convocar um referendo se se chegar � conclus�o da sua necessidade para cumprir �um formalismo constitucional�.
+N�o desvalorizou, por�m, a convoca��o do Conselho de Estado por entender que �n�o deve subsistir a menor d�vida sobre os formalismos constitucionais a cumprir nem sobre a vontade pol�tica de ades�o ao Tratado�.
+Eurico de Melo defende que as decis�es de Maastricht �s�o de grande import�ncia para o pa�s� na medida em que reflectem uma linha de �mais compromisso pol�tico com a CE�.
+Segundo a �Comiss�o de Afectados pela Barragem do Lindoso�, entidade promotora da concentra��o, o objectivo perseguido mant�m-se: prosseguir no seu protesto contra o que entendem ser �uma atitude de chantagem da EDP�.
+Em declara��es ao P�BLICO, um dos elementos da �Comis�o dos Afectados� manifestou esperan�as na possibilidade de reatamento do di�logo interrompido a 16 de Dezembro �ltimo, remetendo para os resultados de uma reuni�o que ir� juntar no Porto o governador civil de Ourense, o presidenta da C�mara do concelho de Lovios (Ourense) e Marques Seabra, respons�vel da EDP, e que est� prevista para sexta-feira, embora sujeita ainda a confirma��o.
+A mesma fonte afirmou que os afectados pela barragem estar�o dispostos a permitir que prossigam os trabalhos de remo��o da igreja de Aceredo, que ir� ficar submersa pela albufeira, logo que a �arbitragem acordada entre as partes seja assumida em documento assinado perante um not�rio�.
+Em v�speras de Carnaval, a Associa��o Portuguesa para a Defesa do Consumidor (DECO) mostra-se preocupada com a utiliza��o de explosivos nas brincadeiras carnavalescas, respons�vel todos os anos por in�meros acidentes, sobretudo com crian�as em idade escolar.
+Em comunicado, a DECO considera essencial a informa��o dos consumidores sobre este assunto, causador tamb�m de ru�do e perturba��o da ordem p�blica, especialmente nas escolas.
+As bombas de Carnaval pertencem ao conjunto de explosivos tecnicamente designado como bombas de arremesso, cuja venda -- regulamentada por lei -- s� pode ser feita a pessoas, com mais de 18 anos, que tenham autoriza��o das autoridades competentes para as comprar.
+Mas de faltas de correspond�ncia entre a lei e a realidade dos factos est� o pa�s cheio.
+Por exemplo, lembra tamb�m a DECO, entre os �brinquedos� preferidos pelas crian�as nos festejos carnavalescos encontram-se os estalinhos, considerados brinquedos ou artif�cios pirot�cnicos que podem rebentar por choque ou atrav�s de um detonador.
+Encontram-se em todo o lado e, no entanto, o seu fabrico � proibido por lei.
+Hoje, Bush tem na agenda as cidades de Baidoa e Bali-Dogle e mais visitas a soldados, orfanatos e organiza��es humanit�rias.
+O �nico incidente registado ontem em Mogad�scio ocorreu na embaixada francesa quando um somali tentou entrar nas instala��es e n�o obedeceu � ordem de parar de um sentinela, anunciou o comando franc�s da opera��o �Oryx�.
+O soldado disparou para o ar, mas o indiv�duo continuou a avan�ar e foi atingido mortalmente.
+Lawrence Summers evitou atacar os pa�ses do G7, mas n�o p�de deixar de falar no Jap�o, sublinhando que os EUA �n�o manipulam artificialmente as taxas de c�mbio� e que � seu desejo e de todo o mundo que este pa�s �volte a ter um crescimento r�pido�.
+O Jap�o respondeu que concorda com as sugest�es norte-americanas e que apoia uma ac��o concertada para travar a valoriza��o do iene, considerada como brutal a amb�gua -- j� que se pode conduzir � diminui��o do excedente comercial japon�s pode tamb�m p�r em causa o crescimento mundial, na opini�o do secret�rio do Tesouro dos EUA, Lloyd Bentsen.
+As multid�es n�o acorreram � abertura, domingo, da exposi��o �A Idade do Barroco em Portugal�, organizada pela National Gallery de Washington e a Secretaria de Estado da Cultura portuguesa, atrav�s do Instituto Portugu�s de Museus (IPM).
+Mas a exposi��o estar� patente at� 6 de Fevereiro e � prov�vel que, nas suas visitas de rotina aos museus, grande parte dos washingtonianos venham a visit�-la.
+�Isto � maravilhoso, � um verdadeiro tesouro, que n�o imaginei que existisse�.
+Mary Sue, 55 anos, entrou com a amiga na exposi��o, por acaso.
+Na realidade, vieram ao museu por causa da exposi��o sobre os �P�ssaros da Am�rica�, que est� nas salas ao lado.
+Mas n�o se arrependeram.
+�� um povo e uma �poca fascinantes.
+Estamos mortas por visitar Portugal�.
+Estas novas atitudes correspondem � c�lebre frase de Robert Fillion .
+�a arte � o que faz a vida parecer mais interessante que a arte�.
+Esta exposi��o nasceu, ainda no dizer de Jean de Loisy, da leitura do c�lebre artigo de Allan Kaprow: �A heran�a de Jackson Pollock�.
+Kaprow pretende que Pollock n�o teve tempo de levar as suas obras at� �s �ltimas consequ�ncias, as quais teriam sido, n�o limitar o quadro � tela posta no ch�o, mas de nele integrar o pr�prio ch�o do atelier, os objectos, os ru�dos da rua, em suma, a vida.
+Este artigo, escrito em 1957, posiciona a arte numa nova direc��o, da qual a exposi��o tenta ser o reflexo.
+Accionismo, novo realismo, �happening�, poesia sonora, Fluxus, �performance�, arte corporal, �environnements�, s�o alguns dos nomes que, segundo os per�odos e os pa�ses, foram dados a estas novas formas de arte.
+O Jap�o quer continuar a negociar com os EUA tendo em vista resolver o diferendo comercial que op�e os dois pa�ses, apesar de a amea�a norte-americana de aplica��o de san��es, caso as autoridades n�p�nicas n�o abram os seus mercados p�blicos num prazo de 60 dias.
+O ministro das Finan�as de T�quio, Masayoshi Takemura, afirmou lamentar a atitude de Washington, mas adiantou que o Jap�o continuar� a tentar ultrapassar as diferen�as entre as duas partes, em cada um dos sectores em negocia��o: autom�vel, seguros e mercados p�blicos nas �reas das telecomunica��es, equipamento m�dico, computadores e sat�lites.
+O ultimato para a abertura dos mercados p�blicos nip�nicos foi imposto por Washington no �ltimo fim-de-semana, depois do fracasso das conversa��es bilaterais.
+Em Portugal desde 23 de Maio, Manuel Ant�nio viveu em Lisboa, cerca de tr�s meses, deslocando-se depois para o Porto.
+Pelas 3h55 da passada quinta-feira foi surpreendido nas escadas interiores da 15� Esquadra da PSP, na Foz do Douro, sem ter usado a porta de entrada.
+Fonte policial admite que o jovem tenha saltado o muro das traseiras das instala��es.
+Na altura, Manuel Ant�nio alegou sede e que queria beber �gua, justificando desta forma a sua entrada na esquadra atrav�s do muro.
+Indocumentado, deu um nome trocado -- Manuel Carneiro --, disse ser mo�ambicano e que tinha os documentos numas obras.
+Esta informa��o n�o foi confirmada pelos agentes da PSP que o acompanharam a estaleiros sitos na Foz, Boavista e no Carvalhido.
+As dilig�ncias da pol�cia terminaram com a sua entrega ao Servi�o de Estrangeiros e Fronteiras (SEF).
+Junto do Consulado de Mo�ambique, o SEF apurou ser falsa a nacionalidade indicada pelo Manuel Ant�nio e descobriu os documentos no interior de uma pasta na posse de um tal Albertino.
+As autoridades verificaram que o Manuel Ant�nio tinha dois passaportes: um angolano, verdadeiro; e outro portugu�s, falsificado.
+Este �ltimo documento ter� sido adquirido no Centro Comercial Dallas, no Porto, por 15 contos e destinava--se a ser usado numa viagem a Fran�a.
+Acusado de falsifica��o de documentos, Manuel Ant�nio viu a deten��o confirmada por um juiz de instru��o, que n�o atendeu ao facto de o jovem ter menos de 16 anos, em fun��o do passaporte angolano de que era portador.
+Quatro dias mais tarde, o Minist�rio P�blico detectou o erro e ordenou a transfer�ncia do processo para o foro do Tribunal de Menores, a quem foi entregue ontem.
+A Fran�a alcan�ou, em 1995, um excedente comercial recorde de 104,5 mil milh�es de francos (3,1 mil milh�es de contos), o que representa um acr�scimo de 27 por cento em rela��o aos resultados de 1994.
+O ministro do Com�rcio Externo, Yves Galland, anunciou ainda que, no ano passado, as exporta��es francesas atingiram o montante recorde de 1427 mil milh�es de francos (cerca de 43 mil milh�es de contos), reflectindo uma alta de 9,2 por cento relativamente ao ano anterior.
+As importa��es apresentaram uma progress�o de 7,9 por cento, o que equivale a 108,6 mil milh�es de francos (3,25 mil milh�es de contos).
+O �ltimo saldo recorde do com�rcio externo havia sido atingido em 1993, com um excedente de 87 mil milh�es de francos (2,6 mil milh�es de contos).
+A Comiss�o Europeia aguarda uma notifica��o das autoridades italianas sobre o plano de reestrutura��o da companhia a�rea Alitalia.
+O principal accionista, o grupo p�blico italiano Iri, adoptou na �ltima quinta-feira as grandes linhas do plano, que prev� uma injec��o de capital na ordem dos 937 milh�es de d�lares (cerca de 142 milh�es de contos) e o refor�o da estrat�gia de alian�as.
+A Comiss�o Europeia dever� em breve tomar uma decis�o sobre a segunda parte da ajuda p�blica acordada com a companhia a�rea grega, Olympic Airways.
+Espera-se ainda a aprova��o, por parte da Comiss�o, da terceira parte do plano de ajuda � companhia francesa Air France, cujo montante global � de cerca de quatro milh�es de d�lares (cerca de 605 mil contos).
+Raramente as institui��es de planeamento falam claro.
+Mas, �s vezes, acontece.
+em segundo lugar, e trata-se de uma conclus�o menos taxativa, � de admitir que tenha diminu�do a posi��o relativa da Regi�o Norte no contexto do espa�o s�cio-econ�mico portugu�s�.
+Desde que o ministro do Planeamento e Administra��o do Territ�rio, Valente de Oliveira, se reuniu em Setembro e Outubro, em Vila Real e no Porto, com as c�maras do interior e do litoral da Regi�o Norte, que a CCRN se tem desdobrado em contactos para discutir a forma de aplica��o do pr�ximo quadro de apoio.
+Deste esfor�o, um contra-rel�gio que se destina a ouvir tanto os �rg�os de poder local como os agentes de desenvolvimento e essa entidade gen�rica a que se chama �sociedade civil�, dever� nascer a proposta nortenha para o novo Plano Regional de Desenvolvimento, � espreita dos fundos acrescidos proporcionados pela aprova��o em Edimburgo do Pacote Delors II.
+A equipa masculina concorre � Primeira Liga, no decatlo, em Helmond, na Holanda, contra as forma��es da Alemanha, Bielor�ssia, Est�nia, R�ssia, Holanda, Pol�nia e Sui�a.
+Sobem � Superliga (para o ano) as duas primeiras equipas e descem as duas �ltimas e Portugal dificilmente escapar� � despromo��o.
+� que, de facto, s� por puro acaso equipas como a alem� e a russa est�o nesta divis�o.
+Portugal alinha com M�rio Anibal Ramos (Benfica), que recentemente deixou o recorde nacional em 7614 pontos, F. Nuno Fernandes (FC Porto- 7381 pontos), Jos� Dur�o (CN Rio Maior- 6614 pontos) e Lu�s Her�dio Costa (Sporting- 6585 pontos).
+Ao mesmo tempo decorre a Superliga feminina (heptatlo), envolvendo a Bielor�ssia, Fran�a, Gr�-Bretanha, Alemanha, Holanda, Pol�nia, R�ssia e Ucr�nia, com as russas favoritas para manterem o trof�u.
+A selec��o feminina portuguesa, por seu lado, compete no heptatlo da segunda liga em Dilbeek, na B�lgica, com belgas e sui�as.
+A equipa vencedora subir� � primeira liga, mas para essa posi��o � favorita a Sui�a.
+Portugal alinha com Sandra Turpin (Benfica- 5218 pontos), S�nia Machado (individual- 5061 p), M�nica Sousa (GD Cavadas- 5122 pontos) e Catarina Rafael (Bairro Anjos- 4602 pontos).
+A maior perspectiva da parte portuguesa � a hip�tese do recorde nacional (� de 5228 pontos) por Sandra Turpin.
+�Desejava que tivessem visto a express�o no rosto do dr. Steve�, disse ao centro de controlo da miss�o Ken Bowersox, comandante do vaiv�m, escreve a ag�ncia Reuter.
+� ali no por�o do Discovery que o Hubble vai ser reparado no decurso de quatro passeios no espa�o, com seis horas de dura��o cada um, que ser�o feitos pelos astronautas Mark Lee, Steve Smith, Greg Harbaugh e Joe Tanner.
+O primeiro, o mais importante desta miss�o, estava previsto precisamente para a noite passada.
+Se tudo correu como estava previsto, fez-se a substitui��o de dois instrumentos de observa��o do Hubble -- o Goddard High Resolution Spectrometer e o Faint Object Spectrograph -- e a instala��o de dois novos instrumentos destinados a ampliar significativamente as suas capacidades de observa��o: o Space Telescope Imaging Spectrograph (STIS) e o Near Infrared Camera and Multi-Object Spectrometer (NICMOS), cada um do tamanho de uma cabine telef�nica.
+A perspectiva de as duas empresas competirem entre si sem se terem em conta os inc�modos resultantes para os aveirenses acabou por ditar uma solu��o de consenso, tendo a autarquia funcionado como intermedi�ria para o bom entendimento entre as partes.
+E a possibilidade de as ruas de Aveiro se transformarem num estaleiro permanente, com um operador a abrir valas onde o seu concorrente as tinha fechado uns dias antes, n�o seria de todo inadmiss�vel, at� porque a legisla��o permite a livre concorr�ncia entre os concession�rios da TV por cabo.
+Segundo o acordo estabelecido, a TV Cabo Mondego, que entrou j� em for�a no mercado local de assinantes, utiliza as fibras �pticas da rede telef�nica j� instaladas pela Portugal Telecom, enquanto a Pluricanal se servir�, mediante uma comparticipa��o financeira, das condutas do g�s natural.
+Para o presidente da autarquia, Celso Santos, este acordo, que ser� formalizado atrav�s de um protocolo alargado � EDP e � Lusit�niag�s, permitiu � C�mara �controlar o processo de instala��o das novas tubagens e evitou que as ruas permanecessem intransit�veis durante muito tempo�.
+O vice-presidente Andriessen participa na Reuni�o de Coordena��o da Ajuda Internacional � CEI, no Centro Cultural de Bel�m.
+A reuni�o termina amanh�.
+Reuni�o informal dos ministros da Agricultura, na Curia.
+Termina no dia 25 deste m�s.
+Ora existem muitas fun��es (ordenamento territorial, incentivos ao desenvolvimento, ambiente, turismo, cultura, vias de comunica��o, educa��o, etc.), para as quais os munic�pios s�o demasiado pequenos e o Estado demasiado distante.
+� para isso que em todos os pa�ses, grandes ou pequenos, existe uma autarquia territorial (ou mesmo duas) entre o Estado e os munic�pios.
+N�s pr�prios, desde a revolu��o liberal at� 1974, sempre tivemos acima do munic�pio ora o distrito, ora a prov�ncia.
+As regi�es administrativas n�o s�o mais do que a restaura��o da figura das prov�ncias, com atribui��es mais centradas no ordenamento territorial e no desenvolvimento.
+� evidente que as regi�es administrativas ter�o tamb�m condi��es para reivindicar uma mais equilibrada reparti��o dos recursos or�amentais, mesmo nas fun��es que h�o-de continuar a ser do foro da administra��o central.
+Nas minhas desloca��es de comboio a Lisboa n�o posso evitar um quase sentimento de revolta, quando comparo a escandalosa indig�ncia da esta��o de caminhos-de-ferro de Coimbra, mais pr�pria de um apeadeiro terceiro-mundista, com a sumptuosidade megal�mana da nova Gare do Oriente, que pelos vistos corre o risco de vir a ser o mais oneroso dos apeadeiros de luxo do mundo.
+E sou levado a pensar que a exist�ncia de regi�es poderia contribuir tamb�m para evitar estas gritantes disparidades de tratamento regional ...
+Que a posi��o do Vaticano possa ser entendida deste modo por um intelectual desta craveira -- sobretudo a ideia de um malvado voluntarismo de Deus que lhe est� subjacente -- deveria fazer reflectir os argumentadores oficiais da doutrina da Igreja.
+Parece-me que nela se continuam a misturar alhos com bugalhos e a n�o hierarquizar adequadamente nem as convic��es de f� nem as raz�es.
+Neste sentido, espero que o interessante documento de trabalho do Conselho Pontif�cio da Fam�lia, �Evolu��es demogr�ficas: dimens�es �ticas e pastorais�, Lisboa, 1994) leve uma grande volta.
+Se, como nele se diz, �a Igreja deseja encetar um di�logo construtivo com os que continuam convencidos da necessidade de realizar um controle imperativo da popula��o e com os governos e as institui��es que se ocupam de pol�ticas da popula��o, j� que existem problemas demogr�ficos reais, apesar de frequentemente serem vistos a partir de uma perspectiva errada e de se proporem solu��es depravadas para os resolver� (n�24), n�o pode favorecer os mal-entendidos.
+Esta � uma posi��o fundamental que n�o pode ser trocada por nada, sejam quais forem as chantagens dos patr�es deste mundo na Confer�ncia do Cairo ou fora dela.
+Numa reuni�o com comerciantes e moradores da vila velha, realizada ter�a-feira � tarde, os vereadores da CDU refutaram as acusa��es feitas pela presidente do munic�pio de estarem a atrasar obras importantes, garantindo n�o estar �contra ou a favor da constru��o do silo na Volta do Duche� -- que se encontra em �banho-maria� --, mas fizeram depender uma posi��o favor�vel da realiza��o de um estudo mais vasto para toda a vila que prove a necessidade daquele investimento.
+Todos os partidos pol�ticos com assento na Assembleia Municipal de Leiria contestam o Plano Estrat�gico do Eixo de Leiria-Marinha Grande e do Sistema Urbano da Alta Estremadura.
+Contactados pelo P�BLICO, os l�deres das concelhias do PP, do PSD- com a maioria da Assembleia Municipal-, do PS e o deputado municipal da CDU s�o un�nimes em considerar que o documento mandado elaborar pela Associa��o de Munic�pios da Alta Estremadura (AMAE) carece de um importanle debate p�blico, havendo mesmo quem n�o Ihe reconhe�a o estatuto de �plano estrat�gico�.
+Apesar disso, as diferentes for�as partid�rias est�o dispostas, na pr�xima reuni�o da Assembleia Municipal, sexta-feira, a criarem as condi��es para que sejam aprovados os projectos a candidatar pela C�mara de Leiria e pela associa��o de munic�pios ao Prosiurb (Programa de Consolida��o do Sistema Urbano Nacional e Apoio � execu��o dos PDM).
+Ant�nio Guterres foi o primeiro convidado de uma s�rie de debates com l�deres pol�ticos que o Inesc est� a promover.
+Ao falar no Porto, mas ligado a outros pontos do pa�s gra�as �s inova��es tecnol�gicas, o dirigente socialista defendeu ontem as suas ideias na mat�ria: ideias orientadas para �uma ci�ncia e uma investiga��o com qualidade e relev�ncia para os problemas do pa�s� e contra a �alian�a entre a mediocridade e a inveja�.
+Ainda n�o me tinha lembrado da conveni�ncia de levar uma sandesitas e uma garrafa de �gua, quando fui confrontado com a not�cia de que tal ousadia era proibida.
+Antevi-me, com ou sem sandes, a ser interpelado -- ou mesmo revistado -- por um qualquer seguran�a imbu�do de uma autoridade disparatada.
+Imaginei a quantidade (e a qualidade!) de conflitos que essa rasqueirice iria provocar.
+E n�o gostei.
+Depois, contaram-me que, em horas normal�ssimas, o acesso a certos locais -- como o passeio mar�timo, que me dizem ser dos mais apraz�veis -- passava a ser proibido, e guardado pelos correspondentes seguran�as, sem que os pr�prios soubessem explicar porqu�.
+Conhe�o o g�nero, e n�o aprecio.
+Esta medida foi conseguida com a colabora��o activa da Associa��o dos Operadores Portu�rios do Sul (AOPS) e da Associa��o dos Agentes de Navega��o e Empresas Operadoras Portu�rias (Anesul), e insere-se na pol�tica de moderniza��o do sistema de informa��es da comunidade portu�ria em Set�bal.
+Sporting-Vit�ria de Set�bal e Mar�timo ou Ovarense-FC Porto.
+Ontem, em Lisboa, foi este o resultado do sorteio das meias-finais da Ta�a de Portugal, a jogar dia 9 de Maio.
+Uma tiragem de sortes que afastou os dois grandes ainda em prova, mas que leva o Sporting a receber uma equipa que lhe roubou, em Alvalade, um ponto para o �Nacional�, enquanto o FC Porto poder� ir � Madeira defrontar a �nica equipa com quem sofreu uma derrota para o �Nacional� nesta temporada.
+Ainda assim, no final todos ficaram contentes.
+J� antes do s�culo XIV se havia iniciado o cap�tulo da hist�ria que une Portugal � famosa Flandres, cuja refer�ncia povoa os nossos livros de Hist�ria do secund�rio, marcando a mem�ria com nomes quase m�ticos, como o de Antu�rpia.
+Muito mais fant�stico do que essas men��es soltas da realidade actual � saber que as cidades m�gicas ainda existem, mesmo que por detr�s de arquitecturas e modas no trajar que j� nada t�m a ver com os tempos em que se bailava a compassos diferentes e em que as ruas se pavimentavam com pressas diversas das nossas.
+O plano tem vindo a ser executado gradualmente, atrav�s de interven��es que integram ou reintegram fun��es antigas.
+� exemplo disso a inaugura��o de uma nova resid�ncia paroquial em Novembro de 1996, o primeiro passo no sentido da reafecta��o de espa�os do monumento.
+Fundado no s�culo VI, em pleno dom�nio suevo, o Mosteiro de Santa Maria de Tib�es foi posteriormente arrasado, datando a sua reconstru��o do �ltimo ter�o do s�culo XI.
+A partir da�, foi sucessivamente remodelado e aumentado e, tal como hoje existe, � o resultado de campanhas de reconstru��o e amplia��o levadas a cabo nos s�culos XVII e XVIII, respondendo actualmente a uma linguagem mista entre o primeiro barroco e o �rocaille�, explica o Ippar.
+P.S. -- �PROJECTO DE DIPLOMA� e �Portaria� v�m em mai�sculas no texto, mas �escola� vem em min�sculas!
+A �ESCOLA de Excel�ncia� escreve-se em mai�sculas.
+O vosso subconsciente anda a trair-vos!
+Os gestores de topo das empresas japonesas est�o a aceitar o corte dos seus sal�rios, em face da s�bita recess�o que se abateu sobre a economia do pa�s, impondo-se um teste pr�tico quanto � solidariedade que caracteriza a imagem das empresas japonesas no exterior, e tentando contribuir desta forma, para que as firmas que administram possam suportar a crise actual.
+Uma vez mais o p�blico primou pela aus�ncia nas bancadas do Aut�dromo do Estoril, tornando quase �secretas� as quatro corridas ontem disputadas.
+Como que a sentirem a falta de �testemunhas�, as provas tamb�m n�o foram particularmente emocionantes, com as vit�rias decididas muito cedo.
+No Trof�u Renault Clio 16V V�tor Lopes comandou de princ�pio a fim, construindo uma s�lida lideran�a.
+Lopes terminaria com 10,017'' de avan�o sobre Jos� Jo�o Magalh�es e 12,378'' sobre Celestino Ara�jo.
+Na F�rmula Ford, Carlos Azevedo (Swift) regressou �s vit�rias, terminando com uma vantagem de 2,328'' sobre Rui �guas (Van Diemen) e 3,040'' sobre Frederico Viegas.
+Quanto ao Trof�u BMW 320iS, Jorge Petiz ultrapassou o seu irm�o Alcides a meio da corrida para obter uma vit�ria f�cil, com 2,382'' de avan�o, deixando o 3�., Ant�nio Barros, a 4,788''.
+O portugu�s Pedro Chaves (Lola / Cosworth) terminou ontem a corrida de F�rmula 3000 disputada em Hockenheim no 13�. lugar, parando logo a seguir � meta com o motor partido.
+O piloto portugu�s fez um bom arranque do 15�. posto da grelha, ganhando quatro lugares.
+�Depois comecei a ter problemas com a embraiagem e com o motor que aquecia muito.
+A partir de meio da corrida, o motor come�ou a perder pot�ncia e partiu na �ltima volta, j� perto da meta�, contou Chaves.
+Ca�do, entretanto, no 12�. lugar, o portugu�s acabou mesmo por perder uma posi��o na derradeira volta.
+A vit�ria foi para o italiano Luca Badoer (Reynard / Cosworth) que dominou a corrida de princ�pio a fim, assumindo tamb�m o comando do campeonato.
+�ltimo teorema de Fermat: agora � de vez?
+Andrew Wiles, o investigador brit�nico que anunciou prematuramente, no ano passado, ter demonstrado o �ltimo teorema do matem�tico franc�s do s�culo XVII Pierre de Fermat, talvez tenha finalmente levado este trabalho a bom termo -- noticiou o di�rio �New York Times� na semana passada.
+Rui Vilar, ex-comiss�rio da Europ�lia 91 e ex-presidente do conselho de administra��o da Caixa Geral de Dep�sitos (CGD), vai ter uma tarefa dif�cil.
+O patrim�nio da Gulbenkian � valios�ssimo, est� estimado em cerca de 260 milh�es de contos, a valores de 1994, mas a desvaloriza��o do d�lar nos �ltimos tr�s anos tem afectado significativamente os rendimentos que permitem a sua sobreviv�ncia.
+A administra��o sempre foi bastante conservadora nas suas aplica��es de capital, que se traduzem principalmente por investimentos no sector do petr�leo e pela gest�o de uma carteira de t�tulos no valor de dois bili�es de d�lares nos Estados Unidos, composta por ac��es e obriga��es.
+S� no ano de 1994, o patrim�nio da Funda��o diminu�u, em valor, cerca de 30 milh�es de contos por causa das perdas cambiais do d�lar contra o escudo.
+Recorde-se que as receitas s�o obtidas na sua quase totalidade em d�lares, que depois � necess�rio converter para a moeda nacional.
+Com �O Fim�, as r�plicas cruzadas formam uma rede ainda mais inextric�vel, um verdadeiro concerto desconcertante, mas afinado, em que, de vez em quando, as palavras parecem disparos, explos�es, rajadas de metralhadora.
+Parecem.
+Mas quem n�o se fiar nestas apar�ncias descobrir� que, nesta am�lgama de sonoridades informes -- ou nesta acumula��o de nuvens (cinzentas) com formas caprichosas -- toma forma um discurso feito de premoni��es, amea�as, trov�es e rel�mpagos, mortes anunciadas.
+Um clima abafado, pren�ncio de cat�strofes.
+Quando, na primeira cena, Tom� -- amigo e c�mplice de Mateus no assass�nio da mo�a -- fala do macaquinho que um amigo lhe vai trazer de Angola, est� a falar da arma do crime, do �macaco� com que Mateus matar� Sandra.
+E, a prop�sito do macaco, Mateus sugere que, em vez disso, tragam um tigre.
+A�, come�am a instalar-se nos di�logos as alegorias da ca�a, do ca�ador e da presa, do predador, dos carn�voras e dos omn�voros, da comida que se devora -- alegorias que dominam toda a pe�a, evocando �a priori� ou �a posteriori� o crime que � o ponto de chegada do espect�culo.
+Assim, todas as palavras com que Mateus (Manuel Wiborg) contribui para as conversas de caserna anunciam ou recordam o estupro e o assass�nio de Sandra, bem como o suic�dio com que ele se vai autopunir no final do espect�culo.
+Apesar dos esfor�os desenvolvidos pela presid�ncia luxemburguesa para conseguir o acordo da totalidade dos estados membros, a It�lia manteve a sua oposi��o ao acordo final em sinal de protesto pela redu��o da quota do leite.
+N�o possui hoje Vendas Novas pra�a de toiros, raz�o por que a corrida aconteceu numa desmont�vel, mas Ant�nio Morais, no seu livro �A Pra�a de Toiros do Campo Pequeno�, ao referir-se �s outras pra�as do pa�s, diz que Jos� Val�rio construiu uma pra�a em Vendas Novas em 1862, tendo nela havido festas de toiros at� 1875, ano em que soldados da Escola Pr�tica de Artilharia, por descuido, provocaram o seu inc�ndio.
+Regista ainda na terra a exist�ncia de uma pra�a nos anos 20 com capacidade para 3400 espectadores, que julgamos ter sido a que existiu at� h� cerca de 20 anos.
+Desta feita, redimiu-se a Escola Pr�tica, ajudando � montagem da pra�a instalada no campo da feira e nela fizeram as cortesias Jos� Maldonado Cortes, Nuno Pardal, o praticante Jos� Francisco Cortes e o amador Jos� Soudo, a quem saudamos o regresso ap�s convalescer do grav�ssimo percal�o que lhe aconteceu na pra�a da Malveira.
+Lidou-se um curro de toiros de Jos� Lu�s Sommer de Andrade, que o Grupo de Forcados amadores da Moita pegou.
+Acompanhando o comportamento dos principais mercados asi�ticos, a Bolsa de Sidney terminou a sess�o de ontem em alta pronunciada.
+Para os operadores, tamb�m aqui as valoriza��es se ficaram a dever � reentrada de novos investidores, nomeadamente fundos de investimento internacionais.
+O �ndice AOI encerrou nos 2034 pontos.
+Nesta classifica��o, destaque para nova subida do sueco Stefan Edberg, que surge na 20� posi��o, gra�as � presen�a na final de Queen's (700 mil d�lares).
+Recorde-se que Edberg, antes de Roland Garros, estava no 45� lugar, mas as excelentes exibi��es na terra batida parisiense e na relva londrina permitiram-lhe o salto.
+O alem�o Boris Becker, vencedor do Queen's, ao derrotar Edberg por 6-4, 7-6 (7-3) 11 anos depois de ter conquistado no mesmo local o seu primeiro t�tulo em relva, manteve a quarta posi��o mas diminuiu a diferen�a pontual para Andre Agassi.
+Em Rosmalen (500 mil d�lares), na Holanda e tamb�m em relva, o vencedor foi o norte-americano Richey Reneberg, que bateu na final o franc�s St�phane Simian por 6-4, 6-0.
+Espect�culo de um jovem grupo italiano, integrado no esp�rito das mais novas tend�ncias teatrais europeias.
+Cerim�nia e combate em que a m�stica grotowski-barbiana, as artes marciais orientais, a m�sica e a poesia italiana e um projecto intercultural se cruzam em boa harmonia.
+ Depois de Montemor, �vora, Beja, Coimbra e Braga, � a vez de a Grande Lisboa o ver na Damaia.
+Ocasi�o �nica.
+SALA D. JO�O V Largo da Igreja.
+Dom., 15, �s 21h45.
+O actor e realizador norte-americano Clint Eastwood, 63 anos, presidente do j�ri do Festival de Cannes (ver p. 28), defendeu ontem a entrada de uma maior variedade de filmes estrangeiros no mercado dos Estados Unidos.
+Em confer�ncia de imprensa dada conjuntamente com a vice-presidente do j�ri, a actriz francesa Catherine Deneuve, 50 anos, Eastwood confessou que, em vez de �san��es de proteccionismo�, preferiria que �uma maior variedade de filmes franceses, asi�ticos, russos� entrasse nos Estados Unidos.
+�Para alargar o horizonte do p�bico norte-americano�, explicou.
+O actor disse estar a produzir �The Stars Fell on Henrietta�, um filme realizado por James Keach, com Robert Duvall, e projecta rodar �Golf in the Kingdom�, que tem por tema o golfe, um desporto de que se confessa adepto.
+Como actor dever� desempenhar o papel de um fot�grafo em �The Bridge of Madison County�, uma pel�cula a realizar pelo australiano Bruce Beresford com base no �best seller� hom�nimo de Robert James Weller.
+Os Utah Jazz voltaram a provar que s�o os visitantes mais antip�ticos da Liga Norte-Americana de Basquetebol profissional (NBA), ao averbarem na jornada de sexta-feira o seu 11� triunfo fora de casa, melhorando o seu registo da Liga na presente temporada para 11v/4d.
+Desde Maio deste ano que Joaquim Correia, escultor natural do concelho da Marinha Grande- que doou os seus trabalhos para integrar o n�cleo do museu no Convento de Santo Agostinho- chegou mesmo a amea�ar desvincular-se do compromisso com a autarquia, caso o processo n�o fosse desbloqueado at� ao final do ano.
+Em declara��es ao P�BLICO, Joaquim Correia afirma manter a sua posi��o, apesar de considerar positiva a iniciativa agora tomada pelo IPPAR, que lhe ter� sido comunicada pela pr�pria C�mara de Leiria na passada sexta-feira.
+Joaquim Correia justificou a sua posi��o cautelosa sobre o assunto, adiantando que o processo de reconvers�o do Convento de Santo Agostinho em museu se arrasta h� oito anos.
+�Quem estraga velho paga novo�, disse V�tor Mel�cias, afirmando pretender que sejam repensadas as situa��es onde �j� houve entregas apressadas� de antigos edif�cios das Miseric�rdias, ocupados at� h� bem pouco tempo por unidades hospitalares do Estado.
+Segundo V�tor Mel�cias, at� aos anos 70, as Miseric�rdias dedicavam cerca de 90 por cento da sua actividade a ac��es na �rea da Sa�de.
+A ocupa��o dos edif�cios das Miseric�rdias por unidades hospitalares do Estado conduziu, nas �ltimas d�cadas, � reconvers�o das �reas de interven��o das Miseric�rdias portuguesas, que t�m actualmente na assist�ncia social o seu principal campo de ac��o.
+�Ainda se podem queixar se quiserem ...�, gracejou Raposo, ap�s entregar o documento.
+Foi por entre elogios un�nimes dos deputados � sua actua��o e a manifesta disponibilidade para dela �retirar os necess�rios ensinamentos� que M�rio Raposo deixou ontem o Parlamento onde foi formalizar e explicar as raz�es da sua ren�ncia ao cargo de Provedor de Justi�a.
+Poucos minutos depois do final da reuni�o que, durante cerca de tr�s horas, juntou o Provedor demission�rio e os deputados da Comiss�o Parlamentar de Direitos, Liberdades e Garantias, era divulgado um comunicado onde se real�a �a actua��o de elevado m�rito do dr. M�rio Raposo, marcada por crit�rios de independ�ncia na defesa dos direitos dos cidad�os perante a Administra��o, dela se devendo retirar os necess�rios ensinamentos para que sejam criadas as adequadas condi��es institucionais q [...]
+Jean Alesi obteve ontem a �pole position� provis�ria para o Grande Pr�mio de Espanha, que se disputa amanh� no circuito da Catalunha (Barcelona).
+O piloto franc�s foi o mais r�pido na primeira sess�o de treinos de qualifica��o, superando por 35 cent�simos de segundo o seu companheiro de equipa Gerhard Berger.
+A Ferrari conseguiu assim colocar os seus dois carros na primeira linha da grelha provis�ria, um resultado que motiva toda a equipa.
+Depois do segundo e terceiro lugares no GP de San Marino, h� duas semanas, e dos bons resultados conseguidos nos testes realizados nos dias seguintes, tamb�m no circuito de �mola (Berger foi o mais r�pido dos 16 pilotos que estiveram em pista, batendo Schumacher e Hill), a Ferrari mostra em Barcelona que a sua competitividade est� a aumentar, refor�ando a ideia de que se poder� bater em igualdade de circunst�ncias com a Benetton e a Williams, as duas equipas motorizadas pela Renault.
+�O Gerhard tem raz�o, ele tem os p�s bem assentes no ch�o.
+� preciso n�o esquecer que estes s�o os treinos de sexta-feira, n�o os de s�bado.
+Entretanto, Bernardo Vasconcelos, m�dico do Benfica, confirmou ontem que, na sequ�ncia do ocorrido no �ltimo Benfica-Sporting, Jo�o Pinto ser� operado, embora ainda n�o se conhe�a exactamente a extens�o da les�o.
+Como �h� d�vidas se existem roturas totais ou parciais dos ligamentos�, o futebolista far� agora uma artroscopia e s� depois se saber� se h� necessidade de fazer uma ligamentoplastia.
+Tamb�m n�o se sabe quando ou onde ocorrer� a interven��o cir�rgica.
+� �informa��o reservada, a pedido do jogador�.
+Um guineense e uma portuguesa que se dedicavam a angariar trabalhadores para o Kuwait, cobrando uma taxa de dez contos, foram detidos pela Pol�cia Judici�ria.
+Assim, �a Liga nem sequer pode comprovar que de facto est� a ser feito um inqu�rito�, sublinhou Pedro Vieira.
+Quanto aos 75 mil contos que a Secretaria de Estado diz agora ter disponibilizado para a opera��o de salvamento do peixe, �ningu�m sabe� onde, como e por quem foram utilizados, at� porque das centenas de toneladas ali existentes apenas se salvaram cerca de duas dezenas.
+Cumprindo a promessa de se deslocar ao Porto na primeira ter�a-feira de cada m�s, para reuni�es de trabalho com os respons�veis aut�rquicos e os agentes culturais da cidade, o ministro recebeu ontem, entre outros, o director do Teatro Nacional S. Jo�o, Ricardo Pais, a respons�vel do Teatro Rivoli, Isabel Alves Costa, e ainda representantes do Teatro Art'Imagem e do Museu de Imprensa, cuja primeira fase ser� inaugurada pelo Presidente da Rep�blica na pr�xima sexta-feira.
+�s j� habituais cr�ticas � pol�tica cavaquista no dom�nio da cultura, que considerou �economicista� e �fr�vola�, Carrilho acrescentou alguns ataques ao actual l�der do PSD.
+�Neste �ltimo ano, o que � que o PSD prop�s em mat�ria cultural?�, perguntou o ministro, aludindo ao primeiro anivers�rio da lideran�a de Marcelo Rebelo de Sousa, que se celebrava � mesma hora em Santa Maria da Feira.
+A Food and Drug Administration (FDA) -- a ag�ncia americana que fiscaliza os medicamentos e os alimentos -- pode passar a regulamentar o tabaco, em virtude dos efeitos biol�gicos desta subst�ncia.
+Quem o decidiu foi o juiz William Osteen, na sexta-feira passada, naquela que foi considerada �a mais devastadora derrota da ind�stria tabaqueira nos Estados Unidos�.
+A decis�o significa que vai vingar a inten��o da FDA de proibir a venda de tabaco a menores, obrigando as lojas a pedir os bilhetes de identidade para o efeito.
+Ser� tamb�m � FDA que caber� regulamentar, a partir de agora, as m�quinas de venda de tabaco.
+Num pa�s em que -- ainda segundo estat�sticas das Na��es Unidas -- mais de metade da popula��o tem menos de 15 anos, foi estranho para muitos observadores que o n�mero de eleitores potenciais se estimasse em 5,8 milh�es, para uma popula��o total calculada em dez.
+V�rios ajustamentos t�m sido feitos ao longo desta maratona, como no Bi�, em que o n�mero potencial de eleitores passou de 577 para 700 mil.
+Tamb�m em Benguela e no Cunene houve altera��es.
+O que se sabe, concretamente, � que neste momento o n�mero de portadores do cart�o de eleitor em Angola j� ultrapassou os 4,3 milh�es e o alargamento do prazo de registo at� 10 de Agosto permitir� recensear as pessoas que vivem nos lugares mais remotos do territ�rio.
+�Esta situa��o demonstra que o tema ambiente est� na moda.
+Quando a obra foi feita pela primeira vez, h� uns cinco anos, n�o aconteceu nada ao respons�vel.
+Agora que s� se veio agravar um mal que j� estava feito � que as pessoas se interessaram�, afirmou Jo�o Mendon�a, adiantando que o Posto 7 n�o tem nada que ver com a destrui��o da duna e esclarecendo que o bar n�o est� a ocupar ilegalmente a praia -- como sup�s um funcion�rio da Junta de Freguesia do Castelo, a que a praia do Moinho Baixo pertence.
+A licen�a de ocupa��o de dom�nio p�blico mar�timo foi emitida pela DRARN de Lisboa e Vale do Tejo em Maio passado.
+O Governo Civil tamb�m autorizou o funcionamento do bar.
+Neste momento, a DRARN n�o est� a conceder novas licen�as de ocupa��o de dom�nio p�blico, mas tem renovado licen�as antigas, segundo Vit�ria Bruno da Costa.
+Isto porque �v�o ser aplicados Planos de Ordenamento da Orla Costeira (POOC), pelo que n�o vale a pena estar a autorizar investimentos que podem vir a estar em desconformidade com os POOC�, explicou a directora.
+No cantinho da bancada central que estava �habit�vel� havia algumas caras conhecidas.
+Todas do Benfica, claro, que divide muitos jogadores com o seu clube-sat�lite.
+M�rio Wilson foi o primeiro a chegar.
+Depois, sentaram-se o guarda-redes Veiga e o belga Lucien Huth, treinador dos guardi�es �encarnados�.
+O Alverca foi feliz e conseguiu o golo logo aos 4'.
+Foi na sequ�ncia de uma jogada bonita de Ra�l, que subiu bem e deu para Ramirez.
+O mais novo refor�o do Benfica -- do Alverca, no caso -- centrou bem e, na �rea, Akw� cabeceou para dentro da baliza.
+Se a primeira volta das elei��es foi marcada por um inenarr�vel caos organizativo, a segunda correu relativamente bem.
+As chuvas atrasaram as opera��es em algumas regi�es.
+Mas o grande motivo do atraso no an�ncio dos resultados parecia ser a proximidade dos n�meros.
+Na segunda-feira � noite, segundo os dados parciais da CNE, Nino tinha uma vantagem tangencial.
+a vontade de mudan�a dividiu ao meio o eleitorado, algo de impens�vel meses atr�s.
+Um �tunnelier� � uma aut�ntica f�brica debaixo do solo.
+� composto por uma cabe�a escavadora, parecida com uma roda dentada, com pontas de carboneto de tungst�nio.
+� este material super-resistente que permite roer a rocha � m�dia de duas a tr�s voltas por minuto e avan�ar � velocidade vertiginosa de 200 metros por semana.
+Em cada volta desta roda avan�a-se 10 cent�metros e s�o trucidadas cinco toneladas de rocha.
+Depois da cabe�a vem uma aut�ntica f�brica ambulante que tritura as pedras e as envia para o exterior, ao mesmo tempo que vai enfiando uns aros em bet�o armado, imperme�veis e resistentes a uma press�o de 200 bars, que revestem o t�nel.
+O tempo da dinamite, p� e picareta j� acabou h� muito.
+Estes monstros s�o comandados e controlados por meios electr�nicos e inform�ticos.
+�Tinha que ser bonito�, invocaram os respons�veis pelo projecto Y, a nova arma compacta da Lancia para o fim do s�culo, mais um dos refinados produtos do �design� italiano, ainda que sa�do dos esbo�os de um engenheiro.
+Maior que o seu antecessor, o Y10, o Y assume formas tridimensionais e curvil�neas, sempre exclusivas e eternamente femininas.
+Esta cumplicidade de �conversa constante entre v�timas da mesma sociedade� -- na defini��o de Liberto Cruz -- doou ao teatro de marionetas o infort�nio e o orgulho da marginalidade.
+Bonecos e bonecreiros foram desde sempre perseguidos e interditados, e � sintom�tico que o dramaturgo Ant�nio Jos� da Silva, �O Judeu�, tivesse morrido na sacra fogueira da Inquisi��o -- condenado pelas pe�as para marionetas que escrevia e representava no seu Teatro do Bairro Alto.
+Os esbirros da Inquisi��o apareciam sempre aos pares, perturbando o S�culo das Luzes, a julgar pela Sala Ant�nio Jos� da Silva do Museu da Marioneta, em Lisboa.
+Um par de presen�as enlutadas fazem aqui as vezes de personagens das trevas, no meio de uma vitrina de bordados barrocos, azuis debruados e rendas de brilho transparente: os personagens das pe�as �Vida do Glorioso D. Quixote de La Mancha e do Gordo Sancho Pan�a� e �Guerras do Alecrim e da Manjerona�.
+Ambos os textos foram recuperados para a cena pela companhia Marionetas de S�o Louren�o (seguindo textos do �Judeu�), cujos fundadores s�o tamb�m os respons�veis pela constitui��o do Museu -- Helena Vaz e Jos� Gil.
+Foi anteontem empossado o grupo de trabalho interministerial que vai preparar a articula��o entre as bibliotecas escolares e as da rede de leitura p�blica.
+Chefiada pela escritora Isabel Al�ada, a equipa, constitu�da por representantes dos minist�rios da Cultura e da Educa��o, tem tr�s meses para analisar a situa��o e fazer propostas.
+�A administra��o p�blica n�o tem capacidade do ponto de vista veterin�rio para fiscalizar e punir os prevaricadores�.
+Nos termos do novo regulamento, devem ainda ser destacadas algumas inova��es.
+Estabelece-se que �n�o h� lugar � suspens�o ou demiss�o do m�dico veterin�rio coordenador ou dos executores no decurso do programa sanit�rio anual, a n�o ser por motivo de for�a maior, devidamente justificado e aceite pela direc��o regional de agricultura, com posterior homologa��o pela Direc��o Geral de Veterin�ria�.
+Consagra-se tamb�m o direito desta �ltima �efectuar visitas de inspec��o e auditoria t�cnica �s OPP e explora��es nelas integradas, bem como aos produtores individuais, impor as correc��es tidas como necess�rias ou propor medidas sancionat�rias�.
+Antes do relator-geral, falaram o cardeal Angelo Sodano, prefeito da Congrega��o para os Religiosos e os Institutos Seculares, e o arcebispo Jan Schotte, secret�rio-geral do S�nodo, que fizeram a s�ntese dos trabalhos preparat�rios.
+Durante esta semana e parte da pr�xima, os cerca de 350 participantes -- bispos, superiores de congrega��es religiosas masculinas e femininas, peritos e auditores -- estar�o reunidos na �aula sinodal�, ou sess�o plen�ria.
+As interven��es s�o feitas com base nos temas e conte�dos do �documento de trabalho�, elaborado precisamente em resultado do processo preparat�rio.
+Depois desta fase, os participantes dividem-se em �c�rculos menores�, por grupos lingu�sticos, de modo a produzir sugest�es para os documentos finais.
+�Perdoa-nos, Erich Honecker� era o t�tulo de primeira p�gina do antigo jornal do Partido Comunista sovi�tico, �Pravda, segundo o qual os actuais dirigentes da R�ssia teriam aberto um precedente para si pr�prios ao entregar o seu �antigo amigo e camarada�.
+A mulher de Honecker, Margot, abandonou ontem a embaixada chilena em Moscovo, tendo seguido directamente para Santiago do Chile, onde vive a sua filha Sonya, e n�o para Berlim como chegou a ser anunciado.
+Margot, a quem os alem�es de leste chamavam de �bruxa�, foi ministra da Educa��o da ex-RDA e actualmente est� a ser investigada por suspeita de ter for�ado v�rios dissidentes pol�ticos a entregarem os seus filhos para adop��o.
+O professor Marcelo Rebelo de Sousa, ao exigir o referendo nacional, est� a facilitar imenso a resolu��o de t�o momentoso problema.
+N�o estamos esquecidos de que o eng� Guterres fez da sua promessa da cria��o das regi�es administrativas uma das principais bandeiras da sua campanha eleitoral.
+Queremos que cumpra a promessa, mas de maneira digna e sensata.
+O tempo, quente e h�mido, impediu a obten��o de grandes marcas, mas n�o foi obst�culo ao dom�nio f�cil de Pinheiro sobre os seus mais directos advers�rios -- Pedro Pessoa, do Bairro Santiago (31m31s), Lu�s Vieira, da AA Peniche-�ptica 2000 (31m55s) e Carlos Almeida, da UR Dafundo (31m57s).
+Em senhoras, Paula Laneiro tamb�m n�o teve dificuldades em bater Umbelina Nunes, do Casal Privil�gio (267� da geral e primeira veterana), e Lu�sa Almeida, do Cruz Quebradense (295�).
+inicialmente, apareceu na lista de classifica��o como sendo Oct�vio Sousa, do Super Estrelas.
+Isto deve-se possivelmente ao facto de Pedro Pessoa ter feito a prova com dorsal trocado.
+Casos semelhantes a este s�o frequentes nas corridas de estrada que se efectuam em Portugal e dificultam e falseiam a elabora��o das classifica��es individuais e colectivas.
+Magalh�es Mota, que se confessou �muito perturbado� ao tomar conhecimento da morte do seu �grande amigo�, evocou um encontro de ambos, h� cerca de um m�s, na Buchholz, livraria lisboeta que o professor Miller Guerra frequentava assiduamente.
+R. -- N�o.
+Pe�o-lhes para desligar e tentar nova liga��o porque est�o a invadir a privacidade do meu auscultador.
+P. -- Um amigo precisa de emprego e concorre � sua empresa.
+Outra pessoa com melhor curr�culo candidata-se ao mesmo emprego.
+D� o emprego ao seu amigo?
+P. -- O �pay-tv� chega tarde a Portugal?
+R. -- Chegou um pouco atrasado porque Portugal come�ou mais tarde com a rede de cabo.
+Mas se tivermos em conta a nossa dimens�o, ningu�m em t�o pouco tempo cobriu o pa�s como n�s.
+Se estamos atrasados, � um ano ou dois.
+O incidente de ontem � grave mas n�o � o primeiro.
+V�rias vezes, nos �ltimos dois meses, afeg�os e iranianos trocaram acusa��es sobre viola��es do territ�rio ou do espa�o a�reo, alega��es nunca confirmadas por fontes independentes.
+Em Agosto, por exemplo, Teer�o anunciou uma troca de tiros, que ter� sido ef�mera e com um n�mero indeterminado de v�timas, sem conseguir no entanto provar uma agress�o.
+Lu�s de Matos apresenta 45 minutos de magia com �Pequenas Grandes Ilus�es�.
+A m�sica do Chile pelo grupo Alturas .
+A nossa participada A. DE MATOS atingiu uma factura��o de cerca de 1.150.000 contos, mais 48% do que no ano anterior.
+Na sua actividade tomou um peso importante a venda de produtos transformados, na linha de estrat�gia que fix�mos.
+�O mau tempo atrasou consideravelmente as opera��es�, disse Ajit Vardi, um dos respons�veis da coordena��o das opera��es em Bombaim, capital do estado de Maharashtra.
+Entretanto, a �ndia rompeu com a sua tradi��o, aceitando com gratid�o a ajuda material estrangeira.
+Recusou, contudo, as propostas dos governos estrangeiros de enviar para o pa�s equipas de especialistas.
+O Governo indiano considera que disp�e de meios suficientes para realizar com sucesso as opera��es de socorro, mostrando-se reticente, segundo um respons�vel da distribui��o da ajuda internacional, no que respeita a receber pessoal estrangeiro.
+ Equipas propostas pela R�ssia, �ustria, It�lia, Alemanha e Su��a estavam a postos para partir para a �ndia.
+Mas o estado de alerta foi suspenso no s�bado com a justifica��o de que j� era demasiado tarde para encontrar sobreviventes e que a �ndia n�o fizera um apelo oficial, considerando suficientes os seus pr�prios recursos.
+Em rela��o � situa��o da Fonte da Telha e �s demoli��es que ali j� se efectuaram, Mar�al Pina afirma n�o saber explicar por que � que parte do entulho se mant�m no local, remetendo a explica��o para o Minist�rio do Ambiente.
+Minist�rio que, ali�s, acusa de ser culpado de parte dos problemas que se vivem na Costa da Caparica.
+O vereador do Ambiente da C�mara Municipal de Almada, CMA, Jos� Lu�s Leit�o, por seu turno, refere que a limpeza das matas que envolvem a zona da Fonte da Telha, que � feita, dentro do poss�vel, pela autarquia, deveria ser compet�ncia do Instituto de Conserva��o da Natureza, ICN, pois a �rea est� sob a sua al�ada, atrav�s da tutela do Minist�rio do Ambiente.
+-- �The Edge�, realizado por um neozeland�s em Hollywood, Lee Tamahori, e baseado numa pe�a de David Mamet, acabou de estrear.�The Edge�, realizado por um neozeland�s em Hollywood, Lee Tamahori, e baseado numa pe�a de David Mamet, acabou de estrear.
+Junta Anthony Hopkins e Alec Baldwin no Alaska, depois de um desastre de avi�o.
+Hopkins � um bilion�rio, e pensa que Baldwin, um fot�grafo de moda, est� a dormir com a sua mulher, Elle MacPherson, �top model� a tentar que a sua carreira cinematogr�fica pegue.
+Hopkins diz que, ao contr�rio dos seus filmes anteriores, que �eram t�o divertidos quanto ver tinta a secar� e �bons exerc�cios de disciplina e economia e tudo isso�, este � um filme do tipo que lhe apetece fazer agora -- ac��o.
+-- �U-Turn�, de Oliver Stone, com Sean Penn, tamb�m acabou de ser distribu�do nos ecr�s americanos.
+Penn tem um problema no seu carro que o obriga a parar numa pequena cidade de loucos no meio do Arizona.
+De l� n�o vai conseguir sair.
+N�o vai ser um filme que obrigar� Stone a pedir desculpa a dezenas de pessoas nas pr�ximas dezenas de anos, como ele ainda hoje anda a fazer depois de �J.F. K .� e de �Assassinos Natos�.
+Os procedimentos de registo, complicados e morosos, devem s� por si afugentar muitos ciganos, j� de si com baixos n�veis de alfabetiza��o.
+E os interessados t�m de apresentar �s autoridades checas documentos obtidos em tr�s reparti��es eslovacas diferentes.
+�O objectivo da lei � livrarmo-nos de habitantes inconvenientes�, explica Michal Pulo, presidente da maior organiza��o de ciganos da Rep�blica Checa.
+Jorge Sampaio formaliza a sua candidatura � Presid�ncia da Rep�blica, na Reitoria da Universidade Cl�ssica de Lisboa.
+Terceiro anivers�rio da assinatura do Tratado de Maastricht pelos ministros dos Neg�cios Estrangeiros e das Finan�as da Comunidade Europeia.
+A solu��o encontrada parece ser a do encaminhamento da popula��o escolar para os centros de sa�de, entregue aos cuidados do respectivo m�dico de fam�lia, mas capacitando estes servi�os do seu papel crescente na sa�de escolar.
+Se n�o houver, de facto, uma substancial altera��o na conduta dos centros de sa�de nesta �rea, a sa�de escolar tender� a ser o que j� � fora dos concelhos de Lisboa, Coimbra e Porto: a medicina curativa das urg�ncias hospitalares, a que os pais recorrem quando o filho est� doente.
+Representantes das 60 fam�lias que ocupam ilegalmente casas na urbaniza��o do Vale de Arcena, delegados do Centro Regional de Seguran�a Social, respons�veis da C�mara Municipal de Vila Franca de Xira e da empresa propriet�ria dos fogos, a Eurocapital, reuniram-se, na sexta-feira, naquela cidade, para discutirem a forma como ser� feito o levantamento individual que determinar� quais os moradores que ali poder�o continuar a residir no caso de terem dispon�veis as verbas pedidas por cada casa.
+Nesta avenida de luxo, cortada longitudinalmente por um canal, perfilam-se, de um lado, os bancos (al�m dos alem�es, cerca de cem bancos estrangeiros operam na capital da Vestef�lia); do outro, os estabelecimentos comerciais -- Cartier, Armani, Gucci, Van Cleef, etc.
+Ourivesarias que exp�em nas suas montras conjuntos de pulseiras e an�is com pre�os superiores a meia d�zia de milhares de contos, casas de moda com casacos de homem a custar mais de duas centenas de contos.
+Cheira a dinheiro em D�sseldorf, uma cidade em que, mesmo nas horas de ponta (e apesar do tr�fico intenso), as �nicas bichas s�o as dos imigrantes turcos junto ao seu consulado.
+Uma cidade onde at� h� carros de lixo com monitores de televis�o em vez de espelhos retrovisores.
+Jos� Greg�rio (�Grego�) � o novo campe�o nacional de surf, t�tulo conquistado durante a II Semana Radical da Beira Litoral, que terminou no �ltimo s�bado na Figueira da Foz.
+O surfista da APSSOC ganhou a final�ssima, onde estiveram os oito primeiros no conjunto das duas mangas.
+A APSSOC, com tr�s atletas nos quatro primeiro lugares, � a campe� nacional de clubes.
+Entretanto, est� a disputar-se a Ta�a de Portugal, competi��o por equipas, na praia Grande, at� o dia 12 de Agosto.
+Paulo Silva (�Fluorescente�) foi o vencedor do Campeonato Nacional de bodyboard, que tamb�m se realizou na Figueira da Foz.
+Apesar do quinto lugar nas duas mangas, Paulo Silva, da Quinta dos Lombos, venceu a final�ssima.
+O Surfing Clube de Portugal, de S�o Pedro do Estoril, venceu por equipas.
+Na praia Grande, j� terminou a Ta�a de Portugal de bodyboard, disputada por equipas e a vit�ria coube ao Surfing Clube da Caparica.
+Subir ao monte Garizim, no primeiro dia de neve do ano, quando as pedras e os abetos se escondem sob um manto de neve, � ascender ao para�so, t�o grande � a beleza celestial da paisagem.
+Mas Shalom Cohen quer descer do cume da �montanha dos bem-aventurados� para um lugar mais terreno.
+O seu desejo � sentar-se no Conselho Palestiniano, onde Yasser Arafat reservou uma cadeira para um dos tr�s candidatos do �mais pequeno e mais antigo povo do mundo� -- os samaritanos.
+S�o cerca de 600 almas, 284 a viver em Nablus, a maior cidade da Cisjord�nia, e 297 a residir em Holon, pr�ximo de Telavive.
+Arafat sempre defendeu um Estado multi�tnico e, ao apoiar os descendentes de uma das tribos de Israel, a de Levi, filho de Jacob, ele mostra que tamb�m os judeus ser�o bem-vindos numa futura Palestina.
+Al�m disso, ao proteger os samaritanos de Nablus, mostra que vai mais longe do que o Estado hebraico, que n�o d� aos de Holon a oportunidade de estarem representados no Knesset (parlamento).
+A Espanha vai investir at� ao fim do ano 67,6 mil milh�es de pesetas em infra-estruturas ferrovi�rias.
+Mas, ap�s o esfor�o destinado ao Sul, com a linha de alta velocidade Madrid-Sevilha, em 1992, as prioridades de 1995 apontam para o Leste e Norte.
+O chamado �corredor mediterr�neo� (Valencia-Barcelona) e a linha de alta velocidade Barcelona-Narbonne representam metade do investimento total em infra-estruturas de transporte ferrovi�rio.
+A UNITA declarou ontem que o seu l�der, Jonas Savimbi, n�o aceitar� o cargo de vice-presidente que lhe � proposto nos acordos de paz de Angola, preferindo viabilizar um Governo de unidade nacional desempenhando o papel de l�der de uma oposi��o leal, com direito a ser consultado pelo Presidente sobre todas as quest�es nacionais.
+�As pessoas no MPLA temem que a UNITA queira romper o processo, estando deliberadamente a provocar atrasos at� Novembro, data em que expira o mandato do Governo�, disse � Reuter o chefe dos negociadores da UNITA em Luanda, Isa�as Samakuva, explicando que tinha entregue ao Governo e � ONU um documento com propostas concretas.
+Mas �os atrasos correspondem �s dificuldades que temos enfrentado�, acrescentou Samakuva.
+�O Governo tem capitalizado sobre o que n�s n�o fizemos, o que s� contribuiu para enfraquecer a lideran�a da UNITA, dificultando-lhe o cumprimento das tarefas restantes�.
+�A seguran�a � a antecipa��o a todos os n�veis para ver, ao longe, e ser-se capaz de tudo prever -- um mal-estar, uma sacanice dos outros.
+N�o conhe�o um s� piloto que n�o conduza � sua velocidade de seguran�a.
+Segundo o artigo 1� do C�digo da Estrada, � aquela que permite ter o dom�nio da viatura e de parar ao m�nimo imponder�vel.
+Baseado na sua experi�ncia e em testes realizados sob controlo m�dico, Jean-Pierre Beltoise afirma que a maior parte dos condutores n�o tem a menor necessidade de parar de duas em duas horas.
+Antes pelo contr�rio: uma paragem ma�adora pode fazer baixar a vigil�ncia.
+Em contrapartida, se se sentir com dores nas costas ou se um enorme desejo de ir apanhar malmequeres o estiverem a massacrar, uma pausa relan�ar� a energia.
+Uma cochilada?
+Tudo bem, se for capaz de passar em poucos minutos da vig�lia ao sono e vice-versa.
+Especialistas em luta anti-terrorista e no complexo �puzzle� de grupos e grup�sculos afeg�os identificam � partida duas organiza��es como as mais suscept�veis de fornecerem treino a jovens idos de outros pontos do mundo.
+E isto porque possuem um conceito �internacionalista� da �guerra santa� contra o Ocidente.
+ o outro � o Markaz Al-Dawat, organiza��o extremista wahabita.
+Os �le�es� de Faro e de Lisboa lutaram ontem muito no Est�dio de S�o Luiz, mas n�o conseguiram marcar.
+Os da casa est�o ainda muito longe de uma boa forma f�sica e a partir dos 25 minutos do primeiro tempo refugiou-se quase sempre na defesa.
+O Sporting dominou, alguns jogadores voltaram a mostrar pormenores interessantes, mas faltou-lhes intelig�ncia no futebol ofensivo.
+Um empate que acaba por ser justo, para duas equipas que precisam ainda de ser muito trabalhadas.
+O Sporting iniciou o jogo com o habitual 4x4x2 de Oct�vio, mas com algumas novidades em rela��o � temporada passada.
+Saber ocupou a direita da defesa, enquanto l� mais na frente Yordanov jogou na esquerda do ataque, cabendo a Hadjy alinhar no meio no apoio ao avan�ado Leandro, ficando a direita a cargo de Pedro Barbosa.
+Lang subiu muito no terreno, ficando Oceano sozinho no apoio � defesa, facto que valorizou o futebol ofensivo dos �le�es�.
+Alberto Jo�o Jardim foi a Porto Santo para conferir posse ao delegado do Governo Regional na ilha, Jos� Rosado.
+O dirigente do PSD da Madeira aproveitou a ocasi�o para dizer que, �at� 1985, n�o houve qualquer desenvolvimento empenhado (no arquip�lago) por iniciativa de Lisboa�.
+Para Alberto Jo�o Jardim, �s� a autonomia pol�tica da Madeira e de Porto Santo juntos permitiu grandes saltos nos �ltimos anos2 e seria �suic�dio para Porto Santo se essa solidariedade e unidade fossem quebradas�.
+O chefe do Governo regional fez apelo ao investimento privado para assinalar que �ao sector p�blico n�o cabe resolver todos os problemas da ilha�.
+Jo�o Alberto da Rocha P�ris, um dos dezanove diplomatas �despromovidos� pelo recente Acord�o do Supremo Tribunal Administrativo, que p�s em causa as promo��es efectuadas em 1987, apresenta hoje as credenciais ao Presidente angolano Jos� Eduardo dos Santos.
+O novo embaixador de Portugal em Luanda tem 46 anos, iniciou a carreira diplom�tica em 1969 e desempenhava as fun��es de director-geral para a Coopera��o.
+N�o � que o s�bio das matrizes encontrou, no PSD, entusiastas seguidores?
+A morgue do Hospital de Beja foi privatizada!
+J� s� falta a Paz eterna ...
+Ainda por cima em Beja, um dos ber�os do colectivismo em Portugal!
+aposto que ele n�o partilharia os m�todos seguidos pelos seus disc�pulos do Baixo Alentejo e era bem capaz de organizar concursos p�blicos abertos ...
+O aval de 600 mil contos do Governo � UGT foi posteriormente abordado por Cunha Rodrigues.
+O procurador adiantou que j� lhe foi entregue um parecer sobre o caso, mas s� dever� pronunciar-se dentro de 15 dias.
+Foi perempt�rio, no entanto, ao afirmar que, se o Supremo Tribunal Administrativo declarar a anula��o do empr�stimo, a central sindical tem de devolver o dinheiro, mesmo que este j� tenha sido gasto.
+A selec��o nacional de futebol e as eventuais agress�es a prostitutas durante o est�gio efectuado antes do jogo com a Irlanda, no final de 1995, tamb�m foi abordada.
+Cunha Rodrigues diz que tudo est� nas m�os do procurador do Tribunal de Cascais, de quem aguarda um relat�rio para breve.
+A extrema direita fez uma campanha eleitoral �moderna�, servindo-se do processo �direct mailing� americano enviando os seus panfletos xen�fobos, do tipo �Hamburgo tem de permanecer um estado alem�o�, pelo correio, em especial a milhares de jovens eleitores.
+E, por ironia do destino, fazem-no a coberto da lei eleitoral hamburguesa que autoriza os servi�os municipalizados a fornecerem as moradas de qualquer habitante, por cinco marcos cada.
+Esta legisla��o, altamente contestada pelos servi�os de protec��o de dados, n�o foi alterada a tempo pelas autoridades da cidade.
+�Uma das coisas que mais me impressionaram foi a desertifica��o do pa�s no interior.
+E todos os esfor�os que t�m sido feitos n�o t�m dado resultado�, disse o Presidente, que desculpou a actua��o dos seus governos ao afirmar que �hoje temos melhores condi��es e melhores t�cnicos do que t�nhamos�.
+Valente de Oliveira, que falou antes do Presidente, foi h�bil ao fazer uma interven��o que quase esvaziava o discurso de Soares.
+�mas estes problemas est�o a ser resolvidos�.
+Quando Borges morreu, a criada apareceu com outro testamento, que era o que desaparecera e que ele tinha procurado antes de partir para Genebra.
+Por isso, Borges fez novo testamento.
+Um e outro dizem a mesma coisa, n�o h� qualquer problema jur�dico quanto a isso, a �nica diferen�a � que, no primeiro, ele deixa uma determinada soma a Fanny e, no segundo, essa soma � um pouco menor.
+Voc� fala constantemente daqueles que se dizem �amigos de Borges�, dos que lhe vinham pedir que �escrevesse os seus pr�logos�, e garante que a est�o a acossar.
+Quem s�o?
+Os primeiros judeus s�rios autorizados a emigrar pelo Presidente Assad j� chegaram aos EUA.
+Em Damasco, outros querem seguir o exemplo, aproveitando este novo privil�gio que amanh� lhes pode ser negado.
+Tchernomirdin est� a cozinhar um Governo com o Parlamento.
+Sabe que se n�o chegar a acordo, � chumbado.
+A R�ssia caminha para uma coliga��o.
+A maioria comunista ter� uma palavra a dizer.
+A nata dos reformistas foi exclu�da ou auto-excluiu-se.
+Os mercados inquietam-se.
+Ieltsin garantiu a Clinton que as reformas ser�o uma prioridade.
+Mas o Presidente est� cercado por pedidos de ren�ncia.
+O Parlamento Europeu condenou ontem a decis�o anunciada pela multinacional Seagate de encerrar a f�brica de Palmela, que ir� provocar o despedimento de 850 trabalhadores.
+A resolu��o aprovada pelos deputados europeus foi apresentada pelo grupo da coliga��o de esquerda e subscrita pelos tr�s deputados comunistas, Barros Moura, Miranda da Silva e S�rgio Ribeiro.
+Considerando que a f�brica est� tecnologicamente bem apretechada, com elevados n�veis de produtividade e com uma excelente situa��o financeira, os parlamentares insistiram na necessidade da Comunidade e dos estados-membros exigirem aos parceiros comerciais, respeito das condi��es sociais m�nimas, tendo por base as conven��es e recomenda��es da Organiza��o Internacional do Trabalho (OIT).
+A Fran�a est� em vias de tomar medidas para a concretiza��o de um embargo comercial ao Haiti, tendo j� pedido aos seus parceiros comunit�rios que acompanhem Paris na tomada destas medidas.
+Um porta-voz do minist�rio franc�s precisou ainda que o Governo est� em vias de congelar todos os bens p�blicos haitianos.
+O ministro dos Neg�cios Estrangeiros, Roland Dumas anunciou ontem, que a Fran�a vai aplicar as decis�es tomadas pela Organiza��o dos Estados Americanos (OEA).
+Decis�es estas referentes � imposi��o de um embargo comercial, at� que o presidente do Haiti, Jean-Bertand Aristide volte ao poder.
+a �m�sica� verbal parece pedir a correspond�ncia com a m�sica do violoncelo, e uma e a outra s�o associadas a ideias e sentimentos de ru�na e ruptura, embora a sua produ��o e perfei��o impliquem, talvez, um al�vio e uma catarse.
+e a sua forma, ou a sonoridade que produz, � expressa por imagens bem vis�veis ou concretas: pontes, arcos, arcadas, bala�stres; rio, caudais, sorvedouro; barcos, lemes, mastros; urnas, blocos de gelo.
+6 Ali�s, h� no poema uma alus�o expl�cita � cor (�brancos os arcos�, para n�o falar na impl�cita dos �alabastros� e dos �blocos de gelos�) e v�rias imagens cin�ticas: �de que esvoa�am�, �por baixo passam�, �se despeda�am�, �caudais de choro�, �tr�mulos astros� ...
+O rond�, ou �rondel� (como lhe chamou o poeta), �Viola chinesa� imita e fala de um som lento, mon�tono, fastidioso, id�ntico ao de uma �parlenda� ou de uma �lengalenga�, que todavia repercute subitamente na consci�ncia do enunciador, ou no seu cora��o, tornando vis�vel uma sua �cicatriz melindrosa� e permitindo a distens�o das suas �asitas�.
+A rela��o cambial entre o escudo e a peseta n�o tem constitu�do mat�ria pac�fica nos �ltimos meses.
+Por um lado est�o os empres�rios, defensores de uma maior estabilidade cambial entre as duas divisas.
+Opini�o diferente t�m por vezes as autoridades monet�rias nacionais, empenhadas em fazer passar a ideia de que o escudo n�o tem sempre de seguir a evolu��o da moeda espanhola.
+Mas os agentes econ�micos e o mercado consideram que h� uma liga��o inevit�vel entre as duas moedas, enquanto o Banco de Portugal e o Minist�rio das Finan�as v�o reafirmando a autonomia cambial do escudo, embora na pr�tica sigam parcialmente a peseta.
+Este tema � alvo de uma an�lise no �ltimo relat�rio da SAER -- Sociedade de Avalia��o de Empresas e Risco, entidade ligada ao ex-ministro das Finan�as Ern�ni Lopes.
+Para a SAER, a recente aprecia��o do escudo face � peseta poder� n�o ter sido a decis�o mais correcta.
+Isto porque, �nestas condi��es, e atendendo ainda a que Portugal e Espanha produzem e vendem, grosso modo, a mesma gama de bens e servi�os, deixar o escudo apreciar-se relativamente � peseta reduz a capacidade concorrencial das nossas empresas no pa�s, em Espanha e em terceiros mercados, ao mesmo tempo que facilita a penetra��o das exporta��es espanholas no mercado nacional e em mercados nossos clientes�.
+Os principais opositores do Presidente russo, Boris Ieltsin, v�o reunir-se hoje em Moscovo para definirem uma estrat�gia que os leve ao poder.
+Foi pelo menos essa a inten��o do ex-vice-Presidente Aleksandr Rutskoi e do l�der do Partido Comunista, Guennadi Ziuganov, quando convocaram o encontro que vai juntar, no centro parlamentar em Moscovo, a oposi��o nacional-comunista e as for�as conservadoras que recusaram assinar, em Abril, o pacto de paz civil de Ieltsin.
+A peseta continuou no topo do mecanismo de taxas de c�mbio do SME onde a divisa portuguesa ocupa agora o terceiro lugar.
+O Banco de Portugal n�o interveio no mercado interbanc�rio.
+A taxa m�dia do �overnight� desceu para 4,949 por cento.
+No mercado da d�vida p�blica foram colocados 15 milh�es de contos de Bilhetes do Tesouro a 182 dias, � taxa m�dia ponderada de 4,4593 por cento, contra 4,4887 conseguidos no �ltimo leil�o da mesma maturidade, realizado a 15 de Janeiro.
+A cria��o da comiss�o de inqu�rito da AMS foi motivada por uma den�ncia de moradores de S�o Marcos, freguesia de Agualva-Cac�m, de altera��es efectuadas nas cartas de ordenamento do PDM, ap�s a sua aprova��o pelos deputados municipais, em que espa�os antes urbanos passaram a zonas industriais.
+Funcion�rios da fiscaliza��o municipal foram tamb�m acusados de aconselharem residentes a venderem as suas propriedades.
+Na proposta de relat�rio elaborada pelo relator, Silvino Teixeira (CDU), constata-se que desde 1982, in�cio da elabora��o do PDM, at� Mar�o de 1993, a autarquia �esteve totalmente alheada da g�nese, defini��o, estudo e acompanhamento� do plano.
+Mas a situa��o n�o melhorou ap�s a aprova��o pela AMS, em Abril de 1994, pois as linhas mestras do ordenamento do concelho n�o foram enviadas para ratifica��o governamental, como a presidente anunciara, mas ter�o ficado �nas gavetas ou prateleiras da C�mara�.
+Mas para al�m dos temas desse �lbum, e em particular da vers�o minimalista de �All apologies�, um original de Kurt Cobain que Sin�ad adoptou j� depois da morte do l�der dos Nirvana, espera-se ainda a interpreta��o de algumas das outras can��es que a tornaram notada, em particular as do �lbum �I Do Not Want What I Haven't Got� (que continha �Nothing Compares 2 U�), j� que as do trabalho que se seguiu, �Am I Not Your Girl?�, constitu�do apenas por vers�es de velhos cl�ssicos da m�sica popu [...]
+Num pa�s com mais de sete mil ilhas e onde os golpes de Estado se foram sucedendo quase mensalmente nos anos 80 e onde a guerrilha fundamentalista isl�mica avan�a na zona meridional do arquip�lago, os incidentes de ontem permitiram � comiss�o de elei��es dizer que �o dia foi relativamente calmo�.
+Os cinco mortos e 26 feridos foram v�timas aparentes da guerrilha independentista mu�ulmana, que atacou com fogo de morteiro v�rias localidades nas ilhas de Mindanao, Sulu e Jolo, de maioria mu�ulmana.
+Nas elei��es locais de 1988, ocorreram 149 mortos, e nas presidenciais de 1992 o balan�o foi de 63 mortos.
+No Porto, a gare do aeroporto de Pedras Rubras era ontem pequena demais para albergar a confus�o de gente que a inundou.
+Havia bichas para tudo, desde a tabacaria at� aos bares, onde as provis�es mostravam uma perigosa tend�ncia para se esgotar.
+As pessoas viam-se obrigadas a improvisar lugares para esperar os avi�es que se iam atrasando.
+Atrasos que, segundo as informa��es recolhidas junto dos balc�es da TAP e da ANA, n�o se deviam � greve dos pilotos, mas ao elevado volume de tr�fego e a um elemento que tem mais poder sobre os c�us que os pilotos: o nevoeiro.
+Os passageiros que se preparavam para partir, na sua esmagadora maioria, para f�rias n�o se mostravam muito preocupados com os atrasos que nalguns casos ultrapassavam as tr�s horas.
+E quanto � t�o falada quest�o de seguran�a, pelo menos Andreia Silva, de partida para Dublin, garantia que �n�o havia meio de transporte mais seguro� e aludia ao acidente da madrugada na portagem dos Carvalhos para exemplificar como viajar por terra � mais perigoso do que pelo ar.
+Mesmo o �Gil�, a mascote da EXPO 98, de bra�os abertos junto � zona das partidas, parecia concordar com ela.
+Sem os necess�rios e competentes requisitos t�cnicos e a adequada experi�ncia para o exerc�cio daquele complexo e dif�cil cargo, dadas as suas ex�guas habilita��es, e sem um m�nimo de conhecimentos de gest�o, este mesmo senhor tem vindo a �gerir� a dita institui��o de maneira desastrada, discricion�ria e in�bil, transformando mesmo aquela j�ia do patrim�nio duriense numa casa arruinada!
+� confrangedor olharmos hoje em dia para os meios de comunica��o social e verificarmos o caos financeiro em que se encontra atolada aquela casa de t�o nobres tradi��es.
+S�o not�cias de penhoras, pelos tribunais, execu��es fiscais, montes de d�vidas � banca e � Seguran�a Social, esbanjamento de v�rios milh�es em neg�cios pouco claros com a Real Companhia Velha, etc. etc.
+Uma gest�o calamitosa e incrivelmente ruinosa que os vinicultores durienses deveriam responsabilizar criminalmente!
+Mas o que � mais espantoso no meio de tudo isto � que toda esta bagun�a provocada pelo senhor Mesquita Montes tem contado com a complac�ncia dos sucessivos governos, nomeadamente os seus v�rios ministros e secret�rios de Estado da tutela!
+Tem contado tamb�m com o completo alheamento do Parlamento, que t�o pressuroso se mostra, por vezes, a discutir assuntos de �lana caprina� e nada faz para esclarecer este monstruoso esc�ndalo!
+E o mau exemplo dos Estados Unidos impede que a maior pot�ncia militar possa herdar a tarefa de controlar os conflitos que persistem ou estejam a caminho.
+A partir de agora, as guerras como a de Angola ou da B�snia s�o praticamente imposs�veis de travar de fora e qualquer interven��o -- cir�rgica ou n�o, em nome da ONU ou n�o -- acarreta um grau de incerteza t�o grande que, ao ser accionada, n�o s� pode eternizar os fogos que se pretendiam extinguir como ainda desencadear inc�ndios em zonas dantes poupadas.
+Foi j� em tom de festa que a Comiss�o Central da Queima das Fitas de Coimbra anunciou o pr�-programa da grande celebra��o dos estudantes deste ano, que ter� pela primeira vez uma mascote: um morcego de ar matreiro, que em sete �poses� diferentes retrata o estudante ao longo dos sete longos dias da Queima.
+a Comiss�o Central continua a contar com a participa��o da Sec��o de Fado, como exige o regulamento, afirmando desconhecer �oficialmente� a decis�o daquela de n�o participar no programa da Queima das Fitas/96.
+Tudo por causa da atribui��o a outro grupo acad�mico, a Fan-Farra, de uma actividade que os Fados consideram sua: o Festival de Tunas (ver P�BLICO de 23/2).
+Nuno Guerra, presidente da Comiss�o Central, minimizou o diferendo entre as duas estruturas e afirmou que �a linha orientadora � trabalhar com a Sec��o de Fado�, pelo que o convite vai ser feito, como � h�bito.
+S� se o Fado n�o aceitar � que a Comiss�o assume a realiza��o dos dois eventos.
+Problema ser� conseguir fazer uma Serenata Monumental sem os fadistas da Academia, que entretanto prometem fazer uma serenata paralela, no mesmo dia e � mesma hora, e em local onde os estudantes n�o poder�o ignorar.
+No entanto, ap�s esta corrida ao d�lar durante a primeira metade da semana, o mercado optou por uma posi��o mais defensiva que provocou um ligeiro recuo do d�lar face � generalidade das restantes divisas.
+Os factores que contribu�ram fortemente para este movimento de correc��o foram, por um lado, os receios de interven��es por parte da Reserva Federal travando a r�pida aprecia��o da paridade d�lar / iene e, por outro lado, o facto de o Bundesbank ter deixado uma vez mais as taxas de desconto e lombarda inalteradas respectivamente nos 5.75 por cento . e 6.75 por cento.
+LLoyd Bentsen, secret�rio do Tesouro norte-americano, afirmou, num discurso proferido na quinta-feira, que um iene fraco n�o � solu��o aceit�vel para os problemas econ�micos no Jap�o e insistiu na necessidade urgente de um est�mulo efectivo que atenue o enorme excedente comercial japon�s.
+Foi com duras cr�ticas ao presidente da C�mara de Vila Real, Manuel Martins, que o �ex-n�mero dois� desta autarquia, Caseiro Marques (PSD), renunciou esta semana ao seu mandato.
+Na sua opini�o, Manuel Martins, que � tamb�m l�der da Concelhia do PSD local e que agora se apresenta novamente como candidato � C�mara, �afunilou o PSD� a n�vel local e �n�o est� � altura de conduzir este partido para o futuro�.
+At� � conclus�o deste complicado processo, a CGD dever� continuar a suspender a execu��o das d�vidas, excepto para as situa��es de abandono.
+Este acordo surge ap�s anos de press�es pol�ticas e sociais para resolver a situa��o irregular das 234 fam�lias que podiam ficar na rua, uma vez que a Caixa Geral de Dep�sitos tinha iniciado autos de penhora.
+A Urbaniza��o da Bela Vista foi constru�da no in�cio dos anos 80, no �mbito de um contrato de desenvolvimento de habita��o celebrado entre o antigo Fundo de Fomento da Habita��o (actual IGAPHE), a Caixa Geral de Dep�sitos e a empresa construtora Amadeu Gaud�ncio, tendo na altura a C�mara Municipal do Montijo favorecido a aquisi��o de um terreno e procedido � isen��o de taxas por ser um contrato de habita��o social com custos limitados.
+O processo de execu��o fiscal vai passar a ser exclusivamente aplicado na cobran�a de d�vidas ao Estado e a outras pessoas de direito p�blico, de acordo com o decreto-lei 241/93 de 8/7/93.
+O fim desta norma � evitar a execu��o fiscal de d�vidas de organismos sujeitos a um regime de gest�o privada, como a Caixa Geral de Dep�sitos.
+Mas n�o ser� aplic�vel aos processos pendentes.
+Relativamente ao FEF do presente ano (um �bolo� global de 253 milh�es de contos), as autarquias portuguesas dever�o, assim, receber no pr�ximo ano mais quase 18 milh�es de contos.
+Um crescimento que, todavia, n�o deixa a ANMP satisfeita.
+�Lamentamos profundamente que as verbas para as autarquias voltem a ser atribu�das em fun��o de uma lei desadequada e injusta, mas infelizmente os partidos n�o se entenderam quanto � promulga��o de uma lei que cumpra a Constitui��o da Rep�blica e que promova uma distribui��o equitativa dos dinheiros do Estado�, criticou M�rio de Almeida.
+Pelo seu lado, Jos� Augusto de Carvalho preferiu sublinhar que o aumento supera em muito o valor previsto para a infla��o no pr�ximo ano (dois por cento), acrescentando que o actual Governo tem cumprido a Lei das Finan�as Locais em vigor.
+�A Associa��o Nacional de Munic�pios Portugueses pode ter outra leitura, mas esta realidade � inequ�voca�, acrescentou.
+Das v�speras da institucionaliza��o do Estado Novo � queda da cadeira que afastou Salazar, meio milhar de cartas trocadas entre o ditador e o seu delfim, que �ilumina zonas vitais de um longo e complexo relacionamento que foi feito de solidariedade e de conflitualidade, de intensidade e de distancia��o, de sintonia e de diverg�ncia�.
+Com pr�vio enquadramento hist�rico.
+Nos pavilh�es da FIL, os visitantes poder�o apreciar a variedade da oferta de embarca��es a motor -- a lancha Campion Marine, de 18 p�s, por 3200 contos; a Status 180, um arrojado exemplar de 18 p�s, com atrelado, a 4730 contos; ou a sofisticada linha italiana Cranchi, especialmente a de 36 p�s e a Azimute, o mais luxuoso �fly bridge� do mundo, que custa 50 mil contos.
+Entre os extremos, v�-se tamb�m uma colec��o de embarca��es da tradicional Bayliner, a refinada Caravelle, Celebrity, Searay, Argus, Chaparral, Rinker, Fairline, Gobbi, Sealine, Nimbus e Cobalt, a �ltima considerada o �Rolls Royce� dos mares.
+Como atrac��es extra, apresenta-se o Balt, um pequeno barco de 5,30 metros, com motor de 50 cavalos, vocacionado para o uso em rios, por 3679 contos, e a Cobia, uma linha de barcos para pesca com consola central.
+Nos veleiros, as op��es n�o s�o tantas, sendo a principal novidade representada pelo Jeanneau 24, de 24 p�s, que se pretende organizar em classe nacional para futuros campeonatos, al�m dos populares e pr�ticos veleiros McGregor e dos conhecidos GibSea, barcos de cruzeiro seguros e confort�veis.
+Em termos de navega��o, h� uma oferta vasta e actual de equipamentos de conv�s e electr�nicos, especialmente sondas (com um novo modelo que permite a visualiza��o do fundo, al�m da proa, o Echopilot Incastec) e GPS (Global Posiotioning System).
+Desde o Ver�o de 1993 mais de sete mil xiitas iraquianos fugiram para o Ir�o, atravessando imensos lama�ais, para escapar �s persegui��es e ataques do Ex�rcito do presidente Saddam Hussein.
+Calcula-se que mais de 20 mil pessoas tenham abandonado as suas vilas e aldeias e se dirijam para �reas pr�ximas da fronteira.
+J� em territ�rio iraquiano contam -- em relatos que t�m surgido nos �ltimos meses, sobretudo na imprensa brit�nica -- o pesadelo que deixaram para tr�s: quil�metros e quil�metros de terrenos lamacentos onde anteriormente existiam p�ntanos, rios, lagos, culturas agr�colas; aldeias destru�das, popula��es massacradas, animais e plantas mortos em consequ�ncia do que se suspeita serem ataques com armas qu�micas.
+As bases de dados piratas existentes em Portugal dedicam-se � afixa��o e difus�o de c�pias ilegais de programas sujeitos a protec��o, de resultados de experi�ncias de penetra��o em sistemas inform�ticos, de informa��o acerca de protec��es, de �portas de acesso� e de �passwords� de entrada nas redes Telenet, Telepac, Itapac e, em alguns casos, n�meros de cart�es de cr�dito.
+Algumas das chamadas �portas de entrada� vendidas eram da Lusa, do Laborat�rio Nacional de Engenharia Civil (LNEC) e da empresa Aeroportos e Navega��o A�rea (ANA).
+A heterogeneidade dos alunos que nos dias de hoje chegam � escola determina diferentes atitudes e, obviamente, diferentes tipos de indisciplinas.
+Durante o debate foram claramente distinguidos dois deles: aquele que � t�pico do ensino preparat�rio e das classes baixas e aquele que se verifica mais no secund�rio e que � protagonizado em conjunto por jovens das classes m�dias-altas, que se sentem protegidos pela sua origem.
+Momentos antes deste verifica��o, Albino Esteves, professor da Secund�ria de Clara de Resende, enunciava medidas preventivas para a indisciplina na escola: o olhar reprovador do professor, o sil�ncio t�ctico, a sobrevaloriza��o dos pequenos sucessos dos alunos mais fracos e o acompanhamento discreto das crises dos alunos.
+De um memorando de Jorge Ferreira sobre a elei��o anual dos l�deres dos grupos parlamentares.
+Poder� dar alguns exemplos concretos do programa que tinha apresentado?
+N�o foi verdade, por exemplo, que os pr�prios livreiros se manifestaram contra a realiza��o de concertos e debates, considerando que isso era desmobilizador dos potenciais compradores?
+Isso n�o pode ser dito fora de contexto.
+fizemos um palco suspenso e as pessoas que convid�mos tamb�m eram um chamariz para o imenso p�blico que acorreu ao local.
+Protestaram, com alguma raz�o, porque os concertos foram previstos tardiamente, visto que as elei��es para a APEL foram muito em cima da data da feira.
+Mas as pessoas acabaram por admitir que t�nhamos feito o nosso melhor.
+Numa reuni�o no final da feira, pedimos desculpa pelo ocorrido e fomos perdoados e ovacionados, porque foi considerada uma iniciativa interessante.
+Na altura, assumi o compromisso de este ano ser dialogante com as pessoas; de preparar a feira com tempo, de tomar nota de um conjunto de cr�ticas.
+Mas pensava que esse modelo seria de manter em 1996.
+Por exemplo, uma das minhas propostas inclu�a um concerto com M�rio Laginha e Pedro Burmester, e tamb�m a vinda de Eunice Mu�oz ...
+O Comiss�rio europeu para o audiovisual apelou anteontem a uma �mudan�a de atitude� da ind�stria europeia do sector, de modo a adaptar-se � globaliza��o mundial das redes de distribui��o.
+A preparar um Livro Verde sobre a pol�tica europeia para o sector, Jo�o de Deus Pinheiro, falando num semin�rio sobre os media, considerou que a Uni�o Europeia devia �encorajar o desenvolvimento de novas e mais eficazes redes de comunica��o transeuropeias�.
+A TVI-Televis�o Independente e a TDF-Telediffusion de France fundaram uma sociedade de teledifus�o para operarem em Portugal, a RETI, que ter� sede social em Lisboa.
+O acordo foi assinado no in�cio do m�s e, na nova empresa, a TVI det�m 55% do capital e a TDF 45%, podendo o capital vir a ter mais participa��es portuguesas.
+Est� prevista a cobertura de 80% do territ�rio portugu�s, atrav�s de 19 esta��es repetidoras.
+As associa��es servem para prestar servi�os � sociedade empresarial e aos fins dos empres�rios.
+Mas, se me pergunta se o parque de feiras de Lisboa ser� mais rent�vel que o do Porto, eu pessoalmente digo-lhe que estou convencido que sim.
+Desde logo por causa do grande afluxo de p�blico ...
+O nome diz tudo.
+O nome do disco, e o nome do grupo que o assina.
+A colec��o completa dos singles dos Pet Shop Boys � um daqueles registos que o consumidor �vido de m�sica pop n�o pode deixar de receber de bra�os abertos.
+Imaginem s� :18 can��es de puro sumo pop, sem bafio nem conservantes.
+Se o formato � sint�tico -- tanto na f�rmula da can��o utilizada pelo grupo como pela sonoridade que a envolve --, neste caso o palavr�o s� tem conota��es positivas.
+Porque Neil Tennant e Chris Lowe t�m, de facto, uma capacidade extraordin�ria para surpreender e agradar.
+Porque sabem ser concisos, sem falsas simplicidades, porque s�o inteligentes, sem armar grandes espalhafatos, porque conseguem ironizar e autoparodiar-se sem fazer figuras tristes.
+�A luta � de todos, o que prejudica os colegas, prejudica-me a mim�, disse Maria Teresa Feio, educadora de inf�ncia de Albergaria, enquanto se preparava para participar na manifesta��o organizada pela Federa��o Nacional de Professores (Fenprof), ontem, em Lisboa.
+Acabou a �America's Cup� de 1992, viva a de 1995.
+Os americanos conseguiram defender a posse do trof�u, mas os desafiantes prometem regressar mais fortes, daqui por tr�s anos, � ba�a de San Diego.
+os cubanos podem simplesmente n�o acreditar que os Estados Unidos os retenham perpetuamente em Guantanamo, ou que os enviem para outros pa�ses, e ent�o a fuga continuar�.
+O est�mulo imediato para a mudan�a de pol�tica foi o receio de que a Florida fosse invadida por uma vaga de refugiados -- uma possibilidade que traria s�rios perigos pol�ticos.
+a incapacidade de Jimmy Carter em conter a vaga de embarca��es vindas de Mariel em 1980 contribuiu para a sua derrota a favor de Ronald Reagan.
+Clinton, cuja pol�tica externa tem uma pobre reputa��o, ficaria seguramente mais vulner�vel �s cr�ticas.
+� claro que nesta perspectiva as autoridades espanholas tudo tentar�o fazer para integrar o pelot�o da frente, cen�rio sem d�vida mais favor�vel a Portugal.
+Mas as coisas em Espanha n�o est�o nada f�ceis.
+A derrapagem da economia acentua-se.
+um em cada cinco espanh�is est�o sem emprego.
+O Governo socialista previa que essa fasquia fosse atingida apenas no final deste ano, depois de um forte aperto na economia, mas, supreendentemente, foi j� alcan�ado.
+Com este estado de coisas, iniciou-se uma forte press�o dos mercados sobre a peseta.
+O Banco de Espanha foi chamado a intervir e come�arem os rumores de que a peseta poderia tomar o caminho da libra e da lira e abandonar o SME.
+Calcula-se o que poderia posteriormente suceder ao escudo, sabendo-se, como se sabe, que os t�cnicos do Comit� Monet�rio aconselham uma desvaloriza��o substancial da moeda nacional.
+Como em Fran�a o clima � tamb�m de incertezas, agravado pelas elei��es marcadas para meados do pr�ximo m�s, a Europa passa nas pr�ximas semanas por um per�odo periclitante.
+T�o periclitante, que se sobrevier nova tempestade cambial, o SME muito simplesmente poder� acabar e dar o lugar � tal UEM acelerada para as economias mais fortes, e mais lenta para os que ainda t�m muito que convergir.
+cada unidade � uma unidade.
+O processo do desejo e da sua imposs�vel miss�o.
+Ao entregar um cargo equiparado a minist�rio � representante de um partido historicamente avesso aos compromissos pol�ticos, Itamar Franco abriu uma crise sem precedentes no Partido dos Trabalhadores (PT).
+�Eu curvei-me �s raz�es do Presidente�, disse Luiza Erundina, que contrariou o veto p�blico de Lula � participa��o do PT no Governo e o pr�prio partido, que j� havia declarado oficialmente a sua oposi��o a Itamar Franco.
+Eleita para suceder ao ex-presidente J�nio Quadros na Prefeitura de S�o Paulo em 1988, Luiza Erundina, de 58 anos, geriu at� ao final do ano passado o terceiro maior or�amento do Brasil.
+�Fui oposi��o a vida toda.
+�s vezes cheguei a ser radical e sect�ria, mas aprendi que a gente s� interfere se estiver dentro do barco�, declarou a ex-prefeita � imprensa paulista, ao justificar a sua presente decis�o.
+O mundo est� a tornar-se �amigo das mulheres�, mas as salas dos conselhos de administra��o n�o.
+Menos de tr�s por cento das empresas ocidentais s�o dirigidas por mulheres, revela um estudo da Organiza��o Internacional do Trabalho (OIT), ontem divulgado em Genebra.
+O rep�rter da �R�dio Elmo� de Pinhel, Craveiro Lopes, foi condenado em tribunal a 14 meses de pris�o e 60 dias de multa por ter criticado a actua��o da PSP, informou ontem a Lusa.
+A pena foi suspensa por 28 meses, atendendo � conduta do arguido anteriormente aos factos e por ter demonstrado �algum arrependimento por os haver praticado�.
+O rep�rter foi accionado por, em cr�nicas emitidas em Julho de 1992, ter criticado o comandante do posto da PSP, subchefe Amadeu Eiras e o guarda Ant�nio Soares, cr�ticas que o Minist�rio P�blico considerou difamat�rias.
+De acordo com o processo, Craveiro Lopes fora detido, meses antes, por aqueles pol�cias �por factos suscept�veis de integrarem il�cito criminal�, ap�s o que ter� dito aos microfones ser �inconceb�vel que, nas m�os de homens sem qualquer escr�pulo, se deposite poder�.
+Em tribunal, o rep�rter afirmou n�o ter tido inten��o de difamar, �mas t�o s� de informar�, n�o pretendendo �atingir a honra e considera��o dos visados�.
+O tribunal considerou, por�m, que as afirma��es de Craveiro Lopes �s�o objectiva e subjectivamente difamat�rias e suscept�veis de atingir sen�o a ess�ncia da personalidade destes, pelo menos o patrim�nio do seu bom nome, do cr�dito e da confian�a por eles adquiridos enquanto pessoas, cidad�os e agentes da autoridade numa comunidade pequena onde todos se conhecem�.
+Em declara��es � Lusa, Craveiro Lopes disse que se limitou a tornar �p�blicas situa��es de persegui��o que ainda perduram�.
+�Se hoje tiver de fazer e dizer o que disse, n�o hesitarei�.
+O presidente da Associa��o Portuguesa de Surdos advertiu ontem que o desenvolvimento da maioria da popula��o surda �deixa muito a desejar�.
+Falando � Lusa do I Congresso Nacional de Surdos, que decorrer� durante o fim de semana em Coimbra, Jos� Bettencourt considerou que �os m�todos de ensino t�m feito tudo ao contr�rio e os resultados est�o � vista�.
+Chamou a aten��o para o facto de �sendo a l�ngua gestual pura e rica do ponto de vista gramatical e lingu�stico, e constantemente ignorada pela sociedade, importa alertar para a sua import�ncia na educa��o, forma��o e na plena integra��o na vida activa�.
+Sem rejeitar o m�todo oralista, Jos� Bettencourt defende que o sistema educacional da crian�a surda deve contemplar l�ngua gestual, escrita e falada.
+A isto, precisou, �damos o nome de comunica��o total�.
+A quest�o dos televisores estereof�nicos tem agora alguma actualidade, uns vez que muitos dos consumidores que compraram aparelhos com essas caracter�sticas antes do in�cio das emiss�es em est�reo s�o agora confrontados com o facto de os seus televisores n�o emitirem em est�reo o sinal que recebem das emissoras.
+Vindo directamente do Algarve, de avi�o, chegou �s instala��es da Igreja ao volante de um Mercedes e vestindo uma �t-shirt�.
+Vestiu depois um fato azul escuro, com gravata vermelha e camisa branca de riscas cinzentas.
+E �culos escuros -- porque estava �muito cansado� -- que tirou para o fot�grafo.
+J� se definiu uma vez como �mo�o de recados� de Deus.
+Qual foi o primeiro recado que recebeu?
+Dentro dos filmes (curtas e m�dias metragens de 17 pa�ses) sobre tem�ticas actuais destaca-se a presen�a de �Black Harvest� de Bob Connoly (Austr�lia), �Aspen� de Frederick Wiseman (EUA), �Face Value� de Johan Van Der Keuken (Holanda) e �Das Ungehobelte Pack� de Nana Swiczinsky (�ustria).
+A retrospectiva / concurso de cinema documental asi�tico inclui ainda a homenagem ao cineasta sueco Stefan Jarl, � qual a Cinemateca Portuguesa adere apresentando duas pel�culas suas.
+Outras quatro obras de Jarl estar�o presentes no festival, simbolizando �a met�dica abordagem de um dos cineastas suecos mais experimentais, mais radicais, tra�ando sempre novas vias no documentalismo�, como salienta Manuel Costa e Silva.
+Tamb�m Portugal se encontra representado com a estreia mundial de �Cren�as e Lendas� de Jo�o Soares Tavares, obra inscrita na retrospectiva RTP, homenageada nestes III Encontros.
+�Nas d�cadas de 60 e 70, a produ��o documentarista em Portugal tinha um ritmo diferente e raizes de produ��o diversas, devido � estrutura das sess�es de cinema que permitia o visionamento de um bloco de filmes curtos e de notici�rios antes da apresenta��o das obras ficcionais do programa.
+Assim, as empresas apostavam em pequenos filmes de prest�gio, longe dos esfor�os de canaliza��o de publicidade que a televis�o hoje mobiliza, para serem projectados nas salas de cinema e esses document�rios passaram a ser, para essa gera��o, a grande escola inici�tica para quem desejava seguir a aventura do cinema�, relembra Manuel Costa e Silva.
+A fase II do terminal �multipurpose�, o novo porto de pesca com a respectiva lota, o sistema de comando e controlo dos terminais petrol�fero e petroqu�mico, diversos acessos rodo-ferrovi�rios e a beneficia��o e tratamento paisag�sticos da zona marginal de Sines, investimentos rondando os 4,7 milh�es de contos, foram as infra-estruturas que Azevedo Soares inaugurou.
+O ministro -- que referiu representarem as receitas do porto de Sines 12 por cento do produto interno bruto do Alentejo, assegurando cerca de 4200 postos de trabalho -- elogiou a �boa colabora��o� entre a autarquia local, a C�mara de Sines, e a Administra��o do Porto de Sines, APS, afirmando que �em Sines disse-se pouco e fez-se muito�, numa indirecta aos problemas entre a Administra��o do Porto de Lisboa e a c�mara da capital.
+Porque � que isto aconteceu?
+N�o � porque todos os condutores fiquem subitamente atacados de uma febre Fittipaldi quando se sentem como que rodeados por algod�o doce.
+O que acontece, explica Robert Snowden, que estuda a percep��o visual do movimento h� mais de dez anos, � que quando est� nevoeiro vemos o mundo com menos contraste.
+ �Quanto maior o contraste detectado pelos nossos olhos, mais r�pido as coisas parecem andar�, diz o investigador.
+Isto porque os nossos olhos confundem facilmente uma mudan�a de velocidade com uma altera��o do contraste.
+� que as c�lulas localizadas numa zona do c�rebro chamada ��rea visual 5� s�o t�o sens�veis a uma como ao outro e a linha de fronteira entre estas duas situa��es � t�nue.
+�As pessoas que t�m esta �rea do c�rebro danificada n�o s�o capazes de ver o mundo em movimento.
+Para elas, est� tudo sempre parado, numa imagem muito difusa�, explicou Snowden.
+-- de toda a maneira, n�o aceito, enquanto coordenador, pela Antena 1, dos referidos programas, as suas afirma��es de �academismo�, �montagem que deixa muito a desejar�, �sopor�fero�, e a refer�ncia �s �bibliotecas� como espa�o a evitar por quem trabalha em r�dio -- refer�ncia, no m�nimo, curiosa.
+Com participa��o de elementos do grupo, neste caso T� Pereira, ser� lan�ado pela Tribal um CD s� com trabalhos da Kaos (os temas inclu�dos dever�o repartir-se entre os primeiros lan�amentos e os mais recentes) misturados numa �megamix� de T� Pereira.
+Ainda deste, aqui sob o nome de DJ Vibe, ir� ser lan�ado j� em Janeiro um maxi-single j� anunciado pela Kaos, a que se juntar�o os dos Urban Dreams e T� Ricciardi.
+Depois disso, a Kaos tem prevista a estreia do LL Project, do DJ Lu�s Leite, de um novo projecto do Porto designado Algo R�tmico, um novo maxi dos Ozone, e �lbuns destes dois �ltimos projectos.
+Em finais de Fevereiro, princ�pios de Mar�o, ser� editada uma segunda compila��o da Kaos que vai incluir todos os lan�amentos n�o contidos em �Totally Kaos�, mais tr�s edi��es exteriores � editora -- um projecto chamado Duplex de um portugu�s radicado na Alemanha, e mais dois em negocia��o -- e ainda uma remix pelos USL para �Bottom heavy� de Danny Tegnalia.
+A explora��o do �aparthotel�, que ter� 134 quartos -- todos eles com �kitchenete� e distribu�dos ao longo de oito pisos (entre o segundo e o nono) -- caber� � cadeia Orion, pertencente aos s�cios franceses do grupo Amorim, maiorit�rios na Inogi.
+J� a explora��o das �reas comerciais, que se v�o distribuir por tr�s pisos -- o r�s-do-ch�o, o primeiro piso e um �mezzanino� -- est� ainda por definir a quem ir� ser atribu�da.
+�Estamos em negocia��es com v�rias empresas, entre elas a FNAC, mas n�o h� ainda neg�cio fechado com nenhuma delas�, disse ao P�BLICO Almeida Guerra.
+O Decreto Regulamentar 19/91 (B) descobriu que a factura e a guia de remessa permitem controlar o efectivo car�cter, p�blico ou particular, do transporte, pelo que acabou com a guia de transporte.
+Menos papelada � sempre bom, ainda que saiba a pouco.
+Se quiser telefonar -- servi�o manual -- para o Transkei prepare-se para pagar 51000 por minuto, o mesmo que, por exemplo, para o Afeganist�o, Qatar, G�mbia, Brunei ou Honk-Kong.
+J� se quiser falar, ainda por servi�o manual, para Trindade e Tobago, Tortola, Estados Unidos, Jamaica ou Austr�lia, o custo por minuto ser� de 34000.
+Um H�rcules C 130 da For�a A�rea Portuguesa seguiu ontem para Mo�ambique e poder� vir a servir, no s�bado ou no domingo, para o transporte de S�o Tom� para Lisboa de mais um grupo de estrangeiros que entretanto sejam retirados do Huambo.
+Na sua viagem para Maputo, o avi�o militar portugu�s transportou 27 militares mo�ambicanos, dois cabo-verdianos e seis s�o-tomenses, que participaram em ac��es de forma��o em Portugal, para al�m de algum material destinado aos portugueses do batalh�o de comunica��es (BT4) que integra a miss�o das Na��es Unidas em Mo�ambique (Onumoz).
+O gabinete do primeiro-ministro vai apresentar queixa na Alta Autoridade para a Comunica��o Social contra o jornal �Seman�rio�, que ontem titulou em primeira p�gina que �Cavaco deu 2,5 milh�es de contos ao PSD�.
+De acordo com a nota oficiosa emitida pelo gabinete de Cavaco Silva, o referido t�tulo � �completamente falso e abusivo� porque �o primeiro-ministro n�o d� dinheiro aos partidos�, e associa o teor da not�cia a um procedimento legal � luz da lei de financiamento dos partidos.
+Em termos de unidades vendidas, se se venderam cerca de 145 mil LP de pre�o m�ximo, em CD, na mesma escala de pre�o, venderam-se quase 405 mil -- uma discrep�ncia que se acentua na factura��o (176 mil contos em LP de pre�o �top� contra cerca de 817 mil contos em CD tamb�m de �top�), devido ao segundo formato ser vendido sensivelmente pelo dobro do primeiro.
+A cassete do mesmo escal�o continua a n�o ir muito bem.
+Vendendo menos que o vinil: pouco mais de 103 mil, o que corresponde a uma factura��o de cerca de 124 mil contos.
+Os discos mais vendidos s�o os que chegaram ao fim do ano passado j� com maior n�mero de galard�es.
+� o caso por excel�ncia de �Waking Up The Neighbours�, de Bryan Adams, que agora chegou a sextuplo de platina (cada disco de platina equivale � venda de 40 mil unidades).
+As colect�neas de �xitos dos Bee Gees, Queen e Tina Turner tamb�m recolheram mais um galard�o de platina.
+O �nico grupo portugu�s que alcan�ou este estatuto foram os Onda Choc com �Ela S� Quer, S� Pensa Em Namorar�.
+O �lbum de estreia dos Resist�ncia tamb�m j� � disco de platina, mas ainda n�o consta nas contas do trimestre, porque s� o alcan�ou em Abril.
+P. -- Face aos dados de que disp�e, continua a acreditar em poder alcan�ar a maioria absoluta?
+R. -- Continuo.
+P. -- J� percebemos a sua cautela.
+Define a maioria absoluta como um objectivo, mas, se n�o a atingir, isso tamb�m n�o ser� para si uma derrota ...
+O Comit� Central do PCP est� reunido para analisar a situa��o na Uni�o Sovi�tica.
+Em cima da mesa, ainda a situa��o interna do partido.
+Mas os dirigentes ir�o tamb�m poder ler um documento sa�do de uma reuni�o em que o in�dito aconteceu.
+Militantes comunistas encontraram-se em p�blico e exigiram a antecipa��o do XIV Congresso.
+Querem novos dirigentes, nova ideologia, novo programa, novos estatutos.
+Em suma, um novo partido.
+Est�o dispostos a lutar por isso, mas v�o fazer uma pausa at� 6 de Outubro.
+Com uma carteira de t�tulos obrigat�ria para o estudo da obra nemesiana -- �Vitorino Nem�sio.A Obra e o Homem�, ed. Arc�dia, 1978, �Temas Nemesianos�, ed. Angra, 1981, e �Vitorino Nem�sio -- � Luz do Verbo�, ed. Vega, 1988, al�m de ter prefaciado v�rios livros para a Imprensa Nacional-Casa da Moeda, que est� a publicar as Obras Completas --, Martins Garcia encontrou-s pela primeira vez com Vitorino Nem�sio num exame.
+Foi em 1960, na Faculdade de Letras de Lisboa.
+Actualmente a leccionar Teoria da Literatura e Literatura Portuguesa na Universidade dos A�ores (doutorou-se, em 1985, com a tese �Fernando Pessoa: ' Cora��o Despeda�ado ' �), tem sido -- a par de David Mour�o-Ferreira, Ant�nio Manuel Machado Pires, F�tima Morna e Maria Margarida Maia Gouveia -- um dos investigadores que mais tem contribu�do para que Nem�sio n�o caia no esquecimento.
+Contactado pelo P�BLICO, o m�dico confirma ter entregue os medicamentos �a um senhor que se calhar facilitou e despejou [ o lixo ] de qualquer maneira�.
+Esta �, para David Paiva, uma justifica��o suficiente para n�o se considerar �directamente respons�vel� pela situa��o criada.
+Mesmo depois de ter sido informado que, ao in�cio da tarde de ontem, o P�BLICO observou tr�s crian�as -- o Ant�nio, de 8 anos, o Tiago, de 12 anos e o Lu�s, de 10 anos --, todos residentes no bairro camar�rio da Pasteleira a brincar no meio do lixo.
+�Est�vamos s� a mexer nas latas.
+N�o tocamos nos rem�dios� afirmou Lu�s.
+Mas o Bruno, de 10 anos, e tamb�m residente no Bairro da Pasteleira, revelou que o lixo �tinha l� umas seringas que depois pusemos a arder�.
+Apesar da evid�ncia do perigo, David Paiva, limita-se a notar que �n�o se tratam de lixos do tipo hospitalar.
+S�o coisas secas que j� est�o fora da validade�.
+Mas para o respons�vel pelos Servi�os de Fiscaliza��o da C�mara do Porto, Jos� Adriano, os produtos encontrados �representam sempre perigo, mesmo que n�o estejam no prazo de validade�.
+�S�o coisas que j� est�o de tal maneira fora de validade que j� n�o t�m ac��o farmacol�gica�.
+Ou seja, podem n�o cumprir o fim para o qual s�o indicadas mas n�o deixam, por isso, de constituir perigo se forem ingeridos.
+�Penso que n�o porque o tempo tamb�m desactiva os medicamentos�, remata, sem certezas, o m�dico.
+Lu�s Afonso -- Quando comecei a fazer o Bartoon, em 1993, tinha essa ang�stia.
+Mais do que isso, vivia aterrorizado com o facto de ter de arranjar uma ideia todos os dias.
+Hoje vivo em paz com esta rela��o.
+P. -- Criaste condi��es para isso, j� que vives em Serpa.
+Rodeaste-te de algum material especial, como foi?
+O cineasta americano Steven Spielberg apresentou na quinta-feira, em estreia mundial, num p�tio de Liceu Sophie Charlotte, em Berlim, na presen�a de uma plateia de jovens alunos impressionados e atentos, um document�rio bastante emotivo sobre os sobreviventes do holocausto, um CD-Rom, intitulado �O Futuro da Educa��o�.
+�� uma maravilha estar aqui convosco� disse o autor da �Lista de Schindler�, que foi agraciado com a Grande Cruz de M�rito, a mais alta distin��o alem�.
+Porque o �debate� foi um pantanal de lugares-comuns e de manobras de divers�o, no qual os oradores, em vez de encararem objectivamente a realidade dos factos, procuraram, antes de mais, a minimiza��o do papel dos outros.
+At� o moderador do debate contribuiu para o arraial, dando ares de perdi��o que sugeriam a imagem dos negociadores comunit�rios face � impossibilidade de um cessar-fogo na B�snia.
+�E a quota do tomate�, perguntou, a dada altura do caos em que se transformou a discuss�o sobre uma reforma que apenas contempla os sectores das culturas arvenses, do leite, da carne bovina e do tabaco.
+mostrou que, apesar das boas palavras, a capitaliza��o de protagonismo pol�tico est� acima da discuss�o s�ria e prof�cua.
+e o homem da CNA, repetiu a confrangedora ideia que o documento inicial da reforma protegia as explora��es extensivas e que o documento final as penalizava, quando, afinal, poucas linhas se alteraram nesta quest�o.
+A conquista de mercados parece ser, no entanto, a preocupa��o imediata da Soporcel para os pr�ximos tempos.
+At� � data, a rede de distribui��o da Wiggins Teape �demonstrou que a nossa op��o estrat�gica tem sido a mais correcta�, considerou �lvaro Barreto, real�ando a necessidade de um parceiro externo que opere nos mercados mundiais do papel.
+ Al�m disso, a Soporcel est� em vias de adquirir uma �importante posi��o� numa distribuidora espanhola, divulgou o presidente do Conselho de Administra��o da Soporcel.
+Os mercados da Europa do Sul s�o para j� os alvos estrat�gicos para a distribui��o do papel produzido pela empresa, principalmente os dos pa�ses Ib�ricos �que s�o os que mostram mais elevada taxa de crescimento�, cerca de 4,5 por cento ao ano.
+Os produtos fabricados s�o, fundamentalmente, o papel de c�pia, o papel de impress�o �offset� e os pap�is de computador, que representam os mercados mais promissores.
+Lu�sa Senos, chefe de Divis�o de Sismologia do Instituto de Meteorologia (IM), relatou os �ltimos progressos da rede nacional de sism�grafos.
+Gerida pelo IM, a rede funcionava desde os anos 70 com nove esta��es anal�gicas em Portugal continental, 11 nos A�ores, uma na Madeira e outra em Macau.
+�Em cerca de 600 sismos no continente, s� em dez a 12 por cento se conseguia calcular os par�metros s�smicos�.
+Raz�o que levou o IM a decidir a instala��o de duas redes digitais.
+Uma das redes digitais, em fase de instala��o entre 1994 e 1997, custar� 150 mil contos, ter� 14 esta��es (12 no continente e duas na Madeira).
+Neste momento, conta j� com seis esta��es no continente.
+Os registos falam por si, pois as esta��es anal�gicas n�o detectavam sismos de magnitude inferior a tr�s na escala de Richter.
+�Em 1995, duplicou o n�mero de sismos detectados.
+Com a rede digital detectaram-se sismos de magnitude inferior a dois�.
+A segunda rede digital, que ser� instalada nos A�ores entre 1997 e 99, ter� 12 esta��es e custar� cerca de 200 mil contos.
+O Trif�sica � um bar da 24 de Julho, isto �, longe das minuscularias do Bairro Alto.
+� muito engra�ado.
+As portas e janelas s�o no n�mero das letras do bar, ostentando, uma a uma, cada letra da palavra T-R-I-F-�-S-I-C-A, em torno do gaveto das Escadinhas da Praia com a 24 de Julho, num vidro fosco, em que s� transparece o corpo da letra.
+Nove corpinhos bem feitos.
+A decora��o tamb�m � divertida.
+Detr�s do balc�o, as bebidas protegidas pela barreira vis�vel de um vidro est�o ligadas a mangueiras, que entornam os preciosos l�quidos computadorizadamente para os nossos copos.
+Isto equivale a n�o poder servir melhor o �whisky� de um amigo, a n�o ser que lhe mangueire dois para dentro do copo.
+E a� tem de se haver com o computador, que lhe cobra os dois!
+Enfim, este conv�vio com computadores custa-nos um bocado.
+� que s�o uns antip�ticos que nunca oferecem copos.
+A n�o ser que os homens, seus donos, lhes ordenem!
+Boa!
+Mas, para al�m de n�o nos oferecer copos, a n�o ser que a obriguem, esta m�quina � muito bonita.
+� t�o gira que a empresa que a comercializa se chama Coisas Giras.
+R. -- Vamos l� a ver.
+nos primeiros tempos, a minha vis�o era a de um homem que vivia numa boa casa, com ar condicionado, boa comida, comodidades.
+Nesses tempos at� olh�vamos com alguma sobranceria para as popula��es, eu ainda n�o me tinha apercebido de que os negros talvez n�o fossem afinal uns selvagens, tinham era uma cultura diferente ...
+Depois, foi na guerra, quando tive de penetrar no mato e andar pelas bolanhas da Guin� ou pelas savanas de Angola, quando descobri povoa��es isoladas, etc., que passei de facto a conhecer �frica.
+P. -- Onde estava quando come�aram os massacres em Angola?
+A anomalia criou dificuldades, nomeadamente, em f�bricas e restaurantes locais.
+Tamb�m, ontem, em entrevista � RDP-Madeira, Alberto Jo�o Jardim, presidente do governo regional madeirense, surgiu em defesa de Jorge Sampaio, numa aut�ntica tr�gua, face � aproxima��o da presid�ncia aberta de Sampaio no arquip�lago.
+Mostrando d�vidas quanto � interpreta��o da lei que revogou a obrigatoriedade do PR ouvir o CEMGFA na recondu��o de um chefe de ramo, Sampaio, disse o l�der do PSD-Madeira, ter� agido bem, j� que, sublinhou, �uma carta daquelas n�o se escreve ao Presidente da Rep�blica�.
+At� porque Fuzeta da Ponte, nas palavras de Alberto Jo�o, n�o era �grande espingarda� e �antip�tico�, devendo por isso ter sido �demitido logo�.
+Depois, Jardim aproximou-se da tese defendida pelo PCP, da exig�ncia da demiss�o de Veiga Sim�o, ao defender que o actual ministro da Defesa, com Marcelo Caetano j� dera �cabo da Educa��o�.
+ TEATRO NACIONAL DE D. MARIA II.
+Hoje, �s 21h30; amanh�, �s 16h00 e �s 21h30; dom., �s 16h00; 2�, �s 21h30.
+TEATRO NACIONAL.
+Dias 5 e 7, �s 18h00.
+A Gr�-Bretanha estar� preparada para impor o seu dom�nio directo sobre Gibraltar, para controlar a banca e o sistema legal respectivos, numa tentativa para acabar com as alega��es de que a col�nia se transformou num centro de lavagem de dinheiro, informou ontem o jornal Sunday Telegraph.
+Funcion�rios do Governo brit�nico confirmaram que a Gr�-Bretanha foi aos arames com as posi��es das autoridades gibraltinas sobre a concretiza��o de directivas da Uni�o Europeia, bem como sobre o tema da lavagem de dinheiro.
+O jornal acrescenta que os gibraltinos receiam que a medida seja um primeiro passo para os brit�nicos desistirem da soberania sobre o rochedo, situado num promont�rio no sul de Espanha, devolvendo-o aos espanh�is.
+O jornal garante, embora um porta-voz oficial do Minist�rio dos Neg�cios Estrangeiros n�o o tenha confirmado, que o Governo brit�nico est� pronto para, �antes do Ver�o�, tomar medidas para que Gibraltar deixe o seu estatuto de col�nia com governo pr�prio, passando a depender directamente da Coroa brit�nica.
+Um mi�do de tr�s anos cujas pernas foram cortadas por uma ceifeira-debulhadora foi operado para a reimplanta��o dos dois membros e recupera no Hospital Rei Eduardo VIII em Durban, �frica do Sul, informou ontem fonte hospitalar.
+Amos Mosea brincava no meio de um milheiral, sexta-feira, numa quinta perto de Underberg, a 200 quil�metros da cidade portu�ria de Durban, no Oceano �ndico, quando foi atropelado pela ceifeira-debulhadora e lhe cortou as pernas abaixo dos joelhos.
+Os gritos da crian�a alertaram um vizinho que a conduziu a uma cl�nica local, em estado de coma e sangrando abundantemente.
+Mas o mi�do saiu de coma para dizer: �Estou doente�.
+Conduzido de helic�ptero para Durban com os cotos das pernas mergulhados em gelo, foi imediatamente operado durante oito horas.
+Pouco a pouco, foi-se esquecendo em grande parte o objectivo primordial que para ali levara norte-americanos, paquistaneses, italianos e outros soldados de meio mundo: distribuir comida a muitos africanos que estavam perigosamente � beira da morte.
+E a situa��o passou a ser, em primeiro lugar, a de um confronto cada vez mais agudo entre a for�a expedicion�ria estrangeira e as mil�cias locais, muito em particular a de Aidid, que se tem feito passar por um nacionalista e um paladino da cultura isl�mica.
+O ataque a�reo de ontem, em que participaram nove helic�pteros, e a opera��o terrestre que se lhe seguiu fizeram com que somalis furiosos tivessem morto um fot�grafo da ag�ncia Reuter, o anglo-americano Dan Eldon, de 22 anos, e outro da Associated Press, o alem�o Hansi Krauss, de 30, estando ainda por confirmar a morte de mais um fot�grafo, Hos Maina, e de um operador de som da Reuters Television, Anthony Macharia, ambos quenianos.
+Para medir a largura da regi�o emissora da radia��o, os astr�nomos utilizaram, como se se tratasse de uma aut�ntica lente de aumentar, a bolha de g�s em expans�o que envolve o pulsar.
+O g�s tamb�m constitui um dos restos da supernova e encontra-se em expans�o sob o efeito da onda de choque gerada pela explos�o.
+Visto que o pulsar era demasiado pequeno para a sua imagem ser medida desde a Terra, mesmo pelos telesc�pios mais potentes, a ideia consistia em captar �imagens� do g�s para, a partir da�, extrair a �imagem� do pulsar.
+ A bolha de g�s �fornece uma resolu��o equivalente � de uma ' lente ' do tamanho da �rbita da Terra�, explica um comunicado ontem emitido pela Associa��o Astron�mica Americana.
+Para o presidente da CML, �o objectivo da UCCLA n�o � ter dinheiro a prazo nos bancos�, mas �servir as popula��es das cidades que s�o membros�.
+E disse tamb�m que a transforma��o da UCCLA em funda��o, proposta por Pinto Machado, �� uma novidade absoluta para todos os membros da Assembleia Geral�.
+O autarca classificou as suas rela��es pessoais com Pinto Machado como sendo �as melhores� e considerou n�o haver qualquer ruptura no plano institucional, porque �o secret�rio-geral acaba de apresentar a sua demiss�o�.
+Negou ainda que as suas diverg�ncias com Pinto Machado resultem de um conflito pol�tico-partid�rio no munic�pio lisboeta (onde o PP est� na oposi��o) que tenha sido transposto para a UCCLA, o que foi confirmado pelo secret�rio-geral demission�rio.
+Os desafios da globaliza��o e as respostas para o desenvolvimento de �frica na viragem do s�culo s�o o tema de um f�rum de dois dias, a come�ar amanh�, em Lisboa.
+Organizado pela Sociedade de Geografia de Lisboa, em colabora��o com o ISCSP-Instituto Superior de Ci�ncias Sociais e Pol�ticas, o encontro conta com a participa��o de representantes das empresas portuguesas com investimentos em �frica e dos respectivos benefici�rios.
+A C�mara de Palmela vai submeter � aprecia��o da Assembleia Municipal uma proposta que reduz em 0,1 por cento a contribui��o aut�rquica, fazendo com que, em 1995, esta taxa, em vez de representar 1,3 por cento sobre o valor dos pr�dios urbanos, passe a ser de 1,2 por cento.
+Outra declara��o, desta vez vinda do Governo indiano, voltou a deixar preocupadas as cinco pot�ncias nucleares (EUA, Fran�a, R�ssia, China e Gr�-Bretanha).
+Um respons�vel indiano pelo projecto de fabrico de m�sseis, Abdul Kalam, disse que o Agni (Fogo) -- um engenho com capacidade para transportar uma carga nuclear a uma dist�ncia de 1500 metros -- est� pronto a ser produzido em s�rie.
+E outro, com maior alcance, vir� a caminho.
+Segundo o assessor cient�fico do primeiro-ministro indiano, Atal Behari Valpayee, trata-se de um Agni melhorado, com um alcance de 2500 metros, num �estado avan�ado de desenvolvimento� e cuja produ��o j� recebeu a aprova��o do Governo.
+�As san��es n�o nos afectar�o neste dom�nio�, disse ainda Kalam, referindo-se �s penaliza��es econ�micas impostas contra a �ndia pelos EUA, Jap�o e Canad�.
+�O nosso programa nuclear � 100 por cento indiano.
+Um inc�ndio de grandes propor��es deflagrou ontem, cerca das 20h00, na escarpa da Serra do Pilar, em Vila Nova de Gaia, numa f�brica de estatuetas localizada na margem esquerda do rio Douro.
+Consciente da debilidade partid�ria no relacionamento com a sociedade, Gonz�lez apostou forte em independentes de prest�gio.
+O �efeito Garz�n�, a inclus�o na lista de Madrid do magistrado que mais reconciliou os espanh�is com a Justi�a, n�o teve, no entanto, o efeito duradouro que o l�der socialista pretendia.
+T�o-pouco o tiveram as promessas de um �novo impulso democr�tico�, necessariamente regenerador, uma das chaves da sua campanha, porque n�o foram completadas com um lavar da roupa suja a n�vel partid�rio.
+O c�rculo vicioso, entre o enunciado da mudan�a e a prud�ncia aconselhada pelo momento eleitoral, deixou pouco sabor na boca.
+O mau momento seria agravado pelos resultados do primeiro debate televisivo, com um Felipe Gonz�lez acabrunhado pela avalanche das cr�ticas dos conservadores.
+O programa televisivo Falar Claro viveu, na segunda-feira, um dos seus mais acalorados momentos.
+Isto porque o jornalista Joaquim Furtado, sem peias nem concess�es como � seu timbre, decidiu esta semana tomar o pulso ao estado do futebol portugu�s.
+Espetou duas l�minas na cara da boneca.
+Deitou-lhe para cima sangue ou tinta vermelha, n�o se percebia bem.
+Come�ou a bater-lhe com um martelo.
+ Cortou-lhe a cabe�a com uma navalha.
+Depenou-a.
+Deve ter passado meia -hora.
+Vera apagou a luz.
+Vestiu o casaco.
+Bebeu �gua.
+Pegou no papel e na caneta e foi-se embora.
+Mais tarde disse que teve a sensa��o que as pessoas estavam � espera do climax que n�o aconteceu.
+De facto, estavam.
+Vera quis mostrar a �usurpa��o da carne� e chamou � sua performance �Foda de Morte� porque num ensaio de Angela Carter sobre Sade, a foda levava � morte das mulheres.
+� a �death fuck�.
+Os automobilistas, esses, continuam � espera ...
+Tr�s centenas e meia de pessoas a ver, ouvir e aplaudir o espect�culo montado pela Associa��o Timorense Lafaek.
+Ontem � noite, num clube desportivo de Darwin, muitos australianos sentaram-se ao lado dos estudantes e levantaram-se para gritar �Viva a miss�o de paz!�, �Viva Timor-Leste�, �Viva Xanana�, quando est� pr�xima a chegada do Lusit�nia.
+Crian�as e adultos da Associa��o dan�aram, vestidas com os trajes e panos coloridos que h� s�culos os timorenses fabricam.
+Leu-se o poeta Borja da Costa e o c�ntico que Xanana Gusm�o escreveu na montanha para a �Mulher Timor�.
+Rui Marques, coordenador da miss�o que quer ir a D�li, foi chamado para ler a carta enviada clandestinamente pelos estudantes timorenses presos na Indon�sia.
+�Obrigado pela vossa visita a Timor-Leste�.
+Depois, num c�mulo de m�sica e emo��o, cantou-se �Peace Mission�, o hino composto e ensaiado para o primeiro dos dois dias de festa.
+�Venham e juntem-se � miss�o de paz.
+(...) Duzentos mil j� morreram.
+Acreditam?
+Acreditam?
+Num local da enorme sala, um jovem timorense vestido de guerreiro solu�ou e chorou abra�ado �s pessoas.
+O movimento ecologista b�lgaro Ecoglasnost pediu aos pa�ses da CEE para fornecer gratuitamente energia � Bulg�ria com o objectivo de possibilitar o encerramento da central nuclear de Kozlodoui, situada no Dan�bio e considerada perigosa pelos peritos b�lgaros e internacionais.
+Os quatro reactores ainda em funcionamento da central, de 440 megawatts cada um, foram constru�dos entre 1974 e 1975 e inspeccionados recentemente por uma miss�o da Ag�ncia Internacional da Energia At�mica.
+Dois deles est�o num estado extremamente envelhecido.
+�Como � que explicamos todas estas medidas israelitas?
+Chamamos a isto terrorismo de Estado organizado�, acrescentou Arafat.
+Enquanto a viol�ncia de palavras entre Israel e a OLP aquece, a viol�ncia continua a marcar a cena no L�bano, onde ca�as israelitas lan�aram ontem mais dois raides contra for�as hostis ao processo de paz.
+No bombardeamento da base da Frente Popular de Liberta��o da Palestina- Comando Geral, na fronteira l�bano-s�ria, ficou ferido um guerrilheiro.
+No ataque anterior contra uma base do Hezbollah, no vale de Bekaa, L�bano, morreu um guerrilheiro fundamentalista e outros tr�s ficaram feridos.
+Ramos Horta n�o veio a Portugal s� para receber os parab�ns pelo Pr�mio Nobel da Paz com que foi distinguido juntamente com o bispo Belo.
+13 anos depois da �ltima vota��o sobre Timor-Leste na ONU, chegou o momento de voltar a levar o tema ao julgamento da Assembleia Geral daquela organiza��o.
+A defesa desta estrat�gia de �guerra total� no plen�rio das Na��es Unidas n�o assenta exclusivamente no impulso que a quest�o timorense ganhou com a atribui��o do Nobel a Horta e a Belo -- Ramos Horta j� tinha sugerido esse passo num memorando que fez chegar ao Pal�cio das Necessidades meses antes de se conhecer a escolha de Oslo --, mas o certo � que o Nobel da Paz timorense constitui um refor�o importante.
+� para o aproveitar que Ramos Horta vem agora pessoalmente insistir na sua proposta.
+�Os que querem passar uma esponja sobre o passado impedem a reconcilia��o�.
+Ele sa�da a esperada vinda do Presidente israelita, Ezer Weizman, �s cerim�nias do 8 de Maio, como um gesto de reconcilia��o por parte da Alemanha.
+Mas, acrescenta, �o ritmo e a forma desta reconcilia��o n�o devem ser ditados do ponto de vista dos culpados, mas pelo das v�timas�.
+Em vez da esperada barracada musical, em Alvalade houve outro tipo de cenas bem menos divertidas.
+Por causa do �perigo de morte� representado pela pala que amea�a que cai mas n�o cai, alguns milhares de pessoas pagaram para assistir ao concerto num local e acabaram noutro, por falta de espa�o.
+�Est� tudo cheio�, diziam elementos da organiza��o, �tente na bancada do lado�.
+O mais conhecido dos ardinas de Lisboa, Carlos Francisco dos Santos, foi ontem a enterrar no Cemit�rio do Alto de S�o Jo�o.
+�Carlos dos jornais�, como todos lhe chamavam, tinha 77 anos e faleceu no domingo, num hospital da capital, em consequ�ncia de uma prolongada doen�a pulmonar.
+Com 57 anos de profiss�o, Carlos dos Santos, tornou-se popular pelos preg�es que usava na venda de jornais e de lotaria, na zona da Baixa e Bairro Alto, e pela maneira af�vel com que se relacionava com toda a gente.
+Fernando foi a figura do jogo de ontem entre o Rio Ave e o Sporting ao marcar tr�s golos ao clube lisboeta.
+Peixe acabou por ser o her�i sportinguista, ao defender, no �ltimo minuto e sobre a linha de golo, um remate que daria o empate ao clube de Vila do Conde.
+No ano passado, os organizadores pensaram em acabar com a corrida, devido � insufici�ncia de apoios.
+os Sinos � uma das provas de estrada com maiores pergaminhos no nosso pa�s.
+Falta agora que os corredores contribuam, com a sua presen�a.
+�N�o Roubar�s�.
+O �roubo� neste epis�dio n�o � de bens materiais.
+Tudo circula � volta de uma crian�a, abanadonada pela m�e natural e criada por outra mulher que reconhece como m�e.
+A primeira vem �roub�-la�.
+A crian�a ir� depois escolher entre elas.
+A Comunidade Econ�mica, por outro lado, dever� preparar as condi��es para a �transi��o gradual para a livre circula��o de mercadorias e servi�os� em todos os Estados que a ela aderirem.
+Mais tarde, quando estiverem criadas condi��es econ�micas, proceder-se-� � cria��o de um �mercado comum de m�o-de-obra�.
+No cap�tulo das bases econ�micas, os membros dever�o acordar ac��es nos dom�nios da pol�tica monet�ria, financeira, alfandeg�ria e de regulamenta��o de impostos.
+Tamb�m se prev� a cria��o de uma moeda �nica, embora, quem o deseje, possa ter a sua pr�pria divisa.
+Ao sair da penumbra, arrastando uma coisa parecida com n�voa cinzenta e oleosa, suspensa a dois palmos da cabe�a, Jo�o Carlos andou tr�s ou quatro passos e parou com um suspiro, como se as suas pernas utilizassem um sistema de suspens�o a ar.
+ Via-se que, por v�rias causas, Jo�o Carlos se cansa com facilidade, pois tamb�m baixou os ombros e inclinou o queixo para o peito, sem nunca olhar para os lados das grandes janelas do tribunal.
+E depois a ju�za chamou-o, vendo um homem com uma gabardina enorme, ainda maior que ele, um homem com cabelos grossos e brilhantes como crina de cavalo que tem andado � chuva, sem acompanhamento veterin�rio e com pouca forragem.
+Os olhos dele sa�am para fora da cara e quase chegavam ao n�vel da cana do nariz, no caso de se apanhar de perfil a figura perdida e magr�ssima de Jo�o Carlos, que tanto podia ter vinte e poucos anos (estragados), como muitos mais, e de facto tinha.
+Com 37 anos, Jo�o Carlos acabara de ser preso mais uma vez por furto num supermercado.
+passou as caixas registadores trazendo escondidas nos bolsos 12 embalagens de 10 l�minas de barbear.
+Mas poderia vende-las.
+� ali�s certo que as venderia num reles mercado negro, com tanto orgulho como, h� umas d�cadas, uns senhores vendiam numa ruela da Baixa de Lisboa os esticadores de colarinho, e parece que disso viviam (disso e de preservativos clandestinos).
+Madrid tamb�m fechou em alta de 0,55 por cento (mais 2,05 por cento), com o seu �ndice geral a atingir 376,82 pontos.
+A alta madrilena foi causada pela abertura em alta de Wall Street e pela subida dos mercados obrigacionistas, que compensaram os sustos com o an�ncio da demiss�o do general russo Alexander Lebed.
+O volume de neg�cios da sess�o de ontem 31,2 milh�es de contos.
+Em Frankfurt, registou-se uma subida de 0,05 por cento, com o �ndice DAX a fechar a 2.176,26 pontos, devido � baixa que atingiu Wall Street na quarta-feira.
+Nos mercados orientais, T�quio foi a excep��o e, ao meio da manh�, a bolsa tendia para uma alta marginal, com o �ndice Nikkei a marcar 12,07 pontos no fim da sess�o da manh�.
+As baixas dos demais mercados marcaram uma tend�ncia.
+Em Hong-Kong, no entanto, houve uma quebra e o �ndice Hang-Seng atingiu 61 pontos negativos, com uma subida posterior para 46 pontos.
+Singapura tamb�m fechou com uma ligeira baixa de 2,54 pontos.
+Vance destacou a aceita��o pelos beligerantes do envio de uma for�a de manuten��o da paz e sustentou que o novo cessar-fogo �indica, ao contr�rio dos anteriores, os procedimentos necess�rios para um acordo espec�fico�.
+�O ponto mais cr�tico reside agora nas possibilidades em manter o cessar-fogo.
+Agora vou para Nova Iorque, onde relatarei a Butros-Ghali [ secret�rio-geral da ONU ] o que aconteceu nos �ltimos dias�, afirmou antes de entrar apressadamente no Mercedes preto, j� com o motor a trabalhar.
+Antes, tinha-se recusado a comentar quais as suas pr�ximas iniciativas caso o acordo n�o resulte.
+Vincent Askew, com 20 pontos, liderou os Seattle SuperSonics na sua vit�ria sobre os Los Angeles Clippers no �nico jogo da Liga Norte-Americana de Basquetebol profissional (NBA) realizado no domingo.
+Terry Dehere, dos Clippers, foi o melhor marcador do encontro, com 24 pontos.
+Os SuperSonics alcan�aram o seu terceiro triunfo na �poca e est�o em quarto lugar na Divis�o Pac�fico.
+Quanto aos Clippers, ainda n�o conheceram o sabor da vit�ria e seguem com cinco derrotas.
+A China quer organizar em 1997 um Grande Pr�mio de F�rmula 1 no circuito de Zhuhai, que tem 4,83km de per�metro e est� situado perto de Hong Kong.
+Nas obras previstas para o complexo, que incluir� um campo de golfe e seria terminado em 1996, as autoridades pretendem gastar cerca de 30 milh�es de contos.
+Se a Federa��o Internacional do Autom�vel, que j� aprovou os planos de altera��o ao circuito, aceder aos sonhos dos chineses, estes garantem a presen�a de 200 mil espectadores.
+ Folha marcou em Chaves o seu segundo golo do campeonato, que foi considerado pelo P�BLICO como o melhor desta jornada.
+Mais um golo conseguido em jogada individual.
+O portista pegou na bola na esquerda, progrediu pelo centro do terreno, entrou na �rea e, com dois toques preciosos, sentou os dois centrais flavienses, Manuel Correia e Amarildo.
+Depois, perante a sa�da de Orlando rematou de p� esquerdo j� a curta dist�ncia da baliza.
+Depois desta jornada, que rendeu 18 golos, Juskowiak e Marcelo continuam a liderar a lista dos melhores marcadores, ambos com oito remates certeiros.
+Os dois jogadores voltaram a marcar no fim-de-semana, curiosamente ambos em bonitos remates de cabe�a.
+�Diatriba de amor contra un hombre sentado� assinala amanh� a estreia na escrita teatral do colombiano Gabriel Garcia Marquez.
+Romancista, contista, cronista e guionista, faltava a �Gabo� experimentar, em livro, o teatro, um meio para o qual outros, por ele, verteram muito dos seus contos.
+A tiragem inicial da obra, um mon�logo, ser� de 50 mil exemplares.
+Segundo o jornal de Bogot� �El Tiempo�, este texto de �Gabo� estar� nos palcos em breve, numa encena��o a cargo de Ricardo Garcia.
+A obra, redigida em 1987, esteve sete anos na gaveta antes de ser editada.
+Sete b�ias de sinaliza��o da barra do Douro v�o ser hoje recolocadas, marcando o percurso da Cantareira at� Vila Nova de Gaia.
+Cinco desses aparelhos est�o equipados com um sistema de ilumina��o, para que a entrada nocturna de embarca��es se possa fazer com toda a seguran�a.
+As b�ias tinham sido arrastadas durante as cheias de Dezembro e Janeiro passados e deviam ter sido repostas logo no in�cio do m�s de Mar�o.
+Uma fonte da Capitania do Porto do Douro disse ao P�BLICO que �o sistema vai entrar imediatamente em funcionamento�, colmatando assim o processo de recoloca��o das b�ias destru�das pelas cheias, que ocorre desde Fevereiro.
+Fica assim totalmente sinalizado o percurso entre a barra do Douro e a Ponte de D. Lu�s I, para o qual foi necess�ria a substitui��o das b�ias originais por outras de maior dimens�o, enviadas pela Direc��o de Far�is.
+As novas unidades ficar�o seguras por blocos de bet�o.
+Iliescu tinha anunciado que se deslocaria amanh� ao vale de Jiu, mas ontem os servi�os presidenciais disseram que a viagem fora cancelada.
+Os tr�s atletas portugueses em ac��o no quarto dia de provas n�o foram felizes.
+A sportinguista Teresa Machado foi a primeira a entrar em ac��o, lan�ando no segundo lugar do grupo A qualificativo do disco.
+a portuguesa, muito nervosa, falhou por completo o primeiro lan�amento.
+No segundo ficou a mais de dez metros do seu recorde nacional (63,70m) com 53,60m, e s� no derradeiro intento (a qualifica��o s� tem tr�s) se aproximou um pouco mais do que vale, com 56,02m, mas n�o chegou.
+Foram precisos 61,22m para ir � final.
+Quem brilhou de manh� foi Lucr�cia Jardim (Benfica), na eliminat�ria de 200m.
+Correu a s�rie 4 e ganhou � russa Natalya Voronova (23,45s) e � americana Gwen Torrence, a campe� ol�mpica (23,46s), com 23m45s.
+De tarde, por�m, tudo acabaria em frustra��o.
+No terceiro quarto-de-final precisava de uma posi��o nas quatro primeiras para ir �s meias-finais e perdeu esse precioso quarto posto para a jamaicana Dhalia Duhaney mesmo nos metros finais, com 23,11s contra 23,16s.
+Ganhou a francesa Marie-Jos� P�rec com 22,73s.
+Resta a pequen�ssima consola��o de ter feito o melhor tempo nacional de 1993.
+De que tem medo o IPACA?
+O Instituto Portugu�s das Artes Cinematogr�fica e do Audiovisual (IPACA, ex-IPC) tamb�m n�o respondeu �s perguntas formuladas pelo P�BLICO.
+Respons�veis seus come�aram por pedir que as perguntas fossem enviadas por fax, para serem respondidas por escrito -- �para n�o haver mal entendidos�.
+O fax foi enviado, tal como pedido, � presidente do Instituto, Zita Seabra.
+Quarenta por cento da �gua pot�vel usada em sua casa vai pela sanita abaixo.
+O Presidente da Rep�blica, M�rio Soares, enviou no in�cio da semana um telegrama � Coreia do Norte, atrav�s do embaixador de Portugal em Pequim, apresentando condol�ncias pela morte de Kim Il Sung, soube o P�BLICO junto de fontes coreanas em Lisboa.
+Portugal tem rela��es diplom�ticas com a Coreia do Norte desde 1975 e � representado em Pyongyang pelo chefe da sua miss�o diplom�tica na capital chinesa.
+O telegrama � dirigido a Kim Jong Il, filho do falecido Presidente da Coreia do Norte e seu presum�vel herdeiro.
+O primeiro-ministro, Cavaco Silva, fez apresentar na embaixada norte-coreana em Lisboa, atrav�s do seu adjunto diplom�tico, �sentidos p�sames� pela morte de Kim Il Sung, falecido h� uma semana, com 82 anos.
+Entretanto, ontem, a Coreia do Sul, encorajada pelos sinais tornados at� agora p�blicos sobre o que se vai passando na semana de luto no vizinho Norte, indicando uma aparente passagem calma de poderes de pai para filho, decidiu levantar parte do alerta especial em que tinha colocado as suas tropas, na sequ�ncia do falecimento do �Grande L�der�.
+P -- Mas as Assembleias Municipais s� t�m parecer vinculativo numa segunda fase ...
+R -- Sem d�vida.
+Mas n�o faria muito sentido, se as Assembleias Municipais se pronunciarem neste ou naquele sentido j� nesta fase, vir a ser aprovada uma lei de cria��o das regi�es contrariando uma vontade que se sabe que vai ser manifestada ulteriormente.
+Mas o sistema de inscri��o foi alterado h� cerca de um ano.
+Deixou de ser for�oso as pessoas deslocarem-se � sede para se inscrever em turnos de 15 dias num dos centros de f�rias.
+Agora podem tratar directamente, em contacto telef�nico ou por fax, com o centro onde pretendem passar f�rias.
+O sistema antigo s� se mant�m para os Centros de Entre-os-Rios, S�o Pedro do Sul e Foz do Arelho.
+�N�s, dentro de Espanha, sempre manifest�mos que deveria haver interesse por Portugal, que sempre teve um certo receio, porque a hist�ria foi o que foi�, comenta ao P�BLICO Jordi Pujol, �se temos uma identidade pr�pria e n�o temos medo, muito menos os portugueses, que t�m um Estado pr�prio�.
+� desta forma directa que o presidente da Generalitat, sem d�vida um dos dirigentes pol�ticos mais influentes de Espanha, equaciona as rela��es entre os dois pa�ses peninsulares.
+Pujol recebe oficialmente pela primeira vez o Presidente da Rep�blica de Portugal -- uma anterior visita de M�rio Soares, no Outono de 1987, foi privada --, mas n�o esquece o ocorrido h� quase nove anos.
+�A visita de Soares foi privada mas muito positiva para n�s�, assegura, relembrando tempos de busca de protagonismo que, nos �ltimos tr�s anos, com socialistas e conservadores no poder em Madrid, deram lugar � condi��o de parceiro indispens�vel.
+Ainda quanto a altera��es na Liga de Clubes, a direc��o ser� reduzida de quatro para tr�s membros, passando apenas a haver um presidente (eleito em AG), um director-executivo e outro elemento eleito pela direc��o da Liga, onde t�m assento onze clubes.
+Este �ltimo ser� o �nico que n�o coincide com o OA.
+De resto, decidiu-se ainda aumentar os elementos da Comiss�o Arbitral da Liga, que ser�o nove em vez de sete.
+Todas estas decis�es, disse Dam�sio, foram tomadas por unanimidade.
+A FPF confirmou ontem a repeti��o do Benfica-Sporting para amanh�, no est�dio do Restelo, negando assim provimento � contesta��o do Sporting.
+A hora do jogo, referente � 30� jornada do campeonato nacional e repetido por alegado erro do �rbitro na expuls�o do benfiquista Caniggia, ficou dependente do clube de Alvalade.
+A d�vida � entre as 18h30 ou as 20h30, esta �ltima proposta pelo Benfica, para permitir a transmiss�o televisiva.
+�Confundir identidade e fisionomia: um sonho tenaz�.
+Uma vez em interac��o com os outros, a gest�o das nossas express�es faciais representa um capital simb�lico que implica um dom�nio e um controle socialmente constru�dos.
+Os presentes autores, seguindo de perto as coordenadas te�ricas e os par�metros metodol�gicos e cronol�gicos de Foucault, prop�em-nos uma hist�ria do modo como a emerg�ncia da racionalidade moderna instituiu uma tens�o constante entre o que � org�nico, e como tal objecto de um saber exacto, e o que � expressivo, logo pass�vel de uma hermen�utica que valida processos cont�nuos de reclassifica��o social.
+Trata-se de uma separa��o politicamente prof�cua, j� que, ao longo de os s�cs. XVI a XIX, o homem ter� interiorizado a necessidade de vigiar a maior ou menor conformidade entre rosto, preceitos �ticos e poder social, ao mesmo tempo que lhe pedem que seja singular e aut�ntico.
+Uma verdade subjectiva incorporada atrav�s de normas sociais e, inversa e complementarmente, pr�ticas sociais que avaliam do grau de integra��o de cada um.
+O �cuidado de si� como inscri��o do poder.
+No termo de um Ver�o em que a febre dos div�rcios e esc�ndalos delapidou o patrim�nio de prest�gio da coroa brit�nica, foi a vez de a libra cair em descr�dito.
+Tal como a rainha Isabel II, John Major limitou-se a acompanhar os factos.
+Mas se o sil�ncio pode favorecer a recupera��o da imagem da coroa, a indecis�o � mortal, tanto em pol�tica como em economia.
+ O primeiro-ministro brit�nico n�o pode querer assinar Maastricht e manter a libra fora do SME e da sua correspondente disciplina.
+Como n�o pode continuar a afirmar a prioridade da luta contra a infla��o deixando a libra desvalorizar-se.
+Menos pode ainda permitir que o seu ministro das Finan�as escreva aos colegas de Gabinete notas prevendo que a libra ir� continuar fora do SME por meses, talvez anos.
+O presum�vel autor de 15 assaltos ocorridos recentemente em resid�ncias de Set�bal e do Pinhal Novo, atrav�s da extra��o do canh�o das fechaduras, foi detido pela PJ daquela cidade e viu a pris�o confirmada pelo tribunal local.
+Trata-se de um jovem de 20 anos, que ter� confessado os crimes e a venda dos objectos roubados para a compra de hero�na.
+O detido saiu pouco antes do in�cio da s�rie de assaltos de uma institui��o de recupera��o de toxicodependentes, aparentemente reabilitado, e come�ara a frequentar um curso de forma��o profissional financiado pela Uni�o Europeia.
+O facto de os assaltos ocorrerem, por norma, � hora do almo�o ou em per�odos a que o jovem faltava �s aulas p�s a PJ na sua pista.
+N�o foi divulgado o valor dos objectos em ouro e electrodom�sticos roubados.
+Os dois autores de um assalto a uma bomba de gasolina da Cepsa em Vila Franca de Xira, ocorrido na madrugada de ontem, foram detidos cerca de uma hora depois na Pra�a de Espanha, �s 4h25, em Lisboa.
+Os mesmos indiv�duos s�o acusados de outros tr�s assaltos ocorridos em Coimbra entre o dia 14 de Agosto e a tarde da �ltima quinta-feira.
+A PSP sabia que os assaltantes se deslocavam numa carrinha Renault Express, facto que a levou a mandar parar o ve�culo onde seguiam.
+Levados de volta a Vila Franca, os assaltantes, de 28 e 31 anos, foram identificados.
+Nos quatro assaltos, preferencialmente feitos a postos de gasolina, foram roubados mais de 300 mil escudos.
+A viatura em que seguiam era igualmente roubada.
+Cerca de metade dos resultados apurados, concretamente sete milh�es de contos, vai ser distribu�da ao accionista (Estado).
+Quanto ao �cash flow� da EDP, atingiu os 128,3 milh�es de contos em 1991.
+�Estes resultados foram positivamente influenciados pelo acr�scimo da procura de electricidade, pela estabilidade dos pre�os dos combust�veis importados para produ��o de electricidade e pela conten��o dos restantes custos de explora��o�, refere a mesma nota.
+Como dividir os minist�rios?
+Os dirigentes turcos Mesut Yilmaz (ANAP, direita) e Necmettin Erbakan (Refah, islamista) iniciaram ontem o processo de forma��o de um Governo, decidindo criar uma comiss�o encarregue de repartir os minist�rios entre os dois partidos.
+Esta quest�o esteve na origem das diverg�ncias que surgiram durante as negocia��es do fim-de-semana e que quase puseram em causa os esfor�os para um acordo.
+A comiss�o mista dever� iniciar amanh� os seus trabalhos que, espera-se, estejam terminados no fim da semana.
+�Cheg�mos a acordo sobre a maior parte das quest�es, s� restam algumas diverg�ncias menores�, explicaram Yilmaz e Erbakan na confer�ncia de imprensa conjunta que deram ontem.
+Segundo a ag�ncia Anat�lia, o Refah aceitou que Yilmaz seja primeiro-ministro at� ao in�cio do pr�ximo ano.
+Em seguida, se for aplicado como previsto o princ�pio da rotatividade, a Turquia dever� passar a ter o seu primeiro chefe de Governo islamista.
+O �nico empate de ontem faconteceu no jogo que op�s o Friburgo ao Bayern Leverkusen (1-1).
+Anteontem jogaram Werder Bremen-Nuremberg (2-3) e Hansa Rostock-Wolfsburg (3-3).
+Os marginalizados do sistema de ensino: que solu��o?
+Os desfavorecidos: que a solu��o para os marginalizados do sistema de ensino?
+O Governo admite criar uma linha de cr�dito bonificado para ajudar a atenuar os preju�zos do temporal que nos �ltimos dias afectou as regi�es do interior Norte e Centro.
+Quem o diz � o secret�rio de Estado da Administra��o Interna, Armando Vara, que passou o dia de ontem a inteirar-se dos efeitos da intemp�rie em Bragan�a, o distrito mais atingido, mas onde j� n�o h� aldeias isoladas.
+�Um Vestido para cinco mulheres� � a pe�a encenada por Diogo Infante que transformou Margarida Pinto Correia numa loura.
+Um texto em torno do �sagrado matrim�nio� no qual se discute as possibilidades de encontrar um homem que n�o seja casado, nem maricas e tenha emprego.
+Um teatro assumidamente comercial.
+O Hospital de Santo Ant�nio, no Porto, dever� pedir ainda esta semana � Lipor para ali depositar as v�rias dezenas de toenaladas de cinzas resultantes da queima de lixos hospitalares que, h� um ano e meio, se acumulam em contentores de pl�stico.
+� o resultado de um of�cio enviado na semana finda pelo Minist�rio do Ambiente, o qual atesta a inocuidade daquelas cinzas e aconselha a sua deposi��o do aterro de Ermesinde.
+As esc�rias do Santo Ant�nio v�o, assim, passar a conhecer um fim semelhante � generalidade das cinzas produzidas pelas incineradoras hospitalares.
+Mais complicada parece a situa��o do Hospital de Guimar�es.
+Desde o encerramento, em Julho, da incineradora do Hospital de S. Marcos, de Braga, que ali s�o tratados os res�duos de todas as unidades de sa�de do distrito, estando a laborar perto do seu limite.
+Toxicodepend�ncia que o levara a contrair o v�rus da sida.
+Cauteleiro de profiss�o, era, portanto, seropositivo, embora a doen�a estivesse ainda numa fase controlada.
+Nascera na freguesia de Miragaia, no Porto, e estava em Santa Cruz do Bispo.
+Na cadeia era bem-comportado e ocupava-se em trabalhos de faxina.
+Suicidou-se a 7 de Agosto.
+Sem deixar qualquer sinal que exteriorizasse o seu estado de esp�rito, uma conversa com algu�m.
+Sa�ra da cela �s 8h30, pedindo ao guarda para voltar pouco depois, para descansar.
+�s 9h30, encontravam-no enforcado, com um len�ol, nas grades da cela.
+O caso est� a ser analisado pela Direc��o Regional de Educa��o do Centro e o Coordenador da �rea Educativa (CAE) j� admitiu n�o haver �condi��es objectivas para que os professores executem as suas tarefas� e continuem as actividades lectivas, sobretudo enquanto o director do Col�gio do Mondego se encontrar ausente no estrangeiro.
+Mas o Sindicato dos Professores da Regi�o Centro (SPRC) reagiu, pedindo a demiss�o da direc��o do instituto de reinser��o.
+A situa��o dos professores deste �reformat�rio� � complicada e complexa, o que motiva a maior parte dos protestos do SPRC.
+� que os professores colocados no Col�gio do Mondego pertencem � escola b�sica do 2� e 3� ciclo de Santa Clara, na Guarda, institui��o para a qual concorrem.
+S� que esta EB 2,3 tem um protocolo com o Col�gio para a disponibiliza��o do seu corpo docente.
+Sendo assim, v�o parar ao instituto de reinser��o social professores �sem experi�ncia profissional e sem forma��o adequada � situa��o�.
+ No outro dia, apareceram l� dois guerrilheiros do PAIGC.
+Um deles olhou para mim e reconheceu-me logo.
+Era Oto, um ex-controlador de tr�fego a�reo em Bissalanca que se juntara � guerrilha.
+Deu-me um ma�o de cigarros e sossegou-me.
+ No dia seguinte, um domingo, cheg�mos a Conakry.
+Levaram-me ao Minist�rio da Defesa para encontrar algu�m a quem me entregar, mas o edif�cio estava vazio.
+Demos imensas voltas pelos corredores at� os gendarmes decidirem levar-me para uma esquadra de pol�cia.
+No dia seguinte, abriram-me a porta da cela, fizeram-me subir umas escadinhas e entrar para uma sala.
+Sentados a uma mesa trapezoidal encontravam-se cinco indiv�duos de �grand boubou� [ vestimenta mu�ulmana ] at� aos p�s, com os seus gorros t�picos, todos muito grandes e com ar s�rio.
+Transportado para os Estados Unidos, foi mantido nove meses numa cela sem luz, vigiado 24 horas por dia por uma c�mara de televis�o e por guardas prisionais.
+�Lavagem ao c�rebro�, press�o do advogado na altura que lhe deu conta da inevit�vel cadeira el�ctrica na falta de uma confiss�o, alega o presum�vel homicida de Luther King.
+James Earl Ray, reza a publicidade da Thames, �reclamou a sua inoc�ncia e pediu julgamento.
+Ningu�m ouviu.
+At� agora�.
+Os bastidores do neg�cio que levou � entrada do Banco Central Hispano (BCH) no capital do Banco Comercial Portugu�s foram a gota de �gua.
+Ali�s, muito possivelmente v�o ser expostos em tribunal.
+Tudo porque Neto da Silva, ex-secret�rio de Estado do Com�rcio Externo e depois l�der da Socifa, exige receber uma comiss�o alegadamente prometida por Am�rico Amorim caso conseguisse vender o lote de ac��es do empres�rio no banco portugu�s.
+O que Am�rico Amorim n�o confirma, recusando-se a pagar qualquer presta��o de servi�os.
+Tudo come�ou quando o �rei da corti�a� concedeu a Neto da Silva um mandato para procurar um comprador para as ac��es que detinha no BCP.
+n�o havia limite de tempo para a concretiza��o do neg�cio, a comiss�o a receber seria de tr�s por cento do montante total envolvido, incluindo impostos, e deveria ser observado o mais rigoroso sigilo sobre a transac��o.
+Marius Weiers, alto funcion�rio do minist�rio sul-africano do Com�rcio e Ind�stria, disse ontem em Joanesburgo que �Portugal re�ne condi��es para vir a constituir a porta da �frica do Sul na Europa�.
+apenas 4,6 por cento dos patr�es s�o licenciados e s� cerca de 30 por cento t�m habilita��es equivalentes ao secund�rio.
+Uma das preocupa��es claras do manifesto � esclarecer que o congresso n�o concorre no plano dos partidos pol�ticos.
+Como ali � afirmado, �n�o cabe exclusivamente � classe pol�tica� dar respostas sobre as interroga��es que o futuro levanta, �antes exige a participa��o activa da sociedade civil, que tem igualmente responsabilidades e deveres de que n�o pode nem deve demitir-se�.
+Gomes Motta, respondendo �s perguntas dos jornalistas, iria mais longe ao afirmar que os �partidos n�o esgotam a actividade c�vica do pa�s� e, afastando qualquer mal-estar que o congresso possa provocar nas hostes socialistas por eventualmente ofuscar alguns dos seus projectos, esclareceria ainda que esta iniciativa e as do PS �em certa medida completam-se�.
+Este manifesto ser� publicado na �ntegra nos jornais, sendo acompanhado por uma ficha de inscri��o para participar nos trabalhos do congresso que decorrer�o no pavilh�o da FIL, de 8 a 10 de Maio pr�ximo.
+At� l�, ser�o organizadas sess�es em v�rias cidades publicitando a iniciativa e recolhendo sugest�es sobre a incorpora��o de temas regionais.
+No esquema apresentado, s�o seis os pain�is em debate, que abrangem temas que v�o desde a cultura, o ensino e a estruturas econ�micas at� ao Estado democr�tico, solidariedade e o papel de Portugal na Europa e no mundo.
+Estes s�o alguns dos dados da avalia��o externa feita pelo Instituto de Inova��o Educacional (IIE).
+O estudo representa um dos mais importantes indicadores das aprendizagens dos alunos, tendo inaugurado a era das �provas aferidas� no sistema educativo portugu�s.
+recentemente (ver P�BLICO de 27/03/97), o ministro da Educa��o anunciou que seriam lan�ados �testes de aferi��o de �mbito nacional� nos 4�, 6� e 9� anos.
+ A inten��o figura no documento estrat�gico para a Educa��o entregue na Assembleia da Rep�blica, no qual, curiosamente, se retira a mat�ria da al�ada do IIE e se entrega ao Gabinete de Avalia��o Educacional (Gave).
+O relat�rio adianta que �todos os relatos de tortura e maus tratos se referiam a pessoas que tinham sido detidas sob suspeita de terem cometido delitos criminais, tendo em muitos dos casos sido libertadas sem serem inculpadas�.
+ R. -- � l�gico que os directos s� podem existir se houver regras claras.
+E t�m de existir porque, neste momento, � imposs�vel produzir um filme sem ser em co-produ��o e sem dar contrapartidas a realizadores estrangeiros.
+Mas acredito que as preocupa��es que os realizadores t�m, tamb�m a direc��o do IPACA as tem.
+P. -- Est� a falar pelo IPACA?
+O espanhol Jesus Montoya (Amaya) venceu ontem a 16� etapa da Volta � Espanha em bicicleta, disputada entre Santander e o Alto de Campoo, na dist�ncia de 173,4km e subiu ao 4� lugar da classifica��o geral individual.
+O Banco Nacional Ultramarino (BNU) tinha aprovado at� Dezembro de 1991 cerca de seis milh�es de contos de financiamentos para linhas dedicadas a pequenas e m�dias empresas.
+A dota��o global das linhas espec�ficas para as PME � de 12,5 milh�es de contos, oriundos do Banco Europeu de Investimentos, do Fundo de Regulariza��o da D�vida P�blica e da Caixa Geral de Dep�sitos.
+Este apoio foi decidido em 1990 e inclui o refor�o de capitais pr�prios, linhas de cr�dito em colabora��o com a CGD, tal como os fundos de apoio �s iniciativas de jovens empres�rios.
+Por outro lado, at� ao final de 1991 o BNU tinha prestado 292 garantias a favor do Instituto de Apoio �s Pequenas e M�dias Empresas, para a liberaliza��o pr�via de incentivos concedidos por este instituto.
+Toxicodepend�ncia: falar claro ou confundir?
+A toxicodepend�ncia, pelo desespero que provoca, � um terreno prop�cio a todo o tipo de manipula��es e os �mass media� n�o t�m fugido a esta tenta��o.
+As not�cias revelam, salvo raras excep��es, um cariz sensacionalista, que alimenta sobretudo a ideia de �cura m�gica� ou o desejo de encontrar �bodes expiat�rios� para a explica��o do fen�meno, permitindo, por exemplo, que se fale de efic�cia do(s) tratamento(s) de formas pouco honestas, por ignor�ncia ou por manipula��o deliberada.
+� �bvio que quem lucra com esta situa��o n�o s�o os toxicodependentes e as suas fam�lias.
+Um exemplo recente de como se pode confundir a opini�o p�blica ocorreu no programa de TV Falar Claro, no passado dia 22 de Junho.
+O relat�rio foi elaborado por tr�s dos mais respeitados peritos da Europa: John Wilesmith, director de epidemiologia no Laborat�rio Central do Governo brit�nico, Bram Schreuder, do Instituto de Ci�ncia e Sa�de Animal da Holanda, e C. Straub, do Centro Federal Alem�o de Pesquisas sobre V�rus e Doen�as de Animais.
+O que espanta, por�m, n�o � que se pretenda fazer marcha atr�s relativamente a um referendo que devia ter sido realizado h� j� muito tempo.
+Bem vistas as coisas, os portugueses dificilmente compreenderiam o facto de serem agora chamados a pronunciar-se sobre algo que h� muito foi decidido � sua revelia.
+Pelo que, muito provavelmente, voltariam a abster-se de ir �s urnas.
+Talvez prefiram ir comer castanhas para qualquer lado.
+O que realmente impressiona � que ningu�m pare�a inclinado a defender -- por simples analogia -- que uma absten��o em massa no referendo das regi�es por� tamb�m em causa, e de modo irremedi�vel, o processo de regionaliza��o.
+Exceptuando o PCP e alguns perigosos radicais socialistas, o referendo das regi�es parece ter sido definitivamente aceite como algo de imprescind�vel.
+Circunst�ncia que, tendo em conta a argumenta��o agora utilizada para tirar a castanha da Europa do braseiro da indiferen�a popular, s� pode ser lida de uma forma: a tend�ncia dominante da classe pol�tica j� tra�ou o destino � velha quimera da descentraliza��o.
+Na melhor das hip�teses, o povo, devidamente aterrorizado pelo fantasma do separatismo, chumba o processo.
+No pior dos cen�rios, o pa�s volta a abster-se e a regionaliza��o tem assegurados mais 25 anos de perman�ncia no fundo falso da gaveta da democracia.
+H�, mas todas elas est�o j� transpostas para o direito nacional.
+Referem-se a normas de seguran�a e de sa�de e higiene, regras de ilumina��o, etc.
+As quest�es mais complicadas est�o ainda sobre a mesa.
+� o caso da organiza��o do tempo de trabalho e disposi��es sobre mulheres gr�vidas, cuja aprova��o n�o se espera para a presid�ncia brit�nica, n�o entrando, assim, em vigor a 1 de Janeiro de 1993.
+�s vezes, pensa que isto de imagem de empresa passa tamb�m por uma boa auditoria e que as empresas de auditoria estrangeiras d�o melhor nome.
+ Poder� recorrer a elas, mesmo que n�o estejam sediadas em Portugal?
+Os partidos pr�-governamentais voltam atr�s quanto � prometida revis�o constitucional.
+13 de Maio -- A oposi��o apela de novo � popula��o para que se manifeste contra o Governo a partir do dia 17.
+A forma��o espanhola do Chapela venceu ontem o Torneio Internacional Feira de S. Mateus, que decorreu em Viseu, ao derrotar na final o Valladolid, por 28-27 (13-13 ao intervalo), na quinta e �ltima jornada da prova, na qual participaram ainda FC Porto, Benfica, Sporting e Madeira SAD.
+O Feira de S. Mateus foi uma excelente oportunidade de ver em ac��o estas quatro equipas portuguesas que ser�o, certamente, uma amea�a � hegemonia do tetracampe�o ABC.
+Segundo um alto respons�vel da institui��o, que solicitou o anonimato, o ritmo de recupera��o da economia mundial dever� acentuar-se j� no segundo semestre de 1992, opondo-se, deste modo, �s teses mais pessimistas que prognosticam fortes probabilidades de uma recess�o mundial.
+�Embora se constatem riscos em algumas zonas, as nossas expectativas s�o de uma recupera��o global�, disse a mesma fonte.
+Apesar de n�o ser previs�vel que a Europa abrande ainda mais a sua actividade, a retoma do seu crescimento dever� ser mais fraca do que em outros pa�ses industrializados, devido � insufici�ncia das reformas estruturais, acrescentou.
+Tomando como caso concreto a Fran�a, com um mercado de trabalho considerado pouco flex�vel, o mesmo respons�vel do FMI antev� que a Europa dever� continuar a registar um �crescimento lento�, caso n�o concretize rapidamente as necess�rias reformas estruturais.
+Esta situa��o faz prever que a Cimeira de Lisboa acabar� por aprovar apenas um �acordo pol�tico� sobre o Pacote Delors II que constitua um compromisso formal dos Doze relativamente � sua futura aprova��o detalhada.
+A demiss�o de Hans-Dietrich Genscher e a crise pol�tica que atravessa o Governo de Bona (tendo em pano de fundo uma �derrapagem� econ�mica, resultado da unifica��o, que est� a afectar a Alemanha e a ser inevitavelmente �exportada� para os outros pa�ses europeus) �, talvez, o maior rev�s que a presid�ncia portuguesa tem de enfrentar n�o s� quanto � aprova��o do Pacote Delors (Genscher era um sincero apoiante das novas perspectivas financeiras para a Comunidade) como quanto � generalidade [...]
+A Administra��o da Casa Branca decidiu ontem manter algumas dist�ncias face �s declara��es do antigo Presidente democrata norte-americano Jimmy Carter ap�s a sua visita a Pale -- o basti�o da lideran�a s�rvia da B�snia nos arredores de Sarajevo --, ao recordar que os s�rvios b�snios continuam a ser considerados �os agressores� no conflito que se prolonga h� mais de dois anos e meio nesta rep�blica balc�nica.
+O Concurso de Dan�a de Sal�o para a Terceira Idade � outro projecto destinado aos idosos sintrenses, que, previsto para o in�cio de Fevereiro, levar� a diversas colectividades locais todos os que queiram concorrer ou simplesmente trocar uns passos de dan�a.
+Distribu�dos por v�rios escal�es segundo a idade, aos concorrentes basta pertencer ao universo de cerca de 30 mil reformados do munic�pio, podendo inscrever-se na altura do baile, a divulgar brevemente por todos as associa��es e grupos de idosos.
+Na sequ�ncia de anteriores projectos, como as visitas ao Jardim Zool�gico, os espect�culos de teatro e um passeio de cacilheiro no Tejo, a ac��o deste ano, como explica Jaime da Mata, pretende continuar o trabalho j� iniciado: �Impulsionar uma viv�ncia que � salutar e necess�ria�.
+Ainda segundo Travessa de Matos, �as c�maras da regi�o querem que a estrada passe a ter caracter�sticas de itiner�rio complementar� e os anteriores respons�veis �apenas queriam proceder � pavimenta��o do piso a partir de Pinheiro, P�voa de Lanhoso�.
+A beneficia��o da EN 103 chegou a estar prevista no PIDDAC (Plano de Investimentos e Despesas para Desenvolvimento da Administra��o Central) para 1995, tendo as reclama��es apresentadas pelos munic�pios � Junta Aut�noma de Estradas impedido a sua concretiza��o.
+Agora v�o �exigir� ao ministro da tutela �a urgente rectifica��o da EN 103, entre Braga e Chaves�.
+As t�cnicas de terapia gen�tica -- tamb�m chamada �geneterapia� -- consistem, em termos gen�ricos, em inserir um gene terap�utico nas c�lulas de um doente.
+O gene agora introduzido, de nome p53, � considerado como um dos mais importantes genes �supressores de tumores�, pois comanda a produ��o de uma prote�na-chave da regula��o da divis�o celular.
+Recorde-se que a canceriza��o se produz quando, por alguma raz�o, as c�lulas do organismo come�am a dividir-se de forma descontrolada sem chegarem � matura��o.
+Da� que, quando o p53 � defeituoso, a prote�na n�o � fabricada e surge o cancro.
+Os cientistas estimam que cerca de metade dos casos de cancros esteja associada a muta��es do gene p53.
+ No caso do doente agora tratado, que sofre de uma forma comum do cancro do pulm�o, o tratamento consiste em administrar v�rias injec��es do gene p53 normal -- ou seja, dotado da sua ac��o supressora de cancros -- dentro do pr�prio tumor, �a bordo� de um v�rus.
+Como o v�rus tem uma propens�o para infectar as c�lulas humanas, consegue em princ�pio fazer penetrar o gene dentro das c�lulas cancerosas, servindo de �cavalo de Tr�ia� aos genes p53.
+Os Estados Unidos �reexaminar�o as bases� do acordo de coopera��o econ�mica conclu�do com o Jap�o, se n�o se alcan�arem �acordos cred�veis� entre os dois pa�ses durante a cimeira Clinton-Hosokawa, prevista para 11 de Fevereiro em Washington.
+A advert�ncia foi feita ontem em T�quio pelo secret�rio norte-americano do Tesouro, Lloyd Bentsen, depois de se ter encontrado a seu pedido com o primeiro-ministro Morihiro Hosokawa e o ministro das Finan�as Hirohisa Fujii.
+Os investimentos estrangeiros v�o continuar a afluir este ano � China, mas a um ritmo menos elevado do que em 1993, e o governo vai refor�ar o controlo sobre as �joint-ventures�, segundo o jornal �China Daily�.
+A baixa ir� dever-se principalmente ao arrefecimento da actividade imobili�ria, cujo crescimento exponencial no ano de 1993 se deveu em grande parte a capitais estrangeiros.
+Estes investigadores �vacinaram� quatro macacos com uma estirpe pouco virulenta do v�rus HIV2 humano (o v�rus da sida mais vulgar na �frica), que n�o provoca a doen�a nestes animais e que desaparece rapidamente do seu organismo.
+A seguir, infectaram-nos com o v�rus da sida dos macacos, o SIV.
+Quase quatro anos depois, tr�s dos animais ainda se encontram em boa sa�de, tendo o quarto morrido h� uns meses.
+Todos os elementos de um grupo de animais que n�o tinha sido vacinado com o HIV2 morreram da sida dos macacos, nos meses que se seguiram � sua infec��o pelo SIV.
+�Com os meus colegas, tencionamos agora tentar desenvolver vacinas destinadas ao ser humano, e em particular contra o v�rus HIV2.
+Tratar-se-� de vacinas preventivas que poder�o ser eficazes contra qualquer estirpe do HIV2.
+Por outro lado, j� estamos a colaborar com colegas do Instituto Nacional de Sa�de da Guin�-Bissau, na selec��o de popula��es que possam vir a participar em ensaios cl�nicos, quando tivermos uma vacina potencial.
+�Reais j�ias do Norte de Portugal� � o t�tulo de uma exposi��o que ir� decorrer entre os pr�ximos dias 10 e 26 de Novembro no Pal�cio da Bolsa, no Porto.
+Comissariado conjuntamente por J� T�vora e Manuel Ad�lio Valle Gomes, o certame conta com o �alto patroc�nio� dos duques de Bragan�a, que cederam para a ocasi�o a tiara em brilhantes do s�culo XIX usada por Isabel Her�dia no casamento com Duarte Pio.
+Segundo Virg�lio Folhadela, presidente da Associa��o Comercial do Porto, entidade que promove a iniciativa, a mostra tem como um dos seus objectivos principais revelar a forte tradi��o nortenha nos campos da ourivesaria e joalharia.
+A exposi��o -- formada sobretudo por objectos provenientes de colec��es particulares -- inicia-se cronologicamente no s�culo XVII e vem at� aos nossos dias.
+Entre as j�ias apresentadas contam-se colares de brilhantes e esmeraldas (s�c. XVIII e XIX), pe�as de ouro popular, trenedeiras, cris�litas, la�as, condecora��es das ordens de Malta e de Cristo em minas e brilhantes e uma caixa de rap� do Rei Carlos Alberto.
+Assinale-se ainda a presen�a de um �stand� da Christie's no espa�o da mostra, bem como de um avaliador oficial daquela leiloeira inglesa.
+Mais de um milh�o de contos dever�, segundo a Lusa, custar o seguro das obras expostas.
+Os independentistas tchetchenos prosseguiram o cerco a diversas posi��es militares russas, que responderam ao ataque abrindo fogo sobre �concentra��es de combatentes� tchetchenos, segundo o termo utilizado pelo centro de imprensa instalado em Mosdok (Oss�tia do Norte), quartel-general das for�as de interven��o russas.
+N�o foram fornecidas informa��es sobre o balan�o destes confrontos.
+Segundo a ag�ncia Interfax, a artilharia russa bombardeou na noite de sexta para s�bado as localidades de Samachki e Zakan-Iurt (situadas respectivamente a 30 km e 15 km a oeste de Grozni).
+Ap�s o fim do cerco � capital tchetchena pelas for�as russas, t�m decorrido violentos e incessantes combates a sudoeste da cidade.
+A artilharia e a avia��o russas t�m vindo a atacar regularmente povoa��es situadas nestas zonas, para onde recuaram os combatentes tchetchenos ap�s a queda de Grozni.
+Os confrontos tamb�m est�o a atingir Argun, 15 km a leste da cidade.
+O mar.
+Tudo o que tenha a ver com o mar, no mar, ao p� do mar.
+Quem � que espera n�o encontrar nestas f�rias?
+O meu porteiro.
+A s�tima jornada do campeonato ingl�s de futebol n�o trouxe altera��es ao topo da classifica��o, j� que os tr�s primeiros venceram os seus jogos e mant�m as posi��es relativas.
+O Norwich bateu em casa o Southampton por 1-0 e soma agora 16 pontos, o Coventry foi ganhar ao terreno do Oldham por 1-0 e est� com 15 e o Blackburn Rovers goleou no seu est�dio o Nottingham Forest por 4-1, somando 14 pontos, mas com menos um jogo.
+Belmiro de Azevedo, presidente da Sonae, acredita que nos pr�ximos anos os neg�cios do grupo no Brasil v�o crescer at� chegarem � dimens�o que actualmente t�m em Portugal.
+�Vamos ser t�o grandes no Brasil como em Portugal�, disse ontem, no Porto, o patr�o da Sonae, no decorrer de uma videoconfer�ncia que colocou em di�logo oito personalidades do mundo dos neg�cios dos dois pa�ses.
+S�o muitos os registos, ao longo de s�culos, de in�meras e variadas express�es de p�nico e supersti��es provocadas pela apari��o s�bita de um objecto brilhante, projectado no escuro da esfera celeste.
+A sua forma estranha, a mudan�a de posi��o relativamente �s estrelas e a altera��o de forma sugeriam interpreta��es de almas vagabundas de grandes homens desaparecidos, ou sinais dos deuses anunciando pr�xima a vingan�a de algum comportamento menos ajuizado dos terrestres.
+Ora, os ricos n�o est�o para isto.
+Como dizia o eng. �lvaro Barreto, ser ministro � ganhar mal, perder neg�cios, empobrecer alegremente.
+O pr�prio professor Cavaco, de modesta fortuna, farto de nos aturar, mandou tudo �quele s�tio -- aplicando uma bofetada sem m�o aos que vivem da politiquice -- e retomar� a carreira profissional ganhando o dobro, chateando-se pela metade.
+Resta, por exclus�o, o governo dos pobrezinhos, humildes mas honrados.
+Modelo muito querido ao dr. �lvaro Cunhal, mas completamente ultrapassado.
+�L'Incoronazione di Poppea� foi a �ltima a ser reeditada e este regresso �s discotecas no suporte j� irrevers�vel conquistador do mercado ter� de ser saudado com uma efusividade guardada para, e s� para, as ocasi�es muito, muito, muito especiais.
+Nunca mais se poder� falar, no futuro, da �Poppea de Harnoncourt�.
+Porque, depois deste registo de 1972, ficaria imortalizada em som e imagem -- na realiza��o visual de Jean-Pierre Ponnelle -- uma aproxima��o dos finais dos anos 70 deliberadamente expressionista e, agora, em 1993, em Salzburgo, Harnoncourt surpreende (e abre uma inflamada pol�mica) ao explorar at� aos limites do sustent�vel uma vis�o quase grotesca, comportando a contamina��o por uma componente burlesca surpreendentemente suportada pelo cinismo da narrativa, com inesperada opul�ncia de [...]
+Os investidores estrangeiros est�o de volta ao Brasil.
+ No centro das aten��es est�o as privatiza��es que h� pouco mais de um m�s tiveram in�cio.
+Para concretizar os seus investimentos, os estrangeiros aguardam apenas a conclus�o do acordo entre o Brasil e o Fundo Monet�rio Internacional (FMI) a realizar ainda este m�s, que trar� ao Brasil mais tr�s mil milh�es de d�lares.
+Alguns investidores externos aguardam apenas pela luz verde para a concretiza��o do empr�stimo, interpretando de forma positiva o facto de o FMI estar disposto a aceitar a carta de inten��es do governo de Collor de Mello, e a disponibilidade deste para aceitar o tratamento de choque que ser� imposto � economia brasileira.
+Quantos melhores jogos j� aconteceram neste Mundial?
+O Nig�ria-Espanha e o Inglaterra-Rom�nia, da primeira fase?
+O Holanda-Jugosl�via e o Argentina-Inglaterra dos oitavos-de-final?
+O Brasil-Dinamarca?
+Ou, o at� aqui melhor de todos, o Holanda-Argentina que, por si s�, justifica a exist�ncia de um campeonato e a paix�o que todos temos?
+Este futebol que apazigua os desejos e termina com as nostalgias foi lan�ado desde o primeiro segundo.
+O tempo de estudo, esses aborrecidos, in�teis e intermin�veis minutos iniciais, deram lugar ao jogo claro.
+Uma densidade construtiva, um vocabul�rio variado e extremo, uma intensidade magn�fica, dif�cil de atingir.
+Poder-se-� ir mais longe?
+Quando, aos 38', Veron decidiu uma pequena pausa em dois passes laterais sem progress�o, levou uma monumental assobiadela.
+Ningu�m queria que aquilo parasse.
+O teatro de Beaumarchais.
+A m�sica de Mozart.
+O cinema de Renoir.
+A Regra do Jogo � a s�ntese perfeita do esp�rito dos dois primeiros na arte do �ltimo.
+Toda a com�dia humana numa dan�a � beira do precip�cio, a guerra que se avizinha.
+Um filme premonit�rio, que ao tempo foi proibido e mutilado pela censura.
+Assim, de acordo com informa��o da CML, na Rua Edison inverte-se o sentido, passando a circular-se da Av. de Roma para a Av. de Madrid.
+Na �scar Monteiro Torres, o tro�o compreendido entre a Rua Oliveira Martins e a Av. de Roma volta a ter sentido �nico, circulando-se da Oliveira Martins para a Av. de Roma.
+No �outro lado da guerra�, em Belgrado, a elei��o do escritor Dobrica Cosic, um ex-pr�ximo de Tito, como Presidente da �nova Jugosl�via� foi cumprida, como previsto, pelo parlamento.
+Cosic era o �nico candidato e os diplomatas ocidentais notam que o seu poder �, na pr�tica, fict�cio, pois o homem que �mexe os cordelinhos� em Belgrado continua a ser o Presidente s�rvio, Slobodan Milosevic.
+A contesta��o a este continua a subir de tom e ontem, pelo terceiro dia consecutivo, as ruas de Belgrado foram o palco de manifesta��es exigindo a sua demiss�o.
+Mais de dez mil estudantes exigiram pacificamente o afastamento de Milosevic, a forma��o de um governo de salva��o nacional e a realiza��o de elei��es.
+Decretaram uma greve, ocupam tr�s faculdades e prometem resistir at� � satisfa��o das suas exig�ncias.
+ Hoje de manh�, � o dia do encontro de Fernando Nogueira com o Presidente da Rep�blica e o ministro j� manifestou grande curiosidade quanto ao que Ben Ali ter� para lhe dizer.
+Segue-se a reuni�o com o secret�rio-geral do partido no poder (RCD, Rassemblement Constitutionnel Democratique, Liga Constitucional Democr�tica), e a entrevista com o ministro de Estado e do Interior, Abdallah Kallel, tido como bra�o direito do Presidente e o n�mero dois do Governo.
+Um dos resultados pr�ticos desta fus�o � que Michael Eisner, presidente da Walt Disney, se tornou da noite para o dia o homem mais poderoso do sector.
+Nada mau para quem era acusado de ter demasiadas cautelas na condu��o do seu grupo e de estar tolhido por uma not�vel falta de vis�o global.
+o seu amigo e colega de direc��o Frank Wells morreu num desastre a�reo e, logo depois, o director dos est�dios, Jeffrey Katzenberg (respons�vel pelo �renascimento� da anima��o na Disney), bateu com a porta, insatisfeito por n�o chegar ao topo da empresa, fundando a produtora SKG-Dreamworks com Steven Spielberg e David Geffen.
+Di Matteo recebeu a bola na sua intermedi�ria, progrediu pela zona central e, a 25 metros da baliza, arrancou um �tiro� fulminante que s� parou nas redes.
+Estavam decorridos apenas 42 segundos do jogo da final da Ta�a de Inglaterra em futebol, que opunha o Chelsea ao Middlesbrough.
+No fim, o Chelsea juntou a este golo mais um, de Newton (83'), e conquistou o trof�u pela segunda vez na hist�ria.
+A circular justifica a iniciativa por terem �surgido alguns problemas no tratamento de Testemunhas de Jeov�, quando � imperativo ou h� a eventualidade de se administrar sangue, como medida indispens�vel � manuten��o da vida�.
+Em declara��es ao P�BLICO, Francisco Costa, membro da comiss�o de liga��o do Hospital, um �rg�o de contacto entre os hospitais e as Testemunhas de Jeov�, disse conhecer o documento, que, no essencial, reitera uma pr�tica utilizada anteriormente pela direc��o do hospital, � excep��o do que se passava com o tratamento de crian�as.
+Enquanto at� aqui o HDL tinha em aten��o o parecer dos pais no tratamento de crian�as a necessitar de sangue, de agora em diante os m�dicos est�o livres de o fazer.
+ Francisco Costa, h� tr�s anos na comiss�o de liga��o hospitalar criada pelas Testemunhas, refere que a posi��o do HDL n�o tem semelhan�as com os procedimentos de outros hospitais dos distritos � volta, citando os casos de Coimbra e Santar�m.
+P. -- Como economista, acredita que isso possa acontecer?
+R. -- N�o, n�o acredito.
+Nunca quis assinar uma carta de inten��es com o FMI que colocasse metas que n�o pudessem ser cumpridas.
+Se o acordo for cumprido, tal com ele foi escrito, a consequ�ncia vai ser mais recess�o, maior aperto.
+Os indicadores apontam para uma quebra na actividade econ�mica, para um aumento do desemprego e da infla��o.
+O acordo inclui um maior aperto fiscal e logo a diminui��o dos gastos e da actividade e da oferta dos servi�os de infra-estruturas.
+O acordo indica que vamos continuar num processo recessivo.
+ As metas quanto � infla��o � conseguir que esta, at� ao final de 1992, se situe nos 12 por cento ao m�s.
+P. -- Porqu�?
+A confirmarem-se os confrontos, ser�o o primeiro incidente fronteiri�o grave desde a morte de oito diplomatas e um jornalista iranianos no Afeganist�o, no princ�pio de Agosto, o acontecimento que agudizou a crise entre os dois pa�ses.
+Desde ent�o, Teer�o, que viu derrotados os seus �ltimos aliados na guerra civil afeg�, e Cabul, com a confian�a renovada e o apoio do Paquist�o, envolveram-se numa escalada de amea�as verbais, concentrando ao mesmo tempo frente a frente poderosos efectivos militares -- os maiores desde o fim da guerra entre o Ir�o e o Iraque, em 1988.
+As fam�lias ali residentes, que actualmente sobem a p� cinco andares por escadas de madeira que amea�am ruir, v�o passar a ter elevador e ver�o o interior das suas resid�ncias modernizado, com a instala��o de casas de banho e cozinhas devidamente equipadas.
+O lan�amento do projecto de recupera��o, por concurso p�blico entre arquitectos, ter� lugar em Julho, estando previsto o in�cio das obras, or�adas em 75 mil contos, para o Ver�o de 1997.
+Jo�o Pinto -- Ant�nio Oliveira n�o lhe dever� fazer a mesma surpresa de Carlos Queiroz que, para admira��o de todos, em Junho deste ano o remeteu para o banco dos suplentes no encontro com a Est�nia, de apuramento para o �Mundial� dos Estados Unidos, trocando-o por Abel Xavier.
+O capit�o portista continua numa forma excelente e, apesar dos anos, n�o perdeu a velocidade, percorrendo o seu flanco com grande �-vontade.
+O jovem sportinguista N�lson ter� de esperar.
+H�lder -- O central benfiquista ter� mais dificuldades em conseguir uma �cadeira� no �onze� do seu clube -- onde os lugares parecem estar destinados � dupla brasileira Mozer / Paul�o -- do que na equipa nacional.
+Uma situa��o complicada porque H�lder se tem mostrado, neste in�cio do campeonato, como um dos melhores jogadores do Benfica.
+Se, como se prev�, a Irlanda do Norte jogar com dois pontas-de-lan�a, H�lder ter� fun��es de marca��o e a� deve jogar mais em antecipa��o para evitar descuidos por alguma lentid�o.
+Quem culpabiliza o exterior pelas suas pr�prias falhas est� na verdade procurando desculpas para o seu pr�prio insucesso.
+As etapas boas e m�s por que cada indiv�duo tem de passar fazem parte do sentido delineado para a sua pr�pria vida.
+Isto �, � necess�rio para essa pessoa ter de enfrentar essas circunst�ncias e prosseguir sem apego.
+(Atitude basicamente budista em rela��o a todas as coisas da vida, porque as desagrad�veis causam repulsa, e as agrad�veis tristeza quando j� se as n�o tem).
+O amor deve servir como pano de fundo � vida de cada pessoa.
+Claro que se evoca aqui um amor justo e totalmente abrangente, com base em regras �ticas universais.
+As coisas que ocorrem ao longo da vida consideradas como desagrad�veis devem ent�o ser encaradas como um novo passo na aprendizagem existencial, algo pelo qual temos de passar para evoluir.
+N�o h� que sentir revolta, des�nimo nem tristeza, devendo procurar-se ver sempre �o outro lado da moeda�.
+At� l�, no entanto, o imenso pa�s precisa da ajuda do Ocidente e, se bem que esteja convencido da �irreversibilidade das reformas em curso�, o Presidente russo n�o deixaria de argumentar que, se elas falharem, �a� haveria que pagar dez vezes mais do que os investimentos que agora s�o necess�rios para sustentar a reforma�.
+No encontro com a imprensa, Kohl afirmaria que foi a primeira vez que houve um �debate com total sinceridade e boa vontade, num respeito total por cada um dos parceiros, at� porque a R�ssia entrou na via da democracia, de um Estado de direito e do respeito pelos direitos humanos�.
+Em Castelo de Vide, no Centro Municipal de Cultura, est� patente a exposi��o �Diversidades ... com Mestre�, com obras de pintura de Martins Correia.
+As motos BMW fazem 75 anos de exist�ncia e o Centro Comercial Colombo comemora esse anivers�rio com uma exposi��o hist�rica, na Pra�a Tr�pico de C�ncer (a pra�a central do Centro).
+Ocasi�o para ver 22 modelos antigos e oito recentes de motos BMW.
+Em verdade vos digo que Indiana Jones e os seus �mulos ainda est�o na fase do desmame se os compararmos com alguns her�is do passado, do �serial� ou do filme de aventuras.
+ Neste �ltimo caso, o destaque vai para um filme de guerra, �Jornada Tr�gica�, que ter� menos a ver com o conflito a que se referia do que com o �western� ou a floresta de Sherwood.
+Errol Flynn p�s de lado o arco e as flechas, mas levou o mesmo esp�rito a bordo do bombardeiro.
+Ao vermos filmes como �Jornada Tr�gica�, �Objectivo Burma� e outros �Sargentos Imortais�, podemos interrogar-nos como foi poss�vel que a Alemanha e o Jap�o tivessem resistido tanto tempo a estas pelot�es de �indom�veis patifes�.
+�Vamos para a Austr�lia despachar os japoneses�!
+Promessa cumprida apenas com um desvio na rota, porque a segunda �incurs�o� de Flynn na guerra foi na Birm�nia, onde faz uma razia entre os filhos do Sol Nascente.
+O filme chamou-se �Objectivo Burma� e, embora mais �s�rio�, resultava no mesmo.
+Realce ainda para o mau dia de Marco Pantani, que apenas veio a Espanha para se preparar para o Mundial da Col�mbia.
+O italiano chegou na 66� posto, a 13m45s do primeiro, e � agora 27� na geral, quatro lugares abaixo de Zuelle.
+Hoje, o pelot�o tem mais uma etapa no dif�cil tra�ado dos Piren�us, com in�cio em Naut Aran e final em Luz Ardiden, na parte francesa, ap�s 179km.
+Destaque para a subida do Tourmalet, a 2115m de altitude, que � uma das montanhas m�ticas da Volta � Fran�a.
+Para a Quercus, as 40 incineradoras de res�duos hospitalares existentes no pa�s s�o ilegais.
+Na Procuradoria Greal da Rep�blica deu j� anteontem entrada um processo criminal contra o Hospital J�lio de Matos, de Lisboa.
+Um momento hist�rico � como pode classificar-se a presen�a do Castelo da Maia na �final-four� da Liga das Ta�as em voleibol, que entre hoje e amanh� decorre na cidade de Cuneo, no Norte de It�lia.
+Depois de terem terminado a fase regular s� com uma derrota nos sete jogos realizados, os maiatos n�o dever�o ir muito mais longe.
+Com advers�rios como os gregos do Olympiakos, os espanh�is do Gran Canaria e os italianos do Alpitour Cuneo (actuais detentores do trof�u), ao Castelo da Maia pouco mais resta do que tentar ter uma participa��o digna.
+A cria��o de um Conselho Nacional do Ambiente seria um passo decisivo para criar um f�rum onde se cruzassem todas as for�as que atravessam horizontalmente as quest�es de ambiente e desenvolvimento.
+Contribuiria, a meu ver decisivamente, para um distender de tens�es e limar de arestas entre os planos de desenvolvimento e a correcta gest�o e preserva��o de bens alimentares, e seguramente para o indispens�vel di�logo entre for�as que dele t�m andado arredias.
+As experi�ncias do Conselho Econ�mico e Social ou do Conselho Nacional de Educa��o, com todos os seus eventuais defeitos, s�o sem d�vida um indicador da possibilidade desta �inova��o�.
+ � tempo de desdramatizar o ambiente e de o encarar de uma forma racional e corajosa, n�o basta o bom senso.
+A ver vamos.
+ Em 1990, existiam 800 supercomputadores em todo o mundo, distribu�dos por grandes empresas ou organismos p�blicos que investem nestes equipamentos para os disponibilizar a institui��es cient�ficas que pagam pelo tempo de utiliza��o -- como a FCCN em Portugal (ver �Universidades nacionais sem supercomputador�).
+Para o presidente da FCCN, o n�mero destas m�quinas �� mais ou menos constante�, porque, conforme se v�o desenvolvendo novos equipamentos, outros deixam de se enquadrar na defini��o de supercomputador.
+As duas grandes fam�lias da supercomputa��o s�o as chamadas �m�quinas de multiprocessamento vectorial� (como alguns Cray) e as de processamento paralelo -- de que a mais conhecida � a Connection Machine, da Thinking Machines.
+Ao contr�rio do processamento vectorial, em que v�rios processadores utilizam a mesma mem�ria -- e onde, na opini�o de Heitor Pina, apenas poder�o existir progressos marginais --, as m�quinas de processamento paralelo, surgidas nos anos 60 e generalizadas no final da d�cada de 80, t�m uma mem�ria dedicada para cada processador, acabando com os �engarrafamentos� na partilha da mem�ria.
+Falamos, obviamente, de Nuno Gomes, que marcou quatro golos num jogo do campeonato, selando a vit�ria do Benfica sobre o Varzim.
+O jovem de Amarante, 21 anos feitos em Julho, custou ao Benfica cerca de 600 mil contos e demorou a mostrar servi�o.
+com Souness e o seu futebol de cruzamentos, Nuno Gomes n�o era a primeira op��o mas, provavelmente, acabar� at� por ter mais facilidade em marcar golos.
+O treinador ingl�s liberta mais a equipa, joga com mais unidades na �rea e favorece o ponta-de-lan�a.
+Nuno Gomes marcou quinze golos no Boavista da �poca passada, sobretudo atrav�s de uma segunda parte da �poca -- com M�rio Reis -- muito forte, numa equipa que tinha ainda Jimmy.
+Era uma dupla terr�vel, que se completava pela pot�ncia do holand�s e o jogo mais t�cnico do jovem portugu�s.
+Funcionou �s mil maravilhas e o Boavista acabou por ganhar a Ta�a de Portugal com dois golos de Sanchez e um de Nuno Gomes.
+Deparamos ent�o com uma situa��o caricata.
+Os israelitas dizem que n�o abandonar�o a sua �zona de seguran�a� enquanto os s�rios continuarem a ocupar o L�bano.
+S� que, ao mesmo tempo, pedem aos s�rios que dominem o Hezbollah, reconhecendo implicitamente que s� Damasco pode pacificar a sua fronteira mais vulner�vel.
+� poss�vel que os s�rios tenham permitido, ou at� mesmo fomentado, esta subida de tens�o para obrigar os israelitas a admitir que o L�bano � um protectorado de Damasco.
+�Numa negocia��o � sempre bom ter dois ferros no fogo�, observou o jornalista franc�s Patrice Claude.
+Como corol�rio inevit�vel do processo, e �na impossibilidade de injectar mais capitais pr�prios�, a administra��o da Jotocar �decidiu solicitar em tribunal um processo de recupera��o�, lamentando �os inconvenientes que esta situa��o n�o deixar� de trazer� aos credores.
+Quanto � data da reabertura da empresa, como disse ao P�BLICO um respons�vel da Cuf-T�xteis, �� ainda imprevis�vel�.
+Teixeira da Mota, porta-voz do BFE, limitou-se a dizer, sobre este assunto, que, �ap�s seis meses de efectiva gest�o da actual administra��o�, conclu�dos em Agosto de 1992, �o banco considerou o processo encerrado� e que �o comprador confirmou� ao BFE, nesse mesmo m�s, �a plena viabilidade e efic�cia do contrato celebrado� entre as partes.
+E mais n�o disse.
+E as varia��es dos pap�is n�o foram mais expressivas.
+O vector accionista atravessa, de facto, um mau momento, apresentando permanentes hesita��es.
+Com muitos dos pap�is com a cota��o interrompida em consequ�ncia do per�odo de pagamento de dividendos, o neg�cio continuou pouco expressivo, sem profundidade.
+N�o se prev�, ali�s, que at� ao final do m�s se registem altera��es com significado.
+O �ndice BVL Geral encerrou em baixa, cotando-se nos 973,03 pontos, menos 0,23 por cento, enquanto o BPA Cont�nuo cedeu 0,18 por cento ao fixar-se nos 154,72 pontos.
+Em termos de totais, na Bolsa de Lisboa intermediaram-se 10,592 milh�es de contos, mais 258,62 por cento.
+A Bolsa do Porto encerrou com 5,094 milh�es de contos, mais 215,44 por cento que na segunda-feira.
+Mesmo antes de jogar, o FC Porto j� estava a ganhar com a viagem � Cro�cia.
+Os respons�veis pelo futebol croata gostaram de rever Ivic e, principalmente, da coragem revelada pelos portistas ao aceitarem jogar numa zona de conflito militar premente, e j� garantiram o direito de prefer�ncia ao FC Porto na escolha de futuros talentos.
+Uma bom investimento na terra de onde sa�ram Boksic, Prosinecki, Boban ou Suker ...
+Um lugar na primeira linha da grelha de partida na contrata��o de futuros talentos croatas foi, para j�, o que o FC Porto conseguiu com a visita a esta ex-rep�blica jugoslava.
+Afastado das competi��es internacionais desde 1990 -- a selec��o jugoslava apurada para a fase final do Europeu da Su�cia ficou em casa � �ltima hora e foi substitu�da pela surpreendente Dinamarca --, o futebol croata tenta agora voltar ao circuito europeu.
+E a presen�a do FC Porto faz parte dessa tentativa, com a particularidade dos �drag�es� n�o cobrarem �cachet� para serem a primeira equipa de nomeada a visitar o pa�s.
+Se for aprovada a vers�o final do anteprojecto de lei de liberdade religiosa -- que hoje ser� apresentada publicamente --, os crentes que o desejarem podem passar a indicar que destino pretendem para uma �quota equivalente a 0,5 por cento do imposto sobre o rendimento das pessoas singulares� (IRS).
+De acordo com o texto proposto no ponto 3 do artigo 31, os contribuintes podem indicar qual a �igreja ou comunidade religiosa radicada no pa�s�, a inscrever na declara��o de rendimentos, que desejam que receba aquela dota��o �para fins religiosos ou de benefic�ncia�.
+O valor de 0,5 por cento n�o � aleat�rio.
+Ele corresponde sensivelmente ao valor actual da devolu��o do IVA, que o Estado faz � Igreja Cat�lica, de acordo com a interpreta��o que tem sido feita da Concordata.
+foi essa a op��o em Espanha e It�lia, pa�ses com acordos de Concordata semelhantes a Portugal.
+Desde h� poucos anos, em ambos os pa�ses optou-se por a possibilidade de cada pessoa indicar, na declara��o de rendimentos, qual o fim que pretende dar aquela percentagem -- 0,8 em Espanha, 0,523 em It�lia.
+Destaque: Grande parte da imagina��o da inf�ncia passa-se na descoberta e na contesta��o da mentira das regras que o mundo lhe imp�e.
+noutras encaram-nas como um jogo e respondem-lhes jogando tamb�m, mas � sua maneira.
+O viaduto da Infante Santo ter� uma faixa em cada sentido, dispondo tamb�m de um passeio para pe�es.
+A sua abertura ao tr�fego estava inicialmente prevista para o m�s de Novembro.
+A medida preconizada prende-se com o objectivo de se criaram mais empregos e visa, sobretudo os pa�ses da Comunidade Europeia que, entre 1972 e 1992, viram aumentar o n�mero de postos de trabalho em cinco por cento, contra uma taxa de 37 por cento nos Estados Unidos, Canad� e Jap�o.
+Para combater o mal -- que s� nos 24 pa�ses da Organiza��o para a Coopera��o e Desenvolvimento Econ�mico (OCDE) abrange 36 milh�es de pessoas --, o FMI sugere a promo��o da flexibilidade no mercado de emprego, cuja aplica��o dever� levar � diminui��o de regalias sociais, mas, em alguns pa�ses, � preciso �retirar as restri��es aos hor�rios de trabalho e sobre os empregos em ' part-time '�.
+Defende o Fundo que �h� uma necessidade urgente em quase todos os pa�ses de reexaminar o financiamento e a generosidade global dos regimes de seguran�a social, com o objectivo de eliminar os elementos que desencorajam a cria��o de novos empregos�.
+No relat�rio citam-se os elevados encargos sociais das empresas, a �generosidade� dos subs�dios de desemprego, o �muito elevado� sal�rio m�nimo e as regras de protec��o de emprego, consideradas �muito r�gidas�, como factores desmotivadores da cria��o de postos de trabalho.
+Entretanto, a Junta de Freguesia de Riba de Ave foi recebida anteontem por Agostinho Fernandes, a quem se manifestou preocupada com a situa��o e prop�s o encerramento da ETRSU at� que seja reparada a avaria.
+�O senhor presidente disse-nos que o que se est� a passar � inadmiss�vel e que a popula��o tem raz�o para estar preocupada, mas adiantou-nos que est� convencido de que se trata de um problema t�cnico�, declarou-nos Miguel Lopes, cujo pedido para suspender a labora��o da esta��o at� � repara��o definitiva da avaria n�o recebeu uma resposta concreta.
+O presidente da C�mara, ali�s (na linha do que dissera ao P�BLICO o director regional do Ambiente do Norte, Guedes Marques), afirmou � Lusa que �uma avaria pontual nunca poder� justificar o encerramento de uma estrutura deste tipo [ ETRSU ]�.
+O Conselho de Arbitragem da Associa��o de Futebol de Lisboa apoiou ontem Jorge Coroado e contestou a Comiss�o de Arbitragem da Liga Portuguesa de Futebol Profissional, que excluiu aquele �rbitro do jogo FC Porto-V.Guimar�es depois de ter montado um esquema para culpar Coroado de fugas de informa��o.
+O conselho lisboeta considera �censur�vel� o comportamento de Coroado, mas recorda que n�o est� prevista nas normas qualquer pena para a infrac��o que este ter� cometido, pelo que a comiss�o da Liga n�o podia t�-lo exclu�do do jogo.
+Por isso, condena as �interpreta��es de conveni�ncia� da lei e p�e os seus servi�os ao disp�r de Coroado, para apoiar o �rbitro em �qualquer ac��o disciplinar que porventura lhe seja injustamente instaurada�.
+O volume financeiro em entradas de cinema atinge 5000 milh�es de d�lares, enquanto o mercado dos videojogos vale 13 mil milh�es de d�lares anuais (7000 milh�es gastos em salas de jogos e mais 6000 milh�es em videojogos dom�sticos).
+Por outro lado, segundo dados da editora Capcom citados na revista �Business Week�, foram vendidas 12 milh�es de c�pias de �Street Fighter� em todo o mundo, havendo 25 milh�es de norte-americanos que j� o jogaram.
+A Capcom cr� que, com estes n�meros, n�o ser� dif�cil recuperar os 40 milh�es de d�lares investidos no referido filme.
+Segundo o presidente da empresa, Kenzo Tsujimoto, �o que se ganha na produ��o do filme � o conhecimento e a experi�ncia de que a Capcom necessita para fazer videojogos no futuro�.
+Nem de prop�sito, na Primavera sair� �Street Fighter III: The Movie Game�.
+Mas a dificuldade desta op��o assumidamente militante da organiza��o est� nessa nova voca��o dos portugueses ter de nascer durante o curto per�odo em que a exposi��o estiver aberta ao p�blico.
+Na verdade, a organiza��o da exposi��o tem de juntar duas realidades que sempre andaram afastadas: arquitectura e grande p�blico.
+O desafio posto � organiza��o da exposi��o � ent�o levar os mun�cipes de uma cidade a apaixonarem-se perdidamente por um edif�cio ao ponto de sa�rem para a rua em sua defesa.
+� por isso que a exposi��o �Cassiano Branco e o �den -- Lisboa 1991� resultou na maior opera��o de mediatiza��o da arquitectura jamais vista em Portugal.
+ A grande d�vida que circula entre a comunidade timorense � a raz�o que teria levado esta mulher doente -- Wendi tem esclerose m�ltipla -- a escolher o jornal �Kompas� para prestar tais declara��es, pois �se, em tudo isto, existe um fundo de verdade, um peri�dico indon�sio seria a �ltima escolha para algu�m que quer ter o m�nimo de credibilidade�, comentam.
+[ Jos� Ramos-Horta reagiu j�, acusando Wendi Holland de estar a ser �utilizada pela intelig�ncia indon�sia�.
+Num depoimento telef�nico prestado ao CMR -- o primeiro �rg�o da informa��o estrangeira a divulgar o conte�do da entrevista publicada pelo jornal indon�sio --, Horta disse tratar-se de uma campanha �que se arrasta h� v�rios meses� e foi lan�ada pelo jornalista Petrus Suriadi, que esteve recentemente em Portugal.
+A Comiss�o Europeia divulgou na semana passada as novas previs�es de crescimento para as economias dos Doze em 1994.
+Para o crescimento m�dio � apontada agora uma estimativa de 1,6 por cento, superior em 0,3 pontos � previs�o adiantada anteriormente, no Outono de 1993.
+Mas enquanto a evolu��o m�dia foi revista em alta, os valores para Portugal registaram um recuo ligeiro: dos anteriores 1,4 por cento para 1,25 por cento.
+O trabalho di�rio de Peter Williams � construir mundos e imaginar o modo como as pessoas v�o interagir neles (ou como pensa que isso vai acontecer).
+Deuses em ascens�o, os criadores de realidades virtuais v�o influenciar as sociedades futuras com as imagens que criarem.
+Se j� se d�o conta disso, � algo que n�o deixam transparecer ...
+ Numa casa georgiana com vista para o Tamisa, fora de Londres, coexistem dois mundos.
+Para os diferenciar, tabuletas indicam o �Real World� e, outra, apontada ao est�dio, o �Virtual World�.
+Carlos Cidade, Linhares de Castro, Lu�s Janu�rio, Leal Amado e Moura e S� -- o �nico que n�o � ex-militante do PCP -- s�o os nomes escolhidos e j� ratificados por o N�cleo de Coimbra da Plataforma de Esquerda para integrar as listas do PS � c�mara e assembleia municipais da capital do distrito.
+A Plataforma de Esquerda dever� ainda ficar representada nos concelhos de Montemor-o-velho, Lous�, Condeixa e Cantanhede, Arganil, Figueira da Foz, Miranda do Corvo e Soure.
+Essencialmente no concelho de Coimbra e em v�rios casos, a PE poder� mesmo encabe�ar a candidatura � presid�ncia das juntas de freguesia.
+� a primeira vez que � feita uma persegui��o federal a organizadores de apostas na Internet.
+Nos �sites� das empresas explicava-se como se podia apostar sobre os resultados de jogos profissionais e universit�rios de futebol, basquetebol, h�quei e basebol.
+Os apostadores teriam de abrir uma conta e depositar a� entre 1000 e 5000 d�lares (185 a 925 contos).
+As apostas sobre os resultados do jogo, feitas por telefone ou pela Internet, custavam entre dez e 50 d�lares (1.85000 a 9.25000) -- valores sobre os quais as empresas retinham dez por cento.
+Os eventuais ganhos seriam depositados nas contas banc�rias dos apostadores ou enviados por correio.
+Cada um dos acusados arrisca-se agora a uma pena m�xima de cinco anos de pris�o e a uma multa que pode ir at� aos 250 mil d�lares (45 mil contos).
+Mas Benjamin Brafman -- advogado de Jay Cohen, presidente e propriet�rio de uma das empresas (a World Sports Exchange) -- n�o tem a certeza de que o Governo norte-americano tenha jurisdi��o neste caso.
+Mary Jo White considerou no entanto que as empresas n�o estavam protegidas pelo facto de a sua sede se situar fora dos Estados Unidos.
+Ao longo da semana o Banco de Portugal cedeu liquidez num montante superior ao da semana anterior, tendo mantido as taxas.
+No que concerne � D�vida P�blica corrente, realizaram-se tr�s leil�es de Bilhetes do Tesouro.
+e no terceiro, a 182 dias, foram colocados 25 milh�es de contos, � taxa m�dia de 10,48 por cento, o que acabou por reflectir uma subida das taxas em rela��o aos leil�es anteriores para id�nticos per�odos.
+O abortamento tem sempre uma justifica��o.
+Bem basta sofr�la, bem basta ter de o realizar por causa dela.
+Digase, de uma vez e claramente, o que se quer ou o que se quer mais.
+As reformas a levar a cabo podem conduzir ao desaparecimento do ENDA e � sua substitui��o por outro �rg�o, que englobe as diversas tend�ncias do movimento associativo universit�rio.
+�Quando as coisas n�o correm bem, � preciso mudar alguma coisa�, justificou um dos participantes num encontro de dirigentes associativos que decorreu durante o fim de semana nas instala��es da Universidade de Aveiro.
+A realiza��o de um inqu�rito destinado a apurar o encargo m�dio mensal de cada estudante universit�rio foi outra das medidas sa�das do encontro de Aveiro.
+�Pretendemos definir com o m�ximo rigor os custos m�dios de cada estudante, conforme a regi�o onde se encontra, pois actualmente n�o existem dados concretos sobre o assunto�, esclareceu Miguel Rodrigues, que � tamb�m presidente da Associa��o Acad�mica da Universidade de Aveiro.
+As conclus�es do inqu�rito -- que ser� realizado com o apoio do Conselho de Reitores -- servir�o de base a uma proposta a apresentar ao Minist�rio da Educa��o relativa ao montante das bolsas de estudo e propinas.
+J� no pr�ximo dia 25, os dirigentes associativos ir�o reunir-se com o ministro Couto dos Santos para discutir, mais uma vez, o problema do pagamento das propinas universit�rias.
+De facto, os ind�cios acumulam-se nesse sentido.
+N�o s� por aquilo que se viu ontem nas fortalezas tradicionalmente inexpugn�veis do PCP, mas tamb�m pela receptividade que a campanha de Torres Couto est� a encontrar e, sobretudo, pelo que as sondagens come�am a indicar.
+Segundo o estudo publicado ontem pelo Expresso, a CDU perderia em Set�bal quatro dos sete deputados de que disp�e, passando o PS exactamente para a situa��o inversa.
+Em 6 de Outubro, ver-se-� se estamos mesmo perante uma �d�bacle� comunista em Set�bal.
+N�o � �bvio � primeira vista, mas uma observa��o mais cuidada n�o deixa d�vidas.
+O portal da Quinta do Castro, o �nico que resta dos dois que j� existiram, est� acentuadamente inclinado e pode cair a qualquer momento.
+Um aterro feito durante a constru��o da nova estrada Valen�a-Mon��o est� a pressionar a estrutura para tr�s e ir�, inevitavelmente, provocar o seu desabamento.
+O muro cont�guo ao portal, parte da estrutura original, foi desmantelado durante a implanta��o da nova via, obra que esteve a cargo das empresas Soares da Costa e Monte & Monte.
+No seu lugar, surgiu um muro feito de blocos de cimento.
+Os peda�os da estrutura original foram escondidos pela zeladora nas proximidades, cobertos por mato, prevenindo eventuais furtos da pedra trabalhada.
+De acordo com Manuel Cunha, o actual presidente da Junta de Freguesia, durante as obras, a empresa respons�vel necessitou de retirar saibro da zona onde o muro se encontrava para efectuar um aterro.
+Escavou quanto quis, imediatamente atr�s do muro que deitou abaixo para as m�quinas poderem passar -- diz o autarca que com a promessa de, depois, reconstruir o muro.
+Em 1990, a estrada foi inaugurada e, quatro anos depois, tudo est� na mesma.
+alguns pa�ses, mas sobretudo a Alemanha, pretendem evitar que a moeda �nica assuma logo desde o in�cio o mesmo estatuto legal que as divisas nacionais, neste caso, o marco.
+Se os Quinze conseguirem resolver estes tr�s problemas, resta apenas � cimeira de Madrid escolher o nome da moeda �nica, tendo em conta que a denomina��o �ecu�, expressa no Tratado de Maastricht, � rejeitada pela Alemanha, que prefere �euro�.
+R. -- Porque o simbolismo das escadas est� em todo o lado: servem para descer ao inferno ou subir ao c�u.
+Sobem-se os degraus para chegar ao sucesso ou descem-se, em caso de fracasso.
+P. -- Quais as suas influ�ncias no campo da pintura?
+R. -- O PC serviu-se da extrema-esquerda, como ponta-de-lan�a, mas depois foi v�tima da sua imprepara��o e do seu espontane�smo.
+Quando, � noite, os Comandos atacam a Pol�cia Militar, � outra vez o Costa Gomes que, de madrugada, consegue convencer o Partido Comunista a desistir.
+P. -- Com que contrapartidas?
+Esta � uma das afirma��es principais da posi��o da Igreja Adventista do S�timo Dia sobre o aborto, que considerou ter chegado a altura �conveniente� de divulgar o ponto de vista daquela institui��o religiosa.
+� essa expectativa que poder� esboroar-se totalmente se Jer�nimo e Matos desistirem � boca das urnas, j� que nesse caso o sucessor de M�rio Soares ser� inevitavelmente eleito no dia 14.
+Isto porque, de acordo com a lei, nas presidenciais � eleito o candidato que obtiver mais de metade dos votos expressos -- ou seja, pelo menos 50 por cento mais um --, excluindo-se desta contabilidade os votos em branco e os nulos.
+Nem sempre foi assim.
+A quest�o do peso dos votos em branco no escrut�nio final s� ficou resolvida com uma altera��o da lei eleitoral, de 26 de Novembro de 1985, que veio p�r ponto final ao diferendo que op�s nessa mat�ria o Supremo Tribunal de Justi�a (STJ) e o Secretariado T�cnico dos Assuntos para o Processo Eleitoral (STAPE) � Comiss�o Nacional de Elei��es (CNE).
+Separados por linhas subterr�neas, o Metropolitano de Lisboa e a C�mara Municipal andam �s avessas.
+As solu��es adoptadas para a expans�o da rede est�o a gerar posi��es contr�rias e desta vez � a empresa que escreve a Sampaio para refutar as afirma��es do Munic�pio.
+07.00-09.00 Cafe�na. Os ouvintes da Radical acordam com Pedro Marques.
+Nono m�s do calend�rio lunar isl�mico, durante o qual mil milh�es de pessoas se abst�m de comer, fumar, ouvir m�sica ou de ter rela��es sexuais, o Ramad�o � o m�s sagrado mu�ulmano.
+ Mas para alguns � tamb�m sin�nimo de �jihad� (guerra santa).
+H� onze meses, explos�es na cidade e massacres em zonas rurais causaram pelo menos 400 mortos e traumatizaram a cidade, onde o grupo isl�mico armado (GIA) enfrentou, com sucesso, a apertada malha das for�as de seguran�a argelinas.
+� o que se teme agora, segundo os panfletos, aparecidos sobretudo em mesquitas n�o controladas pelo Estado, dentro das quais as ora��es t�m sido acompanhadas por murm�rios pouco comuns.
+Traumatizado, refugio-me no Canal 1, onde se estreia Tudo pelos Outros, com V�tor Norte a mostrar que estaria muito melhor num palco.
+Aguento, com um estoicismo que eu pr�prio admiro, a inunda��o de lugares-comuns, compar�vel � das �guas do Tejo, por�m menos ben�fica para a agricultura.
+Mas, quando chega o momento de homenagem �s m�ezinhas, n�o resisto a zapar e a saltar novamente para a SIC.
+Perplexidade e desorienta��o: a SIC tamb�m transmite homenagens �s m�ezinhas e aos paizinhos, a �nica diferen�a est� em que, em vez de flores, d�o-lhes m�sica pimba.
+Zapo e rezapo e os momentos de sincronismo repetem-se com enlouquecedora const�ncia: as mesmas l�grimas, as mesmas fungadelas, o mesmo bem-fazer, os mesmos familiares abra�ados uns aos outros.
+J� n�o sei onde estou, talvez j� nem saiba quem sou.
+Gramm ser� o primeiro a declarar formalmente a candidatura, na pr�xima semana, na sua cidade natal de College Station, Texas.
+Na verdade, Gramm planeia a sua campanha h� anos e anda activamente �na estrada� h� meses.
+O ex-governador do Tennessee Lamar Alexander ser� o seguinte a declarar-se, dias depois.
+Pouco conhecido fora do Tennessee e de Washington, Alexander conseguiu ganhar a aten��o do seu partido ao apresentar � sua volta alguns nomes de peso.
+Um documento confidencial do Painel Intergovernamental Sobre Mudan�as Clim�ticas (IPCC), citado na �ltima edi��o do seman�rio brit�nico �Independent on Sunday�, afirma que o globo est� de facto a aquecer e que a polui��o de origem humana � um dos factores respons�veis pelo fen�meno.
+O documento do IPCC vai ser apresentado numa confer�ncia internacional que ter� lugar em Roma em Dezembro.
+O IPCC re�ne nesse documento a opini�o de 2.000 meteorologistas que prev�em no futuro uma maior incid�ncia de doen�as tropicais, o aumento de secas e cheias, a morte de florestas e a diminui��o de colheitas nos pa�ses pobres.
+O anterior relat�rio do IPCC, elaborado em 1990, considerava que a temperatura do globo estava a aumentar mas n�o considerava provado que a causa fosse a ac��o humana.
+Agora, dizem os investigadores, j� n�o restam d�vidas.
+O IPCC � um grupo de peritos que foi estabelecido em 1988 pela Organiza��o Meteorol�gica Mundial e pelo Programa das Na��es Unidas para o Ambiente.
+Mas o Itamaraty tem vindo a sofrer a ac��o corrosiva dos �ltimos governos, que lhe impuseram ministros sem prepara��o, al�m da redu��o de mais de 50 por cento do seu or�amento.
+Os diplomatas brasileiros queixam-se da falta de recursos, j� que s�o obrigados a cobrir muitas despesas com dinheiro do pr�prio bolso.
+Contudo, o Itamaraty apresenta alguns excessos que surpreendem.
+� o caso das embaixadas situadas nos Estados Unidos e em Londres, Paris e Roma, onde as mordomias saltam aos olhos dos brasileiros.
+o diplomata brasileiro vive num apartamento de quase mil metros quadrados na Avenue Foch, junto ao Arco do Triunfo, em Paris.
+Ultrapassadas as considera��es gerais, o documento mergulha quase ponto por ponto nas propostas do minist�rio sugerindo caminhos, apresentando alternativas e recomenda��es.
+Desde logo no que toca ao �conceito de propina�.
+Neste ponto, o CNE avan�a com dois cen�rios sobre a controv�rsia, revelando como o consenso no seio dos conselheiros n�o tem sido f�cil.
+�Todos temos de fazer sacrif�cios e n�s estamos dispostos a colaborar.
+Mas -- acrescentou -- receamos que o sacrif�cio n�o esteja a ser repartido de uma forma justa pela popula��o.
+Est� a ser pedido aos idosos que se sacrifiquem duas ou tr�s vezes mais do que o resto das pessoas�.
+�Primeiro, porque vai aumentar os impostos dos benefici�rios da Seguran�a Social.
+Segundo porque introduzir� mudan�as nos programas de assist�ncia m�dica- Medicare e Medicaid- que, ao baixarem os honor�rios dos m�dicos, far� com que estes n�o queiram assistir tantos doentes, e, terceiro, porque o aumento sobre a energia aumentar� os pre�os do combust�vel dos sistemas de aquecimento caseiro, que � uma parte consider�vel dos or�amentos dos idosos de baixos rendimentos�.
+O segmento accionista da Bolsa de T�quio fechou com reduzida altera��es face � sess�o anterior.
+Operadores disseram que o mercado dever� manter a mesma tend�ncia durante as pr�ximas sess�es devido � forte procura dos investidores dom�sticos e estrangeiros, tal como vem sucedendo nas �ltimas semanas.
+O �ndice Nikkei perdeu 0,16 por cento.
+As audi�ncias m�dias de Fera Ferida e Mulheres de Areia permitem tamb�m analisar o percurso dos notici�rios da noite, que lhe est�o �colados� na l�gica do programador e nos h�bitos do espectador.
+o m�rito do Telejornal � maior se atendermos ao facto de que esse quinto equivale a primeiro ... entre os programas que podem considerar-se fora do pacote de propostas indigentes que continuam a liderar o top).
+Os dados de audi�ncias da semana fornecem outras surpresas, de v�rio tipo.
+Na TV2, a transmiss�o, em hor�rio domingueiro de almo�o, dos Campeonatos Europeus de Atletismo, � hora a que a portuguesa Manuela Machado vencia a maratona feminina, n�o alcan�ou mais do que uns m�seros 3,0 por cento da popula��o.
+a SIC, que investiu uma nota negra na compra dos direitos de cobertura, e outra nota ainda mais negra nos meios necess�rios � cobertura, n�o consegue mais do que um m�ximo de 5,5 por cento de audi�ncia m�dia para as imagens recolhidas por esses meios nunca vistos.
+Tanto barulho para nada, parece.
+Talvez por, nestes dias, canalizar outros clientes, atra�dos pelos nomes de David Lynch e Mark Frost, que tamb�m perderam os seus cr�ditos pela mans�o de Hugh Heffner.
+Uma �ltima constata��o: a TVI est� pulverizada no Top 20 Nacional, sem qualquer proposta acima ou igual aos 10,9 por cento de S�zinhos em Casa do Canal 1.
+O que, nos tempos que correm, talvez n�o seja defeito mas feitio.
+Ao longo de uma obra original em que tem procurado figurar a g�nese e os momentos mais marcantes da nossa modernidade filos�fica e est�tica, o fil�sofo Manfred Frank (ainda desconhecido do leitor portugu�s, mas j� muito traduzido em Fran�a e na It�lia) serve-se, entre outros, do mito do judeu errante, nestas duas vertentes, para seguir um percurso que, na literatura, conduz da viragem para a Idade Moderna e das primeiras viagens pelo desconhecido at� � contempor�nea deriva pelos mares da [...]
+As duas vertentes -- Aasvero e o Holand�s -- cruzam-se e fundem-se a partir do s�culo XVI, com um predom�nio, no plano da mat�ria dieg�tica, da deriva mar�tima sobre a err�ncia terrestre, desde �Os Lus�adas� e as narrativas de viagens inglesas e holandesas, at� ao �Ancient Mariner� de Coleridge, � �L�gende des Si�cles� de Victor Hugo, ao �Bateau Ivre� de Rimbaud e ao �Navio Fantasma� de Wagner, culminando num epis�dio-chave da anti-Odisseia que � o �Ulysses� de Joyce: o dos lot�fagos (co [...]
+A Pol�cia Judici�ria de Coimbra deteve um casal suspeito da autoria de v�rios crimes de burla, dos quais ter�o sido v�timas centenas de pessoas.
+Ao que tudo indica, o modo de subsist�ncia do casal consistia na publica��o de an�ncios -- no �Jornal de Not�cias� e no seman�rio �Expresso� -- de emprego fict�cios, relacionados com actividades agr�colas no estrangeiro, nomeadamente em Inglaterra.
+�Era um pneu que s� test�mos durante tr�s voltas e corremos o risco de o usar, pensando que era o mais indicado.
+Na categoria de 250cc o japon�s Tetsuya Harada, da Yamaha, alcan�ou a sua terceira vit�ria no �Mundial� -- s� perdeu na Mal�sia --, batendo por mais de quatro segundos o italiano Massimiliano Biaggi (Honda) e o franc�s Jean-Philippe Ruggia (Aprilia), respectivamente, segundo e terceiro classificados.
+A corrida foi completamente dominada por Harada, estreante nesta edi��o do Campeonato do Mundo, que assumiu o comando � partida e nunca mais o largou.
+A partir da Zambujeira, as praias favoritas s�o acess�veis atrav�s dos montes, que se atingem atravessando o ribeiro nas traseiras do caf� Fresco.
+O Carvalhal � a mais conhecida, mas outras h� com menos gente e, por isso mesmo, mais escolhidas pelos nudistas.
+J� quase no limite do concelho de Odemira, aparece a praia da Am�lia, mesmo ao lado da pol�mica Odefrutas de Thierry Russel.
+Mais a sul, Odeceixe.
+E na Azenha � imprescind�vel comer um arroz de marisco.
+A data de 4 de Novembro tem sido apontada, geralmente, como o limite para o in�cio da desloca��o.
+No in�cio desta semana, no entanto, um diplomata indon�sio admitiu a possibilidade da visita come�ar em finais de Outubro, o que, a confirmar-se, apanhar� a Comiss�o Eventual praticamente no grau zero da prepara��o da visita.
+�Na �ltima reuni�o [ quarta-feira passada ], entr�mos, tom�mos uma bica, demos umas palmadas nas costas e viemos embora, porque n�o havia informa��es�, revelou ao P�BLICO um membro da Comiss�o.
+�N�o s� desconhecemos os crit�rios para os convites � imprensa, como ignoramos quem vai e de que partidos, pois o PRD desapareceu da actual AR [ tinha um deputado na Comiss�o ] e o PCP e o CDS diminu�ram a representa��o.
+A data aproxima-se, h� pequenas coisas a fazer, como por exemplo vacinas, e n�o podemos estar a vacinar as Cortes inteiras ..�, acrescentou o deputado, manifestando-se preocupado com o atraso na prepara��o de uma visita que �funciona como um jogo na corda bamba, pois tanto pode ser favor�vel a Portugal como � Indon�sia�.
+Uma reac��o semelhante teve o PCP, por interm�dio de Vitor Dias, da sua Comiss�o Pol�tica, que salientou que a base da gest�o camar�ria � a coliga��o entre socialistas e comunistas, continuando esta a manter a sua capacidade, lembrando que o que existira com o CDS fora um acordo pontual entre os vereadores centristas e o PS.
+Mas, na confer�ncia de imprensa -- onde estava ladeado pelo seu assessor de imprensa, Ant�nio Metello --, Jorge Sampaio deixou tamb�m entender estar para �muito pr�ximo� uma decis�o importante.
+A relativa � sua recandidatura.
+Portugal, com 6,7 por cento dos pobres da CEE.
+Um em cada 3 portugueses � pobre, uma em cada 3 casas n�o tem casa de banho e s� um em cada 5 jovens sabe utilizar um processador de texto.
+Qual ser� o pa�s mais jovem da CEE no s�culo XXI?
+P. -- Ainda no que respeita ao n�vel dos rendimentos, h� uma queixa que a oposi��o e as organiza��es socioprofissionais fazem ao Governo e que tem a ver com os custos dos factores de produ��o, principalmente no que respeita �s taxas de juro.
+N�o acha que a actual situa��o, inserida num contexto fortemente concorrencial, est� a penalizar os agricultores portugueses?
+R. -- Considero que temos alguns custos de produ��o mais altos, isso ningu�m poder� negar.
+Mas temos outros mais baratos, como os alimentos para os animais e o pre�o da m�o-de-obra.
+Sampaio visitou, durante todo o dia, as zonas do Pico e Faial mais atingidas pelo sismo do passado dia 9.
+Optou pela pedagogia nos contactos que teve com os sinistrados e nem sequer esqueceu a sua experi�ncia como autarca, recordando in�meras vezes os tempos em que foi presidente da C�mara de Lisboa para explicar as suas teorias -- mais propriamente alguma �insatisfa��o natural� -- sobre realojamentos.
+Ou pelo menos a forma de evitar alguns conflitos entre a popula��o.
+Foi o pr�prio Presidente a pedir para reunir com todas as pessoas atingidas pela cat�strofe.
+Na localidade de Flamengos, onde ainda est�o mais de 200 pessoas a dormir em tendas, Sampaio apelou ao �bom senso e solidariedade t�pica do povo a�oreano� para �compreender que n�o se podem construir todas as casas ao mesmo tempo�.
+�Quando fui autarca e fazia realojamentos era normal que quem n�o conseguia logo uma habita��o ficasse insatisfeito.
+Aqui temos de compreender que n�o se pode ter tudo ao mesmo tempo�.
+A solu��o, para uma melhor harmonia, �� que seja a popula��o a indicar quais s�o os casos priorit�rios�, explicou.
+Com caixotes do lixo, os alunos finalistas do Instituto Superior de Economia e Gest�o de Lisboa (ISEG) fecharam ontem o anexo de Buenos Aires em protesto contra a altera��o dos planos curriculares.
+Sob o mote �40 cadeiras chegam�, os estudantes impediram a entrada no edif�cio a partir das sete da manh�, tentando assim resolver o problema depois de esgotadas �todas as vias diplom�ticas�.
+Com os olhos na R�ssia, tecnicamente em viola��o do texto assinado em 1990, os pa�ses-membros do tratado sobre as armas convencionais na Europa voltam a reunir-se a partir de hoje, em Viena, para tentar limpar definitivamente o continente de armas convencionais.
+Mas ningu�m aposta num sucesso f�cil.
+-- Contradit�ria.
+Ao regime n�o interessava o desenvolvimento cultural do pa�s.
+O que interessava era beber vinho, beber vinho era de comer a um milh�o de portugueses, era dar �s crian�as sopas de cavalo cansado, era ir a F�tima .
+-- Ao futebol -- Bom, s� para ver o Benfica!
+Outra ideia exposta por Soltwedel foi a de que o mercado de trabalho, apesar de envolver pessoas, �deve funcionar como qualquer outro mercado�, acrescentando que �os europeus t�m de se livrar da ideia que os sal�rios n�o t�m nada a ver com o trabalho realizado�.
+�Se n�o se mudar este curso, os asi�ticos v�o esfregar as m�os�.
+Nesta reflex�o sobre o futuro da economia europeia p�s-GATT, o professor alem�o � perempt�rio em afirmar que �os sal�rios v�o descer�, em fun��o do �aumento da competi��o das empresas�, ao mesmo tempo que �o desemprego provavelmente crescer� ou estabilizar�.
+Na sua �ptica, isto deve-se a todo o tempo em que a ind�stria europeia esteve protegida.
+�Depois de tanto tempo debaixo do guarda-chuva, � evidente que a Europa se vai molhar�.
+O Minist�rio da Sa�de vai lan�ar at� ao final de Maio 15 novas experi�ncias na �rea dos centros locais de sa�de, de modo a �melhorar a acessibilidade� aos cuidados de sa�de prim�rios.
+O projecto Alfa, assim designado, foi ontem apresentado na extens�o de Fern�o Ferro do Centro de Sa�de do Seixal, e passa pela cria��o de grupos interdisciplinares e pelo alargamento do hor�rio de funcionamento destes p�los de sa�de.
+A import�ncia do projecto foi real�ada pela presen�a em Fern�o Ferro do primeiro-ministro e da ministra da Sa�de.
+A titular da pasta da Sa�de disse que �o futuro atendimento nos centros de sa�de ser� mais humanizado e personalizado�.
+�� preciso reganhar a confian�a das pessoas e incentivar os profissionais�, afirmou.
+Os bombeiros n�o conseguiram remover de imediato o corpo, que ficou preso entre as ferragens do conjunto dianteiro de rodas.
+Apenas ao meio-dia -- tr�s horas ap�s o acidente -- chegou � esta��o Socorro uma carruagem com ferramentas, macacos hidr�ulicos e cerca de vinte funcion�rios da Divis�o de Manuten��o do Metro.
+Os mec�nicos levaram ainda uma hora para remover algumas pe�as do conjunto de rodas, antes de levantar a automotora A-78 e, finalmente, retirar o corpo mutilado de Pedro Alexandre.
+Os pais do jovem, depois de ouvirem a not�cia do acidente no Metro, j� haviam ido �s urg�ncias do hospital S�o Jos� e depois seguiram at� � entrada do Metro no Martim Moniz.
+Ap�s reconhecerem os documentos do filho, foram conduzidos para a esquadra da PSP da Mouraria, acompanhados da mesma multid�o de curiosos, aparentemente insatisfeita com a simples apari��o do corpo numa maca coberta.
+�Isto parece uma manifesta��o�, comentou um oficial da PSP, antes de ordenar a dispers�o da aglomera��o.
+O segundo lugar do torneio � repartido pelo ingl�s D. J. Russel e pelo irland�s Ronan Rafferty, ambos com 67 pancadas.
+Este, vencedor da ordem de M�rito de 89 e 17� do �ranking� mundial, posiciona-se agora como o principal favorito � vit�ria no Open portugu�s.
+Embora ainda faltem tr�s dias de prova, a verdade � que o seu momento de forma � excelente, ali�s como o comprovam os resultados alcan�ados na presente temporada.
+O irland�s foi primeiro em Palm Mcadows e segundo nos Asian Classic, Dubai Deset Classic e Hong Kong Open.
+O seu �score� no final do primeiro dia do Open, surge na sequ�ncia de 16 voltas abaixo do par.
+Um total de 57 pancadas, para uma m�dia de tr�s por volta.
+Actualmente quinto classificado na Ordem de M�rito Europeia, o eventual triunfo no evento portugu�s permitir-lhe-ia ultrapassar Severiano Ballesteros no terceiro lugar da lista europeia dos ganhos monet�rios.
+Aqui reside a principal curiosidade da prova.
+David Silva, o profissional de Vila Sol, por seu lado, foi o melhor portugu�s em competi��o, terminando os 18 buracos em Par do campo.
+No entanto, come�ou mal, fazendo tr�s �bogeys� nos tr�s buracos iniciais.
+Seguir-se-ia uma recupera��o de grande categoria, na qual alcan�ou cinco �birdies�, o que lhe permitiu manter as aspira��es a um lugar entre os 65 finalistas dos dois derradeiros dias da competi��o.
+�Se jogar bem amanh� [ hoje ], um lugar entre os finalistas � inevit�vel�.
+R. -- N�o tenho sentimentos religiosos, n�o acredito na vida eterna.
+Sei que n�o h� vida do lado de l�.
+�le n�ant�, dizem os franceses.
+Tenho dito aos meus que devem preocupar-se com esse dia, n�o por mim, mas por eles.
+� uma bonita palavra esta, saudade.
+Eu deixarei de existir.
+O que ficar de mim � a consci�ncia do que fui e que os outros recordar�o -- ou n�o.
+mas se a lembran�a for de saudade, ent�o acho que pode ser uma boa coisa.
+Ferreira de Almeida disse ao P�BLICO estar a par da situa��o e que vai solucionar o problema brevemente, colocando uma conduta de cimento ligada ao colector.
+Por ora, os moradores desta zona da Ajuda, vivem rodeados de esgotos por todos os lados.
+Mas, afirma V�tor Castelinho, a Junta de Freguesia tem um projecto para aproveitar o espa�o livre que d� para a Rua Eduardo Bairrada constru�ndo um pavilh�o polidesportivo e dois campos de t�nis, para servir a popula��o da Ajuda, � espera de luz verde na C�mara.
+V�tor Mendes, de 40 anos, licenciado em direito e antigo recordista nacional (2,15m) do salto em altura, � desde o ano passado o director da sec��o de atletismo do Sporting e foi confirmado nas suas fun��es j� pela nova direc��o.
+A sua equipa, encabe�ada pelos coordenadores t�cnicos Bernardo Manuel e Abreu Matos, tamb�m se mant�m, e a t�nica � antes a do seu refor�o, com a entrada de Fernando Mamede para o quadro t�cnico, com fun��es ainda para definir, e de um incondicional do atletismo do clube, Ant�nio Frade, como seccionista.
+� um per�odo de euforia, em Alvalade, que tem um significado �extraordin�rio, pois foi o corol�rio de um trabalho desenvolvido desde h� um ano�, como explicou V�tor Mendes.
+�Quando vim para o Sporting nem havia praticamente equipa de pista formada, houve uma redu��o dr�stica do or�amento e estava quase tudo por reconstruir.
+Agora os dois t�tulos quebraram um jejum de sete e oito anos e foram conquistados, para mais, quando o Benfica era dado como favorito�.
+�S� com essa condi��o aceitei ficar � frente da sec��o�.
+Ex-primeiro-ministro, de 1974 a 1978, e ex-ministro dos Neg�cios Estrangeiros da B�lgica, no in�cio da d�cada de 80, Leo Tindemans �, hoje, o presidente do Grupo Democrata Crist�o do Parlamento Europeu, o segundo maior agrupamento parlamentar de Estrasburgo, a seguir aos socialistas.
+Europe�sta convicto, como de resto o s�o todos os governos deste pequeno pa�s, dividido ao meio pela l�ngua e pela cultura, entre um Norte flamengo e um Sul franc�fono, Tindemans defende que o grau de uni�o pol�tica conseguido em Maastricht � insuficiente.
+Seria necess�ria uma nova confer�ncia intergovernamental sobre a Uni�o Pol�tica Europeia -- mesmo antes da revis�o do Tratado, agendada para 1996 -- e, em seu entender, o alargamento da Comunidade aconselharia a que esta se dotasse de uma verdadeira constitui��o com metas e com princ�pios.
+�Por fim j� n�o conseguia remar mais.
+Tinha as m�os cobertas de bolhas, as costas queimadas, o corpo do�a.
+Com um suspiro, mal agitando a �gua, deslizei para o mar�.
+Suficientemente refrescante e estival.
+O segundo livro traduzido nesta colec��o, deste autor sul-africano.
+� um primeiro livro e recria �a Am�rica dos fins dos anos 40� renovando �a grande tradi��o de Hammet e Chandler�.
+Criou a personagem de um detective negro, Easy Rawling que, no entanto, �n�o procura fazer o contraponto de Marlow e Spade mas, sim, erguer uma figura particularmente convincente num livro contado num ritmo impar�vel e com um sabor pungente e aut�ntico�.
+Segundo o escritor e realizador de cinema Nicholas Meyer era nesta percentagem que Sherlock Holmes dilu�a a sua coca�na, antes de ter sido curado do seu mau h�bito pelos amistosos cuidados do dr. Watson.
+Mas o optimismo continuou a dominar nas sess�es seguintes, apesar das indefini��es quanto � evolu��o das taxas de juro.
+O Dow Jones passou novamente a barreira dos 2.900 pontos para se situar pouco acima dos 2906 pontos na sess�o de quarta-feira.
+Ainda com tend�ncia positiva, a Bolsa de Nova Iorque evoluiu de forma hesitante at� aos 2910 pontos, para no dia seguinte voltar a descer.
+Isto apesar de os indicadores econ�micos at� agora divulgados indicarem um final r�pido para a recess�o no pa�s.
+Na quinta-feira foi o indicador das encomendas de bens duradouros, que em Abril cresceu cerca 2,9 por cento, melhorando as perspectivas de crescimento industrial.
+Na sess�o de sexta-feira, ao subir para os 2913,91 pontos (mais 0,45 por cento do que na sess�o anterior) o �ndice Dow Jones permitiu ganhos da ordem dos 0,95 por cento na semana.
+Boris Ieltsin continua a dar sinais contradit�rios sobre a forma como vai utilizar a sua vit�ria pol�tica no referendo de domingo passado para acelerar o ritmo das reformas e imprimir-lhes uma orienta��o mais clara.
+O cabelo ter� ficado grisalho mas o seu bom aspecto e o ar agaiatado n�o deixam transparecer os seus 50 anos e, muito provavelmente, milh�es de f�s continuar�o a am�-lo quando ele tiver 64.
+O ex-Beatle Paul McCartney iniciar� em breve mais uma extenuante �tourn�e� de um ano pelo mundo, para promo��o do seu novo �lbum.
+A campanha para as elei��es de dia 19 no Punjab come�ou sob o signo das armas e do terror dos separatistas sikhs, que tendem a controlar o curso dos acontecimentos naquele estado indiano.
+Ontem foram assassinados cinco militantes do partido Bharatiya Janata, da direita hind�.
+Ontem, o FC Porto voltou a mostrar fibra de campe�o e deu a volta ao jogo com o Mar�timo, mais uma vez gra�as ao �olho� do treinador, que tamb�m erra, mas acerta mais vezes do que os outros, e a uma equipa que nunca desiste.
+Os campe�es s�o assim e este, ao cabo de 17 jornadas, leva 11 pontos de avan�o sobre o segundo, o Guimar�es, que ontem empatou no Bonfim e j� sonha com a Liga dos Campe�es.
+O Rio Ave, outra surpresa, foi derrotado em casa pelo Farense e tem j� muitos candidatos nas suas costas dispostos a roubar-lhe o terceiro lugar.
+O Estrela da Amadora continua a fazer um campeonato � Fernando Santos, ou seja, um campeonato certinho e sem crises, e at� venceu o Salgueiros na Reboleira.
+O Boavista est� a melhorar e empatou no que come�a a ser muito complicado est�dio do Campomaiorense, numa partida que tamb�m terminou empatada nas cr�ticas dos treinadores ao �rbitro.
+O Chaves, com �lvaro a treinador, �comprou� um bal�o de ar com a vit�ria em Coimbra.
+O Belenenses foi perder ao campo do Varzim e continua a sua marcha a passo acelerado para a Divis�o de Honra.
+H� mais de uma semana que o cessar-fogo entre s�rvios e mu�ulmanos est� a ser mais ou menos respeitado e, � semelhan�a das restantes frentes de combate, a situa��o nesta regi�o tem-se mantido calma.
+Apesar de em Brcko, 54 quil�metros a noroeste, se terem registado violentos confrontos no fim-de-semana.
+Uma espessa sopa de feij�o com peda�os de carne, p�o e beterraba � o almo�o destribu�do no s�bado aos soldados s�rvios.
+A refei��o � subitamente interrompida pela apari��o de um jipe NIVA 600 de onde saem dois soldados armados -- andar armado nesta regi�o, seja-se civil ou militar, � medida obrigat�ria -- com os boletins de voto para o referendo dentro de um grande envelope.
+Na Videoteca de Lisboa, �s 22h30, realiza-se o II Encontro da Associa��o de V�deo, Arte e Novas Tecnologias Interactivas PT.
+No Centro Cultural da Malaposta � inaugurada, tamb�m no �mbito das comemora��es da Revolu��o dos Cravos, a mostra colectiva �Artistas de Abril�, com obras de Jo�o Vieira, Jos� Santa B�rbara, Lu�s Ralha, Maria Keil, Vespeira e Rog�rio Ribeiro.
+�s 21h.
+A entidade patronal da Guial, a empresa t�xtil de Barcelos cujos trabalhadores se mant�m em greve desde a passada quinta-feira, requereu ao Tribunal Judicial de Barcelos a ilegalidade da medida tomada recentemente pelos trabalhadores e que consistiu no impedimento da sa�da de quatro carrinhas da companhia carregadas de mercadoria.
+Na sess�o de ontem do Mercado Monet�rio Interbanc�rio, que deu in�cio a um novo per�odo de constitui��o de reservas de caixa, o Banco de Portugal voltou a n�o anunciar as taxas directoras de interven��o e a manter suspensa a facilidade di�ria, anunciando, no entanto, uma ced�ncia de fundos at� ao montante de 300 milh�es de contos, a seis dias, em sistema de leil�o de taxa de juro contra a recompra de Bilhetes de Tesouro.
+JOS� CAL�ADA -- As rela��es entre o sindicato e o minist�rio n�o existem.
+Est�o, em bom rigor, como sempre estiveram.
+Desde 22 de Dezembro de 1993 que vimos solicitando � senhora ministra uma audi�ncia.
+Isto j� foi feito por variad�ssimos faxes, atrav�s de contactos pessoais em Janeiro de 94, mas n�o tivemos, at� agora, resposta.
+Os contactos -- de natureza informal, nem sequer oficiosos -- t�m sido com o sr. subsecret�rio de Estado-adjunto da ministra.
+ Em termos estritamente oficiais, este sindicato ainda n�o conseguiu ser recebido pela ministra, o que n�o deixa de ser espantoso tendo em conta o tempo decorrido desde a primeira audi�ncia.
+P. -- E porqu�?
+Quanto a Portugal, Ferreira do Amaral reafirmou ontem que as vias priorit�rias de liga��o � Galiza s�o a conclus�o da auto-estrada Porto-Valen�a- e bem assim a conclus�o do IP1, entre a ponte que ontem inaugurou com Jos� Borrell e a ponte que ambos tamb�m inauguraram h� dois anos no Guadiana-, a auto-estrada Famalic�o-Guimar�es e o restante trajecto do IC5 at� Vila Pouca de Aguiar e da� at� Chaves e Verin (IP3), e ainda a conclus�o do IC1 (Porto-Valen�a) ligando alguns tro�os dispersos j [...]
+Tudo terminou na Pousada de S. Teot�nio, na zona hist�rica de Valen�a, entre um matraquear de perguntas dos jornalistas.
+Mas teve a compensa��o de ver, ao lado do seu hom�logo, na larga varanda da pousada, os primeiros ve�culos n�o oficiais a atravessarem a nova ponte.
+E avistou, do lado contr�rio, a constru��o met�lica de Eiffel, sucumbida a mais um engarrafamento.
+Talvez o �ltimo.
+Sem entrar em pormenores, devo recordar que as empresas portuguesas a trabalhar na Guin�-Bissau rondam a meia centena, das quais seis ultrapassam, em investimentos e responsabilidades, o milh�o de contos.
+Por si s�s, aos numerosos pequenos investidores (que n�o est�o inclu�dos nos 50) cabe uma fatia de 2,5 milh�es de contos.
+O Grupo Champalimaud, a Petrogal, a TAP-Air Portugal, a Marconi, a Salvador Caetano, a Tertir, a Somec e a Soares da Costa s�o algumas das que mais investiram e, presume-se, maiores interesses t�m a defender.
+Se a isto acrescentarmos o que est� a ser perdido -- passageiros e mercadorias na TAP, mercadorias na Portline e na Transinsular, dormidas nos hot�is (Hotti e 24 de Setembro), opera��es na banca (Banco Internacional da Guin�-Bissau e Totta & A�ores), turistas nos clubes de ca�a (como o de Cap�) --, as verbas perdidas causar�o vertigens.
+Quando iam a caminho do Pa�s Basco, jornalista e operador de c�mara receberam a not�cia da morte, em Bilbao, de Luiz Andr�s Sampiero, um tenente da Guardia Civil encarregue sobretudo de assuntos ligados � toxicodepend�ncia.
+Esperavam, pois, encontrar uma cidade em estado de s�tio, mas depararam-se com uma �estranha normalidade�.
+Ainda chegaram a tempo de registar a sa�da do funeral da Capela do Rei, �que n�o � o dos bascos�, acompanhado por muitas flores e palmas.
+Quando se trata de falar � que j� � mais dif�cil.
+Os bascos n�o gostam de falar da ETA ou das suas ac��es.
+�N�o falo, choro�, declara um transeunte perante as c�maras.
+� um povo apaixonado, o povo basco.
+Mas tamb�m �um povo adormecido� pelas atrocidades cometidas pela �Espanha federalista contra a individualidade de um povo�.
+E a maior dessas atrocidades passou-se h� 60 anos, em Guernica, para sempre imortalizada no famoso quadro de Pablo Picasso.
+Fra�sto da Silva, ex-ministro da Educa��o e actual presidente do Instituto Nacional de Administra��o, admitiu ter sido sondado para o lugar de Antero Ferreira nos conselhos directivo e de administra��o da Funda��o das Descobertas, tal como noticiou a edi��o de ontem do P�BLICO.
+�N�o falei com o primeiro-ministro, mas fui sondado nesse sentido�, disse.
+Claro, o tom j� � outro.
+Jacques Chirac fala de �terrorismo�.
+Chama �b�rbaros� aos s�rvios.
+Finalmente utiliza a �nica linguagem que eles conseguem compreender: a da firmeza.
+�Fale, senhor presidente!
+Procure falar j�!
+Talvez baste uma palavra mais forte, e que cale fundo, para fazer recuar os fora-da-lei� --, trata-se de um progresso que se imp�e saudar.
+Contudo, uma �nica pergunta: o novo Presidente ficar�-se-por aqui?
+Ou ir� ao ponto de dizer: mais �intoler�vel� ainda que a humilha��o de um soldado � a humilha��o de 300.000 homens, mulheres e crian�as, bombardeados quotidianamente, desde h� tr�s anos?
+Ele sabe que n�o lhe dei o meu voto.
+Mas tamb�m sabe que goza de um estado de gra�a que confere a todas as suas palavras uma imensa repercuss�o.
+Oxal� aproveite esta oportunidade.
+Oxal� seja o primeiro chefe de Estado a tomar finalmente, e de forma clara, o partido da democracia e do direito.
+Como gaullista que �, seria esta a ocasi�o hist�rica para se mostrar fiel � �ideia da Fran�a� de que se diz herdeiro.
+Mil�o.
+Confer�ncia de imprensa de Jean-Pierre Elkabbach comentando os resultados da cimeira das televis�es p�blicas europeias que reuniu na v�spera em Paris.
+O que � ao certo uma �televis�o p�blica�?
+ E o que a distingue de facto das televis�es ditas �comerciais�?
+No fundo, e de acordo com Elkabbach, tr�s apostas: o primado da �produ��o� sobre a �difus�o�; a recusa em opor �divertimento� a �cultura�; e por fim a vontade de conciliar o imperativo do mercado (o famoso �n�vel de audi�ncias� ao qual � definitivamente absurdo pensar que qualquer televis�o possa escapar) e a exig�ncia de qualidade (� qual � perfeitamente escandaloso acreditar que uma �s�rie�, mesmo �popular�, deva ser estranha, por natureza).
+Trata-se de facto de apostas.
+Ou desafios.
+Quer sejam formulados com nitidez, quer sejam formulados, sobretudo, � medida de essa rede de televis�es tecida � escala do continente -- esta � a boa nova da semana.
+Parte significativa, mas n�o integral, do seu report�rio gravado para essa companhia foi reeditado numa compila��o tem�tica de seis LP, primeiro editada pela CBS (actual Sonny) em1986.
+Essa mesma colect�nea deu mais recentemente origem a um reedi��o em dois CD duplos, que se encontram esgotados no nosso mercado.
+RESPONS�VEIS pol�ticos do I�men e da Ar�bia Saudita tentavam ontem fazer diminuir a tens�o entre os dois pa�ses devido a uma velha disputa fronteiri�a que provocou tr�s mortos em combates perto de uma ilha contestada do Mar Vermelho.
+O vice-primeiro-ministro e ministro dos Neg�cios Estrangeiros iemenita, Abdel Kader Bajamal, dever� deslocar-se hoje � Ar�bia Saudita para tentar desbloquear a querela, disse � AFP um diplomata em Sanaa, capital iemenita.
+Foi ministro de Jacques Chirac (1986-1988) e agora � considerado o seu principal rival, embora ainda n�o se tenha candidatado oficialmente �s presidenciais.
+�douard Balladur avisou que n�o tomaria nenhuma decis�o antes de Janeiro e acusou Chirac de usar o RPR como uma fortaleza.
+As diferen�as pol�ticas entre os dois s�o menos �bvias do que o contraste nos estilos.
+Chirac, o extrovertido; Balladur, o taciturno.
+Balladur nasceu em Esmirna (Turquia), em 1929, e formou-se na Escola Nacional de Administra��o, de onde saiu a elite da fun��o p�blica francesa.
+Foi colaborador de Pompidou, de 1969 at� 1974, depois administrou tr�s empresas e, em 1986, entrou no parlamento como deputado do RPR.
+No Governo, apesar da subida do desemprego, continua a ser popular.
+A �ltima sondagem d�-lhe o apoio de 53 por cento dos franceses, contra 47 para Chirac.
+O presidente cessante da Comiss�o Europeia, Jacques Delors, n�o se apresentou ainda como o �candidato da esquerda� � sucess�o do seu correlegion�rio Mitterrand.
+Mas ningu�m duvida de que tem a inten��o.
+Se no in�cio as sondagens lhe atribuiam o �ltimo lugar, agora � mais popular do que Balladur.
+Delors, 69 anos, aderiu ao Partido Socialista em 1974.
+Antes foi militante da Juventude Oper�ria Crist�, l�der sindical, funcion�rio do Banco de Fran�a e professor de Gest�o na Universidade de Paris-Dauphine.
+Foi tamb�m ministro da Economia e �maire� de Clichy.
+Na CE, quis ser um pioneiro, defendendo o ideal de uma �verdadeira federa��o europeia at� ao fim do mil�nio�.
+A equipa do Sporting apresentou-se no Municipal de Chaves com Capucho a ocupar a posi��o de Cadete.
+Da substitui��o n�o resultou, entretanto, qualquer ganho ofensivo para os �le�es�.
+A falta de profundidade atacante, a azelhice no remate -- aspecto em que apenas Juskoviak esteve diferente para melhor -- foram raz�es para o nulo que se verificava na primeira parte.
+Os onze remates do Sporting contra apenas um do Chaves, durante a metade inicial, d�o bem a ideia da superioridade atacante dos homens de Alvalade e n�o abonam em nada a capacidade concretizadora dos seus avan�ados.
+Na segunda parte, a t�nica do jogo continuou a ser a mesma.
+Henrique Calisto decidiu-se, por isso, a mexer na equipa trocando Saavedra por Omer.
+Face � passividade de Saavedra, o novo treinador do Chaves procurou injectar sangue novo na dianteira, esperando que a sua equipa se tornasse mais agressiva na frente de ataque.
+apenas o finland�s Tarkki (outra estreia) incomodava, aqui e ali, a defesa sportinguista.
+A comercializa��o e instala��o do servi�o de televis�o por cabo em Valongo e Ermesinde � da responsabilidade da TV Cabo Porto, uma empresa do grupo TV Cabo Portugal, que prev�, at� ao fim de 1995, estender este servi�o a 75 mil habita��es na �rea do Grande Porto.
+A caravela Boa Esperan�a chegou � doca do Jardim do Tabaco ter�a-feira, ao entardecer.
+Terminava uma rota oce�nica de 10 mil milhas n�uticas atrav�s dos portos de Lisboa (Portugal), C�diz (Espanha), Las Palmas (Can�rias), San Juan (Porto Rico), Nova Iorque e Boston (EUA) e Liverpool (Inglaterra) na Regata Colombo 92, uma viagem comemorativa dos 500 anos de descoberta do Novo Mundo.
+Foram quase 4 meses de navega��o em que 60 velejadores portugueses filiados � Aporvela fizeram as vezes de marinheiros sob o comando dos skippers e irm�os Jo�o Lucio Costa Lopes e Jos� In�cio Costa Lopes.
+O estado dos �sectores produtivos� -- agricultura, ind�stria --, dos servi�os e da �complexidade da m�quina administrativa�, bem como a sua �articula��o� com os problemas da �pobreza� e da �modernidade� � outra das �reas.
+�A pobreza e a Solidariedade� � assim o tema de um col�quio a que Soares assiste em Set�bal, no dia 6, onde intrev�m o presidente da C�mara, o bispo de Set�bal e Bruto da Costa.
+Por outro lado, Soares encontra-se com agricultores em A-dos-C�es, no domingo, 31, em Manique do Intendente, na ter�a-feira seguinte, e ao fim da tarde do mesmo dia, em Vila Franca.
+A 5, no Barreiro, visita a Sociedade Agr�cola Lavradiense, a 9, na Azueira, Mafra, encontra-se de novo com agricultores, na Central Fruteira e a 10 visita a Adega Regional de Colares.
+J� no dom�nio da ind�stria, inaugura as instala��es da Dan&Cake, em Alverca, e visita uma empresa de flores e o Centro Tecnol�gico da Corti�a, a 6, no Montijo.
+�Prefiro n�o fazer progn�sticos pois, como disse ao Presidente [ Felipe Gonzalez ] e ao meu colega, at� que recebamos respostas definitivas aos convites de participa��o j� enviados, n�o gostaria de fazer qualquer antevis�o�, sublinhou James Baker.
+�O senhor Baker est� quase a conseguir a paz�, sentenciou, por seu lado, Gonzalez.
+Como o P�BLICO noticiou, a Unicre remeteu para a DGCP a fundamenta��o da nova taxa, onde se refere que os custos do servi�o, por opera��o, � de 15850.
+Um valor bastante superior � soma da �taxa de cliente�, de 100 escudos, com a nova taxa de 30 escudos.
+Isto �, a Unicre sustenta que os 130 escudos a cobrar diminuem, mas n�o eliminam os preju�zos do servi�o.
+ Pais Antunes, director-geral da Concorr�ncia e Pre�os, n�o ficou �inteiramente convencido� com os argumentos avan�ados pela empresa gestora da rede Visa.
+E por isso pediu a suspens�o preventiva da taxa.
+S�o v�rias as d�vidas suscitadas � DGCP pelo processo da taxa.
+Pais Antunes entende, antes de mais, que o modo como a Unicre quer taxar as gasolineiras � �discriminat�rio�, na medida em que, em toda a rede Unicre, as �bombas� s�o as �nicas sujeitas a um valor fixo, e n�o a uma percentagem sobre o valor das transac��es.
+Num sector como o dos combust�veis, em que as margens de comercializa��o s�o esmagadas e sujeitas a pre�os fixados administrativamente, uma taxa fixa poder�, na opini�o de Pais Antunes, contribuir para que as companhias se encostem sistematicamente aos pre�os m�ximos.
+Por outro lado, o director-geral destaca o facto de Portugal ser o �nico pa�s que conhece em que os portadores de Visa est�o obrigados a pagar uma taxa de 100 escudos.
+Quanto ao peso dos encargos financeiros no valor das vendas, 57 por cento das empresas declara que s�o inferiores a cinco por cento, ao passo que tr�s por cento revela encargos superiores a 20 por cento.
+Analisando estes dados, a AIP constata que, �se � um facto que em todos os subconjuntos (empresas industriais, de constru��o, com�rcio e servi�os, exportadoras e n�o exportadoras) se verifica uma maioria de empresas em que os encargos financeiros s�o inferiores a cinco por cento do volume de vendas, � bastante significativo o facto de um quarto das empresas apresentar uma rela��o encargos financeiros-vendas entre os cinco e os dez por cento (29 por cento no caso das industriais) e 18 por [...]
+Com a partida, para terras de outras gentes, de Buddy Guy, o blues ficou mais pobre.
+Com o regresso, � nossa terra, de Buddy Guy, o blues est� mais rico.
+Se o seu �ltimo disco, �Feels Like Rain�, j� garantia a proximidade da ressurrei��o, �Slippin' In� confirma-a.
+E, mesmo ao cair da folha, 1994 tornou-se um ano de alegria.
+Mestre da guitarra de doze cordas e emblema do blues do Piedmont, Blind Willie McTell � um dos guardadores da mem�ria da genu�na m�sica popular da Am�rica.
+A come�ar em Outubro de 1929 e acabar em Setembro de 1933, totalizando 41 pe�as, esta compila��o � um disco de cabeceira.
+A C�mara Municipal de Gaia vai demolir todas as constru��es de raiz do acampamento cigano de Francelos.
+ A decis�o foi tomada ontem, depois de um encontro com uma representa��o dos moradores que se queixam do tr�fico de droga na zona, atribuindo-o � comunidade cigana ali instalada h� mais de 20 anos.
+Esta � uma das quatro reivindica��es que a autarquia contemplou para agrado dos moradores que afirmam agora ir aguardar pacificamente a actua��o das autoridades.
+Paralelamente, circulou tamb�m na localidade um panfleto apelando ao boicote �s aulas nas escolas da zona.
+Recorde-se que as mesmas pessoas amea�aram j� n�o votar nas pr�ximas elei��es se o problema n�o for resolvido.
+A Pol�cia Judici�ria est� a investigar a morte de Jos� Ant�nio Tom�, de 39 anos, cujo corpo foi encontrado, ontem de madrugada, em Trov�es, S. Jo�o da Pesqueira.
+Ramos Lopes, um �hist�rico� do PSD local, que foi presidente da primeira concelhia e � actualmente director da Funda��o Gulbenkian, encabe�a uma lista sem nomes sonantes da actual gest�o aut�rquica, que integra, entre administradores de empresas, o escultor �scar Guimar�es e Maria Jo�o Vieira, assessora do ex-ministro Diamantino Dur�o.
+Aos 15 anos, Pedro, �Meloman�, �Sapito� para o �people� da Arrentela e do Monte da Caparica, j� fora acusado e absolvido de assalto a um estabelecimento.
+J� passara muitas noites nas esquadras da PSP e nos postos da GNR e j� apanhara �muita porrada da b�fia�.
+Mais qualquer discurso fez que n�o recordo.
+Mas mais espantoso ainda � que nos esperou depois � porta de sa�da, n�o para cumprimentar os clientes -- como � de bom tom! --, mas de novo agredir verbalmente o meu anfitri�o, dizendo-lhe �N�o volte c� mais!� ... e atirando-lhe com a porta na cara.
+N�o pretendo fazer ironia nem obter resultados.
+mamem a pastilha el�stica caladinhos ... sen�o rua !!!
+n�o precisa de se preocupar.
+Mas n�o lhe ensinaram aquela antiga regra do neg�cio que diz que um cliente descontente � um cliente perdido?
+E at� pode n�o saber quem � ...
+s� estava presente o a�oreano.
+Nos vinhos a coisa melhorava um pouco: uns borbas e uns reguengos razo�veis, um Paulo da Silva (Colares) e alguns Bairrada, vinho e espumante.
+Azeite, creme de castanha, frutos secos e algumas ovelhas conclu�am o que Portugal tinha para mostrar.
+Foi pouco.
+Depois de tr�s dias a visitar a feira as opini�es dividiam-se um pouco, sobretudo entre professores e alunos.
+Os primeiros estavam desencantados com o facto do Sal�o deste ano n�o ostentar o brilho, nem as dimens�es, de edi��es anteriores.
+Foi um dos primeiros sintomas do efeito que as decis�es de Maastricht ter�o na ultra-subsidiada agricultura francesa -- e o primeiro sinal de retra��o.
+Vitimado por um ataque card�aco, morreu no passado dia 3 de Julho, em Nova Iorque, o guitarrista Johnny Copeland, um dos �ltimos nomes carism�ticos do blues texano.
+A not�cia da sua morte s� ontem foi conhecida, atrav�s da revista americana �Variety�.
+A maior �rea queimada verificou-se no distrito de Santar�m, com oito mil hectares, enquanto em Bragan�a arderam apenas 64 hectares.
+No per�odo de 1 de Junho a 7 de Julho os bombeiros foram chamados a 7249 fogos florestais.
+Entretanto, o violento inc�ndio que deflagrou ter�a-feira no concelho do Fund�o, entre as povoa��es de Barco e Lavacolhos, voltou a reacender-se na madrugada de ontem, ap�s ter sido considerado extinto na quarta-feira, segundo informa��o dos bombeiros locais.
+Ontem, discursando no Washington Institute for Near East Policy, no �mbito da sua terceira visita oficial aos EUA, Netanyahu voltou a propor o reatamento do di�logo com a S�ria sem condi��es pr�vias.
+Isto �, ignorando tudo o que foi negociado nos �ltimos cinco anos, o que Damasco considera inaceit�vel.
+Netanyahu garantiu ao Presidente s�rio, Hafez Assad, que encontrar� nos israelitas �parceiros razo�veis e cooperantes� se �escolher o caminho da paz.
+O problema � que, para Netanyahu, esse caminho passa primeiro pelo fim dos ataques do Hezbollah no L�bano, enquanto para Assad passa primeiro pela devolu��o dos Gol�.
+�N�s dizemos que queremos retirar-nos do L�bano, mas na imprensa s�ria respondem-me: n�o se retirem�, queixou-se Netanyahu.
+�A S�ria procura ostensivamente a nossa retirada mas, na pr�tica, impede-a�.
+Para os actores � uma terap�utica, para os espectadores � uma aprendizagem e uma li��o.
+Os oitos actores do Grupo Teatro Terap�utico do Hospital J�lio de Matos entregam-se com tal verdade e liberdade, falam t�o desassombradamente de temas-tabus, que os espectadores se sentem em desvantagem.
+Quem s�o os doentes: eles ou n�s?
+Rua do Ouro: reparado at� hoje.
+Rua da Prata: reparado at� hoje.
+... ela atende a Academia Sueca ...
+Ele, em Frankfurt, ela em casa.
+�Foi 15 minutos antes do an�ncio.
+Eu estava na cozinha, onde quase tudo de importante acontece naquela casa, e telefonaram-me.
+Atendi e creio que falou uma senhora em ingl�s -- eu n�o sei ingl�s --mas quando ouvi ' Academia Sueca ' n�o ouvi mais nada, fez-se um vazio na cabe�a, n�o sei se agradeci, se fui cort�s, se dei um grito.
+Porque toda a gente ia para o aeroporto de Madrid � espera dele, era o que estava previsto.
+A minha preocupa��o era essa, ter mandado toda a gente para o aeroporto ...
+Quando lhe perguntamos quais s�o as poss�veis raz�es desta ascens�o aparentemente irresist�vel das mulheres, Whipp responde-nos que acha que existem dois factores principais.
+Talvez n�o seja de estranhar, ali�s, que as novas campe�s da corrida sejam origin�rias da China, o pa�s mais populoso do mundo -- onde o universo da escolha � potencialmente maior do que em qualquer outro pa�s do Mundo.
+O ritmo de treino das novas campe�s chinesas, por exemplo, � absolutamente espectacular.
+Entre outras coisas, elas correm uma maratona por dia em terreno acidentado -- mais de mil quil�metros por m�s! -- e durante cinco a seis vezes por ano o seu treino decorre na meseta tibetana, a cerca de cinco mil metros de altitude.
+Os et�opes est�o este ano ausentes da pista das A�oteias, o que retira algum fulgor � prova masculina.
+Na corrida feminina, o n�vel � superior, pese embora a falta da americana Lynn Jennings, a tricampe� mundial de crosse.
+... e municipais na Tun�sia.
+Quanto � Federa��o Nacional de Professores, que ser� recebida na pr�xima ter�a-feira pelo Presidente da Rep�blica, a quem vai falar sobre a situa��o no sector da educa��o, mant�m um dos dois dias de greve marcados durante o mandato de Diamantino Dur�o.
+Os dirigentes da Fenprof avisam no entanto desde j� que se Couto dos Santos insistir nalgumas das directrizes dos seus antecessores arranjar� lenha para se queimar.
+A mudan�a de ministro n�o basta, diz a Fenprof, disposta a mostrar o descontentamento dos professores na manifesta��o do pr�ximo dia 27, data da greve.
+Depois de uma anterior fase de conversa��es, terminada h� 11 dias, a It�lia conseguiu persuadir a comunidade arm�nia do Nagorno-Karabakh a deixar de boicotar as negocia��es de Roma.
+Mas depois da nova ofensiva azerbaijana, com centenas de tanques, helic�pteros e avi�es de ataque, os dirigentes arm�nios do enclave disseram que n�o conseguiam sair de l�.
+As conversa��es de ontem, presididas por Mario Raffaelli, o mesmo que dirige o processo de paz mo�ambicano, come�aram sem eles, embora a pr�pria Arm�nia tenha estado presente.
+Mantendo uma rela��o evidente com a estrutura hist�rica dos encontros, Albano da Silva Pereira, organizador principal, articula as escolhas entre o nacional e o internacional, a consagra��o e a revela��o, o pedag�gico e o experimental.
+por outro, corre riscos est�ticos.
+Sendo a fotografia um lugar de registo dos corpos, parece que, este ano, do conjunto de exposi��es se desprende mais uma atitude de registo, a participa��o do fot�grafo na constru��o e selec��o da imagem do que a afirma��o das pr�prias imagens, do seu corpo.
+estabelecer uma ordem na multiplicidade ca�tica das imagens pode ser uma tarefa exterior e desligada desse real mas tamb�m pode, ao procurar entender as inten��es dos fot�grafos, enriquecer o conjunto fornecer-lhe um novo sentido.
+Foi este homem, marginalizado at� em decis�es que diziam respeito ao seu pelouro, que em Janeiro decidiu partir a loi�a.
+N�o queria ver o seu nome ligado a uma gest�o altamente duvidosa e pediu ao secret�rio de Estado dos Mercados Agr�colas que mandasse investigar as irregularidades de que tinha conhecimento.
+No essencial, tratava-se da adjudica��o de servi�os de promo��o de imagem e de obras sem concurso p�blico nem visto do tribunal de contas (ver P�BLICO de 21 de Abril).
+Chegadas � Comiss�o Parlamentar de Agricultura atrav�s de uma not�cia do P�BLICO, as acusa��es em quest�o levaram os deputados a ouvir Pedro Rodrigues, agora na prateleira do Iroma, e Branco Rodrigues, agora presidente do Conselho de Administra��o da PEC-Alimenta��o, a empresa que controla as quatro PEC regionais entretanto criadas.
+Conclu�da a audi��o parlamentar, o deputado comunista Lino de Carvalho, relator do processo, n�o teve d�vidas em propor que a Comiss�o Parlamentar assumisse a iniciativa de inqu�rito �s �irregularidades e ilegalidades� detectadas no cap�tulo da promo��o da imagem das PEC e da publicidade da sua privatiza��o.
+O PS anu�u de imediato e o PSD, atrav�s do seu coordenador para a �rea da Agricultura, Carlos Duarte, admitiu claramente a hip�tese de vir a subscrever a proposta de Lino de Carvalho.
+�Depois de ouvirmos o depoimento do eng. Pedro Rodrigues fic�mos com algumas d�vidas relativas aos concursos p�blicos e penso que h� aqui algumas quest�es que o pr�prio Governo n�o conhecia�, afirmou Carlos Duarte ao P�BLICO h� duas semanas.
+Anteontem, por�m, o PSD rejeitou a proposta de inqu�rito pretextando com o facto de estar em curso uma inspec��o da iniciativa do pr�prio Minist�rio da Agricultura.
+�Escrevemos ao PS pedindo um envio r�pido da contra-proposta, passadas tr�s semanas ainda n�o conhecemos oficialmente a posi��o do PS.
+Procurei na �ltima semana contactar o presidente da concelhia socialista, mas, encontrava-se no estrangeiro, espero vir a concretizar esse contacto ainda esta semana� -- conclui Carlos Arrojado.
+As celebra��es do tricenten�rio da morte do (assim convencionado) primeiro her�i nacional do Brasil, Zombi dos Palmares, tem destaque especial na edi��o de hoje do Acontece, na TV2.
+mais de 300 mil c�pias vendidas desde o Natal, �shows� continuamente esgotados no Canec�o, por onde passaram j� cerca de 50 mil pessoas, recep��o calorosa por parte do p�blico.
+O regresso �s mem�rias do passado coloca nos olhos de Joaquim Afonso um brilho de indisfar��vel saudade.
+�Dantes, a vida era cheia de felicidade.
+A aldeia tinha mais liberdade�, garante.
+A agricultura foi, desde pequeno, a sua �nica profiss�o.
+�Mas fiz o exame da quarta classe com distin��o�, assegura, com evidente orgulho.
+Tem raz�es para isso, j� que as aulas exigiam-lhe diariamente um sacrif�cio de mais de 22 quil�metros, feitos a p�, qualquer que fosse a �poca do ano.
+� medida que vai desfiando parte do seu passado, Joaquim Afonso olha fugazmente o c�u, abre mais os olhos, esbo�a um sorriso e l� vai continuando a sua conversa.
+Desta vez, � para lembrar uma prenda do seu pai, que, num dos invernos rigorosos de Busteliberne, resolveu fazer-lhe uma surpresa, oferecendo-lhe umas botas.
+ �Eu gostava tanto delas que, quando passava em s�tios onde sabia que n�o encontrava ningu�m, tirava-as e ia descal�o.
+ Para as poupar�, justifica.
+A cota��o de Lenine nunca foi t�o baixa.
+As filas gigantescas � entrada do imponente mausol�u na Pra�a Vermelha desapareceram.
+Os russos preferem agora passar horas numa fila para um s�mbolo do capitalismo: o McDonald's e os seus hamburgers.
+�O pr�ximo czar da R�ssia vai ser colocado no trono pelo dinheiro�, proclama Alexandre Melnik, um diplomata russo.
+As crian�as que abandonam a escola na ilha de S�o Jorge, A�ores, para ajudar os pais e a economia familiar s�o ainda em grande n�mero, segundo informou o presidente da Comiss�o de Protec��o de Menores de Velas.
+Carlos Noysan acrescentou que o absentismo escolar constitui o principal motivo de queixas apresentadas � comiss�o.
+Entre as raz�es apontadas pelo mesmo respons�vel para a fuga � escolaridade obrigat�ria, em S�o Jorge, est�o dificuldades no pagamento do almo�o das crian�as nas cantinas das escolas, a falta de dinheiro para a compra de livros e a car�ncia de transportes.
+Problemas que, de acordo com Carlos Noysan, o Instituto de Ac��o Social e o Centro de Presta��es Pecuni�rias da Seguran�a Social t�m procurado solucionar.
+O Governo Regional dos A�ores comprometeu-se a pagar os juros de um empr�stimo a contrair pela C�mara de Santa Cruz da Graciosa e empres�rios locais, visando a compra de um barco de passageiros or�ado em 54 mil contos.
+A embarca��o, de 30 lugares e 14 metros de comprimento, destina-se ao tr�fego entre as ilhas do grupo central do arquip�lago, viajando prioritariamente entre a Graciosa e a Terceira.
+O secret�rio regional da Economia, Duarte Ponte, justificou o apoio governamental com o envolvimento camar�rio no projecto, desafiando outras autarquias das ilhas a iniciativas semelhantes.
+A partir de Maio, outra embarca��o com capacidade para 150 passageiros ser� alugada pelo governo regional para ligar as ilhas dos grupos central e oriental.
+Para o desempenho das suas atribui��es espec�ficas no campo social, a Santa Casa da Miseric�rdia de Lisboa (SCML) assegura a obten��o de meios financeiros pr�prios atrav�s, sobretudo, da organiza��o e gest�o, a n�vel nacional, da lotaria nacional, das apostas m�tuas (totobola, totoloto e joker) e, agora, tamb�m, da lotaria instant�nea, em cujos lucros comparticipa.
+Foi D. Maria I que, por real decreto assinado a 18 de Novembro de 1783, concedeu a lotaria � Santa Casa da Miseric�rdia, com o objectivo primordial de sustentar os �hospitais reais de enfermos e expostos�.
+A medida acabaria por imprimir ao jogo uma credibilidade que, de outro modo, talvez n�o tivesse.
+Actualmente, de acordo com a lei vigente, a SCML recebe um ter�o dos lucros l�quidos da lotaria e o Estado fica com dois ter�os.
+Em Monte-O-Novo � feriado municipal.
+�s 15h00 sai � rua a prociss�o em honra de S. Jo�o de Deus.
+Uma hora mais tarde o Movimento Democr�tico de Mulheres promove um conv�vio no gin�sio municipal.
+No Cine-Teatro Curvo Semedo, Mafalda Veiga, Francisco Fanhais, entre outros, participam, �s 21h00, num espect�culo produzido pelo grupo �Porta Aberta�.
+O grupo de teatro Maizum apresenta a pe�a �Florbela�.
+Silvina Pereira interpreta o papel da poetisa Florbela Espanca.
+�s 17h00, no Museu Nacional de Arte Antiga, �s Janelas Verdes.
+No lado oposto desta realidade est�o o Barcelona e o Real Madrid.
+Os catal�es, em baixa de forma que n�o poupa mesmo Figo, suavizam as suas presta��es com o ataque mais realizador -- 37 golos -- e o terceiro posto.
+Lugar de p�dio, mas a sete pontos do comandante e a dois do rival local, o Espanyol, outra das revela��es.
+Est� longe a constitui��o no Nou Camp de um novo �dream team�, como o de Rom�rio e Stoichkov, o que j� enerva a direc��o.
+Consequ�ncia imediata: as diatribes do t�cnico Johan Cruyff, consentidas com os �xitos, apresentadas mesmo como sinais de genialidade, passam agora a ser criticadas.
+J� no Real Madrid, apesar de a equipa continuar mal -- foi humilhada pelo Deportivo da Corunha na quinta-feira com tr�s golos de velocidade de Bebeto --, de n�o reencontrar a frescura que a levou na �poca passada ao t�tulo, o treinador Jorge Valdano continua com o apoio da direc��o.
+Adivinha-se que a prazo, enquanto as d�vidas do clube concentrarem a aten��o dos dirigentes e as contas n�o possibilitarem o pagamento da rescis�o do contrato.
+N�o � de literatura do que aqui se trata, embora a hist�ria, densa e tr�gica, nos envolva desde a primeira p�gina.
+Uma hist�ria bem real, t�o real e violenta que o livro foi j� posto fora de circula��o oficial no Brasil, tal as ondas de choque que provocou o seu testemunho-den�ncia, vinte anos j� passados sobre o desaparecimento sem rasto de S�nia Maria de Moraes Angel Jones.
+Os Estados Unidos devem subir substancialmente as taxa de juro durante o pr�ximo ano, de forma a manter a infla��o sob controlo e manter um ritmo de expans�o sustentada da economia.
+O conselho foi ontem dado pelos t�cnicos da OCDE, no seu mais recente relat�rio sobre o estado da economia norte-americana, em que se avisa as autoridades do pa�s a reduzirem os gastos com a Seguran�a Social com o objectivo de assegurar, a longo prazo, a sa�de financeira federal.
+A organiza��o prev� que os Estados Unidos cres�am 2,9 por cento em 1995, contra 3,8 por cento este ano, enquanto a infla��o passar� de 2,1 por cento em 1994 para 2,8 por cento no pr�ximo ano.
+A OCDE considera que nos dois �ltimos anos a economia norte-americana teve um bom desempenho, mas receia o aumento das tens�es inflacionistas e novas quedas do d�lar, caso as taxas de juro n�o venham a ser aumentadas.
+Ao que tudo indica, a decis�o de uma nova reuni�o com o ministro foi j� tomada, embora a data ainda n�o tenha sido estabelecida.
+Enquanto isso, os universit�rios do Norte t�m estado a reunir-se e a estudar cada um dos princ�pios apresentados pelo titular da pasta da Educa��o.
+� o caso do n�cleo do Porto do Movimento Nacional Contra o Aumento de Propinas e da Associa��o de Estudantes da Universidade de Aveiro, que t�m estado a discutir as mesmas propostas com vista a elaborar um documento onde seja patente a sua tomada de posi��o sobre o assunto.
+Para Miguel Dias, dirigente estudantil da Universidade de Aveiro, as palavras de Couto dos Santos no Porto deixaram �muitas d�vidas em rela��o � justi�a social�.
+Concretizando, aquele universit�rio referiu a convic��o de que �n�o devem ser os estudantes a pagar pelos erros de certos reitores que n�o t�m feito bem a gest�o dos dinheiros�.
+A inten��o dos estudantes de Aveiro �, agora, saber como � feita a gest�o das universidades e, por outro lado, saber �qual � o fim do aumento das propinas�.
+Para os habitantes do Sul, observou a Reuter, a discuss�o da guerra civil nas Na��es Unidas d� credibilidade ao seu Estado, embora este seja apenas reconhecido pela Somalil�ndia, outro territ�rio secessionista, da vizinha Som�lia, que tamb�m ningu�m reconheceu.
+�H� dois tipos de reconhecimento na lei: um claro e outro impl�cito�, comentou Abdul-Moneim Abdullah, professor de Direito na Universidade de �den.
+�Por enquanto, estamos felizes s� com o reconhecimento impl�cito�.
+Enquadrada no local mais terciarizado da capital, a exposi��o Habita��o Lisboa/92, organizada pela autarquia no Terreiro do Pa�o, pretende, durante a semana de 16 a 23, n�o s� lan�ar o debate sobre os problemas no sector como relembrar a fuga de habitantes para fora da cidade, substitu�dos pelos servi�os e o com�rcio.
+Com excep��o para o Instituto Nacional da Habita��o e do Instituto de Gest�o e Aliena��o do Patrim�nio Habitacional do Estado (os dois organismos estatais ligados ao sector) -- que declinaram o convite feito pela C�mara -- estar�o presentes no certame diversas entidades com responsabilidades na Habita��o.
+Assim, entre os 28 expositores contam-se as autarquias, as empresas construtoras, os bancos, as imobili�rias, as seguradoras e as associa��es como a dos inquilinos e a dos propriet�rios, entre outras.
+11 de Setembro -- O Bundesbank interv�m em defesa da lira, sendo seguido pelo banco central da B�lgica.
+13 de Setembro -- O Comit� Monet�rio da Comunidade opta por um realinhamento do SME, com a desvaloriza��o da lira italiana em 3,5 por cento e a valoriza��o das restantes divisas do Sistema em 3,5 por cento.
+ Carlos do Carmo interpreta temas � capella, outros em que � acompanhado por apenas um instrumento e apresenta uma can��o in�dita.
+� o primeiro concerto do programa �Vozes�.
+ Espect�culo do Ballet Folcl�rico Mexicano, no �mbito da programa��o oficial do pa�s.
+Um m�s ap�s a Declara��o anglo-irlandesa sobre o futuro do Ulster, continuam long�nquas as hip�teses do Sinn Fein, o bra�o pol�tico do IRA vir a aprovar os seus termos.
+O documento � pouco favor�vel aos republicanos que s� na P�scoa se pronunciaram oficialmente sobre ele.
+Interessa antes de mais decidir quem preferimos para representar a Rep�blica, a que somos e a que queremos vir a ser.
+A representa��o depende muito da qualidade do ser do representante.
+A este n�vel o estilo � o homem.
+Nenhuma d�vida me sobra ao comparar Cavaco, s�mbolo de um economicismo rasteiro e de uma extrema escassez cultural e espiritual, com Sampaio, pessoa de outra espessura humana e promessa de que a pol�tica continuar� a predominar em Bel�m.
+E quem n�o sentir� que o v�nculo entre entre Sampaio e as liberdades pol�ticas � n�o s� mais antigo mas mais visceral?
+A ag�ncia Nova China informou que para redigir este dicion�rio de 34 470 entradas em l�ngua chinesa, foi necess�rio o trabalho de 300 especialistas durante tr�s anos.
+A enciclop�dia �� considerada o primeiro grande instrumento de trabalho exaustivo e sistem�tico para o mestudo do marxismo-leninismo a ser publicado depois do nascimento da doutrina marxista�, explicou a ag�ncia.
+A primeira edi��o, de 11 mil ecxemplares, est� j� reservada na sua totalidade.
+A FRENTE Polis�rio acusou ontem Marrocos de violar pela terceira vez o cessar-fogo no Sara Ocidental ao enviar avi�es para sobrevoar a povoa��o de Mijek, no sudeste do territ�rio.
+�Para que possa n�o responder �s viola��es marroquinas, a parte sarau� exige que a comunidade internacional lance um alerta a Marrocos para que cesse as provoca��es e para que se comnporte de forma respons�vel, respeitando os seus compromissos�, declarou a Polis�rio, organiza��o que luta pela independ�ncia do territ�rio do Sara Ocidental, num comunicado divulgado em Argel.
+O norte-americano Pete Sampras foi afastado pelo seu compatriota Jim Courier (24� ATP) pelos parciais de 7-6 (7-5), 6-4, o que significa que o n�mero um do mundo vai chegar � �catedral da terra batida�, Roland Garros, com duas derrotas em outros tantos encontros disputados sobre o p� de tijolo.
+Michael Chang, n�mero dois do �ranking�, foi eliminado pelo argentino Hernan Gumy (54� ATP) com os parciais de 6-3, 6-2.
+Surpreendentes foram tamb�m as derrotas dos finalistas do torneio de Hamburgo, na semana passada, ambos batidos por australianos.
+Andrei Medvedev, vencedor do torneio alem�o, foi afastado por Scott Draper por 7-5, 6-3, e F�lix Mantilla (cabe�a de s�rie n�13) foi eliminado por Patrick Rafter por 6-1, 3-6 e 6-4.
+Quando a advogada lhe perguntou porque � que o carro dos portugueses n�o teria sido interceptado, Cardiell afirmou que pensava �estarem combinados com a pol�cia, tendo em conta a forma como as coisas se passaram�.
+�Se nos tinham localizado, podiam ter actuado quando ainda est�vamos com os portugueses�, afirmou.
+�A pol�cia n�o quer que eu preste declara��es.
+N�o lhes interessa saber mais dados.
+V�-se que este trabalho foi preparado pela pol�cia, que ter� facilitado a entrega da droga para nos deter�, rematou o arguido, sugerindo que pode perfeitamente identificar os portugueses envolvidos se as autoridades mostrarem interesse nisso.
+Para um utilizador experiente ou para um principiante, uma liga��o r�pida, eficaz e econ�mica � Internet em Portugal s� � assegurada atrav�s da Teleweb ou da Esot�rica, dois dos cinco fornecedores que operam em Portugal.
+Esta � a conclus�o a que chegou a revista de consumidores �Pro Teste�, que publicou na sua edi��o de Setembro um estudo sobre esta mat�ria.
+Fora das fronteiras do Imp�rio, consegue pouco sucesso no Oriente, mas seduz os germanos atrav�s da �heresia� do arianismo, que considerava Jesus apenas como um homem, excluindo a sua dimens�o divina.
+Mais tarde, os povos que aderiram e esta �deriva��o� ser�o duramente reprimidos.
+Inicialmente religi�o dos pobres, a nova mensagem depressa se estende a todas as camadas sociais e ser� vigorosamente divulgada pelos pequenos grupos iniciais, que depois d�o origem a novas comunidades.
+A devo��o ou o arrependimento de pessoas ricas implica, em simult�neo, a concess�o de doa��es �s autoridades crist�s, que as enriquecem.
+Trezentos anos ap�s a morte de Jesus, o cristianismo iria tornar-se a religi�o oficial do Imp�rio.
+ Um grupo autodenominado �Combatentes pela Liberdade do L�bano� reivindicou ontem o rapto de um oficial da For�a A�rea norte-americana e do seu filho, que desapareceram na Turquia, e amea�ou execut�-los caso n�o seja libertado um dirigente do Hezbollah, revelou ontem em Ancara a ag�ncia Anatolia.
+A ag�ncia referiu que um indiv�duo n�o identificado e que se exprimia mal em turco telefonou de um pa�s estrangeiro para afirmar que o grupo tinha em seu poder o tenente-coronel Mike Couillard, 37 anos, e o seu filho Matthew, dez anos.
+Os dois norte-americanos desapareceram h� tr�s dias no Ocidente da Turquia.
+Segundo apurou o P�BLICO, o ministro Ferreira do Amaral nomeou o presidente do Conselho Superior de Obras P�blicas, Arm�nio Faria, para representar o MOPTC nas negocia��es tendentes a um acordo.
+Um gesto que revela disponibilidade do Governo para �um acordo de cavalheiros� que o presidente da C�mara de Fafe, Parc�dio Summavielle, classifica de �francamente positivo�.
+ A hist�ria remonta a 1983, altura em que o Governo do Bloco Central, atrav�s do Minist�rio do Equipamento Social, titulado pelo socialista Rosado Correia, assinou com a C�mara de Fafe um protocolo no qual se comprometia a custear um centro coordenador de transportes at� ao montante de cem mil contos e a financiar em 90 por cento a constru��o de uma via circular � cidade, como contrapartidas � desactiva��o da linha f�rrea entre Guimar�es e Fafe.
+O XVIII Festival Internacional de M�sica da P�voa de Varzim abre hoje a sua programa��o com um concerto por o Coral de Letras da Universidade do Porto e da Orquestra Esproarte, �s 21h30, na Igreja Matriz da cidade.
+O espect�culo conta com a participa��o de Rui Taveira (tenor), Oliveira Lopes (bar�tono), Thomas Gal (piano), Isabel S� (harpa) e Helena Sofia Pereira (t�mpanos).
+Os m�sicos ir�o interpretar, sob a direc��o do professor Jos� Lu�s Borges Coelho, a cantata BWV 4 de J. S. Bach, bem como a �Sinfonia Simples� e a cantata �Misericordium�, ambas de Benjamin Britten.
+Em Pa�os de Brand�o inicia-se tamb�m o XIX Festival de M�sica de Ver�o.
+Na sede do C�rculo de Recreio, Arte e Cultura actua, pelas 21h45, o Quarteto L�rico do Real Teatro de Queluz, que � formado por Elsa Saque, Carlos Guilherme, Ana Ferraz e Wagner Dinis.
+Os cantores ser�o acompanhados ao piano por Armando Vidal.
+Hoje, a �ltima oportunidade de assistir ao espect�culo de dan�a cl�ssica indiana, com coreografia e interpreta��o de Mallika Sarabhai, uma das mais eminentes bailarinas indianas, especializada nos estilos Bharata Natyam e Kuchipudi, e cuja participa��o na obra de Peter Brook, Mahabharata, no papel de Draupadi, lhe concedeu fama internacional.
+O treinador portista optou por uma marca��o individual e muito atenta sobre os homens do Corunha.
+E logo aos 4', as cerca de 500 pessoas que se deslocaram ao Pavilh�o das Antas assistiram ao primeiro golo apontado por Pedro Alves.
+Durante a primeira parte, a constante rota��o defensiva operada pelos portistas conseguiu anular a rapidez dos espanh�is.
+A concentra��o defensiva, e consequente n�mero de bolas recuperadas, viria a originar o segundo e terceiro golos portistas, ainda na primeira parte.
+Primeiro por Pedro Alves, num inesperado remate � meia volta, e a dois minutos do intervalo por T� Neves que, de costas, conclui uma bonita jogada de envolvimento.
+Para terminar: Francisco Paula de Oliveira merece hoje o protagonismo e a import�ncia que lhe foi negado.
+O reconhecimento que tarda.
+O respeito que se imp�e.
+Os New Jersey Nets perderam a invencibilidade no campeonato, pois foram derrotados pelos Chicago Bulls, por 99-86.
+Os Bulls, com 21 pontos de Steve Kerr, regressaram �s vit�rias e somam agora quatro triunfos -- todos eles conquistados em casa -- e duas derrotas, que lhe d�o o terceiro lugar na Divis�o Central.
+Apesar da derrota, a equipa de New Jersey pode orgulhar-se de ter obtido, nesta �poca, a melhor s�rie vitoriosa num in�cio de campeonato (quatro triunfos consecutivos) desde 1976, altura em que se estreou na NBA.
+A privatiza��o do BFE processar-se-� por concurso p�blico de 65 por cento do capital do banco.
+O pre�o m�nimo por ac��o � de 1980 escudos, valor que se encontra pr�ximo da oferta p�blica de aquisi��o final feita pelo Banco Portugu�s de Investimento, em Fevereiro, o que atribui ao BF um valor global de 158,5 milh�es de contos.
+A opera��o de venda, que incide sobre 52 milh�es de ac��es, permitir� ao Estado um encaixe m�nimo de 103 milh�es de contos, destinado ao Fundo de Regulariza��o da D�vida P�blica.
+O concurso p�blico para a aliena��o do BFE � aberto �a investidores, individualmente ou em grupo, que observem os requisitos de dimens�o de activos, fundos pr�prios e capitais, estipulados no caderno de encargos�, a aprovar por Resolu��o do Governo.
+Stuart Eizenstat, sub-secret�rio de Estado do Comercio norte-americano, lan�ou um apelo �s autoridades do Estado da Calif�rnia para que reveja a sua decis�o de boicote aos bancos su��os, aparentemente sem efeito.
+O Governo su��o reagiu criticamente a este boicote, amea�ando recorrer � Organiza��o Mundial do Com�rcio (OMC).
+A tens�o entre os EUA e a Su��a vai seguramente crescer com a entrevista dada ao �L'Hebdo�, de Genebra, pelo vener�vel universit�rio Jean-Fran�ois Bergier e publicada quarta-feira.
+A Oliva informa os obrigacionistas da emiss�o Oliva/89 que o valor l�quido do juro por obriga��o � de 34 escudos e est� dispon�vel a partir de 20 de Junho na sede da empresa.
+A Indelma-Ind�strias Electro-Mec�nicas SA estar� cotada durante 30 dias no mercado sem cota��es.
+A perman�ncia no mercado come�a a ter efeito a partir do dia 31 de Maio.
+ O ANTIGO chefe de Estado do Uruguai, Julio Sanguinetti, foi eleito para a Presid�ncia da Rep�blica nas elei��es gerais de domingo, segundo a projec��o de um instituto privado ap�s o escrut�nio de 15 por cento dos votos.
+Segundo o instituto Cifra, o Partido do Colorado (liberal), de Sanguinetti, ter� vencido a corrida eleitoral com 33,5 por cento dos sufr�gios, contra 31 por cento do Partido Nacional (conservador), do Presidente cessante Lu�s Alberto Lacalle, e 30 por cento da Frente Ampla (esquerda).
+Sanguinetti, um advogado de 58 anos, que dirigiu pela primeira vez o pa�s entre 1985 e 1990, ficar� no poder por um mandato de cinco anos.
+ Arrigo Sacchi, seleccionador nacional italiano de futebol, indicou ontem os 22 jogadores que v�o representar o pa�s no Mundial de Futebol dos Estados Unidos.
+Os atacantes Gianluca Vialli e Gianluigi Lentini ficaram de fora.
+O Milan -- finalista, com o Barcelona, da Liga dos Campe�es Europeus -- � o clube que mais jogadores fornece � selec��o transalpina, com sete futebolistas, seguindo-se o Parma, com cinco, a Juventus e o L�zio de Roma, ambos com tr�s.
+Na manh� seguinte, com surpresa minha, o telefone tocou e falei tudo o que era necess�rio.
+Espantada com o facto, resolvi aproveitar e fazer algumas chamadas que se encontravam pendentes h� dois dias.
+S� consegui fazer duas, � terceira tudo voltou ao princ�pio.
+Chegada ao meu local de trabalho, tentei desta vez falar com a assistente do meu n�mero de telefone.
+Finalmente algu�m me entendeu, pois de imediato me disse que deveria tratar-se de ... uma avaria na central!
+Como fiquei contente por, ao fim de dois dias, ter conseguido fazer-me entender pela Portugal Telecom.
+E confesso que a minha l�ngua-m�e � o portugu�s ...
+Frei Bento Domingues, O.P.
+Diz-se que um documento do Vaticano ou provoca uma grande pol�mica -- e torna-se um sucesso editorial -- ou vai dormir tranquilamente para as bibliotecas eclesi�sticas.
+Jo�o XXIII teria sido a sant�ssima excep��o.
+Frederico Cunha celebrar� o seu 48� anivers�rio no pr�ximo domingo, na companhia dos seus familiares no Brasil.
+Distante, com a pena de 15 meses de pris�o suspensa, pela infrac��o de favorecimento pessoal, o seu afilhado Jos� Miguel Noite completar� 28 anos no pr�ximo dia 24, num pa�s da Uni�o Europeia onde estuda como bolseiro de uma institui��o madeirense.
+Os mercados de ac��es fecharam ontem em alta, com as bolsas de Londres e Paris (Frankfurt esteve encerrada) a beneficiarem de uma onda de compras coincidente com o come�o da cimeira do Grupo dos Sete (G7) pa�ses mais ricos do mundo, em Halifax, Canad�.
+O d�lar subiu para valores acima dos 1,41 marcos, aproximando-se cautelosamente dos 85 ienes e mostrando grande firmeza.
+Um d�lar est�vel ajudou os investidores activos nos mercados de ac��es, na medida em que serve de suporte aos exportadores europeus, disseram operadores.
+O �ndice FTSE-100 da Bolsa de Londres subiu 0,92 por cento, enquanto o CAC-40, de Paris, valorizou-se 1,43 por cento.
+A Bolsa de T�quio, por seu lado, encerrou nos 14867,26 pontos, mais 206,77 pontos, ao contr�rio de Hong Kong, que terminou em baixa ligeira.
+Wall Street, a meio da sess�o encontrava-se em alta, com o �ndice Dow Jones a cotar-se nos 4493,85 pontos, mais 2,77 pontos da v�spera e muito pr�ximo da barreira psicol�gica dos 4.500 pontos.
+�No seu todo o dia foi muito positivo�, afirmou um �trader�, que salientou estarem os mercados bastante activos.
+Analistas disseram que a reuni�o do G7 pode terminar com o diferendo entre Washington e T�quio relativo ao sector autom�vel.
+Dos v�rios tipos de fundos existentes, os que registaram maior procura, proporcionalmente � sua quota de mercado, foram os internacionais.
+Os pouco mais de 3,3 milh�es de contos que geriam no final do ano passado passaram no final de Junho a 21 milh�es de contos, ou seja, um crescimento de 536 por cento.
+A este aumento n�o � alheio o facto destes produtos serem os que actualmente oferecem as maiores taxas de rentabilidade, a par dos fundos de ac��es.
+A �especilaiza��o do mercado� foi, segundo �lvaro Peixoto, a nota mais importante do semestre.
+Para o secret�rio geral da ASGFIM torna-se, no entanto, necess�rio proceder a certas altera��es dentro do sector, por forma a torn�-lo mais competitivo.
+Uma das reivindica��es � a urgente altera��o da actual lei que baliza o comportamento dos fundos.
+A lei � de 1988 e obriga a que as sociedades gestoras tenham pelos menos 25 por cento das suas aplica��es em d�vida p�blica nacional e 75 por cento das aplica��es t�m que ser em t�tulos cotados em Bolsa.
+A vit�ria, por 3-0, do Estrela da Amadora frente ao Desportivo de Chaves � indiscut�vel, como tamb�m � indesment�vel que aquilo que se viu ontem na Reboleira pouco teve a ver com futebol.
+Os visitados, que fizeram uma exibi��o sofr�vel, s� depois de os transmontanos ficarem com menos um jogador conseguiram criar jogadas com princ�pio, meio e fim.
+O jogo dos visitantes descreve-se com uma palavra: paup�rrimo.
+Tudo � poss�vel encontrar no IX Sal�o de Antiguidades e Coleccionismo, desde objectos de ouro e prata, moedas, mob�lias cl�ssicas, armas, tape�arias, pinturas, livros e postais antigos, onde marcam presen�a de destaque as pe�as de Art Deco, �a mais nova das antiguidades�.
+Estes tr�s items foram considerados pelos alunos -- e at� por professores do ensino secund�rio -- como os mais pol�micos da primeira chamada.
+No que diz respeito � analogia �fluido est� para v�treo assim como viscoso est� para transl�cido�, o j�ri considerou que a validade do conte�do da quest�o � muito reduzida, dado que �o grau de dificuldade da rela��o estabelecida horizontalmente, acrescida do facto de os termos terem um significado em F�sica n�o coincidente com o da linguagem corrente�.
+Assim, todos os candidatos recebem os dois pontos atribu�dos a este item.
+O investimento directo portugu�s no estrangeiro elevou-se no per�odo em an�lise a 482 milh�es de d�lares, o que � mais 48,31 por cento do que o verificado no per�odo hom�logo do ano de 1991, e que se explica em grande parte pelo aumento do investimento industrial em Espanha.
+Quase metade dos valores registados destinam-se ao pa�s vizinho e, desse montante, cerca de 95 por cento ter� sido aplicado na ind�stria.
+O investimento l�quido nacional em t�tulos estrangeiros, por seu turno, elevou-se a 298 milh�es de d�lares quando no ano anterior praticamente n�o tinha express�o.
+Um inc�ndio destruiu, ontem � tarde, quatro habita��es de madeira do bairro de pescadores avieiros de Vila Franca de Xira, desalojando um total de dez moradores, que n�o sofreram quaisquer danos pessoais.
+Trav�es -- Discos � frente, tambores atr�s.
+Rodas -- Pneus 185/60 R 14.
+Entre 1 e 4 de Setembro decorreu em D�sseldorf, a edi��o Primavera-Ver�o 1992 da Igedo, certame de vestu�rio, no qual Portugal participou como pa�s-parceiro.
+A Igedo come�ou em 1949 com apenas sete exibidores e hoje cresceu at� aos dois milhares e meio, em representa��o de 47 pa�ses.
+; e a sua relev�ncia � tanto maior quanto o seu calend�rio a torna um bar�metro da esta��o que antecipa.
+Os pa�ses mais representados em termos de exibidores s�o a Fran�a, a It�lia e, obviamente a Alemanha, enquanto os maiores compradores v�m de B�lgica, Escandin�via e Holanda.
+Coordenadas que fazem com que a Igedo seja especialmente atractiva para as empresas portuguesas viradas para a exporta��o, sobretudo quando o processo de ades�o � CEE torna o momento decisivo para o que � um dos sectores produtivos de maior peso na economia do nosso pais.
+Uma senhora, por�m, que n�o resistiu a abra�ar com algum orgulho o vereador Oliveira Dias, chamando-lhe �ex-camarada� (recorde-se que Oliveira Dias foi, durante largos anos, o rosto da CDU na autarquia portuense), abeirou-se de Gomes para lhe dizer que os arm�rios que a C�mara instalou na avenida s�o ex�guos para guardar a mercadoria.
+O presidente desculpou-se, lembrando-lhe, mais uma vez, que tudo aquilo � provis�rio.
+Quem n�o colocou nenhum obst�culo foram as vendedoras de peixe, que saudaram efusivamente o autarca.culo foram as vendedoras de peixe, que saudaram efusivamente o autarca.
+�� filho, d�-me um abra�o, que eu fa�o sempre campanha por ti e eu hoje n�o cheiro a peixe como no outro dia�, afirmava, emotiva, uma senhora de meia-idade.
+Para o autarca vila-franquense, a �hesita��o� do PSD na discuss�o desta mat�ria estar� relacionada com a posi��o assumida pelo ministro das Obras P�blicas, Transportes e Comunica��es, em reuni�o recente com a C�mara e a Assembleia Municipal.
+�Transpareceu a tentativa de evitar discutir este problema e, depois, uma certa vontade de que o assunto fosse dado como definitivamente arrumado.
+Finalmente, o ministro acabou por admitir que esta quest�o ter� que ser analisada futuramente, n�o deixando de considerar que a aboli��o ter� que se verificar, mas sem definir qualquer horizonte temporal�, sustenta o presidente da Assembleia Municipal.
+Carlos Arrojado aguarda que, depois das f�rias parlamentares, o projecto-lei e a peti��o pela aboli��o das portagens sejam discutidos, em plen�rio da Assembleia da Rep�blica, at� ao final deste ano ou princ�pio de 93.
+�Os interesses da Brisa e do Governo, ao n�vel do Or�amento de Estado, n�o podem continuar a contrariar a necessidade de criar melhores condi��es de vida � popula��o�, acrescenta o autarca, que salienta que por haver portagens n�o pode ser decidida a proibi��o da circula��o de pesados no interior de Vila Franca e Alverca.
+�A �nica justifica��o para n�o haver a aboli��o de portagens � de que, ali, se recebe muito dinheiro�, conclui o presidente da Assembleia Municipal.
+O Braga conseguiu ontem em Chaves a sua primeira vit�ria no campeonato nacional, por 2-1.
+Tal como aconteceu na partida frente ao Farense, Wosniak voltou a ser a principal figura do encontro, mas, desta feita, pela positiva.
+Bem pior est� o Chaves, que continua sem ver a cor dos pontos.
+E, ao fim de tr�s jornadas, a forma��o de Jos� Rom�o soma outras tantas derrotas.
+O Presidente da Assembleia da Rep�blica solicitou a v�rios juristas ligados ao seu gabinete que se pronunciem sobre a legalidade da forma como tem sido descontada no vencimento de deputados do PSD a quantia correspondente �s multas aplicadas por faltas de compar�ncia.
+Barbosa de Melo foi sens�vel �s reclama��es de alguns parlamentares da maioria que a ele se dirigiram pondo em causa tais descontos sem deles terem sido previamente informados e, segundo o P�BLICO apurou, admitiu mesmo vir a solicitar um parecer � Procuradoria Geral da Rep�blica.
+Na reuni�o de ontem, o l�der dos sociais-democratas portuenses n�o deixava qualquer ind�cio sobre a forma como ir� conduzir todo o processo face �s candidaturas que se desenham.
+Lembrava apenas que �tudo est� ainda em aberto� e, por isso, insistiu na necessidade de n�o haver declara��es precipitadas de apoio at� porque poderiam fragilizar �eventuais candidaturas que possam vir a aparecer�.
+E contrariando as teses de que o pr�ximo presidente do PSD ser� fatalmente um l�der de transi��o, defendeu a aposta numa solu��o para quatro anos.
+se o PSD interiorizasse a ideia de que seria uma lideran�a a prazo estaria a admitir desde j� que os resultados das pr�ximas elei��es aut�rquicas iriam redundar num novo desaire eleitoral.
+Pelo contr�rio, Menezes considera que os resultados obtidos nas elei��es presidenciais s�o potenciadores de uma vit�ria nas aut�rquicas.
+Sem iludir a derrota, defendeu que em causa esteve ainda o julgamento do cavaquismo conjugado com �um estado de gra�a� do governo socialista, circunst�ncias que lhe permitem concluir que a margem de 46% dos votos obtida por Cavaco Silva �, apesar de tudo, animadora.
+Para al�m do mais, diz Menezes, o PP saiu destas presidenciais �fragilizado, e come�am a ser not�rias as fracturas internas�.
+o PSD tem de se afirmar como uma oposi��o cred�vel, com uma lideran�a forte, at� porque nas previs�es de Menezes tamb�m entram as medidas impopulares do governo socialista que estar�o a� a chegar ...
+Ora a� est� ...
+talvez chegue mesmo � auto-estrada.
+Perigosamente.
+O citado desvio, pela escarpa do Observat�rio, aproxima-se dos acessos � Ponte do Freixo e � auto-estrada.
+Nada mais nada menos que o Itiner�rio Principal n�.1, o eixo vi�rio mais importante e carregado do nosso pa�s.
+Logo que a V.C.I. do Porto f�r dada por concluida, um aut�ntico formigueiro de carros em andamento.
+Sempre na �nsia de, � primeira oportunidade, cortarem pelo caminho mais recto.
+As autoridades paquistanesas criticam a �ndia pelos seus planos de desenvolver m�sseis bal�sticos, sobretudo o Agni, que tem um alcance de 2500 quil�metros, e o Prithvi, de m�dio alcance.
+O ano passado, o Paquist�o enviou uma carta ao secret�rio-geral da ONU depois de o �Washington Post�, citando os servi�os secretos americanos, ter noticiado que a �ndia instalara m�sseis Prithvis pr�ximo da cidade de Jullundur, no estado do Punjab, no noroeste.
+Nova Deli desmentiu a not�cia.
+� �ndia e ao Paquist�o � reconhecida a capacidade de fabricar -- e de j� possuir -- armamento nuclear.
+Ambos se recusaram a assinar, no final de 1996, o tratado de interdi��o total de testes nucleares (CTBT, segundo o acr�nimo ingl�s).
+Helena Vaz da Silva, uma ex-jornalista, manifestou a opini�o de que a �qualidade vende� e que �os ' media ' ganhadores s�o aqueles que conseguem conjugar qualidade, lucro e interven��o c�vica�.
+Menos optimista sobre a rela��o qualidade / venda est� o analista econ�mico Francisco Sarsfield Cabral, que apontou o caso do mercado brit�nico, onde os tabl�ides vendem dez vezes mais do que os jornais de qualidade.
+O tamb�m ex-jornalista da RTP lembrou que se est� a voltar � ideia de que �o accionista deve interferir na vida das empresas�, mas referiu que �o jornal para ser cred�vel, n�o pode ser portador de recados do seu propriet�rio�.
+Para defesa da credibilidade do pr�prio jornal e para permitir aos leitores um julgamento, disse, �a estrutura dos accionistas deve ser conhecida� como, ali�s, acontece na maior parte dos casos em Portugal.
+Sarsfield Cabral recordou que os ganhos dos investidores medi�ticos devem ser financeiros como acontece noutras �reas empresariais, mas n�o devem ser esquecidos os outros tipos de proventos que eles recolhem: �A influ�ncia pol�tica e a influ�ncia social�.
+Freneticamente, o Governo tem-se empenhado em gastar centenas de milhares de contos em propaganda para nos convencer das vantagens do aumento em 50 por cento da capacidade (seis faixas) da ponte em termos rodovi�rios e noutra inquestion�vel vantagem que � a futura implanta��o nela do caminho-de-ferro.
+ser�o 120.000 passageiros luxuosamente transportados todos os dias e que, como � �bvio, deixar�o de circular de autom�vel.
+Depois, exagera desmedidamente com os paternais cuidados com os utentes da Ponte 25 de Abril, ao ponto de, ap�s tantos benef�cios, ainda lhes querer dar mais uma ponte.
+(...) Depois de tantas melhorias, qual a necessidade da nova ponte?
+N�o se estar� a comportar o Governo como aqueles babados paizinhos que estragam os filhos com guloseimas?
+Estar�o os esfor�ados ministros a prever uma explos�o demogr�fica no pa�s?
+Os respons�veis pelas condi��es de trabalho da imprensa, do Minist�rio dos Neg�cios Estrangeiros, j� montaram, entretanto, no CCB, 630 telefones directos, 25 faxes, cinco aparelhos de telex e 350 m�quinas de escrever para os, certamente poucos, jornalistas que n�o usem o respectivo computador.
+� primeira vista, se compararmos os n�meros -- de jornalistas e de telefones, por exemplo --, n�o parece estar garantido de antem�o que, nas horas de ponta, se evitem os �engarrafamentos�.
+Confrontado com o pedido da mudan�a topon�mica, o munic�pio viseense, atrav�s do seu l�der, Fernando Ruas, oficiou a Junta no sentido de dar o nome de Lu�s Martins a uma outra avenida que atravessa a freguesia e que se situa num tro�o da Entrada Nacional 2 gerido pela C�mara -- o que n�o agradou aos elementos da autarquia repesense.
+Jos� Ferr�o, presidente da Junta, referiu que, mesmo assim, a proposta da edilidade �vai ser analisada numa pr�xima sess�o da Assembleia de Freguesia, a realizar em Abril�.
+Um americano de 43 anos, John Esposito, que se entretinha a sequestrar crian�as numa cela met�lica enterrada no jardim da sua casa em Nova Iorque, que vigiava atrav�s de circuitos internos de v�deo, entrega-se � pol�cia.
+O custo de vida no Funchal � superior ao de Lisboa, revelam estudos que apontam as taxas portu�rias praticadas no arquip�lago como causa do desequil�brio observado.
+O aparecimento de filtros e de certo tipo de tabaco mais suave em cigarros, destinados a evitar os cancros das vias respirat�rias, est�o a multiplicar os casos de uma certa forma de cancro do pulm�o, adianta um estudo realizado pela Sociedade Americana de Cancro (ACS).
+At� aos anos 50, o fumo do tabaco usado nos cigarros era extremamente irritante e, em consequ�ncia disso, era dif�cil inal�-lo profundamente.
+A maior parte dos fumadores realizava, assim, apenas uma inala��o parcial e a maior parte do fumo ficava-se pelas vias respirat�rias superiores e depositava-se na garganta e na boca.
+Era a� que se fixava a maior parte das subst�ncias cancer�genas e a maior parte dos cancros dos fumadores surgiam nestas regi�es.
+Foi quando apareceram os filtros e os tabacos mais suaves que os fumadores come�aram a inspirar o fumo mais profundamente, levando-o mesmo at� aos alv�olos pulmonares, permitindo a cria��o de dep�sitos de subst�ncias cancer�genas nas vias mais finas, na periferia dos pulm�es -- o que deu origem ao desenvolvimento de uma forma de cancro chamada adenocarcinoma.
+Pinochet passeou-se por Portugal, tudo numa visita �oficialmente privada�.
+Governo e Ex�rcito fizeram vista grossa � passagem do general.
+Eis quando sen�o um jipe dos Comandos deu o n�o dito por dito.
+Certo � que Pinochet fez do pa�s uma casa portuguesa, com certeza.
+Secreta foi uma reuni�o nos arredores de Lisboa e uma saltada-rel�mpago a Londres.
+O chefe do grupo conservador �Rossia� no Parlamento russo, Serguei Babourine, concordou que n�o existe �nenhuma base legal� para julgar as ac��es de Erich Honecker enquanto chefe de Estado de um pa�s internacionalmente reconhecido.
+O pr�prio Gorbatchov considerara �imoral� a expuls�o de Honecker mas a posi��o pol�tica em que se encontra actualmente o Presidente sovi�tico n�o � a mais prop�cia para fazer fazer as suas opini�es.
+No dia dedicado a debater as quest�es do emprego, o antigo presidente da Comiss�o Europeia brindou-os com as suas quatro condi��es para o �xito da moeda �nica (uma quest�o que divide profundamente os alem�es), entre as quais est�o um �pacto de confian�a� entre os pa�ses que ambicionam participar na terceira fase da UEM e um �novo sistema monet�rio europeu� que crie o quadro das rela��es entre os que aderem numa primeira vaga e os que ficam de fora.
+Falando ao cora��o dos sociais-democratas, numa cr�tica velada ao ministro das Finan�as Theo Waigel, Delors afirmou que �a pol�tica econ�mica n�o � s� uma quest�o de or�amento e de moeda� a cheirar a �thatcherismo defunto�, mas deve ter tamb�m em conta a dimens�o humana, que envolve os sal�rios e o emprego.
+Mas n�o deixou de advertir para o facto de �o pleno emprego n�o ser poss�vel com d�fices�.
+Estas s�o algumas das novas linhas directivas do TPC anunciadas recentemente pelo Ministro da Educa��o brit�nico, David Blunkett, para as escolas p�blicas.
+Estudos recentes revelaram que metade dos alunos da quarta classe n�o estavam a receber TPC compulsivo.
+A GNR de Barcelos deteve esta semana um indiv�duo de 28 anos, solteiro, desempregado, residente na freguesia de Carvalhas, acusado de ter agredido e tentado violar uma vi�va de 48 anos, residente na mesma localidade.
+O alegado violador foi detido em sua casa, depois de a v�tima ter denunciado o caso �s autoridades policiais quando se encontrava no Hospital Distrital de Barcelos para receber tratamento.
+Os factos, segundo a queixa apresentada, ocorreram ao in�cio da noite, num caminho ermo, sem qualquer casa pr�xima, uma passagem pouco utilizada na aldeia.
+No momento em que regressava a casa, o indiv�duo lan�ou-se sobre ela, tapou-lhe a boca, agrediu-a e atirou-a ao ch�o.
+Depois rasgou-lhe a roupa e tentou consumar a viola��o.
+A mulher ofereceu resist�ncia e, alguns minutos mais tarde, apareceu um filho da v�tima, de 16 anos, que pegou num pau e agrediu o alegado violador.
+Na Escola Superior de Educa��o Jean Piaget, de Macedo de Cavaleiros, est� instalado o clima de medo.
+Nesta vila transmontana t�m ocorrido algumas situa��es de confronto entre estudantes e grupos de jovens locais, a �ltima das quais teve lugar na madrugada de s�bado passado e resultou em ferimentos com alguma gravidade num aluno.
+Para resolver o problema, o director da institui��o de ensino universit�rio vai hoje reunir com o presidente da C�mara e o comandante da GNR locais.
+Centros urbanos e candidatos independentes s�o os denominadores comuns dos pequenos partidos na disputa aut�rquica.
+Tudo por causa dos poucos meios financeiros e das restri��es da lei � candidatura de listas de cidad�os.
+Mais uma semana que passou mais, mais uns m�ximos hist�ricos estabelecidos pela Bolsa de Nova Iorque.
+Apesar de a economia norte-americana estar a dar mostras de um arrefecimento, as ac��es cotadas em Wall Street continuam a valorizar-se de uma forma progressiva e continuada.
+Na semana que passou os fundos de investimento foram os investidores mais activos.
+O �ndice Dow Jones fechou nos 4585,84 pontos, mais 1,66 por cento face � semana anterior.
+O director responsabilizou ainda a Redac��o pela degrada��o da situa��o de �O Primeiro de Janeiro�, exigindo maior empenho por parte dos jornalistas.
+�, por�m, percept�vel que s� aparentemente os dois mundos t�m vivido completamente separados.
+Por um lado -- salientando os aspectos positivos --, as produ��es estrangeiras conduziram a um apetrechamento t�cnico dos prestadores de servi�os nacionais, � constitui��o de excelentes equipas, � rodagem de actores que beneficiaram as produ��es nacionais e, nalguns casos, promoveram internacionalmente a nossa imagem.
+No entanto, ao mesmo tempo, elas muitas vezes vampirizam a imagem do pa�s, determinam um aumento nos custos de produ��o em Portugal -- vide, de novo, as declara��es de Cunha Telles --, o que, por sua vez, conduziu a que o IPC deixasse de ter possibilidade de suportar, para o mesmo n�mero anual de produ��es, a totalidade do seu custo e marginalizaram os nossos criadores -- realizadores, guionistas e m�sicos -- que s� muito excepcionalmente foram chamados a intervir nessas produ��es.
+A necessidade de obter financiamentos fora dos apoios do IPC tem levado a uma altera��o estrutural das rela��es produtores / realizadores no �mbito do cinema portugu�s, com o refor�o do papel dos primeiros, que deixaram de ser meros gestores dos subs�dios obtidos pelos segundos para passarem a ser portadores do valor acrescentado que � o financiamento, normalmente externo, que complementa os apoios locais -- IPC e RTP -- e permite a montagem financeira do projecto.
+Por sua vez, o refor�o do papel dos produtores e a presen�a de produtores estrangeiros trazem altera��es decisivas na pr�pria natureza dos projectos a montar, passando naturalmente a existir uma aten��o diferente em rela��o ao espectador e, consequentemente, �s perspectivas comerciais da obra.
+Do alto de uma varanda, microfone na m�o, ele atordoa a pra�a, vibra e faz vibrar todos os que o escutam.
+S� Sampaio parece n�o ser muito admirador do estilo, cultivando uma sobriedade pouco compat�vel com o espect�culo que lhe foi preparado.
+Mas a verdade � que foi em Abrantes que o l�der socialista teve a melhor entrada e apresenta��o da campanha.
+Quando chega ao palco a assist�ncia j� est� rendida.
+Resta apenas dar substrato pol�tico a uma ades�o emocional garantida.
+O respons�vel por este ambiente � um jovem de 28 anos, l�der da JS de Abrantes e apontador de profiss�o.
+Trabalha na C�mara local, mas a sua paix�o � mesmo o espect�culo, a r�dio, o jornalismo.
+�Isto come�ou na escola.
+Logo naqueles espect�culos de escola, mostrei que tinha um certo jeito para aquilo�, explica Manuel Maur�cio.
+Mais tarde inscreveu-se numa escola de m�sica e n�o perdia tudo quanto era �festinha�.
+Solicitado para apresentar, ler poemas, animar a malta.
+Um �entertainer� ribatejano que foi ganhando notoriedade.
+Nestes �ltimos dias, os bracarenses apenas t�m estudado �pormenores do jogo, pois, quando duas equipas de n�vel semelhante se encontram, os pormenores � que decidem�, refor�ou Donner, que vai iniciar o jogo com uma defesa 3x2x1, agressiva quanto baste.
+Depois, na transi��o para o ataque, M�rio Costa e Paulo Faria (ainda em d�vida devido a uma les�o) saem para dar lugar aos atacantes Bolotskih e Dobrescu.
+tem o t�tulo praticamente assegurado.
+Jos� Saldanha mostra-se ainda �muito surpreendido� com a suspens�o de que foi alvo a ETEM por parte da Direc��o-Geral de Armamento (DGA) e acusa este organismo de �n�o assumir as suas responsabilidades�.
+Alegando ter tomado conhecimento da suspens�o atrav�s do P�BLICO, Jos� Saldanha diz que �as responsabilidades t�m que ser assumidas pela DGA que recebeu todos os documentos da opera��o e autorizou a exporta��o�.
+�quem assinou os documentos de exporta��o n�o fui eu�.
+Jos� Saldanha acusa a DGA de ter um comportamento �amb�guo e vago� nesta hist�ria, �dando cobertura a investiga��es sem credibilidade nenhuma que foram utilizadas na Bol�via num contexto de luta pol�tica pelo poder nas �ltimas elei��es presidenciais�.
+Saldanha n�o quer ser o �bode expiat�rio� deste caso e admite meter um processo judicial contra o Estado portugu�s, com pedido de indemniza��o.
+A situa��o dos professores de Portugu�s a leccionar no estrangeiro dominou a reuni�o, na passada quinta feira, da Federa��o Nacional dos Sindicatos da Educa��o (FNE) com as secretarias de Estado da Administra��o Escolar e da Inova��o e Educa��o.
+Uma das vit�rias reivindicadas pela FNE � a integra��o destes professores no Estatuto da Carreira Docente dos educadores de inf�ncia e dos professores do ensino b�sico e secund�rio.
+Uma medida que, no entender da entidade sindical, �s� peca por tardia� e vai alterar o decreto-lei de 1979 que regulamentava a actividades desses professores, estabelecendo-se agora novas condi��es e regras para o exerc�cio do ensino da l�ngua portuguesa no estrangeiro.
+A �exig�ncia de clareza nos concursos para o recrutamento dos professores� a leccionar no estrangeiro foi outra das reclama��es da FNE, que rejeita a pr�tica actual, �sujeita a despachos pontuais�.
+Defende, por isso, que �os concursos para recrutamento de professores passem a ser regulados por decreto�.
+Outra exig�ncias da estrutura sindical foram que o portugu�s como segunda l�ngua fa�a parte integrante dos curr�culos das comunidades emigrantes no estrangeiro, que o sistema de seguran�a social dos professores assegure a protec��o na sa�de e que o vencimento-base seja acrescido de um subs�dio de custo de vida.
+A �guerra das pedras� deu credibilidade internacional � luta dos palestinianos e obrigou Arafat a renunciar ao terrorismo e a reconhecer Israel.
+Mas a frustra��o pela degrada��o das suas condi��es de vida e, sobretudo, a m� orienta��o da lideran�a palestiniana, fizeram com que os palestinianos perdessem quase tudo o que tinham conquistado, quando decidiram apoiar Saddam Hussein na crise do Golfo.
+Os Cocteau Twins fecharam a 18� edi��o do Printemps de Bourges, infelizmente sem aquilo a que se pode chamar �uma chave de ouro�.
+Valeu uma compensa��o chamada Morphine, desconhecida de muitos, mas que s� veio confirmar a excel�ncia do programa deste ano do festival franc�s.
+Os coment�rios de Miguel Sousa Tavares (MST) inseridos no artigo do P�BLICO de 4 de setembro sob o t�tulo em ep�grafe s�o deveras esclarecedores no que concerne aos tributos dos cidad�os.
+tem que ver com a contribui��o aut�rquica no que respeita a bens n�o geradores de rendimento, como � o caso de terrenos classificados de �urbanos�, a partir de 1989.
+Depois, verifica-se uma enorme discrep�ncia de valores na sua avalia��o para efeitos fiscais, sempre gravosa, sem atentar sequer nas flutua��es do mercado e das pr�prias leis.
+mas, se o objectivo foi libertar os terrenos, evitando a estagna��o, seria necess�rio que as autarquias respondessem �s solicita��es, o que n�o acontece.
+Assim, encorajado pela total disponibilidade do meu interlocutor, que igualmente admito, tentaria que entendesse, pelo menos, as vantagens econ�micas, em termos de desenvolvimento local (tur�stico, por exemplo), da salvaguarda da invulgarmente importante, mesmo � escala mundial, jazida com pegadas de dinoss�urios da serra d'Aire, cujo valor cient�fico j� trouxe aqui cientistas de todo o mundo (dos EUA � China), e da transforma��o do s�tio num local p�blico com todos os equipamentos neces [...]
+Quanto a este assunto, tentaria convencer o professor a visitar o local e procurar saber as opini�es dos respons�veis locais e das respectivas popula��es e, ainda, a auscultar o sentir dos v�rios sectores da vida portuguesa sobre este caso, incluindo o do cidad�o an�nimo.
+A prop�sito da exposi��o �Dinoss�urios da China�, em curso no Museu Nacional de Hist�ria Natural, come�aria por insistir no convite a Sua Excel�ncia a visit�-la.
+Aproveitaria para lhe mostrar o estado lament�vel de um enorme casar�o, esventrado e em tosco, na sequ�ncia do inc�ndio da Faculdade de Ci�ncias (h� 18 anos!), de reinstala��o sempre adiada e pomposamente referido como Museu Nacional, que pouco deve � tutela, mas ao qual se reconhece uma obra cient�fica, cultural e pedag�gica not�vel.
+Dir-lhe-ia que reunir aqui esta magn�fica colec��o de verdadeiros f�sseis, alguns gigantescos, dos terrenos mesoz�icos da velha China foi m�rito exclusivo deste Museu, que n�o contou com quaisquer apoios diplom�ticos, numa realiza��o que vai fazer mais pelo estreitamento das rela��es entre os dois pa�ses do que quaisquer outras j� realizadas.
+As centenas de milhares de visitantes esperados assim o permitem concluir.
+ Dir-lhe-ia, ainda, como � lament�vel neste caso a falta de apoio do Minist�rio da Educa��o.
+De resto, os comerciantes espanh�is parecem estar mais atentos ao que se passa no Algarve do que os colegas portugueses.
+Al�m do p�o, tamb�m a pastelaria espanhola passou a ocupar as prateleiras dos supermercados, a pre�os concorrenciais.
+Quanto �s discotecas, parecem ser o mealheiro dos jovens -- � nelas que �investem� a principal fatia do or�amento de f�rias --, apesar de tamb�m neste ramo os espanh�is estarem a descobrir um fil�o.
+Fi�is aos seus �pubs� em Albufeira, continuam os ingleses, mas longe v�o os tempos em que com um libra -- em 1988 valia 285 escudos, hoje fica-se pelos 230 -- faziam figura de reis e senhores.
+A primeira das cr�ticas ontem apresentadas foi para o novo sistema retributivo, que os enfermeiros consideram nunca se ter adaptado � carreira de enfermagem.
+Segundo explicou o dirigente Jos� Azevedo, a entrada em vigor dos novos diplomas trouxe consigo �in�meras anomalias, geradoras de injusti�as gritantes�, como � o caso dos profissionais que, depois de subirem na carreira, ficam a ganhar menos do que antes da progress�o.
+Quanto a horas extraordin�rias, os enfermeiros salientam n�o terem acesso �s �vultuosas verbas� que nelas se consomem e que para alguns profissionais da sa�de funcionam �mais como complementos disfar�ados de vencimento do que como resposta a necessidades efectivas�.
+Outros processos a contribuir para o descontentamento da FENSE s�o os que dizem respeito ao descongelamento de vagas, � cria��o de uma Ordem dos Enfermeiros e �s equival�ncias de t�tulos acad�micos.
+Em rela��o ao primeiro, a desactualiza��o dos quadros das institui��es cria um desajustamento em rela��o �s necessidades que faz com que, na pr�tica -- diz Jos� Azevedo -- nem um ter�o das vagas disponibilizadas seja aproveitado para colocar novos enfermeiros.
+Resultado: h�, ao mesmo tempo, �institui��es com falta de pessoal e enfermeiros no desemprego�.
+O que a FENSE pede � �um regime de excep��o na contrata��o de novos enfermeiros, como o exigem as circunst�ncias�.
+Professor.
+Por que raz�es escolheu fazer f�rias em Portugal?
+Foi uma escolha acertada?
+R. -- � uma constata��o, uma diferen�a de estilo.
+Sempre fui muito claro nas minhas convic��es.
+Acho que a milit�ncia partid�ria toda a gente percebeu que n�o vou ter, nem terei, nem tenho tido.
+Mas n�o faz sentido, em meu entender, no quadro actual da democracia portuguesa, fazer isso hoje.
+Acho que a avalia��o que as pessoas t�m que fazer sobre as personalidades, sobre o perfil e a sua adequa��o � fun��o tem muito a ver com as pessoas e n�o com ter ou n�o ter um cart�o.
+P. -- Mesmo que seja eleito, n�o se desvincular� do partido?
+Muito provavelmente, a resposta oficial ser� positiva.
+Apesar de parecer paradoxal, Milosevic e o seu partido t�m agora boas raz�es para alguma esperan�a, pelo menos para a sua pr�pria sobreviv�ncia pol�tica no poder, se n�o mesmo para um futuro que poder� revelar-se brilhante e colorido.
+Os antigos (e agora �reformados�) comunistas conseguiram regressar ao poder em diversos pa�ses (Hungria, Pol�nia, Bulg�ria) e uma op��o pol�tica similar tem tamb�m agora boas possibilidades nas pr�ximas elei��es presidenciais na R�ssia, marcadas para Junho.
+Por saber que desempenha um papel decisivo na sua nova fun��o de �construtor da paz� na B�snia -- sob os des�gnios da Pax Americana delineada em Dayton, Ohio -- Milosevic espera que ele pr�prio e o seu regime na S�rvia possam estar a salvo.
+Entre as promessas, a mais importante foi a de que tenciona retirar as tropas russas da Tchetch�nia, embora n�o tenha dito quando, mas apenas que n�o ser� na totalidade, e n�o seja a primeira vez que an�ncios semelhantes s�o feitos nas v�speras de novas ofensivas militares.
+�Queremos retirar as tropas at� �s fronteiras da Tchetch�nia�, disse Ieltsin durante uma visita a Tcheliabinsk, nos Urais.
+�Se retiramos totalmente, os bandidos degolar�o imediatamente toda a popula��o civil�.
+�Eles n�o se ficar�o por a�.
+Ir�o at� ao Daguest�o, ao Karachaevo-Tcherk�ssia e a outras rep�blicas do C�ucaso do Norte, para as ocupar, e haver� terrorismo e banditismo internacional�.
+Programa Quadro de Ci�ncia e Tecnologia da Uni�o Europeia.
+Neste cap�tulo, a principal vantagem da nossa ades�o foi a abertura de novos horizontes, o lan�amento de rela��es com grupos de investiga��o e empresas -- a gera��o de um fluxo de informa��o que permitiu abrir novas perspectivas no sistema de investiga��o cientifico e tecnol�gico nacional.
+Jos� Ant�nio relembra que Max era levado da breca para contar hist�rias e para pregar partidas.
+�Um dia, o Humberto Madeira, o Raul Solnado e o meu pai iam em viagem e passaram pelo pinhal de Leiria.
+O Humberto Madeira resolveu contar que esta era uma zona onde, em tempos, tinha havido muitos assaltos.
+No regresso, o meu pai j� vinha a dormir.
+Rebentou um pneu e ouviu-se um grande estrondo.
+A sua distrac��o ia ao ponto de telefonar para casa a perguntar se o filho se chamava Jos� Ant�nio ou Ant�nio Jos�.
+Embora n�o soubesse uma nota de m�sica, trauteava e compunha por intui��o.
+�Tinha um espectacular ouvido musical e era capaz de distinguir os sons de cada instrumento na mais complicada melodia.
+Sou ass�duo na melhor esplanada da cidade, no Molhe.
+H� vinte anos, era arriscado l� ir, por causa dos marginais.
+Depois, tornou-se local de conv�vio da mais alta qualidade.
+Mas receio que esteja outra vez a entrar num mau per�odo, com a explora��o excessiva e de mau gosto.
+ No pr�ximo ano, devido ao princ�pio da gratuitidade dos transportes escolares preconizado na lei de bases do sistema educativo, os alunos do oitavo ano ter�o tamb�m transporte gratuito, o que agravar� em 17 por cento o custo comparticipado pela C�mara.
+Aos alunos de Alqueir�o, Casal das Pimenteiras, Casal da Fonte e Beselga a autarquia assegura a utiliza��o de carreiras p�blicas ou at� de t�xi por considerar que o percurso a p� oferece riscos �s crian�as.
+De Wilde, Costinha, Lu�s Miguel, Gil Baiano, Marco Aur�lio, Beto, Pedrosa, Oceano, Vidigal, Pedro Martins, Peixe, Pedro Barbosa, Afonso Martins, Dominguez, S� Pinto, Yordanov, Ouattara e Paulo Alves foram os futebolistas escolhidos por Oct�vio.
+Ficaram de fora, al�m dos chamados �s selec��es, Vujacic, Tiago e Balajic.
+A cena � tirada de uma p�gina asfixiante de Joseph Conrad: um cad�ver em pijama, deitado numa cama com uma manta verde nos p�s, dentro de uma pequena cabana de madeira a tresandar a luto, flores e formol, velado por a vi�va, por a filha e com uma guarda de honra de cinco jovens soldados Khmer.
+�N�o restam d�vidas que � o corpo dele e que est� morto�, garantiu um dos jornalistas nesse grupo, o americano Nate Thayer, correspondente da �Far Eastern Economic Review� e o estrangeiro que melhor conheceu a hist�ria do fundador dos Khmer Vermelhos.
+Pol Pot morreu oficialmente com um enfarte card�aco, quarta-feira � noite, mas � inevit�vel pensar em causas menos naturais para o seu desaparecimento.
+As reac��es foram cautelosas e os EUA, porque nos maus filmes de terror os mortos-vivos conseguem sempre levantar-se, exigiram uma aut�psia.
+Na passada Primavera, a Assembleia de Freguesia do Carregado solicitou � C�mara de Alenquer, de maioria socialista, o processo de candidatura da eleva��o a vila, mas j� em Setembro, Vasco Miguel, do PSD, entregava no Parlamento um projecto de lei nesse sentido, atitude que o presidente da C�mara classificou de �oportunismo pol�tico�, alegando que o diploma caducaria com o final da legislatura e n�o chegaria a ser discutido, o que de facto aconteceu.
+Agora, o PSD voltou a pegar no assunto, pelas m�os de Duarte Pacheco, que sustenta a retomada desta iniciativa na �vontade das pessoas que nasceram e vivem no Carregado e no concelho de Alenquer�.
+Duarte Pacheco alega que o Carregado � a zona do concelho de Alenquer com maior �ndice de crescimento industrial e �� tamb�m um dos maiores centros populacionais da regi�o, pelo facto de se situar num dos maiores n�s rodovi�rios do pa�s�, acrescido das liga��es fluviais e ferrovi�rias.
+O ministro portugu�s da Administra��o Interna avistou-se ontem com o ministro holand�s da Justi�a para receber os �dossiers� sobre a imigra��o e do Grupo de Trevi que transitam da presid�ncia holandesa da Comunidade para a portuguesa.
+A quest�o da cria��o da Europol (uma pol�cia comunit�ria) foi um dos temas em an�lise no encontro.
+A passagem do testemunho, entre Haia e Lisboa, em mat�ria de pol�tica interna comunit�ria apenas se fez agora por indisponibilidade da agenda do ministro holand�s da Justi�a.
+O Holanda-Inglaterra decide quase tudo neste grupo.
+Quem ganhar fica praticamente com o visto no passaporte para os EUA, enquanto � Noruega basta um ponto na Pol�nia para pela primeira vez participar numa fase final do �Mundial�.
+� claro que as contas podem todas complicar-se, caso holandeses e ingleses empatem e polacos ven�am noruegueses.
+Nesse caso, a Pol�nia permaneceria na corrida.
+A decis�o do Grupo ficaria ent�o muito complicada, com tr�s equipas com hip�teses de chegar ao fim com 14 pontos e a ser preciso recorrer ao �foto-finish� para saber quem se apuraria.
+� o maior grupo, com sete equipas.
+Para j� Rep�blica da Irlanda e Dinamarca ocupam os dois primeiros lugares, mas a Espanha ainda est� na corrida.
+O Rep�blica da Irlanda- Espanha de hoje j� ajudar� a definir posi��es, mas dever� ser o Espanha- Dinamarca de 17 de Novembro que tudo esclarecer�.
+A n�o ser que o Eire ven�a hoje a Espanha e a Dinamarca derrote a Irlanda do Norte, ficando Eire e Dinamarca de imediato com a presen�a nos EUA garantida.
+O objectivo gen�rico destas iniciativas, patrocinadas pela Uni�o Europeia, � o desenvolvimento de uma pol�tica global de redu��o da inseguran�a, � volta de quatro eixos fundamentais: respeito pelos direitos humanos, participa��o activa dos cidad�os na defini��o e aplica��o da pol�tica de seguran�a requerida, aposta em solu��es de partenariado envolvendo os sectores p�blico e privado e preocupa��o com a seguran�a do cidad�o, para evitar que ele se torne autor ou v�tima de crimes.
+�As actividades amadoras v�o ser desinflaccionadas negociando o ordenado dos atletas.
+N�o � o marketing e a sponsoriza��o que resolvem os problemas�.
+Esta nova equipa prop�e-se fazer um trabalho honesto e claro.
+O Benfica j� conta, neste momento, com setenta mil s�cios e sete mil praticantes.
+�Os s�cios v�o poder criar lugares para os seus filhos praticarem desporto�.
+Manuel Monteiro mentiu aos portugueses.
+O que j� escrevemos esta semana excitou as consci�ncias do Partido Popular, ciosas em mostrar que o processo da lista de candidatos por Lisboa correu na maior das lisuras, incluindo o folhetim Lu�s Nobre Guedes.
+no dia 9 de Junho o l�der dos populares disse que j� tinha a resposta de Nobre Guedes ao convite para encabe�ar a lista da capital e torn�-la-ia p�blica na semana seguinte, depois de a comunicar � estrutura distrital.
+At� ontem, ningu�m conhecia, pela boca de Monteiro, a resposta ao convite.
+S�o estes os factos e de nada vale que Nobre Guedes venha desagravar o comportamento do seu l�der -- a crer na �ltima edi��o do �Independente�.
+Diz o jornal, referindo-se � comiss�o pol�tica de ontem, que Nobre Guedes �sublinhar� que o presidente do PP nunca mentiu no decurso do processo de constitui��o das listas�.
+� assim a transpar�ncia do Partido Popular, feita de recados.
+Teresa Vasconcelos -- Digo isto no livro e digo com convic��o: h� muitas �Anas� neste pa�s.
+Mas tamb�m h� muitas outras que poderiam ser como a Ana e se desmobilizaram.
+P. -- Por falta de condi��es, falta de forma��o, falta de acompanhamento?
+P. -- Foi um colega seu ...
+R. -- Fa�am o favor de tomar nota do que vou dizer, e agradecia que fosse transmitido, porque de vez em quando as coisas n�o saem de acordo com a verdade dos factos.
+A EDP tem tido um comportamento impec�vel em todo este processo.
+Quando come�aram os estudos da barragem de Foz C�a, fez um estudo de impacto ambiental.
+Quando se descobriram os vest�gios arqueol�gicos, fez um protocolo com o Ippar [ Instituto Portugu�s do Patrim�nio Arquitect�nico e Arqueol�gico ], porque n�o percebia da mat�ria.
+Depois foi a EDP que vedou e policiou toda a �rea onde foram descobertos os achados arqueol�gicos.
+Por isso, se aquilo est� recolhido, deve-se � EDP.
+Quando os protestos eclodiram, os media internacionais divulgaram para todo o mundo informa��es sobre uma �carnificina em larga escala� da popula��o revoltada da cidade pelas for�as fi�is a Ceausescu.
+Estas not�cias alarmistas -- num relat�rio recente, os servi�os secretos romenos acusaram espi�es h�ngaros e de outras nacionalidades de terem incitado � revolta -- contribu�ram decisivamente para o derrube do regime, mas mancharam a revolta.
+Afinal, os cad�veres nus e alinhados descobertos n�o eram v�timas da repress�o, mas antes de pessoas pobres, doentes e inv�lidas, exumadas de fossas comuns e colocadas no local.
+Hoje, o �monumento aos her�is desconhecidos de 1989� n�o faz qualquer alus�o a este epis�dio e no enorme talh�o continua a ser enterrado quem n�o pode pagar uma sepultura decente.
+�Est�vamos em estado de choque, v�timas de uma alucina��o colectiva.
+Vi o cad�ver de uma mulher com o seu filho no ventre.
+Eram talvez cinco corpos, mas eu via vinte�, contou recentemente, ao enviado do �Le Monde�, Cornel Balint, presidente de uma das seis associa��es de revolucion�rios de Timisoara, que agrupam cerca de 800 pessoas recompensadas por terem sido feridas, presas ou molestadas durante os combates.
+Um dos objectivos dos Encontros -- tal como os definiu h� seis anos Madalena Perdig�o, fundadora do Acarte -- era permitir o acesso dos core�grafos portugueses aos circuitos europeus, dos quais estavam arredados por desconhecimento m�tuo.
+De um lado, pela situa��o geogr�fica de periferia e, do outro, pelo desconhecimento das tend�ncias em vigor nos grandes festivais internacionais.
+Os Encontros Acarte propunham-se responder a esse duplo anseio: informar o p�blico e estimular os autores portugueses, na perspectiva de os despertar para as est�ticas do p�s e neo-modernismo.
+�A linha fundamental [ dos Encontros ] � servir o p�blico, mant�-lo dentro da actualidade coreogr�fica.
+Mas a afirma��o da sua identidade nunca se fez pela adop��o de uma linha est�tica �nica.
+N�o h� uma direc��o �nica [ no conceito de modernidade ].
+O p�blico tem o direito a escolher o que lhe interessa e n�o creio que seja necess�rio conduzi-lo pela m�o como se fosse um indigente�.
+Smidovitch, com um �fomos aonde plane�mos ir�, declinou confirmar, no entanto, se a fiscaliza��o abrangera sedes de minist�rios, quinta-feira vedadas categoricamente ao pessoal da ONU pelo regime de Bagdad.
+Sobre se haviam encontrado algo suspeito tamb�m n�o adiantou pormenores.
+A miss�o de 22 peritos � a primeira a chegar � capital iraquiana desde que, tamb�m sem resultados, foi passada revista ao Minist�rio da Agricultura, a 28 e 29 do m�s passado, miss�o que p�s termo a um impasse de tr�s semanas entre o Governo de Saddam Hussein e a comiss�o da ONU.
+As inspec��es, recorde-se, come�aram em Abril do ano passado, pouco depois do fim da Guerra do Golfo e resultaram dos acordos de cessar-fogo ent�o assinados.
+Ap�s negocia��es com o principal patrocinador da equipa, a Shell, Hardwick entregou um dos �wild cards� (convites) a Felisberto Teixeira, que conta tamb�m com o apoio da Federa��o Nacional de Motociclismo (FNM) e est� convicto que o portugu�s far� bom trabalho, reconhecendo, por�m, que qualquer estreia, e muito especialmente em 500cc, � sempre dif�cil.
+�No pa�s prosseguem os incidentes: de percursos para uns, perigosos para outros� (Notici�rio das 13 horas da �R�dio Nacional�).
+�... Os laborat�rios do Departamento de Estado e do Pent�gono ainda n�o conseguiram dar uma resposta precisa quanto � imprevisibilidade da reac��o do seu �ex� aliado (Jonas Savimbi), que parece ter ingerido demasiadamente ester�ides anabolizantes nos tempos de ajuda encoberta e ter crescido demais� (�Correio da Semana�, 22.9).
+� mesmo de sup�r que se trata de uma das mais importantes exposi��es fotogr�ficas dos �ltimos anos, a n�vel internacional.
+O tema da �gua, ligado aos oceanos como proposta gen�rica da Expo-98, permite abordar esse elemento em todos os seus estados, liqu�do, f�sico e gasoso, e obviamente relaciona-se com o pr�prio processo fotogr�fico.
+Sendo, al�m de cr�tico de �pera e fotografia, professor de qu�mica, trabalhando no dom�nio da termodin�mica dos objectos moleculares e das rela��es entre artes e ci�ncias, Calado estava habilitado como ningu�m a comissariar esta exposi��o.
+O Instituto de Investimentos e Privatiza��es dos A�ores (IIPA) est� a negociar um empr�stimo de 1,5 milh�es de contos, �para poder fazer face aos compromissos assumidos na �rea dos incentivos de base regional�, disse ao P�BLICO o presidente do IIPA, Jo�o Bernardo Rodrigues.
+De acordo com aquele gestor, o IIPA contactou o Banco Comercial dos A�ores, a sociedade Esp�rito Santo- sociedade de investimentos e o Banco Pinto & Sotto Mayor.
+O empr�stimo ser� negociado por tranches, j� que o valor em causa � para o bi�nio 1991-92�.
+O valor total dos compromissos assumidos � de 4,2 milh�es de contos, dos quais 1,5 milh�es s�o da responsabilidade do Governo Regional dos A�ores via IIPA e o restante das comunidades.
+Segundo Jo�o Bernardo Rodrigues foram aprovados 60 projectos, �cujo valor total de investimento � da ordem dos nove milh�es de contos�, distribuidos, entre outros, pela agro-pecu�ria, constru��o civil, basaltos, cimentos e para a realiaza��o de uma nova unidade fabril de cerveja.
+Indignados com a aus�ncia de negros em lugares eleg�veis, os representantes da comunidade imigrante africana aconselharam o voto em branco.
+Fernando K�, 41 anos, presidente da Associa��o dos Guineenses desde 1987, n�o compreende como num pa�s com uma presen�a multissecular africana, n�o h� um pol�tico, um deputado, um presidente de c�mara, sequer um vereador negro.
+Deputado suplente do PS com entradas fugazes no Parlamento, acusa Ant�nio Guterres de trair promessas feitas em Julho a representantes da comunidade e prepara uma campanha internacional contra uma situa��o que torna os negros portugueses em cidad�os de segunda classe.
+As cidades candidatas � organiza��o dos Jogos Ol�mpicos de 2004 v�o conhecer hoje o regulamento a que dever�o obedecer, durante a sua campanha de prepara��o e promo��o.
+Ser� durante um encontro a realizar no Museu Ol�mpico, em Lausana, na Su��a, onde os representantes das 11 cidades se ir�o reunir pela primeira vez.
+Jogo grande foi o que juntou, em Phoenix, os Suns e os Portland Trail Blazers, e que terminou com a vit�ria dos donos da casa, por 118-109.
+A.C. Green, ex-jogador dos Lakers, marcou 31 pontos para liderar os Suns, num jogo em que �Sir� Charles Barkley averbou apenas 21 pontos.
+Discreto no campo, o temperamental Barkley voltou a ser not�cia fora dele.
+Agora, vai ter de responder em tribunal a uma queixa de agress�o interposta por um f�.
+Barkley � acusado de ter agredido Edward Durham num bar, depois de este o ter criticado por empurrar uma senhora que apenas pedia um aut�grafo.
+Barkley n�o comenta o incidente, e o mesmo faz a sua equipa.
+Os Suns, finalistas da �poca passada, est�o mais concentrados no campeonato, onde ocupam o segundo lugar da Divis�o Pac�fico, liderada pelos invictos Seattle Supersonics.
+Um m�todo muito comum de movimentar ficheiros na Internet.
+O FTP � uma forma especial de fazer �login� noutro local da Internet com o objectivo de carregar ou enviar ficheiros.
+H� muitos servidores da Internet que cont�m imenso material que pode ser acedido por qualquer pessoa atrav�s de um acesso an�nimo, pelo que s�o chamados �servidores FTP an�nimos�.
+Um m�todo bastante utilizado para realizar menus de material dispon�vel na Internet.
+Gopher � um programa de utiliza��o cliente-servidor, pelo que requere que o utilizador disponha de uma vers�o cliente do Gopher.
+Os Gopher difundiram-se muito rapidamente atrav�s da Internet nos �ltimos anos, mas est�o a ser suplantados pelo hipertexto suportado atrav�s da World Wide Web.
+Contudo, existem ainda milhares de Servidores Gopher na Internet, que dever�o subsistir por mais algum tempo.
+� semelhan�a de muitas outras igrejas evang�licas, o pagamento do d�zimo na Universal n�o � mais do que a adop��o de uma pr�tica b�blica.
+Ainda hoje o c�digo cat�lico de Direito Can�nico refere que �a igreja tem o direito, independentemente do poder civil, de exigir aos fi�is o necess�rio para o culto divino, para a honesta sustenta��o dos seus ministros e para satisfazer outros fins pr�prios� ...
+� o que faz todos os meses a Igreja do Reino de Deus.
+Em Portugal, o dinheiro dos crentes � colocado num envelope, que os obreiros da IURD, sempre atentos, recolhem e controlam.
+No Brasil, existe uma caderneta individual onde cada seguidor regista, religiosamente, as suas contribui��es para a igreja de Macedo.
+Se a escrita sobre arquitectura n�o abunda entre n�s, Manuel da Gra�a Dias tem dado um excelente contributo para alterar essa situa��o.
+N�o s� as �mem�rias descritivas� dos seus projectos s�o, s� por si, criativos textos sugerindo, com especial efic�cia, uma multiplicidade de imagens a partir dos desenhos que apresenta, como a sua produ��o para a imprensa revela id�ntica qualidade.
+Pelo uso de uma linguagem despojada de excessivas conota��es t�cnicas inibidoras da aproxima��o do leigo, pela originalidade dos sentidos que ajuda a desvendar nos temas do quotidiano..
+Os portugueses est�o em forma no surf.
+Pelo menos os jovens que est�o a participar no Campeonato Europeu de juniores, a decorrer at� domingo em Newquay, Inglaterra.
+no surf, Jo�o Macedo, Andr� Pedroso (em sub-18), Ruben Gonz�lez e Pedro Monteiro (em sub-16) est�o j� nas meias-finais.
+N�o haver� greve nem bloqueio � �poca da NBA, a Liga Norte-Americana de Basquetebol profissional, independentemente de ainda n�o existir acordo sobre o tecto salarial, decidiram os propriet�rios e jogadores.
+Este an�ncio vem sossegar aqueles que temiam que sucedesse no basquetebol o que j� aconteceu no basebol e no h�quei no gelo -- a interrup��o ou adiamento dos campeonatos.
+Sendo assim, e uma vez que o acordo para o contrato colectivo de trabalho, cujas negocia��es se arrastam h� meses, est� previsto para breve, tudo indica que a NBA come�ar�, como planeado, a 4 de Novembro, ou seja, na pr�xima sexta-feira.
+Ainda no �foyer�, p�de-se desde logo perceber que no Servi�o de M�sica da Funda��o Calouste Gulbenkian se est� em momento de reflex�o, j� que foi distribu�do ao p�blico um question�rio onde, entre outros pontos, se pede uma opini�o sobre a periodicidade de apresenta��o da m�sica contempor�nea.
+Precisamente no concerto dessa noite se propunha um percurso pela m�sica mais recente intersectado de forma exemplar por sonoridades do passado.
+A enquadrar as obras de Wolfgang Rihm �La lugubre Gondola / Das Eismeer� e de Luigi Nono �No hay caminos, hay que caminar .. Andrei Tarkovsky�, tivemos a m�sica de Giovanni Gabrieli (Canzone a tre cori e In Ecclesiis) em orquestra��es da autoria de Bruno Maderna.
+A rela��o entre Luigi Nono e Giovanni Gabrieli ultrapassa a simples passagem e viv�ncia na cidade de Veneza, situando-se principalmente ao n�vel da explora��o das potencialidades musicais inerentes � espacializa��o sonora.
+Apesar de n�o se ter procedido a qualquer distribui��o de m�sicos no espa�o do Grande Audit�rio Gulbenkian, � imagem do que se fazia com as obras do m�sico renascentista na Catedral de S�o Marcos, a orquestra��o de Maderna reflecte um total respeito pelas estruturas de di�logo entre �coros� com uma constitui��o t�mbrica diferenciada.
+Desde o primeiro dia do ano, entre os palestinianos procurados, 13 foram mortos pelas for�as de seguran�a e 15 feridos mas 40 foram libertados pelo Ex�rcito, disse Yatom durante uma visita de inspec��o � regi�o de Bel�m (sul da Cisjord�nia).
+Quatrocentos outros s�o ainda procurados pelas for�as de seguran�a.
+Informa��es do Ex�rcito, em termos globais, d�o conta de 23 palestinianos mortos e 211 feridos por soldados, desde o dia 1 de Janeiro, na Cisjord�nia.
+O n�mero � confirmado pelos palestinianos.
+Danny Yatom disse que a �fraca propor��o de palestinianos mortos em rela��o ao n�mero de palestinianos presos prova que o Ex�rcito n�o dispara sen�o em casos excepcionais, contrariamente a certas alega��es�.
+R. -- N�o h� � uma circula��o do pensamento.
+H� muitos poucos debates, as ideias que s�o objecto de discuss�o internacional -- as quest�es relacionadas com a realidade virtual, a com o p�s-humano, ou com as auto-estradas inform�ticas -- n�o se discutem em Portugal.
+Por outro lado, n�o h� radicalismo.
+Sem radicalismo n�o h� qualquer evolu��o da sociedade.
+P. -- Uma quest�o �bvia: em ano de elei��es, e embora o debate sobre a situa��o portuguesa seja feito em Fevereiro, a iniciativa n�o se colar� demasiado �s for�as pol�ticas -- socialistas-Plataforma de Esquerda -- que est�o � frente da C�mara de Cascais?
+Apesar dos elevados montantes que o Banco de Portugal tem vindo a injectar no sistema ao longo do actual per�odo de constitui��o de reservas de caixa, a sess�o de ontem do mercado monet�rio interbanc�rio voltou a apresentar-se bastante pressionada pela procura de fundos, com os primeiros n�veis de oferta no curto prazo a serem superiores ao fecho da sess�o de ontem.
+H� medo no Sul dos Estados Unidos.
+As igrejas frequentadas pela popula��o negra est�o a ser incendiadas e os l�deres das comunidades acreditam que s�o manobras de intimida��o dos que sonham com o regresso da �supremacia branca�.
+Casos como o da Sopete, Lusotur ou Alian�a Seguradora ilustram de forma pouco favor�vel a seguran�a que os investidores podem ter nas previs�es adiantadas por algumas empresas ...
+Na Sopete, os resultados alcan�ados em 1991 n�o chegam sequer para cobrir os dividendos de 125 mil contos estimados no primeiro semestre, isto apesar de no coment�rio ao balan�o do primeiro semestre de 1991 ser referido que �se aponta para o segundo semestre um real crescimento da actividade do todo da empresa, e em especial da �rea do jogo e da �rea hoteleira�.
+Enfim, e esse ter� sido o ponto decisivo da sua interven��o, Ant�nio Tanger sublinhou que a ocorr�ncia de um novo problema poderia criar a Portugal algum embara�o diplom�tico.
+No quadro pol�tico actual, em que Portugal ocupa a presid�ncia das Comunidades, era de todo indesej�vel que a diplomacia portuguesa tivesse de intervir para protestar junto de qualquer pa�s dessa regi�o.
+Isso poderia revestir-se de particular gravidade se o ataque ocorresse dentro de Angola, onde h� poucos dias dois padres e uma freira foram mortos num ataque de �bandidos�.
+Ap�s esta interven��o, os membros da expedi��o reuniram-se com a organiza��o da viagem para deliberar sobre o que fazer.
+Foi na sequ�ncia desse encontro que Pedro Villas Boas decidiu dar por finda a expedi��o, inviabilizando desta forma o prosseguimento da viagem anteriormente definido por uma pequena coluna expedicion�ria.
+Wayne Barker, cineasta arbor�gene da Austr�lia, fez a apologia duma cultura viva, �que conserve a heran�a dos nossos pais mas com os olhos postos no futuro�.
+Walter Saunders, outro realizador e produtor arbor�gene, sublinhou a pilhagem cultural que a sua comunidade continua a sofrer, por exemplo no dom�nio das imagens: dos motivos arbor�genes utilizados para ornamentar lou�a e tecidos, os s�mbolos que ornam os avi�es australianos, os boomerangs fabricados e vendidos por australianos brancos, sem que os arborigenes recebam quaisquer direitos ..
+�� um disco que estava j� meio feito no Canad� com can��es compostas antes do 25 de Abril.
+Acabou por ter uma vertente mais pol�tica em consequ�ncia dos acontecimentos imediatos, por uma quest�o de sensibilidade � linguagem da �poca e da vontade de participar nela.
+(...) Mas a pesquisa formal � basicamente a mesma.
+Podem dizer-se coisas muito v�lidas politicamente com uma linguagem pobre e que n�o entra nas pessoas.
+Cecilia Bartoli cantou em S�o Carlos, em Lisboa, no passado 24 de Mar�o.
+N�o numa �pera de Rossini, como reclamaria a miss�o do teatro, no ano do bicenten�rio.
+Mas oferecendo um conjunto de pe�as para voz e piano do compositor, numa memor�vel demonstra��o da vitalidade da redescoberta do canto rossiniano.
+Se h� est�tica de canto que foi deteriorada quase irreversivelmente com os novos conceitos de drama musical que imperaram durante a segunda metade do s�culo passado e a primeira metade do presente, essa foi certamente a perseguida por Rossini.
+Tourabi foi tamb�m o art�fice de uma alian�a �nica, entre um pa�s �rabe sunita -- o Sud�o -- e um pa�s n�o �rabe xiita -- o Ir�o, selada quando o Presidente iraniano, Ali Akbar Hashemi Rafsanjani, visitou Cartum em Dezembro de 1991.
+�� imposs�vel saber se foi Rafsanjani quem manipulou Tourabi, ou vice-versa, mas � seguro que Sud�o obteve vantagens pol�ticas e financeiras, porque se transformou no coordenador de v�rios movimentos islamistas�, observou Balta, enumerando a FIS, na Arg�lia, a An-Nahda, na Tun�sia, a Irmandade Mu�ulmana, na Jord�nia, o Hezbollah, no L�bano, a Gama'at Islami, no Egipto, e o Hamas, na Faixa de Gaza -- uma �internacional de barbudos�.
+A faixa et�ria e as horas do dia mais perigosas coincidem em todos os distritos, mas Beja foi o que registou menor incid�ncia de condutores alcoolizados em Dezembro: apenas 2,4 por cento.
+Durante o �ltimo m�s do ano passado a Brigada de Tr�nsito aplicou o teste de alcool�mia a 25.093 condutores, em todo o continente e detectou 1.578, ou seja, 6,2 por cento do total, que conduziam sob o efeito do �lcool.
+N�o foi por isso, contudo, que a Vidisco se sentiu menos ofendida, de modo que acabou por decidir meter o caso em tribunal, porque �aquilo n�o se faz.
+A televis�o � paga por todos n�s, para qu�?
+Para eles fazerem publicidade privada, como se fosse um programa pago?
+N�o � digno de um programa de um organismo oficial, que na verdade � de uma associa��o de multinacionais.
+Tent�mos encontrar logo um advogado que estivesse dentro da legisla��o, o que foi dif�cil por causa da quadra de festas.
+Agora, temos um advogado e se ele considerar que h� suficiente mat�ria legal para isso, vamos a tribunal�.
+Resta saber quem poder� ser processado.
+Mas ser� que vai ficar por aqui?
+O que houve entre a primeira e a segunda metade desse ano parece, assim, uma esp�cie de intervalo -- um tempo fora da Hist�ria.
+Enquanto o poss�vel medo de repres�lias fazia com que as acusa��es dos jogadores contra Oct�vio nunca se tivessem concretizado, ele, em contrapartida, teve a coragem de �dar� a cara em alguns momentos de grande pol�mica.
+Assim, no seu primeiro Porto-Sporting como adjunto, com a rivalidade Pinto da Costa-Jo�o Rocha estava no auge e com Artur Jorge a n�o conseguir ganhar nas Antas, Oct�vio seria acusado de agredir � cabe�ada e a murro alguns respons�veis sportinguistas, indo assim mais al�m do que o pr�prio treinador principal que, imp�vido e sereno, comandou a sua equipa sem se envolver nas �guerras� marginais em que Pinto da Costa e Oct�vio mostravam estar como �peixes na �gua�.
+Quando Quinito foi contratado para o FC Porto e se prop�s recuperar, entre outros, Gomes e Madjer, o homem de Palmela afastou-se e, nas suas costas, come�aram a aparecer muitas das hist�rias a seu respeito que, at� ent�o, tinham ficado escondidas nos segredos dos bastidores.
+Mark Miller � o treinador-jogador, mas � poss�vel que Toni Formosa venha a ser uma esp�cie de �manager� da equipa, com assento no banco, se aceitar a proposta que lhe fez o presidente do clube.
+Ter� que dar uma resposta at� � pr�xima semana.
+Nos jogos com o Ekranes, os dois golos foram marcados pelo l�bero Buttigieg, um jogador elegante e talvez o melhor do pa�s.
+Os outros internacionais s�o Cluett (guarda-redes), Caucchi e Buttigieg (defesas) e Buhagiar (normalmente defesa-esquerdo).
+R. -- Parecer-me-ia l�gico, num pa�s pobre, de recursos escassos como o nosso, que, no m�nimo, as fam�lias e os estudantes suportassem uma percentagem correspondente � dos ingleses.
+P. -- O que iria tornar o ensino superior um ensino mais elitista, dos ricos.
+Abriu ontem ao p�blico a primeira iniciativa da galeria do Instituto de Arte Contempor�nea (IAC).
+No Pavilh�o Branco do Museu da Cidade, em Lisboa, a exposi��o de Adriana Varej�o inaugura uma nova �rea de interven��o do IAC.
+Os objectivos pol�ticos s�o clarificados no texto do director: criar �um local de reflex�o�, �enfatizando a cidade como sede do pensamento e de problematiza��o de temas que fazem parte do nosso tempo�, num campo que �, nesta fase, �dedicado � multiplicidade de formas e conceitos que se originam a partir do conhecimento de culturas e geografias que nos s�o de menor adjac�ncia�.
+Assim, a galeria pretende acolher imagens ligadas ao que se convencionou designar por multiculturalismo.
+A inten��o descentralizadora do programa �Rotas�, assumido pelo Minist�rio da Cultura e que o IAC integra, � pois confrontada com uma iniciativa tendente a centralizar a discuss�o art�stica e a capitaliz�-la.
+Iniciar a programa��o atrav�s da apresenta��o da obra de Adriana Varej�o � uma op��o politicamente perfeita: porque � uma artista brasileira, porque � uma artista que recorre (entre outros) ao manancial infinito das imagens deixadas no Brasil pelo colonialismo portugu�s, porque est� integrada nos circuitos mais din�micos da internacionaliza��o art�stica actual.
+Adriana Varej�o situa-se no campo das reflex�es sobre o cruzamento de culturas.
+Nesse sentido, cruza imagens dos diferentes tempos e civiliza��es, criando imagens de s�ntese e procurando problematizar, atrav�s do objecto art�stico contempor�neo, os resultados de uma experi�ncia hist�rica universal.
+Um cat�logo com textos de Isabel Carlos (do pr�prio IAC) e Paulo Herkenhoff (comiss�rio para a Bienal de S�o Paulo) completa a apresenta��o.
+Leonor Beleza, presidente da mesa do Congresso do PSD, foi convidada a colaborar com o Governo no debate sobre as reformas do sistema eleitoral.
+Os jornalistas tinham perguntas agressivas para o Presidente, mas ficaram algo desarmados com o ar triunfal de Clinton.
+Um ainda indagou se ele tencionava mesmo lan�ar a invas�o, contra a opini�o da maior parte dos americanos e do Congresso.
+Clinton, que afinal de contas n�o teve de lan�ar invas�o nenhuma, deu exemplos de outras interven��es militares, como em Granada ou no Panam�, em que os Presidentes n�o pediram autoriza��o ao Congresso.
+�Nem todas as decis�es podem ser populares.
+N�o se pode conduzir a pol�tica de acordo com as tend�ncias das sondagens.
+N�o creio que o povo americano queira que eu fizesse isso�.
+como se pode dizer aos americanos que C�dras � um cr�pula e dois dias depois afirmar que se trata de um honrado militar?
+Clinton balbuciou qualquer coisa sobre a inutilidade dos ressentimentos e vingan�as, citando Jean-Bertrand Aristide, o Presidente exilado que dever�, at� 15 de Outubro, regressar ao Haiti, e de quem a CIA diz ser um doente mental com perigosas ideias anti-americanas.
+O mesmo documento afirma que a �forma jur�dica e os crit�rios de gest�o nela impl�citos� se revelaram �inapropriados� � miss�o que foi cometida ao Teatro de S�o Carlos, quando da sua transforma��o em EP em1980: �organizar de forma permanente espect�culos de m�sica, �pera e bailado e para os divulgar pelo pa�s, para dar a conhecer as obras de autores nacionais e para formar e manter um corpo de cantores / actores e de m�sicos�.
+�Ao longo de quase 200 anos, a hist�ria do Teatro Nacional de S�o Carlos acompanhou a vida cultural portuguesa, sendo imperioso criar condi��es para que, no novo renascimento que Portugal atravessa, as estruturas da �rea da cultura se adequem ao acrescido dinamismo da sociedade.
+Em conclus�o, ser� de natureza privada a entidade respons�vel pela nova gest�o do Teatro Nacional de S�o Carlos�, diz, a terminar, o comunicado.
+Os Mission�rios do Esp�rito Santo v�o alargar � cidade do Porto o trabalho que j� desenvolvem na regi�o de Lisboa, de apoio aos imigrantes lus�fonos que residem em Portugal.
+A decis�o dever� ser concretizada a curto prazo, depois de o cap�tulo provincial (assembleia) daquela congrega��o religiosa ter debatido a ac��o do Centro Padre Alves Correia (Cepac), de Lisboa -- o organismo, da responsabilidade dos mission�rios, que esteve em diversas ac��es de apoio � legaliza��o dos imigrantes clandestinos, e que denunciou a corrup��o existente na delega��o de Faro do Servi�o de Estrangeiros e Fronteiras, investigada pela Pol�cia Judici�ria (ver P�BLICO 20/07/94).
+�Nesse sentido, eu gostaria de ver alargado o trabalho do Cepac a cidades como o Porto, porque ele � um sinal de aten��o � miss�o no nosso pa�s�.
+Da �ltima vez em que estivemos em Paris nossa casa foi um apartamento na Rue de la Pompe, no 16 �arrondissement�.
+Uma alternativa mais barata para hotel, com a vantagem adicional de nos permitir simular uma certa domesticidade francesa.
+Viajamos para fugir da nossa rotina mas somos seduzidos pela rotina dos outros e ter um lugar certo onde comprar a �baguette� para o caf� da manh� passa a ser um indispens�vel prazer parisiense.
+Conhecer n�o a vida mas a vidinha de um lugar depende de estabelecer-se na comunidade por alguns anos.
+Durante dez dias passamos v�rias gera��es na Rue de la Pompe.
+Na nossa vizinhan�a imediata existiam quatro ag�ncias funer�rias e duas lojas especializadas em queijos.
+A concentra��o de funer�rias se deve, imagino, � proximidade do cemit�rio de Passy.
+Lojas s� de queijos s�o comuns em Paris e uma experi�ncia tur�stica interessante e barata � entrar numa delas e respirar fundo.
+Voc� sai com a certeza de que chegou muito perto do cora��o selvagem da Fran�a ou ent�o de uma cura definitiva para a sinusite.
+A coincid�ncia de funer�rias e queijarias na nossa circunst�ncia n�o significava nada, portanto, mas n�o pude escapar da m�rbida observa��o de que est�vamos cercados pelas pompas da morte na Rue de la Pompe.
+O queijo tamb�m � a encomenda��o cerimonial de um morto, no caso uma por��o de leite.
+James Joyce chamou o queijo de defunto do leite, e a qualidade de um queijo � o resultado da ac��o de bact�rias vivas num corpo morto.
+Ou seja, de apodrecimento controlado.
+� o que acontece tamb�m na prepara��o de carnes secas e aves e peixes �faisandes� e no correcto acondicionamento de corpos humanos para a eternidade.
+os dois assimilam o facto que a vida � uma doen�a incur�vel com uma taxa de mortalidade de 100 por cento, e se tornam melhores por isso.
+Rodrigo Mil Homens venceu ontem, na classe Runabout Limited, a segunda etapa do Europeu de Jet Ski, que durante o fim-de semana se disputou no len�ol de �gua do Rio Lima, na praia do Prior, em Viana do Castelo.
+Miguel Valente, por seu lado, foi o vencedor na classe Sport Limited.
+Estes foram os �nicos portugueses a conseguir um primeiro lugar nas v�rias classes que fizeram parte desta etapa do Europeu, que contou com a participa��o de 186 concorrentes, 28 dos quais portugueses.
+ Finalmente, apesar de j� ter sido editado em 1992, merece refer�ncia o cl�ssico norte-americano Spiderman -- o Homem-Aranha --, uma cria��o original de Steve Dikto (desenho) e Stan Lee (texto), que � evocado por ocasi�o do 30� anivers�rio da sua cria��o (1962) numa edi��o em �comic-book� de luxo pela editora espanhola Forum (Planeta-De Agostini).
+�La Saga del Traje Alienigena� tem a assinatura de Ron Frenz, Ruck Leonardi (desenho), Tom DeFalco e Roger Stern (argumento).
+BB -- Em geral?
+[ Risos ...] A fun��o da arte � a de expandir as possibilidades da linguagem visual.
+Por outras palavras, o que a arte nos d� � mais possibilidades de ver as coisas de diferentes maneiras.
+E creio que � o bastante.
+O mesmo se passa em rela��o � literatura.
+Acredito que a fun��o da literatura e da arte � a de expandir as possibilidades do que as pessoas podem dizer, mas n�o necessariamente para fazer delas melhores seres humanos.
+� poss�vel que tenha essa consequ�ncia, mas n�o se deve misturar as coisas.
+Quais � que s�o as prioridades?
+O fundamental, para mim, � que a arte seja livre de constrangimentos.
+P -- H� uma nova moral nos Estados Unidos?
+J� presente no local sob os ausp�cios das Na��es Unidas para garantir o respeito das �zonas de exclus�o a�rea� decretadas na B�snia, a Alian�a Atl�ntica n�o dever� encontrar grandes dificuldades no alargamento da sua miss�o.
+Mas, apesar da imagem de coes�o que ontem se esfor�aram por exibir durante uma reuni�o no Luxemburgo, os ministros dos Neg�cios Estrangeiros da CE e o secret�rio de Estado norte-americano, Warren Christopher, n�o esconderam algumas �nuances� sobre a estrat�gia a seguir face � B�snia.
+N�o existe, pois, um conceito �operacional� de urg�ncia, como bem atestam as urg�ncias dos hospitais.
+�1� Prioridade -- situa��es exigentes: situa��es mais amea�adoras para a vida e que necessitam de interven��o instant�nea.
+Exemplo: fractura da laringe com obstru��o respirat�ria completa.
+2� Prioridade -- situa��es emergentes: situa��es que exigem interven��o imediata num per�odo de poucos minutos.
+Exemplo: pneumot�rax de tens�o [ ar dentro da cavidade pleural a comprimir os pulm�es e o cora��o, eventualmente ].
+3� Prioridade -- situa��es urgentes: situa��es que necessitam de interven��o dentro da primeira hora.
+Exemplo: hemoperitoneu devido a hemorragia intra-abdominal cont�nua [ les�o dentro da cavidade abdominal a sangrar continuamente ].
+4� Prioridade -- situa��es defer�veis: situa��es que podem ou n�o ser imediatamente evidentes, mas que necessitar�o de tratamento subsequente.
+Outro aspecto que suscita preocupa��o � �o uso, por vezes indiscriminado, de antibi�ticos�.
+Entre as resist�ncias a estes f�rmacos provocadas por este tipo de actua��o -- por parte dos profissionais e da popula��o --, inclui-se, como �um dos exemplos mais graves�, a inefic�cia dos medicamentos contra a tuberculose, doen�a cuja incid�ncia continua a deixar Portugal muito mal colocado no panorama europeu.
+A emerg�ncia da sida, � qual est� associada uma s�rie de infec��es oportunistas, o problema da hepatite C (recentemente identificada e ainda sem cura nem tratamento) e o recrudescimento da papeira (mais de onze mil notifica��es durante o ano passado) constituem outras das facetas mais negativas da situa��o portuguesa em mat�ria de doen�as transmiss�veis.
+ Mas a margem de 87 para 10 a favor da Lei de Reautoriza��o dos Institutos Nacionais de Sa�de, de que fazia parte a cl�usula respeitante ao tecido fetal, fez renascer as esperan�as, entre muitos dos interessados em prosseguir as pesquisas envolvendo tecido fetal, de que esta medida tenha for�a suficiente para ultrapassar o veto presidencial.
+A vota��o representa tamb�m uma vit�ria significativa para os que se prop�em efectuar este tipo de investiga��o, uma vez que conseguiram convencer muitos senadores antiaborto que defender a utiliza��o de restos fetais n�o � a mesma coisa que defender a pr�tica que lhes d� origem.
+Voltando aos terrores da Lapa dos Morcegos: fartos de tantas v�timas, os alde�es decidem armar um cavaleiro capaz de defrontar a fera.
+Terr�vel foi o combate e assustadores os gritos da batalha, que o improvisado cavaleiro acaba por vencer, expulsando a fera, que foge por entre o p�blico e vai aterrorizar outras paragens.
+De entre os que assistiam a esta pe�a da companhia Aquilo, as crian�as eram, sem d�vida, as mais fascinadas.
+Algumas choravam, aterrorizadas, outras respondiam aos urros do monstro, e um petiz chegou mesmo a propor aos pais uma incurs�o na gruta, �para mat�-la�, � fera, pois ent�o.
+�N�o � a desvaloriza��o que vai resolver os problemas de fundo do sector t�xtil�.
+� assim que Jorge Seabra, gestor t�xtil, avalia a recente desvaloriza��o da moeda portuguesa que, no entanto, considerou insuficiente.
+Para Jorge Seabra, a quest�o tem de ser posta a dois n�veis.
+Por um lado, a efectiva perde de competitividade dos produtos portugueses face a produtos concorrentes provenientes do espa�o europeu, em especial da It�lia e da Espanha.
+� nesse n�vel que, em sua opini�o, a desvaloriza��o foi insuficiente.
+Mas, por outro lado, n�o � a desvaloriza��o do escudo que vai resolver o problema de fundo de competitividade da ind�stria portuguesa.
+Uma autocombust�o ocorrida ontem de manh� em dep�sitos da empresa de tratamentos pretol�feros Nesta, em Sines, n�o chegou a provocar ferimentos nos oper�rios nem prejuizos avultados.
+A Sociedade Lisboa 94 poder� vir a desvincular-se do apoio, de 25 mil contos, � produ��o da �pera �O Corvo Branco�, de Philip Glass e Bob Wilson, com libreto de Lu�sa Costa Gomes, encomendada pela Comiss�o Nacional para as Comemora��es dos Descobrimentos (CNCDP).
+Marcada inicialmente para o fim da presid�ncia portuguesa das Comunidades, em Junho de 1992, depois anunciada para Abril de 1994, no Centro Cultural de Bel�m (ver P�BLICO de 10/2/1992) a �pera s� poder� ser apresentada em Janeiro / Fevereiro de 1995.
+Vasco Franco, vereador respons�vel pela PM lisboeta, contesta e salienta que a ac��o da pol�cia alfacinha � mais importante do que parece � primeira vista.
+Al�m do controle da venda ambulante, os efectivos acorrem �s situa��es de cat�strofe, fiscalizam bairros degradados, para evitar o aparecimento de novas barracas.
+�Muitas vezes, a PM � a �nica autoridade que passa em determinados bairros�.
+S� este ano, at� Setembro, a pol�cia alfacinha j� retirou da via p�blica 1600 ve�culos abandonados, miss�o �essencial numa cidade com pouco estacionamento�, sublinha.
+ Al�m disto, ainda acompanham as ac��es de despejo.
+A verdade � que se tratam de tarefas executadas noutros munic�pios por fiscais camar�rios.
+Mas, �com menos efic�cia que a PM�, assegura Vasco Franco.
+Entretanto, Couto Ferreira apresentava ontem um alto teor de �cido �rico no sangue e encontrava-se em acelerado emagrecimento, segundo informou o director-cl�nico do Hospital Prisional S.Jo�o de Deus, Manuel Pin�u.
+�Por agora, as altera��es registadas ainda n�o s�o irrevers�veis mas a presen�a de um elevado teor de �cido �rico no sangue pode ter consequ�ncias a n�vel renal�, frisou o citado m�dico.
+Couto Ferreira encontra-se a ser acompanhado pelo m�dico mas recusou qualquer tipo de tratamento, caso entre em coma.
+O l�der do PSD-Porto, Lu�s Filipe Menezes, desafiou ontem o presidente da C�mara do Porto a assumir o mandato de deputado durante a discuss�o do Or�amento de Estado para 1998 para que prove o seu empenho na defesa da regi�o, mas Fernando Gomes, em jeito de coment�rio, passou ao lado do repto.
+��s picardias e jogos de palavras do doutor Lu�s Filipe Menezes n�o tenho rigorosamente nada a dizer, porque tenho mais com que me preocupar�, disse.
+Este grupo religioso j� foi �visitado� pelas autoridades policiais, em fins de Novembro, no seguimento de queixas em mat�ria de urbanismo, contra est�tuas de dimens�es desmesuradas erguidas perto do mosteiro.
+Os �cavaleiros� foram, inclusive, proibidos de construir um �templo pir�mide� de propor��es megal�manas.
+Entretanto, nos Estados Unidos, os sobreviventes do desastroso ataque policial contra os membros do culto davidiano, o �massacre de Waco�, juntamente com os familiares das v�timas mortais, v�o processar o presidente norte-americano, Bill Clinton, e o FBI.
+O exerc�cio do poder nas organiza��es �, por vezes, descrito em termos de �jogos e de jogadores�.
+Os autores de �The Strategy Process� referem o especialista em ci�ncias pol�ticas Graham Allison, que, em 1971, descreveu os jogos nas organiza��es e nos governos como �complexos, subtis, simult�neos e sobrepostos�.
+a coliga��o de regras, em cada momento define o jogo.
+Mintzberg e Quinn identificam uma s�rie de �jogos de poder�, de entre os quais retir�mos e adapt�mos os que apresentamos no quadro junto.
+O essencial, na compreens�o dos �jogos de poder�, � conseguir saber quem est� envolvido e onde est� a for�a em cada situa��o concreta.
+A gest�o de topo -- alvo a atingir em muitos destes jogos -- pode ganh�-los se os conhecer bem e souber �jog�-los�.
+23 de Maio -- Manifesta��es espont�neas em protesto contra o facto de Daniel Cohn-Bendit, l�der do Maio de 68, ter sido proibido de residir em Fran�a.
+A Uni�o Nacional dos Estudantes de Fran�a, o sindicato dos Professores do Ensino Superior, o Movimento 22 de Mar�o e os Comit�s de Ac��o Liceal convocam uma manifesta��o para o dia 24 em solidariedade com Cohn-Bendit.
+cerca de 30 mil jovens manifestam a sua solidariedade com Cohn-Bendit.
+�s 20 horas, o general De Gaulle fala ao pa�s, propondo a realiza��o de um referendo.
+Os manifestantes reagem � interven��o, agitando len�os brancos e gritando �Adeus De Gaulle�.
+O chefe do Estado-Maior da For�a A�rea, general Mendes Dias, decidiu proceder a algumas altera��es nos quadros das Oficinas Gerais de Material Aeron�utico (OGMA), tendo sido determinada, na passada sexta-feira, a substitui��o do general Rui Espadinha, na direc��o daquele estabelecimento fabril.
+Interinamente, o brigadeiro Portela ir� assegurar a chefia das OGMA, mas, nos pr�ximos meses, dever� suceder uma reestrutura��o mais profunda.
+Seria bom que os l�deres partid�rios pr�-referendo fossem mais claros expondo raz�es objectivas, como por exemplo as de foro constitucional, �tico ou consuetudin�rio.
+Os cidad�os gostariam de saber porque se deseja retirar aos deputados o direito que t�m de aplicar a sua sapi�ncia e poder na feitura de leis que a todo o povo e ao pa�s dizem respeito.
+E tamb�m gostar�amos de saber porque se exige a transfer�ncia de tais poderes para o veredicto do povo, sendo sabido que se trata de assuntos de elevada complexidade pol�tica, social e jur�dica, para os quais o cidad�o comum tem menor prepara��o t�cnico-cient�fica.
+A investigadora acha mesmo que n�o h� raz�es para se excluir as prote�nas animais das ra��es, desde que exista uma certifica��o da origem dessas prote�nas que garanta que elas prov�m apenas das partes n�o perigosas das carca�as.
+�Antes de 1988, a contamina��o era impar�vel e, hoje, estamos a tomar precau��es excessivas numa altura em que o risco j� � muito mais pequeno�.
+E diz que, daqui a uns anos, a BSE, que era uma doen�a rara at� h� dez anos, tornar� a ser novamente isso: uma doen�a rara.
+A guerra entre os jornais desportivos est� no ponto de rebu�ado.
+A �Gazeta�, que passa a ser editada cinco dias por semana, parece em vias de alcan�ar o seu objectivo: obrigar o arqui-rival �A Bola� a tornar-se um jornal di�rio, o que, segundo P&N apurou, ser� um facto a partir de Janeiro do pr�ximo ano.
+ E, de caminho, instalou-se na redac��o de �A Bola� um clima de aut�ntico PREC, com a demiss�o do chefe de redac��o, Joaquim Rita, e a venda, por parte de Aur�lio M�rcio, da quota que detinha no jornal.
+M�rcio amealhou 70 mil contos e est� de malas feitas para o �Record�, acompanhado, na sua transfer�ncia, por Jo�o Alves Costa e Norberto Santos.
+ Quem disse que as transfer�ncias eram s� no defeso?
+J.M. -- A� � que est�.
+Temos prestado uma aten��o diminuta � interac��o entre a realidade f�sica, o problema ambiental e o sistema pol�tico.
+Se pegarmos no caso de �frica, a grande quantidade de gente que ali morre de fome n�o resulta do problema ambiental ou de uma agricultura subdesenvolvida, mas dos governos.
+S�o pa�ses mal geridos que, por isso, n�o conseguem fazer chegar os alimentos aos seus povos.
+Curiosamente, na Am�rica do Sul, que dantes estava bem pouco desenvolvida, as popula��es conseguem bastar-se em alimentos e vivem da exporta��o de alguns produtos.
+Portanto, voc� deveria preocupar-se com os sistemas pol�ticos vigentes.
+E.G. -- Continuo a n�o estar convencido de que se trate apenas de um problema pol�tico.
+Porque a produ��o por hectare em �frica tem diminu�do drasticamente, o campo est� a desertificar-se.
+O len�ol fre�tico baixa todos os anos, as pessoas t�m de cavar po�os cada vez mais fundos.
+Muita dessa desertifica��o tem sido disfar�ada com o uso crescente de adubos, n�o tanto em �frica como na �ndia e noutros s�tios.
+� uma causa maci�a de subnutri��o e fome.
+Na semana passada, Fidel Castro avisara que se Washington n�o alterar a pol�tica quanto aos emigrantes cubanos poder� ficar sujeita a uma inunda��o, pois que a Havana deixar� de tentar impedir os seus cidad�os de partir e os parentes de os virem buscar.
+O Presidente levantou o espantalho de um �xodo como o de 1980, ano em que 125 mil cubanos se dirigiram de barco para os Estados Unidos, onde tantos dos seus compatriotas j� se encontravam a viver no estado da Fl�rida.
+Os bombardeamentos podem produzir uma reviravolta na guerra que amea�a, sen�o a integridade territorial do pa�s pelo menos a estabilidade do regime de Mobutu Sese Seko que o domina h� mais de tr�s d�cadas, admitiram analistas citados pela ag�ncia francesa.
+Mas s�o escassas as informa��es sobre as v�rias frentes de combate, desconhecendo-se at� que ponto as for�as governamentais progridem no terreno.
+S� h� rumores, por exemplo que os militares tomaram, com a ajuda de antigos soldados ruandeses (hutus) e mercen�rios, posi��es perdidas h� dez dias na estrada para sudeste, para Walikale, e a cidade de Bafwasende, a Leste, e que a avia��o se prepara para atacar Punia, situada entre Kisangani e Bukavu.
+Assim, o projecto de lei ontem aprovado por unanimidade na Assembleia da Rep�blica, em vez de afirmar que as taxas ficam suspensas at� ao fim do ano (como o texto inicial, da iniciativa do social-democrata Rui Rio, previa), passou a estabelecer como limite para a suspens�o �a entrada em vigor do diploma que regule a utiliza��o de cart�es de d�bito de pagamento autom�tico�.
+E Rui Carp, o vice-presidente da bancada �laranja� respons�vel pelas quest�es econ�micas, foi claro ao afirmar no plen�rio que, �num curt�ssimo per�odo de tempo� o Governo dever� deixar esta quest�o resolvida.
+Outras das altera��es introduzidas no projecto de lei diz, ali�s, respeito � iniciativa legislativa que inevitavelmente se seguir�.
+� que, enquanto o primeiro projecto afirmava que uma vez suspensa a taxa, a Assembleia da Rep�blica apresentaria um projecto de regulamenta��o do uso dos cart�es, o projecto ontem aprovado apenas afirma que �dever� ser aprovada at� 31 de Dezembro do corrente ano legisla��o que preencha cabalmente o vazio legislativo� existente.
+Ou seja, n�o ser� necessariamente o Parlamento a faz�-lo, e a convic��o dominante na direc��o da maioria � que, a partir de agora, �o Governo ir� ter uma participa��o decisiva�.
+A teoria diz que � necess�ria a exist�ncia de uma certa dist�ncia entre o �charme� e o �anticharme� para o Psi ser produzido.
+�Se n�s cumprimimos a mat�ria, n�o deixamos os quarks distanciarem-se o suficiente para produzirem o Psi�, explica Paula Bordalo.
+�H� uma dist�ncia m�nima para o Psi se formar e teoricamento se n�o se conseguir observar o n�mero suficiente de Psis em rela��o � f�sica normal, � porque podemos estar na presen�a do plasma�, continua.
+Desde as v�rias recolhas de dados com a experi�ncia do oxig�nio, passando pelo enxofre e agora com o chumbo, o detector de mu�es -- a part�cula em que o psi se decomp�e e que os f�sicos procuram para saber se o psi foi produzido -- foi sempre mostrando uma diminui��o do psi.
+O zimbabweano admitiu mesmo que Sampras � dos poucos jogadores que costuma observar em ac��o.
+�Ele � t�o suave.
+Todas as suas pancadas s�o quase perfeitas�, justificou.
+Com esta presen�a nos quartos-de-final, o seu melhor resultado em provas do Grand Slam, Black angariou pontos suficientes para subir cerca de 30 lugares no �ranking�, mas n�o � s� isso que o tenista africano leva de Flushing Meadow.
+�O mais importante � saber que sou capaz de bater jogadores do ' top-10 '�, afirmou ele, referindo-se � elimina��o neste Open de Michael Stich (8� ATP) e Thomas Enqvist (9� ATP).
+Mas o dia j� tinha come�ado bem para as cores norte-americanas, com o triunfo de Meredith McGrath e Matt Lucena na final de pares mistos.
+O conjunto norte-americano derrotou a tamb�m norte-americana Gigi Fernandez e o checo Cyril Suk por 6-4, 6-4, conquistando o t�tulo e o cheque de 7500 contos.
+�T�nhamos decidido desde o primeiro encontro que �amos divertir-nos e que n�o ficar�amos zangados se as coisas corressem mal�, afirmou Lucena, que tamb�m revelou que os dois concordaram em jogar apenas cinco dias antes de a competi��o se iniciar.
+McGrath j� tinha sido finalista nesta variante em 1989, o �ltimo ano em que o t�tulo tinha revertido tamb�m para um par da casa.
+Segundo Artur Moreira, a not�ria leu a escritura e s� faltava assinar.
+Depois, continua, �disse que s� o faria na presen�a de duas testemunhas�.
+No dia seguinte, Armanda e Moreira voltaram ao cart�rio, acompanhados das testemunhas.
+�Sem qualquer explica��o, a not�ria disse que n�o fazia a escritura�, recorda o vendedor.
+E as testemunhas n�o chegaram a ser ouvidas.
+Seguiu-se uma troca de palavras menos amistosa e Elvira Maris acabaria por chamar a pol�cia, declarando ter sido maltratada e que o bom funcionamento do cart�rio havia sido perturbado.
+A autoridade chegou e procedeu � identifica��o dos dois indiv�duos.
+�O que eu pergunto � por que � que para a pol�cia a minha identifica��o � suficiente e para a senhora not�ria n�o �, questiona Maria Armanda.
+�Eu estou a ser alvo de discrimina��o.
+Ela est� a brincar comigo, com o vendedor, com as testemunhas, com todos n�s�, conclui.
+Por sua vez, Moreira afirma conhecer outro travesti que comprou uma casa j� h� 18 anos, sem ter tido qualquer problema.
+Contactada pelo P�BLICO, Elvira Maris declarou n�o poder �fazer uma escritura quando aparece uma pessoa com uma identidade diferente da presente no documento comprovativo da sua identidade�.
+A Alemanha n�o lhes concedeu asilo e os seus pa�ses muitas vezes n�o os querem de volta.
+S�o detidos, para evitar tenta��es de clandestinidade.
+Nas �pris�es de deporta��o� alem�s, est�o cerca de quatro mil �indesejados� � espera da expuls�o.
+At� l�, podem passar 18 meses em celas superlotadas.
+Sem terem cometido qualquer crime.
+Sem nada poderem fazer.
+A n�o ser esperar pelo destino de quem nasceu no pa�s errado.
+No in�cio do s�culo, era uma esquadra de pol�cia.
+As celas tinham capacidade para albergar at� 140 detidos por um curto per�odo de tempo.
+Perante as novas necessidades, o velho edif�cio na Kruppstrasse foi �reciclado�.
+Hoje � a mais conhecida �pris�o de deporta��o� berlinense.
+�Portugal � uma rebaldaria.
+E como n�o h� Constitui��o, ent�o vamos todos tomar atitudes arruaceiras�.
+De bandeira branca ao ombro, reclamando o encerramento das grandes superf�cies ao domingo, o comerciante, de face congestionada, resumiu � sua maneira a inten��o dos 150 lojistas presentes na reuni�o realizada anteontem � noite, em Vila Nova de Gaia.
+Depois de discutidos o melhor dia e a melhor hora para a manifesta��o, a maioria pronunciou-se pelo ajuntamento no pr�ximo dia dois de Mar�o, s�bado, a partir das quatro horas da tarde, na sala de visitas do Porto: a Pra�a General Humberto Delgado.
+Um falso m�dico passeou-se durante pelo menos dez anos pelo Hospital de Santa Maria.
+O �doutor Dinis�, segundo a Pol�cia Judici�ria, chegou a dar consultas no estabelecimento de sa�de lisboeta e a encaminhar pacientes para especialistas a quem convenceu da condi��o de colega.
+O director de Santa Maria admitiu ontem ter sido detectado um indiv�duo que se fazia passar por m�dico, imediatamente �entregue� �s autoridades, mas garantiu desconhecer que ele desse consultas.
+O deputado do PSD Guilherme Silva esteve reunido h� dias com as estruturas sindicais dos ju�zes, a quem comunicou o facto de o seu partido, apesar de �estar sensibilizado para as queixas dos ju�zes�, apenas tencionar desbloquear a situa��o num contexto em que se gere na Assembleia da Rep�blica �um amplo consenso democr�tico�, designadamente com a ades�o do PS que, segundo a direc��o parlamentar do PSD, se tem mostrado fechado a considerar o assunto �numa perspectiva de regime�.
+Esta �perspectiva de regime� de que fala o PSD implica a busca de uma solu��o no quadro do Estatuto Remunerat�rio dos Titulares dos Org�os de Soberania, e n�o apenas uma altera��o pontual deste diploma com base na interpreta��o legal avan�ada pelo Sindicato dos Magistrados do Minist�rio P�blico.
+Acredito que se referissem a animais indefesos, pois o termo �cobardia� n�o se aplica a quem enfrenta toiros.
+Apresenta esses dois grandes escritores -- por conveni�ncia --, mas olvida nomes importantes no mundo das letras e das artes.
+N�o lhe diz nada Ernest Hemingway, Pr�mio Nobel da Literatura, um homem cativado pela festa e que escolheu a terra de Espanha para sua �ltima morada?
+E Federico Garcia Lorca, que deixou t�o belas obras e alguns sentidos poemas dedicados a um famoso toureiro?
+Esquece Picasso, Goya, Mariano, Benlliure, que honraram a festa com as suas pinturas e esculturas?
+Quantos artistas, quer estrangeiros quer nacionais, a festa de toiros tem motivado.
+o cinema nasceu em Portugal?
+E o �football�?
+Contudo, somos capazes de fazer isso tudo e bem.
+Por fim, restar� um lote de ac��es que ser� vendido na Bolsa de Lisboa.
+Os pre�os das ac��es para investidores nacionais ser�o �mais vantajosos� comparativamente aos valores para os n�o residentes.
+No Minist�rio da Ind�stria existe o desejo de manter o m�ximo de capital poss�vel em m�os nacionais.
+A justifica��o � que �este � um sector em que podemos dar cartas, pelo que n�o conv�m uma aloca��o excessiva de poder de decis�o para o exterior�.
+No entanto, por definir continuam as parcelas exactas de capital que ser�o alienadas em Portugal e no estrangeiro.
+O Governo ainda n�o delineou os contornos definitivos da opera��o, o que dever� acontecer no final de Abril ou princ�pio de Maio em Conselho de Ministros.
+Actualmente, uma das quest�es colocadas pelos investidores � a de qual ser� o pre�o a pagar por cada t�tulo.
+Mas tamb�m aqui ainda nada est� definido.
+No entanto, o P�BLICO soube que foram realizadas duas avalia��es da empresa de celulose.
+ Uma pela dupla BFE/Salomon Brothers e outra pelo BPI em associa��o com a UBS (Union des Banques Suisses).
+O II Encontro Nacional de aquarofilia iniciou-se ontem com a inaugura��o de exposi��es de filatelia, aqu�rios, artesanato tem�tico e fotografia, estando ainda inclu�das no evento outras iniciativas, como �workshops� sobre fotografia e montagem de aqu�rios, anima��o musical e projec��o de filmes.
+ Esta tarde tem in�cio o col�quio, que conta com a participa��o de especialistas nacinais e estrangeiros em aquariofilia e actividades subaqu�ticas, bem como respons�veis por alguns aqu�rios europeus e o director do Ocean�rio da Expo'98.
+A pensar nos mais novos, o Programa Aquaj�nior permite a utiliza��o de programas multim�dia relativos � vida marinha.
+O Porto tem um novo grupo de teatro que obedece ao princ�pio da diversidade.
+A troca de experi�ncias com outros grupos, a produ��o de espect�culos, a organiza��o de debates e de ateliers de forma��o contam-se entre os objectivos do grupo, fundado por quatro actores profissionais.
+O primeiro �workshop�, orientado por um especialista japon�s em �commedia dell'arte�, Kuniaki Ida, come�ou esta semana.
+A agita��o � volta das filmagens suscitou igualmente um novo interesse por aquele romance.
+A Ilh�u Editora teve de p�r nas bancas mais uma edi��o cabo-verdiana -- a terceira desde que a obra conheceu a luz do dia --, de modo a satisfazer a curiosidade dos novos leitores deste primeiro e bem sucedido livro de Germano Almeida, cuja bibliografia n�o cessa de aumentar desde que decidiu lan�ar-se nesta tarefa de dar corpo �s hist�rias e que, segundo ele, v�o ter consigo todos os dias no seu escrit�rio de advogado, no Mindelo.
+Segundo Francisco Manso, os custos do seu filme dever�o situar-se entre os 250 mil e 300 mil contos, or�amento que ele considera �razo�vel� para os padr�es portugueses.
+Portugal, atrav�s do IPACA, RTP e v�rios outros organismos p�blicos e privados, assegura o grosso do financiamento, cabendo a parte restante ao Brasil e Cabo Verde, e ainda a entidades ligadas � Uni�o Europeia.
+Os juristas franceses est�o algo apreensivos com o novo C�digo Penal, que entra hoje em vigor e que substitui o velho C�digo Napole�o de 1810, ao cabo de um trabalho que durou 18 anos.
+Quando em todo o mundo ca�ram os muros com que alguns tamb�m tentaram defender a irracionalidade dos seus princ�pios, eis que em Portugal se estabelecem quil�metros infinitos de aramados, que esquartejam o pa�s em talh�es, onde uma minoria � autorizada a explorar at� ao exterm�nio o que a natureza a todos oferece.
+Pelo caminho, com o conluio de uns poucos que ocuparam ou ocupam altos cargos na Administra��o (e que envergonham muitos outros que honrosamente se batem por um adequado ordenamento cineg�tico), t�m ficado enxovalhados, espezinhados e chumbados, o direito � propriedade privada, o interesse nacional e o direito � vida.
+Com uma vis�o facciosa e m�ope do que � o desenvolvimento rural (j� que as receitas da ca�a raramente ficam nos locais onde esta se pratica), continua a Secretaria de Estado da Agricultura e do Desenvolvimento Rural (SEADR) a empenhar-se em propalar sem quantificar os proveitos desta actividade.
+Mas quanto perde o pa�s por cada ave de rapina ilegalmente abatida?
+Quanto perde o pa�s por cada predador envenenado?
+Quanto perde o pa�s por cada esp�cie rara morta?
+Quanto gasta anualmente a JAE na reposi��o dos milhares de placas que anualmente servem de alvo a ca�adores sem escr�pulos?
+Em quanto contabilizar os preju�zos causados por centenas de c�es de ca�a que s�o anualmente perdidos ou abandonados pelos ca�adores?
+Em quanto ficaria a limpeza de toda a imund�cie provocada na natureza pelos almo�os e lanches que fazem parte deste ritual de matan�a?
+Quem deve ser responsabilizado pela polui��o dos nossos campos e �guas, provocada pelas toneladas de chumbo anualmente disseminadas por milhares de disparos?
+E as perdas de rendimento ao n�vel regional, causadas pela elimina��o, nas zonas de ca�a, das pr�ticas agr�colas e pastoris consideradas incompat�veis com as actividades cineg�ticas?
+E os postos de trabalho assim eliminados?
+Entretanto, durante toda a manh� de ontem, uma testemunha tentou, na esquadra da PSP, dizer o que tinha visto e ouvido.
+�Mas esse direito foi-me sempre negado�, afirma a testemunha, A. Carvalho.
+�O pol�cia disse-me que o agrediu, mas agora, horas depois, desmente tudo�, adiantou.
+O caso foi entregue � PSP das Caldas da Rainha e remetido ontem ao tribunal, que decidiu que o condutor n�o pode sair da �rea da resid�ncia, embora possa conduzir, tendo sido instaurado inqu�rito para averigua��es.
+Tamb�m em Outubro, quando a C�ria Diocesana denunciou publicamente o fen�meno das profana��es, Aristides Lima, dirigente do PAICV, foi uma das vozes a levantar-se na condena��o dessa pr�tica e a manifestar � comunidade cat�lica a solidariedade do seu partido.
+R. -- Nestes casos, a boa pol�tica � que durma em casa.
+Assim, ao menos sabe-se onde e com quem.
+A vizinha do apartamento da frente insiste em fazer ioga nua, � vista de todos.
+Queixa-se ao senhorio?
+O Teatro Experimental de Cascais apresenta �Invent�rios�, no Audit�rio Mirita Casimiro.
+MONTEMOR-O-NOVO A Unidade de Inf�ncia do Centro Dram�tico de �vora estreia hoje � noite, no Cine-Teatro Curvo Semedo, �O Meu Amigo Rodrigo�.
+Pol�ticos israelitas e da Organiza��o de Liberta��o da Palestina (OLP) afirmam-se dispostos a punir os fundamentalistas isl�micos que no domingo causaram 19 mortos numa povoa��o a norte de Telavive.
+O ministro israelita dos Neg�cios Estrangeiros, Shimon Peres, afirmou ontem que o Governo de Yitzhak Rabin vai pressionar o l�der da OLP, Yasser Arafat, no sentido de que se actue energicamente perante a Jihad Isl�mica, o grupo radical que reivindicou o facto de dois dos seus militantes suicidas -- naturais da Faixa de Gaza -- haverem sido os respons�veis pelo atentado de Beit Lid.
+Cristo ressuscitou?
+Entre metade e um quarto dos estudantes cat�licos acreditam em Deus mas n�o em Cristo.
+� um fen�meno de desintegra��o social.
+As consequ�ncias s�o imprevis�veis.
+Voltando a mostrar-se particularmente � vontade em condi��es de pouca ader�ncia, o brasileiro Ayrton Senna (McLaren / Ford) conseguiu ontem o melhor tempo na primeira sess�o de qualifica��o para o Grande Pr�mio da Europa que amanh� se disputa em Donington (Inglaterra).
+No entanto, a luta pela �pole position� n�o est� ainda terminada, com o franc�s Alain Prost (Williams / Renault) a prometer dar luta ao brasileiro, hoje, na derradeira sess�o de qualifica��o.
+Aquilo que os pilotos temiam confirmou-se, com um primeiro dia de treinos muito chuvoso, a dificultar as suas tentativas de qualifica��o.
+Prost come�ou por ser o mais r�pido mas resolveu recolher �s �boxes�, aguardando por um final de sess�o com menos �gua.
+Pouco depois, era a vez de Senna bater o seu tempo.
+�Embora a pista de Donington tenha uma drenagem melhor que a de Interlagos, acho que correr aqui � mais perigoso�.
+Com a entrada do Tratado de Maastricht na Assembleia da Rep�blica prevista j� para a pr�xima semana-- depois de o Conselho de Ministros ter decidido ontem o envio de uma proposta de resolu��o para ratifica��o parlamentar--, o debate sobre o processo e o �timing� adequado � sua aprova��o passou imediatamente � ordem do dia.
+Muitas quest�es subsistem ainda, nomeadamente sobre a necessidade de proceder a uma revis�o constitucional que compatibilize o texto da Constitui��o com o texto do Tratado.
+Hoje mesmo, o Parlamento ver-se-� confrontado com a proposta de resolu��o que o CDS vai apresentar, visando dot�-lo de poderes de revis�o constitucional de modo a permitir o recurso ao referendo que os centristas defendem para ratificar as propostas de Maastricht.
+Quando no avi�o se acende a luz verde e come�am a encaminhar-se para a porta e a sair ao ritmo de um por cada dois segundos, as batidas do cora��o j� normalizaram.
+J� nem ouvem a voz de �j� do instrutor-largador.
+Sentem a palmada que ele lhes d� no ombro e l� v�o.
+parecem bonecos articulados.
+Por vezes, muito raramente, n�o se ganha para o susto.
+Como foi no quarto salto, em que um deles passou a velocidade mete�rica, com o p�ra-quedas dorsal em vela romana [ m� abertura em que a calote do p�ra-quedas se mant�m na forma de facho, sem superf�cie de sustenta��o ].
+O jovem instruendo tinha aproveitado bem os ensinamentos e abriu o p�ra-quedas de reserva.
+Respirou-se fundo, no ar e em terra.
+J� no ch�o �recriminam-se� entre si, por causa de pequenos descuidos.
+�N�o fui eu!
+A Ligier-Renault, a Lotus-Mugen-Honda e a Sauber-Mercedes suceder�o � Jordan nos referidos ensaios.
+A Ferrari chegar� amanh� ao circuito de Barcelona, onde apresentar� no seu novo modelo uma das inova��es mais interessantes para a nova �poca: a fixa��o das suspens�es directamente incorporadas no �chassis�.
+Paralelamente aos testes de reabastecimento, o McLaren-Peugeot de Mika Hakkinen e o Sauber-Mercedes de Karl Wendlinger foram os mais r�pidos em pista, cronometrados na suas voltas mais r�pidas em 1m19,32s, enquanto o Lotus-Mugen-Honda de Pedro Lamy foi o mais lento, gastando 1m27,52s, muito aqu�m do italiano Alessandro Zanardi, piloto-ensaiador da equipa, que gastou apenas 1m22,95s.
+Frentzen (Sauber-Mercedes), com 1m20,02s, Irvine (Jordan-Hart), com 1m20,14s, Panis e Bernard (Ligier-Renault), respectivamente, com 1m21,64s e 1m22,24s, seguiram-se � dupla mais r�pida, no circuito de Montmel�.
+A maioria das queixas, verdade seja dita, n�o tinham nada de transcendente ou desconhecido.
+Repetiam que as missas s�o uma chatice e que as homilias oscilam entre o horror e a banalidade.
+Algumas pessoas atribu�am � mis�ria lit�rgica a suposta descida na pr�tica religiosa.
+Conclus�o porventura demasiado linear.
+Insistiam em que os jovens dos 18 aos 25 anos j� n�o aturavam o que os pais tinham aguentado.
+O que, sendo talvez verdade, esquece que tamb�m h� jovens para tudo.
+O irrequieto Jos� Magalh�es resolveu brindar deputados, jornalistas e amigos com uma prenda �pirata�.
+Uma disquete com a tradu��o portuguesa do Tratado de Maastricht �com os conhecidos e altamente verberados erros oficiais�, a que juntou uma bem humorada etiqueta: �Disquete sem v�rus (utiliza��o de software preservativo antieuropeu n�o aconselhada).
+Uso e difus�o livres.
+Pirateada e distribu�da gratuitamente no Natal de 1992 por Jos� Magalh�es a todo e qualquer interessado, de boa ou m� f�.
+Consta que enviou uma a Manuel Monteiro e outra a Paulo Portas.
+Uma pirataria completa.
+N�o, n�o � mais uma caso de vida ou telenovela mexicana de televis�o, ou uma hist�ria para adormecer.
+Esta � bem real e um exemplo de que pode acontecer a qualquer m�e que, ao tirar uma ecografia, descobre que n�o � um, mas tr�s filhos que vai dar � luz.
+Uma alegria para alguns e um choque para outros.
+�O choque foi na ecografia, v�o fazer j� 18 anos, e s� suspeit�vamos de um�.
+�Mas reagi muito bem e fiquei a saber que engravidei duas vezes�.
+Esta m�e explica que, em casos como este, �os cuidados s�o todos multiplicados por mil�.
+N�o � caso para menos, pois s�o g�meos, mas neste caso, os chamados g�meos falsos, dizig�ticos na linguagem t�cnica, por serem gerados em �vulos diferentes.
+A In�s e o Jo�o s�o quase insepar�veis, embora, quando pequenos, ela assumisse a fun��o de chefe nas brincadeiras e tivesse um instinto maternal em rela��o ao irm�o.
+S�o muito diferentes, ela pela sua independ�ncia e compet�ncia e ele pela sua pregui�a e boa disposi��o.
+Sempre na mesma turma escolar, separaram-se no 10� ano.
+E foi quando se notou que �era a irm� o grande apoio nos estudos�, conta a m�e.
+A In�s quer ser arquitecta, enquanto que o Jo�o j� mostra tend�ncia para a �rea da sa�de.
+A solu��o definitiva de situa��es como as da central da Av. Casal Ribeiro, do Campo das Ceboulas e de numerosas outras centrais dedicadas aos transportes inter-regionais e inter-urbanos est� por�m dependente de uma obra de grande envergadura, ainda sem localiza��o completamente decidida e cujo in�cio n�o dever� ocorrer antes dos pr�ximos dois ou tr�s anos.
+Uma vez que esta central esteja em funcionamento, passar� a receber tamb�m as carreiras que em Janeiro sair�o do Campo Pequeno para Sete Rios.
+O espa�o a libertar nessa pra�a poder� ent�o ser consagrado aos carros de turismo que agora ocupam o topo do Parque Eduardo VII.
+Tamb�m uma residente h� 30 anos nos Moinhos da Funcheira, Isabel Martins da Silva, considera excessivo o que pagou pela liga��o de esgotos, 181 contos, e da �gua, 81.
+Para esta moradora, a autarquia ainda n�o fez as obras necess�rias porque considera aquela urbaniza��o como �o bairro dos ricos�, como afirma ter sido dito pelo vereador do urbanismo, Miguel Vieira, da CDU.
+V�rias solu��es t�m sido propostas para contornar este obst�culo.
+Uma delas consiste em supor a coexist�ncia de Universos paralelos.
+mas existe num outro, porque a� nunca assassinou a sua av�.
+ Uma outra alternativa consiste em supor que este tipo de situa��es incoerentes nunca se poder� verificar, porque a pr�pria Natureza se encarregar� de o impedir.
+Como?
+Talvez, autorizando apenas as viagens ao passado que n�o d�em origem � temida morte da avozinha.
+Ou talvez, simplesmente, proibindo qualquer tentativa de utiliza��o dos buracos de minhoca para construir m�quinas do tempo.
+Ou ainda, desencadeando a destrui��o imediata das m�quinas que alguma vez vierem a ser constru�das.
+Depois de um come�o de campeonato decepcionante, a equipa de basquetebol do FC Porto confirmou ontem a sua subida de forma, vencendo o l�der do campeonato por 85-84, em Ovar, com um cesto de tr�s pontos conseguido por J�lio Matos a tr�s segundos do fim do jogo.
+Com este resultado, o FC Porto isolou-se no segundo lugar do �Nacional�, enquanto o Benfica igualou a Ovarense no comando.
+�O Dalkon Shield representa um epis�dio horr�vel na hist�ria do controlo da natalidade, mas hoje � claro que os problemas se deviam a esse DIU em particular e n�o afectam todos os dispositivos do g�nero�, diz Jade Singer, respons�vel do Centro de Sa�de de Santa M�nica.
+De facto, � necess�rio uma certa dose de f� para considerar o uso do DIU, admite Debra Bronstein, vice-presidente respons�vel pelo �marketing� na GynoPharma.
+E muitas associa��es de consumidores, incluindo a Associa��o Internacional para a Educa��o das V�timas do Dalkon Shield, continuam a fazer campanha contra o dispositivo, afirmando que n�o existe suficiente informa��o sobre os seus riscos.
+A multinacional farmac�utica desmentiu as not�cias, acusando o jornal de �afirma��es incorrectas� e sublinhando ter �toda a confian�a na seguran�a de emprego e na efic�cia� do sedativo, desde que ele seja �utilizado de acordo com as prescri��es� referidas na literatura que o acompanha.
+Comercializado sob as designa��es de �Midazolam �(inject�vel) ou �Dormicum� (ampolas) e � venda em Portugal, o �Versed� � um sedativo utilizado em pequenas interven��es cir�rgicas, nomeadamente para anestesias locais.
+Segundo a Roche, o medicamento -- utilizado nos �ltimos cinco anos sem problemas por mais de 15 milh�es de americanos -- foi clinicamente testado nos Estados Unidos e na Europa, �n�o tendo sido registado qualquer morte, cujas causas pudessem ser atribuidas � administra��o do ' Versed '�.
+Os casos referidos pelo �New York Times� como sendo imput�veis ao medicamento foram considerados pela Roche como �mortes sem rela��o causal com a aplica��o do medicamento�, tratando-se de �doentes cujo estado de sa�de j� era grave e que apresentavam problemas cardiovasculares e ferimentos muito graves�.
+Os representantes da CNN em Bagdad comunicaram a composi��o do comboio e a cor dos autom�veis � sua sede nos Estados Unidos.
+Estes pormenores seriam fornecidos �s autoridades americanas e, deste modo, talvez o risco de um ataque a�reo aliado sobre o comboio fosse reduzido.
+H� uma dezena de dias que a estrada entre a fronteira iraquiano-jordana e Bagdad � alvo de bombardeamentos quase di�rios por parte de avia��o aliada.
+Encontra-se cortada em diversos pontos e � preciso por vezes tomar estradas secund�rias, tamb�m alvo dos raides, como constatou este enviado ao efectuar o trajecto no dia 30 de Janeiro.
+Para o bem ou para o mal, at� ao momento, quem foi, na sua opini�o, a figura portuguesa do ano de 1995.
+E porqu�?
+Foi um b�lgaro: o Iordanov.
+Porque marcou dois golos magn�ficos ao Mar�timo e, assim, o Sporting ganhou a Ta�a de Portugal.
+J� n�o v�amos o padeiro h� treze anos.
+Por que raz�o � t�o grave este facto?
+Porque -- consideram os relatores, interpretando as diversas contribui��es ao congresso -- �a natureza dif�cil dos problemas, a escala de tempo longa da sua resolu��o, a sua raiz cultural e o facto que o objecto das pol�ticas n�o � outro sen�o a mudan�a de comportamentos das comunidades educativa e cient�fica, imp�em uma implica��o org�nica dessas comunidades nas pol�ticas que lhes dizem respeito�.
+Isto, acrescentam, �sob pena de um est�pido desperd�cio de recursos, de energias e de motiva��o�.
+As �pol�ticas anti-participativas� que os relatores consideram existir nesta �rea s�o por eles julgadas particularmente inadequadas num �per�odo de expans�o e de sustentado aux�lio comunit�rio� como o actual, mas elas s�o, para mais, particularmente vivas no actual momento de crescimento do sistema, devido ao mais vivo confronto entre novos e velhos elementos.
+A primeira dificuldade que se depara �s empresas que conseguem concretizar um processo de investimento em Mo�ambique � o de recrutamento de quadros para perman�ncias prolongadas no pa�s.
+Para al�m de problemas salariais, mais ou menos complicados, de alojamentos dif�ceis de encontrar e caros, a quest�o mais sens�vel acaba por ser de ordem familiar, principalmente as relacionadas com filhos em idade escolar e com a respectiva educa��o.
+Se no Maputo existem escolas particulares e p�blicas suficientemente cred�veis para suprir as necessidades, nas restantes concentra��es urbanas isto j� n�o acontece.
+No segundo centro populacional de Mo�ambique, a cidade da Beira, os portugueses residentes conseguiram, por iniciativa pr�pria, criar uma escola que lecciona o ensino prim�rio e que para o ano vai iniciar-se como escola preparat�ria.
+A organiza��o do 17� Festival Nacional de Gastronomia n�o pensa para j� na sua internacionaliza��o, prevendo-se apenas �que anualmente seja um pa�s da Europa comunit�ria a ser convidado, bem como um PALOP�.
+No primeiro dos casos est� Espanha, que poder� estar representada j� na pr�xima edi��o.
+As �hostilidades� abrem hoje, por tradi��o, com os paladares promovidos pela Regi�o de Turismo do Ribatejo, que entregou a responsabilidade da refei��o a um restaurante do Cartaxo.
+Tal como em todos os restantes dias, o almo�o est� marcado para ter in�cio �s 13h.
+Esta �poca, o Sporting ainda n�o tinha conseguido marcar mais de dois golos nos jogos em Alvalade.
+F�-lo ontem, batendo um Braga pouco atrevido e superando mais um �fantasma� da equipa.
+Mas, na bancada, j� poucos acreditam no t�tulo.
+A transac��o de 117.203 ac��es nominativas e 34.376 ao portador da Ocidental Holding SGPS constitui ontem uma das notas dominantes do mercado accionista nacional.
+Movimentados na Bolsa de Valores de Lisboa (BVL), os lotes foram intermediados pela Comercial Dealer.
+�No princ�pio da nossa rela��o electr�nica, acordava a meio da noite e ficava deitado a pensar se teria correio de Bill.
+Ele parece escrever as suas mensagens � noite, dormir (talvez) e mand�-las na manh� seguinte�, escreve John Seabrook, explicando que a primeira pergunta por E-mail foi sobre o pr�prio meio de comunica��o.
+�Sou a �nica pessoa que l� o meu E-mail, por isso ningu�m se pode envergonhar ou pensar que a mensagem vai circular por a� ...
+E-mail n�o � uma boa forma para nos zangarmos, uma vez que n�o se pode interagir�.
+Seabrook notou que as mensagens de Bill n�o tinham princ�pio ou fim, coisas como �caro�, �seu� ou �atentamente�.
+Bill nunca se lhe dirigiu pelo nome e despedia-se com um enigm�tico �&�, que em linguagem de correio electr�nico significa �responda�.
+Para os empregados da Microsoft � comum encontrarem Bill Gates electronicamente antes de o conhecerem pessoalmente e, � imagem do chefe -- o �mais esperto de todos� --, passam a vida a descreverem-se uns aos outros como �espertos� e �superespertos�.
+�� um conceito vago.
+H� uma certa vivacidade, uma capacidade para absorver factos novos.
+Para perceber uma nova situa��o e, depois de algu�m lha explicar, dizer imediatamente: ' Que tal assim? ' .
+Fazer uma pergunta pertinente.
+Absorver em tempo real.
+Capacidade para recordar.
+Relacionar coisas que ao princ�pio n�o parecem ligadas.
+O �rbitro Pinto Correia esteve bem durante toda a primeira parte e durante quase toda a segunda.
+No entanto, acabou por perdoar um segundo cart�o amarelo a Rui Neves, condescendendo igualmente com Dimas.
+O referido relat�rio explica que a substitui��o dos pequenos ecr�s pelos multiplexes -- at� ao final de 1998 est� previsto o aparecimento de mais tr�s (Loures Shopping, Norte Shopping e Vasco da Gama) -- dever� aumentar o n�mero de espectadores: de 12 milh�es em 1996 para 18 milh�es!) em 2000.
+Em termos de distribui��o, os filmes norte-americanos det�m praticamente 95 por cento da quota das bilheteiras portuguesas, �a mais elevada da Uni�o Europeia�.
+Segundo o relat�rio, �as principais distribuidoras nacionais ou s�o subsidi�rias norte-americanas ou empresas portuguesas que distribuem os filmes dos grandes est�dios norte-americanos�.
+Dois exemplos: A Castello-Lopes tem acordos com a Fox e a Miramax, a Lusomundo com a Disney e a UIP e adquire filmes da BMG.
+Momento -- Andou boa parte do ano a perder tempo, tendo em conta apenas os seus interesses, a actuar no meio-campo.
+Segundo observadores citados pelo �Times�, a China encontra-se �numa posi��o muito mais fraca do que estava em 1982-84, per�odo em que decorreram as negocia��es que levaram ao acordo para a transfer�ncia de soberania�.
+Nessa altura, recordam, �Deng Xiaoping estava no auge da lideran�a do processo de reforma, era um septuagen�rio vigoroso e o desenvolvimento econ�mico parecia impar�vel�.
+Holanda: Bons jogadores de futebol .
+Falam ingl�s muito bem, n�o parecem ter uma grande cultura pr�pria.
+�s vezes, um pouco parvos, geralmente descontra�dos.
+It�lia: Bons a falar e a fazer neg�cio .
+Pa�s bastante ca�tico, mas muito industrioso.
+Muito convencidos.
+Um dos povos mais agrad�veis da Europa, amistosos.
+Mas muita pobreza.
+Fisicamente, o povo mais belo da CE.
+-- Mas o senhor � t�o cuidadoso com a sua mota que se arrisca a ser preso s� para n�o a deixar na rua?
+-- Mas normalmente � assim!
+As outras coisas n�o s�o t�o ...
+O que os h�ngaros temem � uma onda de nacionalismo nos pa�ses vizinhos que perigue as escolas e institui��es culturais de h�ngaras.
+Isto poder� levar muitos h�ngaros a desses pa�ses a decidir que a emigra��o para a Hungria � a melhor op��o.
+Para resolver os complexos problemas dos h�ngaros que vivem nos pa�ses vizinhos, o Governo criou um gabinete especial chefiado por Geza Entz.
+Os Orlando Magic foram ao �The Omni�, em Atlanta, perder pela primeira vez em dez jogos seguidos da edi��o deste ano da NBA, a Liga Norte-Americana de Basquetebol profissional.
+Os Atlanta Hawks venceram por 107-105 um jogo muito emotivo, que s� ficou decidido a 32s do fim.
+Foi quando John Konkac marcou os dois lances livres que desempataram o jogo, e de seguida impediu que os Magic chegassem ao cesto, obrigando-os a um �turnover�.
+Por outro lado, no caso do haxixe os riscos para a sa�de s�o compar�veis aos de produtos como o �lcool, tabaco, caf� e certos medicamentos.
+Mais um argumento: o consumo do haxixe n�o provoca depend�ncia, ao contr�rio do �lcool ou da nicotina.
+H� ali�s estudos americanos que provam que o haxixe � um bom relaxante, logo, se diminui o �stress� diminui um dos factores de risco do enfarte mioc�rdio.
+A legaliza��o das drogas duras � mais dif�cil, pois tem que ser feita � escala internacional.
+e n�o precisariam de injectar-se nos cantos do Casal Ventoso.
+Isto diminuiria a criminalidade, haveria menos mortes por �overdose� e de outras doen�as, como a sida.
+Domingos -- Afinal n�o estava de rastos, como se quis fazer querer.
+Entrou confiante, integrou-se bem nas movimenta��es e mostrou que, com ele em campo, � bem mais f�cil rentabilizar o jogo ofensivo.
+Fez tr�s remates, todos eles perigosos, e marcou um golo que s� est� ao alcance de um verdadeiro ponta-de-lan�a.
+N�o tinham raz�o de ser as desconfian�as de equipa t�cnica.
+Fernando Couto -- Esteve imperial, sempre muito certo a comandar a defesa.
+H� alturas em que parece ter um �man que atrai a bola na sua direc��o.
+ Os futuros �chips� de 256 megabits poder�o ser usados em aplica��es que exigem grandes quantidades de mem�ria, como o processamento de imagens em supercomputadores e noutras tecnologias de ponta.
+A alian�a entre a Hitachi e a Texas -- que � uma resposta � associa��o da IBM, Siemens e Toshiba anunciada em Julho passado nesta mesma �rea -- poder� ser ainda alargada a outras firmas, como a NEC e a American Telephone and Telegraph.
+Alberto Ralha nunca foi alvo de grandes cr�ticas pela simples raz�o de que raramente algu�m se lembra da sua exist�ncia.
+Beneficiando do facto de �negociar� com uma classe tradicionalmente pouco reivindicativa, a dos docentes do ensino superior, Alberto Ralha � designado por os sindicalistas como �boa pessoa�, mas mau conhecedor das quest�es do ensino superior.
+N�o fora o dinamismo do seu director-geral, Pedro Lince, e o mandato deste secret�rio de Estado ficaria irremediavelmente marcado pela inoper�ncia.
+enquanto as universidades p�blicas amea�am fechar as portas por falta de dinheiro para pagar a docentes e n�o docentes, � cada vez maior o n�mero de estabelecimentos de ensino privados de qualidade duvidosa a ministrar somente cursos que outros recursos n�o exigem sen�o a caneta e o papel.
+Est�o suspensas as buscas do corpo do pescador que, ter�a-feira, foi engolido por uma onda, na zona da Boca do Inferno, no Guincho.
+�O mar continua mau, bate muito ali, e as �guas est�o barrentas, o que torna imposs�vel a ac��o dos mergulhadores�, explicou o capit�o do Porto de Cascais, Fernando Tavares de Almeida.
+Contrariamente ao que esperavam Lu�s Rodrigues e os seus correligion�rios, o caso, por�m, ficou adormecido no minist�rio das Finan�as.
+�A doutora Manuela Ferereira Leite [ que sucedeu a Rui Carp na secretaria de Estado do Or�amento ] disse-nos que quando assumiu fun��es encontrou o processo meio espalhado por aqueles buracos�, confirmou ontem ao P�BLICO o ent�o vice-presidente da concelhia local do PSD, Valdemar Saleiro.
+�Uma das coisas que nos disseram que l� vinha � que havia na C�mara despesas n�o justificadas no valor de 194 mil contos�, refere um dos outros elementos do PSD local que ent�o acompanhou o caso.
+Valdemar Saleiro -- que tinha tamb�m responsabilidades partid�rias ao n�vel do distrito de Beja -- diz que n�o se recorda de ter ouvido falar neste n�mero, mas confirma que a secret�ria de Estado o informou de que �o relat�rio apontava coisas que tinham de ser esclarecidas�.
+ A sess�o de ontem do Mercado Monet�rio Interbanc�rio apresentou-se logo na sua abertura bastante pressionado pela procura de fundos, o que motivou uma subida das taxas de juro, que se foi acentuando � medida que se avan�ava na sess�o.
+Com a finalidade de estabilizar o mercado, o Banco de Portugal anunciou a meio da manh� uma interven��o ocasional de ced�ncia de fundos at� ao montante de 150 milh�es de contos, mediante a coloca��o de BT/TIM a um dia e � taxa de 11,75 por cento.
+Apesar da press�o evidenciada no mercado, apenas foram colocados 53.677 milhares de contos.
+O grupo A. Silva & Silva registou, no ano passado, resultados l�quidos consolidados de 13,5 mil contos.
+Os resultados foram afectados pela realiza��o de provis�es extraordin�rias na sua participada Assiconstr�i -- entretanto vendida � Somague -- no valor de 330 mil contos, �por forma a n�o transferir para o seu comprador riscos provenientes de situa��es pendentes de reclama��es a clientes, eventualmente geradoras de diminui��es patrimoniais futuras�, refere um comunicado da empresa ontem divulgado.
+Em termos individuais, a A. Silva & Silva registou lucros de 444,6 mil contos.
+O volume de factura��o ascendeu a 42 milh�es de contos, os resultados operacionais consolidados foram de 1,1 milh�es de contos (contra 652,4 mil contos registados em 1995), enquanto os resultados correntes totalizaram 342,4 mil contos (mais 159,2 por cento do que no ano anterior).
+S�o pequenas part�culas met�licas e foram descobertas incrustadas nos ossos dos calcanhares do piloto Jorge Albuquerque.
+A sua import�ncia?
+� que ningu�m consegue explicar como � que elas poder�o ter ido a� parar a n�o ser devido � acelera��o provocada por uma explos�o.
+E � poss�vel que o material que as comp�e seja aquele de que era feito o recipiente que conteve a bomba.
+Trata-se de uma das provas fundamentais apurada pela V CEIAC, a par da descoberta dos vest�gios de explosivos em pe�as do avi�o.
+Clinton apressou-se a dizer que tais contactos n�o deviam ter acontecido e a sua porta-voz, Dee Dee Myers, indicou que documentos sobre esses encontros foram compilados e seriam entregues a Fiske, respeitando a intima��o feita por este.
+O procurador especial chamou a depor seis funcion�rios da Casa Branca e quatro elementos do Departamento do Tesouro, para esclarecerem os contactos mantidos.
+Entre os primeiros, conta-se o advogado Bernard Nussbaum, amigo de Clinton e que se demitiu na semana passada, em rela��o com o caso.
+A op��o do Alentejo vai para o chamado Turismo Verde, em zonas de inequ�voca qualidade ambiental.
+�Um turismo que promova a descoberta da diferen�a e n�o a massifica��o�, diz Andrade Santos, presidente da RTE.
+Al�m da costa mar�tima, a melhor preservada do pa�s, o Alentejo oferece turismo rural e agro-turismo, turismo cineg�tico e o aproveitamento de albufeiras, recorrendo sempre � boa qualidade do seu patrim�nio natural.
+E a riqueza do seu patrim�nio constru�do, a cultura musical regional, o artesanato e a gastronomia tradicional completam, em linhas gerais, o quadro da regi�o.
+Ultimamente, a implanta��o de campos de golfe tem surgido como um complemento tur�stico n�o desprez�vel, ainda que assumindo, por vezes, discut�veis propor��es (ver caixa).
+O �romance� da princesa Ana -- segunda descendente e �nica filha da rainha -- com o comandante Tim Lawrence, 37 anos, j� fizera as del�cias da imprensa popular brit�nica, nomeadamente na altura em que o casal apariceu em p�blico, por diversas vezes, ap�s o div�rcio de Ana.
+A data e local do casamento, que dever� realizar-se numa cerim�nia privada, n�o foram divulgadas pelo Pal�cio de Buckingham.
+Face ao impacte dos planos do construtor, a comiss�o de trabalhadores da Renault Portuguesa foi ontem ao Minist�rio da Economia �lembrar ao Governo� o compromisso franc�s de criar 180 novos postos de trabalho em Cacia e investir nesta unidade 12 milh�es de contos.
+A resposta do Governo � que o assunto tem estado na agenda dos contactos com o grupo franc�s, uma posi��o que para os trabalhadores �n�o representa qualquer garantia�.
+Os representantes dos trabalhadores esperam pelo comit� de grupo europeu extraordin�rio para uma decis�o quanto � ades�o a uma eventual greve dos efectivos do construtor.
+ Contudo, o porta-voz da comiss�o de trabalhadores, citado pela Lusa, defendeu que a redu��o de postos de trabalho na Renault Portuguesa � mais significativa, ao ter passado de 3500 trabalhadores em 1992 para 1200 em 1997.
+Em 35 anos, os danos causados ultrapassam dois mil milh�es de pesetas (2400 milh�es de contos), diz o relat�rio.
+Acrescenta que esta estimativa se refere unicamente aos danos directos dos sinistros e n�o inclui os custos da luta contra os inc�ndios -- verbas avultadas, como pode concluir-se pelas destinadas ao ano em curso, que atingem os 130 mil milh�es de pesetas (156 milh�es de contos).
+ Outro aspecto preocupante � o da frequ�ncia crescente de �anos negros�, em que se registaram recordes de zonas ardidas.
+ De 1978 at� agora, Houve quatro anos em que se registou a perda de �reas superiores a 400 mil hectares.
+O ano passado esse valor foi de 432 mil hectares, estando o recorde no ano de 1985, com 484 mil hectares.
+ A Assembleia Parlamentar da Francofonia (APF, ex-AIPLF), reunida em Abidjan, na Costa do Marfim, condenou ontem a insurrei��o desencadeada h� um m�s por �certos elementos� das For�as Armadas da Guin�-Bissau e exortou as partes em conflito ao di�logo �no respeito pela legalidade constitucional�.
+Numa resolu��o adoptada em sess�o plen�ria, a APF �reafirma a legitimidade do actual governo, eleito em elei��es livres e democr�ticas� e �condena sem reservas a rebeli�o iniciada em 7 de Junho de 1998�.
+A Assembleia exprime o seu �apoio total� ao governo do Presidente �Nino� Vieira, e lan�a um �apelo premente a uma cessa��o imediata das hostilidades�.
+Sendo o primeiro de uma nova gera��o de jogos a tr�s dimens�es para consolas de 16 bits que a Nintendo lan�ou no mercado de v�deojogos, Starwing � surpreendente pela qualidade dos gr�ficos que apresenta, pelo som, perfeitamente adequado ao ambiente gr�fico, mas sobretudo pela extraordin�ria velocidade a que se desenrola.
+O jogador -- que desempenha o papel de Fox McCloud -- tem de completar um total de 18 miss�es, sobrevoando planetas ou enfrentando os inimigos no espa�o, onde as dificuldades s�o acrescidas, j� que, para al�m dos opositores, tem de desviar-se de aut�nticas chuvas de aster�ides e outros objectos que vagueiam sem rumo.
+Tem tamb�m de zelar pela seguran�a da sua equipa de pilotos, ajudando-os sempre que est�o em dificuldade.
+Ap�s algum treino, os comandos s�o f�ceis de manejar, permitindo ao jogador executar piruetas ou utilizar o propulsor para acelerar momentaneamente a sua nave.
+H� tr�s formas de visionamento do cen�rio, � escolha do jogador.
+ Nas cenas de espa�o a �vis�o de ' cockpit '�, com o recurso a uma mira, � de grande utilidade.
+O programa de trabalho e o or�amento para os pr�ximos tr�s anos para o Observat�rio Europeu das Drogas e das Toxicodepend�ncias, com sede em Lisboa, foram ontem discutidos numa reuni�o em que participaram representantes de todos pa�ses da Uni�o Europeia.
+ O PCP, esse, est� sempre bem e com pouco trabalho pela frente.
+O que era preciso fazer, est� feito: derrotar uma iniciativa do Governo.
+Distribu�dos os trabalhos, falta perceber como se chegou aqui.
+O que tamb�m n�o � dif�cil.
+Tudo nasce num triplo erro governamental.
+Reduzir a procura do consumo de droga � a prioridade do �rg�o Internacional de Controlo dos Estupefacientes da ONU.
+Um relat�rio divulgado hoje, que d� conta dos receios causados por uma �cultura prop�cia ao consumo� e que condena as vozes mais tolerantes sobre o uso de narc�ticos, revela uma descida do n�mero de consumidores de hero�na na Europa e uma subida do cultivo de �cannabis� e do consumo ocasional de estimulantes e alucinog�neos.
+A hero�na � cada vez mais uma droga fumada e menos injectada.
+A Norquifar refere ter obtido do presidente do Infarmed, Aranda da Silva, um compromisso no sentido de permitir uma distens�o �mais razo�vel, nunca inferior a tr�s anos�, do tempo para a aplica��o do decreto e queixa-se de ter sido surpreendida, pouco tempo depois, com as notifica��es.
+�O cumprimento do decreto [135/95] nos prazos em que agora o Infarmed veio exigir � quase imposs�vel porque o novo licenciamento pede documentos que s�o dif�ceis de obter num curto espa�o de tempo -- como sejam a obten��o de alvar�s camar�rios ou pareceres de seguran�a do Servi�o Nacional de Bombeiros�, diz S�rgio Figueira, da Norquifar.
+�Quando os prazos se esgotarem�, explicou ainda, �os armaz�ns correm o risco de ser fechados por qualquer ac��o de inspec��o� daquele instituto.
+Alguns prazos est�o, segundo S�rgio Figueira, a poucos dias de chegar ao limite.
+A Norquifar, que j� se tinha queixado de se estarem �a criar empregos por decreto�, entende que a contrata��o de um licenciado a tempo inteiro para os armaz�ns � n�o s� demasiado onerosa para as pequenas e m�dias empresas que representa, como injustific�vel, pois um armaz�m grossista n�o vende directamente ao p�blico.
+Afirmou um dia ao P�BLICO que o seu inimigo principal era �o poder pol�tico�.
+H� um ano, achava muito divertidas as not�cias que a davam como poss�vel candidata do CDS-PP �s legislativas.
+ �Nunca pensei entrar na pol�tica!�, jurava Manuela Moura Guedes.
+ao fim de 16 anos, deixava a RTP para ir apresentar o Telejornal da TVI.
+Foi sol de pouca dura..
+Em Junho, afastada dos notici�rios h� meses, era outra vez not�cia.
+Rumores de que estava a caminho da SIC -- apesar das resist�ncias de �pesos pesados� da redac��o do canal de Carnaxide --, enquanto outros a davam com um p� nas listas do renovado PP.
+Est� tamb�m prevista a presen�a dos presidentes das entidades estatais mais ligadas � aplica��o do programa: Instituto Nacional da Habita��o e IGAPHE Instituto de Gest�o e Aliena��o do Patrim�nio Habitacional do Estado.
+A indigna��o dos dirigentes do Sindicato dos T�xteis tem como principal alvo a C�mara Municipal, que alegadamente ter� colocado �entraves� que fizeram baixar o valor do im�vel.
+Em causa estar�, nomeadamente, a impossibilidade de alterar a fachada do edif�cio de quatro andares, cada um deles com cerca de seis metros de p�-direito.
+O pr�dio foi agora vendido � empresa Constru��es Progresso no �mbito de negocia��es particulares.
+Estas foram iniciadas h� meses, depois de se considerarem esgotadas as hip�teses de venda por arremata��o judicial, dado os valores oferecidos -- num m�ximo de 186 mil contos -- estarem muito aqu�m da avalia��o.
+�Foi melhor do que o que eu esperava�, comentava uma mulher no elevador, acerca do Show de Moda Primavera / Ver�o que anteontem � noite teve lugar no Shopping Center Cidade do Porto, no Bom Sucesso, e que se dever� vir a repetir oportunamente.
+Aconteceu numa �passarelle� montada sobre a pista de gelo e rodeada por cadeias dirigidas aos diversos convidados de cada uma das lojas que acederam em participar.
+Os �sem convite� permaneceram atr�s das cadeiras e nas varandas dos tr�s pisos do �shopping�.
+Curioso foi o facto de o �show� se ter dividido em duas partes e das �noivas� surgirem logo ap�s o intervalo e n�o no final da passagem, como � h�bito nestas coisas.
+Antes, ao fim da tarde, os manequins e crian�as passaram modelos da Mango, Union Blue, Authentik, Quiosque, Lanidor, Jacadi, Inquieta��o, Action Sport, Nexus, Petit Patapon, Cenoura, Boxer Shorts, Nastra, Bambini, Tin Tin e Miss Selfridge.
+� noite, numa prespectiva mais cl�ssica, desfilaram colec��es da Pronuptia, Gianonne, Lace, Mac Moda, Pinto's, Cortefiel, Alain Manoukian, Rond�ssimo, Globe, Springfield, MyGod, Vitalis, r�brica, Manel Boutiques, Veneza 5, Nogaret, Acapicua, Sugar, Dec�nio e Paulina Figueiredo.
+Um rol de propostas, �pr�t-�-porter�.
+R. -- A ideia agrada-me e j� fiz, inclusive, algumas experi�ncias.
+De qualquer modo, aquilo que se passa, em termos de electrificar a sanfona ou os outros instrumentos, � que os espect�culos do Realejo s�o na sua maior parte ac�sticos.
+Essas experi�ncias de electrifica��o n�o t�m resultado, at� agora, muito bem, embora no disco haja uma faixa que aponta um bocado nesse sentido, a cantiga de Santa Maria, com um movimento meio arrockalhado.
+Pode ser um ponto de partida ...
+ P. -- Mas por enquanto o Realejo continua a ser um grupo de e para interiores, no duplo sentido da palavra?
+Considerado culpado mas absolvido pela justi�a militar italiana, o ex-nazi Erich Priebke foi de de novo preso mas isso n�o lava o sentimento que cobriu uma maioria da It�lia: a vergonha.
+Pelo veredicto, pela maneira como o processo foi conduzido.
+Para os ju�zes, os crimes de Priebke prescreveram.
+Como disse o Presidente Scalfaro, �nunca prescrever�o na mem�ria dos italianos�.
+A Direc��o do Torreense, cuja a equipa de futebol milita na 2� Divis�o de Honra, demitiu-se em bloco na Assembleia Geral realizada na noite da passada segunda-feira.
+Raz�o: a C�mara Municipal de Torres Vedras (CMTV) n�o tem ajudado o clube financeiramente.
+�Cheg�mos a um ponto em que deixaram de existir condi��es financeiras que garantam uma gest�o respons�vel do clube�, garantem os dirigentes.
+Ant�nio Jos� dos Santos, o presidente demission�rio, prometeu continuar a assegurar a gest�o corrente do clube e admitiu �reconsiderar� a sua posi��o e ficar at� final do mandato, mas �s� depois de saber �com quanto � que C�mara pode auxiliar o clube�.
+Jos� Augusto Carvalho, presidente da CMTV, considera �profundamente injustas� as acusa��es lan�adas sobre a autarquia.
+�A c�mara s� se comprometeu a contribuir com igual montante ao total da soma dos restantes s�cios-empresas e cumpriu�, disse Jos� Augusto Carvalho.
+A Federa��o Portuguesa de Andebol vai analisar amanh�, em reuni�o do executivo, a n�o realiza��o, no passado s�bado, do jogo FC Porto- Gin�sio do Sul, referente � segunda jornada do �Nacional�.
+O executivo federativo est� j� a fazer um levantamento de toda a documenta��o, que dever� ser complementada com relat�rio da equipa de arbitragem nomeada para dirigir a partida.
+Segundo uma fonte da FPA, o jogo foi marcado para o Pavilh�o das Antas e o FC Porto recebeu a notifica��o atrav�s de comunicado, �logo pode-se concluir que o recinto j� n�o se encontrava interdito�.
+A resolu��o do problema pode agora passar por um acordo entre os clubes.
+Caso isso n�o venha a verificar-se, a Federa��o vai instaurar um processo onde o FC Porto ter� de apresentar as raz�es que levaram a equipa a comparecer em Aveiro e n�o nas Antas.
+um pacato cidad�o, livre de toda a suspeita, um homem simp�tico e prest�vel, guarda em casa 24 cad�veres mutilados.
+Descoberto por acaso, revela-se algu�m que encara os crimes como algo natural.
+Para a psiquiatra que o observa, um ser intrigante; para a pol�cia, uma encarna��o do dem�nio.
+Jeff Goldblum e Alan Bates em confronto.
+�J� perdemos algumas cartas altas, ainda assim, temos algumas figurinhas�, desabafou ao P�BLICO um dirigente nacional do PP para justificar o sil�ncio dos �not�veis�.
+�Fernandes Thomaz, Lu�s Queir� e Celeste Cardona s�o as personalidades que restam�, enumerou, para depois concluir que Monteiro n�o pode esquecer isso, caso decida recandidatar-se.
+ A mo��o da distrital do Porto, �A casa comum da direita�, tamb�m procura congregar um maior n�mero de apoiantes.
+S�lvio Cervan conseguiu a ades�o do presidente da distrital de Aveiro, S� Correia, pr�ximo de Paulo Portas, e da concelhia de Vila Nova de Famalic�o, uma estrutura do distrito de Braga, de Ant�nio Pedras.
+A aplica��o aos avi�es comerciais de passageiros dos sistemas electr�nicos de defesa antim�ssil, em uso na avia��o militar, est� a ser equacionada pelo governo norte-americano.
+Ontem, na Casa Branca, reuniu-se pela primeira vez uma comiss�o constitu�da por 21 especialistas em sistemas de defesa e contramedidas electr�nicas.
+Segundo Al Gore, vice-presidente dos Estados Unidos, esta comiss�o deve apresentar um relat�rio de conclus�es na pr�xima ter�a-feira.
+Os dez m�sicos que estar�o em palco, incluindo Bryan Adams, fizeram um �nico pedido -- querem comida vegetariana nos camarins.
+Os promotores do concerto recomendam aos espectadores: levem guarda-chuva n�o v� S. Pedro pregar uma partida.
+ Reconhecendo possuir algumas �saudades do Parlamento�, a autarca afirmou que preside a um �concelho apaixonante�, apesar das contradi��es, mas lamentou que este �tenha estado esquecido pelo poder central e desmobilizado pelo poder local�.
+Da� a meter uma cunha aos antigos colegas foi um passo.
+Muitos aspectos da legisla��o das autarquias, sustentou Edite Estrela, �s�o inibidores da ac��o e da criatividade� e �verdadeiros obst�culos� a vencer pelos autarcas.
+�� prefer�vel haver leis mais flex�veis, para serem cumpridas, do que leis muito restritivas que, por vezes, n�o s�o cumpr�veis�, disse, criticando �a suspei��o� com que os autarcas s�o tratados pelo Estado.
+Em troca defende que os prevaricadores sejam �castigados exemplarmente�.
+Em Paris, a bolsa tamb�m registou alguns ganhos com a abertura em alta do Dow Jones, apesar de uma sess�o calma que resultou num aumento de 0,56 por cento, com o CAC 40 a encerrar nos 2940,89 pontos.
+Em Madrid, o �ndice geral subiu 0,44 por cento.
+Duas outras personalidades que se digladiam, sem d�vida, pelo primeiro plano no c�rculo do poder s�o o presidente da C�mara de Moscovo, Iuri Lujkov, aliado dos banqueiros e das mafias da capital, e Anatoli Chubais, o ex-vice-primeiro-ministro rec�m-nomeado chefe da casa civil da Presid�ncia.
+ Gestor genial, reformista moderado e autor do programa de privatiza��es do Governo, Chubais tem a seu favor o apoio dos governos ocidentais e do FMI, que tender�o a ver nele o garante da continuidade da pol�tica das reformas, quando Ieltsin j� n�o puder impor a sua vontade.
+Mas, tendo tamb�m em conta a hist�ria recente, n�o � um dado adquirido que Ieltsin esteja acabado.
+ No in�cio deste ano, em plena cat�strofe tchetchena e ap�s dois ataques card�acos, a popularidade do Presidente n�o chegava aos dez por cento.
+e, no auge da campanha eleitoral, foi visto a gritar e cantar, a correr e a dan�ar o �twist� e o �rock'n roll�.
+Inauguram no Lagar do Azeite, em Oeiras, pelas 21h30, o X Sal�o Livre de Humor / Oeiras 97 e o VIII Festival Internacional de Humor.
+Patente ao p�blico at� ao pr�ximo dia 26, de Segunda a Domingo, das 14h00 �s 19h00.
+O �Di�rio da Rep�blica� n�o tem o exclusivo de publica��o das boas not�cias legais ou regulamentares.
+Prova indiscut�vel disso � a Circular 3/94/DEP/1 do director-geral dos Servi�os Prisionais que veio regulamentar �o controlo e reten��o da correspond�ncia dos reclusos�, com vista a que o mesmo �esteja em harmonia com o quadro constitucional de previs�o e tutela dos direitos fundamentais�.
+A partir de agora, a correspond�ncia s� ser� aberta na presen�a dos reclusos, para verificar do eventual envio de �bens� n�o autorizados e a mesma s� ser� lida em casos devidamente justificados.
+ O jornal popular alem�o �Bild� anunciou ontem que o construtor automobil�stico alem�o Audi vai pagar 200 milh�es de marcos (cerca de 20,4 milh�es de contos) pela marca italiana Lamborghini.
+A not�cia citava especialistas da ind�stria autom�vel, mas ainda nenhuma das partes envolvidas na transac��o divulgou o montante envolvido.
+A Audi j� tinha anunciado o seu interesse na compra da totalidade do capital da Lamborghini, cujo principal accionista � o filho mais novo do ex-presidente indon�sio Suharto.
+Jorge Petiz, em Porsche Carrera RSR, concorrente ao Campeonato Nacional de Velocidade (CNV) de Cl�ssicos, foi ontem o piloto mais r�pido no conjunto das duas sess�es de treinos para a Rampa da Arr�bida, pontu�vel para o Nacional e trof�us monomarca e �Challenge� FIA de Montanha, prova que esta manh� se disputa.
+Petiz realizou a melhor subida em 1m44,084s, superando o seu pr�prio rubricou ao volante do BMW M3 do CNV (classe N2), fixado em 1m45,887s.
+Na classe at� 2 litros (N1), Carlos Borges (Toyota Carina E) imp�s-se com 1m51,484s.
+Agarrou-se � causa como uma n�ufraga, e seria compensada.
+� maneira do regime.
+Com uma medalha por servi�os prestados � comunidade enquanto �Tr�merfrau� -- �destas, s� foram entregues duas� --, uma certid�o numa capa vermelha com o s�mbolo da RDA a dourado, uma viagem em paquete para os �activistas da primeira hora�, onde viajou ao lado de Erich Honecker, da bailarina Great Paluka �e muitos outros�.
+E a est�tua.
+A maior honra, por�m, ter� sido esse convite para falar em Berlim, frente a 100 mil pessoas no �dia do trabalhador da constru��o�, em 1979.
+Se foi ela a autora do texto?
+�Qual qu�, eles � que prepararam tudo.
+Eu s� tive de ler�, o que, pela maneira como o diz, ter� sido um grande al�vio.
+Poder-se-� dizer que o estilo resulta da sua profiss�o, fotojornalista.
+Trata-se do contr�rio.
+ Ele faz jornalismo fotogr�fico, porque esta profiss�o � a que melhor se adequa ao seu estilo, � sua maneira de olhar o mundo.
+H�, no seu modo de ser, como que uma esp�cie de timidez, que o leva a fazer fotografia n�o intrusiva, uma das maiores dificuldades do of�cio.
+N�o conhe�o fotografia sua onde os sujeitos estejam pouco � vontade, furiosos por estarem a ser fotografados, envergonhados, com o sentimento de estarem a ser violados na sua privacidade.
+Os que olham para a sua c�mara, com mais ou menos prazer, com ou sem indiferen�a, n�o mostram hostilidade ao fot�grafo.
+E os que n�o olham para a c�mara, n�o parece, pelo que est�o a fazer, que reagiriam contra o que os observa.
+N�o conhe�o fotografia sua que revele uma intimidade secreta, algo que ningu�m gostaria que se soubesse ou visse.
+� dif�cil, sendo-se fot�grafo, cultivar sem concess�es este respeito pelos outros.
+Muito mais ainda, sendo-se fotojornalista, profiss�o em que o �voyeurisme� pode ser elevado ao estatuto de virtude, como acontece tantas vezes com as revistas e os jornais sensacionalistas, como praticam t�o obsessivamente muitos jornalistas nas televis�es contempor�neas.
+Nas suas fotografias, Alfredo Cunha revela uma humanidade quase inocente, uma comovente igualdade perante os outros.
+N�o se consegue perceber que o fot�grafo ganha a sua vida � custa dos outros.
+s�o a demonstra��o exacta da ternura t�mida que o habita.
+O jornalista n�o parte do princ�pio que tem direitos sobre os outros, nem sobre os seus sentimentos, nem sobre as suas obras.
+Ele sabe o que � o �direito � informa��o�, mas n�o o reduz aos seus pr�prios direitos, que considera limitados pelos seus deveres.
+N�o conhe�o fotografia sua na qual a dor, o desespero, a c�lera, a vol�pia ou a intimidade sejam explorados.
+E, no entanto, as suas fotografias s�o capazes de nos revelar as pessoas, os sentimentos, as situa��es.
+Tanto mais que Alfredo Cunha tem simpatias humanas, culturais e pol�ticas.
+Fotografa os poderosos por profiss�o, os sem poder por voca��o.
+Visivelmente, prefere os pobres, os que trabalham, os que sofrem.
+Nestas condi��es, � muito dif�cil n�o fazer fotografia empenhada, �engag�, com fun��o e utilidade pol�ticas.
+Ele consegue-o.
+Fotografa um lado da condi��o humana, mas f�-lo sem pragmatismo, sem intuito propagand�stico, sem outra inten��o que n�o seja mostrar-nos o que ele v�, como ele v�, o que ele prefere.
+No encontro, a Sun apresentou a sua nova estrat�gia �The Road to Java� e o projecto Java Centre Program, iniciativas que pretendem promover a adop��o do Java pelas empresas, integrado numa arquitectura de �network computing� (baseada na rede e no conceito de �thin clients�).
+Ao abrigo do Programa de Centros Java (Java Centre Program), a Sun vai continuar a abrir Centros Java (Java Centres), geridos pela pr�pria Sun, e Centros Autorizados Java (Authorized Java Centres), geridos por parceiros, que fornecer�o servi�os profissionais para instala��o de projectos baseados em Java nas empresas.
+Neste momento, j� existem cerca de 225 centros destes em todo mundo, onde cerca de tr�s a cinco mil profissionais oferecem um conjunto de servi�os de consultoria, integra��o de sistemas, forma��o e manuten��o.
+A sua unifica��o, agora, ao abrigo do Programa de Centros Java, � uma tentativa da Sun de reorganizar os servi�os j� existentes na �rea das solu��es de �network computing� baseadas em Java -- explicou a COMPUTADORES Mark Tolliver, vice-presidente respons�vel por Market Development.
+ Ao que o P�BLICO apurou, as raz�es que t�m levado o n�cleo duro da metal�rgica a opor-se � entrada da Cortal / Seldex no elenco dirigente tem pouco que ver com o receio de um eventual controlo da empresa pelo concorrente, e muito mais com a inconveni�ncia de desvendar �estrat�gias empresariais em prepara��o�.
+Com efeito, nos �ltimos anos a F. Ramada tem tomado uma s�rie de medidas de reestrutura��o e moderniza��o.
+� o caso da diminui��o substancial do quadro de pessoal, que passou de quase 1200 trabalhadores em 1987 para os actuais 600, o que custou quase meio milh�o de contos � empresa.
+Tamb�m a perda do mercado angolano levou a administra��o da F. Ramada a preparar, atrav�s da rec�m-formada empresa de �import-export� Ramada Internacional, a conquista de mercados alternativos, nomeadamente os dos pa�ses do Magrebe, da Am�rica do Sul e da Espanha, esta �ltima uma das grandes apostas, precisamente, da Cortal.
+H� ainda a inten��o de avan�ar com um projecto imobili�rio no centro de Ovar -- nos terrenos da antiga sede da empresa --, com o que a administra��o conta arrecadar 2,5 milh�es de contos, para um investimento calculado em um milh�o.
+A Organiza��o dos Pa�ses Exportadores de Petr�leo (OPEP) vai pedir a todos os seus membros, na pr�xima reuni�o, marcada para 13 de Fevereiro, para que reduzam a produ��o de �crude�, de forma a inverter a tend�ncia de queda do pre�o do barril.
+Segundo o seu presidente, o venezuelano Alirio Parra, �s�o necess�rios cortes da parte de todos os Estados�, pelo que se imp�e a sua ades�o �sem excep��es� a esta medida.
+Problemas: Distribui��o e venda de droga.
+�Obviamente que n�o quero dizer que as car�cias de uma m�e possam ser substitu�das de forma mec�nica.
+O contexto onde as coisas se passam � importante e nada disto quer dizer que os elementos psicol�gicos n�o sejam importantes�, observa Uvnas-Moberg.
+�Mas o que eu digo � que, quando h�, por exemplo, uma crian�a que est� numa incubadora e se n�o pode contar com a estimula��o sensorial que a m�e normalmente lhe concederia, � conveniente proporcionar-lhe essa estimula��o.
+A investigadora lembrou que tamb�m existem drogas que t�m o mesmo efeito.
+�Uma droga que nos deixa ligeiramente sedados, que aumenta a nossa capacidade de interac��o social, � o �lcool em baixas doses, que tamb�m faz subir a oxitocina�, continua Uvnas-Moberg.
+ �H� tamb�m drogas que propiciam a liberta��o de oxitocina, como certos neurol�pticos, e existe tamb�m oxitocina em ' sprays ' nasais, que certas mulheres usam para facilitar a amamenta��o, quando h� problemas na produ��o de leite.
+A revista come�ou a ser feita exclusivamente por Lourdes Castro e Ren� Bertholo.
+Os primeiros n�meros eram como que uma carta aos amigos.
+Depois, iniciou-se a colabora��o de todos os elementos do grupo e de muitos outros portugueses, ligados � colabora��o escrita, emigrados ou n�o, como Jo�o Vidal, J. M. Sim�es, Jos� Gil, Helder Macedo, Nuno Bragan�a, Cristovam de Pavia, Alfredo Margarido, Ant�nio Areal, Jos�-Augusto Fran�a; ou estrangeiros, como Andr� Pieyre de Mandriargues, Karl Laszlo, Benjamim Peterson.
+ o 11, Primavera de 1963, homenageia Yves Klein).
+O trabalho fazia-se no �atelier �-casa de Lourdes e Ren�, que possu�am uma m�quina de impress�o serigr�fica onde todas as imagens originais da revista eram realizadas.
+A t�tulo de exemplo, citem-se originais (que os assinantes da revista recebiam devidamente numerados, assinados e datados) de todo grupo e ainda de Vieira e Arpad, Saura, Millares, Peter Saul, Corneille, Tinguely, Klein, um objecto �op� de Soto, postais sobre desenhos originais de Le Parc, Cruz-Diez, Alberto Greco, Aleschinsky, Telemaque, etc.
+Tendo-se tornado dif�cil avan�ar com um colectivo t�o alargado e dispersivo, Lourdes e Ren� resolveram restringir a concep��o da revista a eles pr�prios, Christo e Voss e acordaram em que cada n�mero seria da responsabilidade individual de um deles, com a colabora��o dos restantes.
+Assim concebem e realizam os melhores n�meros da revista (do 9 ao 12, sa�dos entre 1962 e 1963), sob a sucessiva direc��o de Voss, Ren�, Christo e Lourdes Castro.
+ e, finalmente, Lourdes, que iniciava a explora��o das suas �sombras�, concebe uma obra essencialmente visual: mais de 50 postais organizados em p�ginas picotadas, onde o recorte e o contorno s�o decisivos, mas a imagem fotogr�fica se revela tamb�m essencial.
+O secretariado da comiss�o pol�tica distrital de Leiria do PS re�ne-se na pr�xima segunda-feira e a substitui��o do governador civil ser� um dos temas a abordar, como afirmou ao P�BLICO o seu presidente, Jos� Canha.
+Apesar de a escolha para governador civil de Leiria recair sobre Alberto Costa, Jos� Canha informou que a comiss�o pol�tica de Leiria tem mantido contacto com o ministro nos �ltimos dias, com o intuito de lhe transmitir o seu parecer.
+O peso (a dura��o) diferente que d� a cada som cria uma riqueza fascinante na ilus�o de timbres e de din�micas.
+A manipula��o do tempo (do ritmo, da ess�ncia do fen�meno musical) recria todas as outras dimens�es do som, numa capacidade de reinventar cada arquitectura musical que fez deste recital um acontecimento �nico.
+Voluptuoso na suite em r� menor de Louis Couperin, Leonhardt tornou o �pr�lude non-mesur� de Anglebert numa verdadeira disserta��o filos�fica, cujo final, no seu despojamento el�ptico, surgiu como um ponto de interroga��o metaf�sico e perturbante.
+Para cada trecho de Fran�ois Couperin, Royere, Forqueray, um esp�rito mais comedido n�o prescindiu de enorme fantasia, por vezes de transbordante emo��o numa atitude comunicativa e interiorizada, simples e aristocr�tica, reveladora de uma fascinante personalidade de int�rprete.
+O resultado da sua primeira aplica��o poder� ser visto, em breve na Cidade Universit�ria, em Lisboa, local onde ir� ser constru�da, com base neste molde de bet�o, uma galeria t�cnica, cuja concep��o permitir� diversas interven��es no subsolo sem qualquer inc�modo para os transeuntes.
+Trata-se, segundo o projectista Martins de Oliveira, que concebeu o molde, de uma conduta visit�vel, capaz de reduzir �ao limite m�ximo� todos os inconvenientes produzidos pelas permanente opera��es de manuten��o levadas a cabo pela C�mara Municipal, EDP, Telecom e TLP.
+Com este tipo de modelo, a renova��o das v�rias redes, ao n�vel do saneamento b�sico, ser�o executadas sem perturbar terceiros, explicou Martins de Oliveira.
+Embora as ac��es dos trabalhistas, enquanto oposi��o, tenham sido extraordinariamente cautelosas em rela��o � Irlanda do Norte, no poder poder�o enfrentar um cessar-fogo renovado tendo a completa no��o das consequ�ncias de um poss�vel falhan�o.
+E enquanto por um lado a administra��o trabalhista n�o depender� dos unionistas para obter o apoio parlamentar, por outro, necessitar� que eles se mantenham comprometidos para assegurar o avan�o das negocia��es de paz.
+Se o IRA apelar a um cessar-fogo ap�s as elei��es � muito prov�vel que se d� in�cio a um segundo processo de paz.
+Mas ser� que ir� ser muito diferente do primeiro, e quais s�o as suas hip�teses de sucesso?
+O sismo de segunda-feira em Los Angeles n�o poupou ningu�m, nem mesmo as estrelas.
+Michael Jackson fracturou a cabe�a e os dedos da m�os, a cantora Dolly parton ficou sem nariz e os actores Oliver Hardy e Mae West perderam acabe�a- mas n�o literalmente.
+Todas estas trag�dias art�sticas deram-se com as r�plicas daquelas estrelas no museu de cera da cidade, que n�o escapou aos efeitos do sismo.
+Sylvester Stallone foi uma das figuras que se manteve impass�vel e n�o se desmoronou enquanto a terra tremia.
+Em carne e osso como em cera, eis um homem impass�vel.
+O secret�rio de Estado da Integra��o Europeia, V�tor Martins, seguir� hoje directamente de Bruxelas (onde preside ao Conselho do Mercado Interno) para Rabat (Marrocos), para manter conversa��es com o ministro marroquino dos Neg�cios Estrangeiros sobre os resultados do Conselho de Ministros dos Estrangeiros dos Doze que se realizou a 17 de Fevereiro em Lisboa, onde foi aprovada uma proposta visando a conclus�o de um acordo de com�rcio com Marrocos.
+A par de um medicamento antiacidez, o grupo aconselha especificamente dois antibi�ticos de entre a Claritromicina, Metronizadol e Amoxilina, prescritos apenas durante uma semana.
+� que se receia que a escolha de um tratamento incorrecto e disseminado � escala planet�ria fa�a aumentar a resist�ncia da bact�ria aos antibi�ticos.
+O grupo tamb�m recomenda fortemente a elimina��o da bact�ria em doentes com um tumor raro chamado linfoma de MALT (Tecido Linf�ide Associado � Mucosa), quando ainda est� numa fase de baixo grau de malignidade.
+V�rios estudos t�m mostrado que estes linfomas regridem depois de tratamentos dirigidos � HP, o que sugere uma rela��o entre os dois.
+A Assembleia da �rea Metropolitana de Lisboa (AML) continua a debater-se com o problema da falta de instala��es, mas pelo menos j� tem mesa.
+A elei��o decorreu ontem, num acto com algumas perip�cias, mas sem grandes surpresas, j� que se cumpriu o acordo previamente estabelecido entre a CDU e o PS.
+Se n�o se der qualquer fiasco de �ltima hora, � prov�vel que o PDL mantenha uma maioria confort�vel dos 299 lugares da Assembleia.
+�Os coreanos pensam, em geral, que a pol�tica n�o devia ser controlada pelos homens de neg�cios poderosos�, afirma Ahn Byung Joon, professor de Ci�ncias Pol�ticas na Universidade Yonsei de Seul.
+e uma vit�ria do PUN com essa amplitude parece perfeitamente plaus�vel.
+Chung deu voz a uma poderosa corrente de insatisfa��o, especialmente entre os eleitores das classes m�dias.
+Embora a economia da Coreia do Sul tenha crescido cerca de nove por cento no ano passado, s�o muitos os cidad�os que v�em com preocupa��o o d�fice comercial do seu pa�s, que � de 8,8 mil milh�es de d�lares [ cerca de 1250 milh�es de contos ].
+Chung afirma que o problema � haver demasiada interfer�ncia do Governo nas finan�as e poucas medidas contra as barreiras proteccionistas dos japoneses.
+�O p�blico n�o deve ser desinformado por esta campanha publicit�ria�, disse o comiss�rio para os Assuntos Sociais, Padraig Flynn.
+os n�o fumadores devem ter prioridade�.
+Estas declara��es surgem a poucos dias de, mais uma vez, o Conselho de Ministros da Sa�de da Uni�o, discutir uma proposta que visa banir completamente a publicidade ao tabaco.
+Em Portugal e na It�lia esta publicidade j� n�o � poss�vel, mas em v�rios outros pa�ses da Europa ainda persiste, embora com restri��es diversas.
+As propostas abolicionistas t�m encontrado a oposi��o de �lobbys� tabagistas gregos e espanh�is.
+Esperam-se, agora, as oportunas reportagens do golpe de 1974 sem tanques nas ruas, nem cravos, nem vivas ao MFA.
+E mesmo assim de madrugada, com �bola branca� a avisar os mais sens�veis.
+Nessa altura a RTP ter� atingido a perfei��o.
+E n�s teremos esgotado a paci�ncia ...
+Teatro e m�sica, pelo grupo �Meia Preta�, pelas 22h00, na Fortaleza, em Arma��o de P�ra.
+Em Lagoa, � inaugurada a Fatacil -- Feira de Artesanato, Turismo, Agricultura, Com�rcio e Ind�stria.
+�Tomislav Ivic, que foi treinador do Benfica e do FC Porto, esteve na Luz na quarta-feira a assistir ao jogo entre as suas duas antigas equipas [ 1� m�o da Superta�a ].
+ �Penso que o Benfica tem muitos talentos, tem bel�ssimos jogadores, mas penso tamb�m que o FC Porto, pelo que vi na quarta-feira, � mais agressivo sobre a bola do que o Benfica.
+ A exposi��o abre com o quadro de J�lio Pomar �Fernando Pessoa� e inclui, entre outras, obras de Carlos Botelho, Jo�o Abel Manta, Jo�o Hogan, Bernardo Marques, M�rio Eloy e Maluda.
+Inaugura��o de uma exposi��o de escultura de Rui Vasquez.
+�s 19h30 no Hotel Alfa, em Lisboa.
+Patente ao p�blico at� 10 de Mar�o.
+Foi descoberto em 22 de Mar�o deste ano por astr�nomos americanos, quando um rasto da sua cauda foi avistado na atmosfera de J�piter.
+Em Julho do pr�ximo ano, muitos dos telesc�pios do mundo inteiro v�o querer assistir ao seu fim.
+Mais de 20 filmes est�o em competi��o a partir de hoje na 46� edi��o do Festival de Cinema de Locarno, que se realiza at� 15 de Agosto.
+Da lista fazem parte filmes como �Au nom du Christ�, do realizador Roger Gnoan M'Bala (Costa do Marfim), �Beijing Zazhong�, de Zhang Yuan, considerado o primeiro filme �punk-rock� chin�s, �Bhaji on the Beach�, de Gurinder Chadha (Gr�-Bretanha), �Koraku Zaru�, do japon�s Kenchi Iwamoto, �L'�crivain public�, de Jean-Fran�ois Amiguet (Fran�a), �L'Ordre du jour�, do palestiniano Michel Kheifi, �La Ribelle�, do italiano Aurelio Grimaldi, �La Vida Conyugal�, do mexicano Lu�s Carlos Carrera, e [...]
+Para o Leopardo de Ouro e o Pr�mio da Cidade de Locarno, o j�ri deste ano � constitu�do pelos realizadores Chantal Akerman (B�lgica), Olivier Assayas (Fran�a), Kathryn Bigelow (Estados-Unidos), Ferid Boughedir (Tun�sia), Alexei Guerman (R�ssia), al�m de um pintor italiano e de um produtor da televis�o su��a.
+ O festival prop�e ainda um panorama sobre a produ��o su��a recente, filmes sobre filmes na sec��o Cin�ma-Cin�mas, uma homenagem ao realizador italiano Valerio Zurlini e a primeira retrospectiva integral da obra de Sacha Guitry.
+Para ressarcir a C�mara das despesas a suportar, bem como do direito � montagem do equipamento, ir� ser estabelecido um contrato de comodato de um bloco sanit�rio e dois abrigos para passageiros.
+Em princ�pio chegou pensar-se instalar um jornal electr�nico, mas isso implicava a compra dum computador demasiado caro para o munic�pio.
+Apesar de algumas informa��es que d�o conta alguns casos em que o nome da firma aparece citado, o presidente da C�mara de Alcoba�a, Miguel Guerra, referiu que o contrato ainda n�o estava assinado, e que �a empresa tem tido um comportamento impec�vel com a C�mara�.
+P. -- Pode querer fomentar a cria��o de um grupo financeiro, tal como o Banif pretendeu fazer.
+R. -- At� agora, a pol�tica do Governo para as privatiza��es tem sido orientada de modo a obter o melhor encaixe financeiro.
+A forma de o conseguir � p�r � venda as ac��es que ainda det�m nas empresas nas melhores condi��es.
+N�o existem motivos para alterar, agora, a sua estrat�gia.
+� um homem alto e magro, m�os de dedos esguios e gestos largos a abra�ar desenhos, projectos, frases.
+Numa voz grave, falou sobre o Ocean�rio de Lisboa e depois, descontra�do, perdeu-se a contar coisas dos tempos da inf�ncia e juventude, os caminhos que foi tomando e que o levaram a ser considerado o maior especialista do mundo em grandes aqu�rios.
+Depois de ter feito cinco grandes aqu�rios e muitas outras obras de dimens�es monumentais, confessa que a coisa mais importante que fez foi �mexer� no metropolitano de Boston, remodelar toda a rede, torn�-la agrad�vel e simples de entender.
+Fala aqui um pouco de tudo, das cobras da inf�ncia em Nova Iorque �s preocupa��es sobre a corrida aos armamentos nos anos 60, do prazer de ver golfinhos a brincar livremente, de vulc�es e glaciares a esculpir a paisagem da Terra.
+Ao falar de Portugal, come�a subitamente a trautear um fado de Am�lia, de um disco que comprou h� muitos anos, n�o sabe onde nem porqu�, e que gosta de ouvir e de cantar sem saber que palavras s�o aquelas.
+Um destes tr�gicos erros foi o manique�smo.
+Este fazia-nos acreditar que os nossos eram filhos da luz e os advers�rios, das trevas.
+Pens�vamos que tudo o que nos diziam era razo�vel e verdadeiro, mesmo sem o ter meditado, e que os nossos advers�rios estavam sempre enganados.
+Nessa base, prop�nhamos projectos no global, com os quais pretend�amos resolver todos os problemas do pa�s.
+Foram, principalmente, as atitudes sect�rias e exclusivistas que nos levaram � derrocada democr�tica, entre outros factores.
+Em 1993, pensamos de forma diferente.
+Procuramos acordos, tratamos de conciliar posi��es, estamos receptivos para compreender e para aceitar crit�rios diferentes dos nossos.
+A cria��o de um observat�rio com indicadores de converg�ncia, que dever� abranger mais de uma dezena de sectores, � um dos projectos a concretizar com a nova fase de interven��es estruturais da Comunidade.
+Atrav�s deste �rg�o, que funcionar� como base estat�stica, as autoridades portuguesas pretendem acompanhar o percurso de converg�ncia de cada um dos sectores, com base em tr�s grandes grupos de indicadores: estado de situa��o, �performance� conseguida e investimento.
+ Al�m do estudo dos sismos e das caracter�sticas tect�nicas das regi�es onde eles t�m lugar, as explos�es nucleares podem ainda servir outros fins.
+De facto, h� muitos anos que h� quem proponha a utiliza��o de pequenas bombas nucleares em certas grandes obras de engenharia como terraplanagens de montanhas ou abertura de albufeiras.
+ Recorde-se ali�s que, em Dezembro de 1992, quando a China realizou a maior explos�o n�o nuclear de sempre -- arrasando a montanha Paotai, situada na Zona Econ�mica Especial de Zhuhai, na ilha de Sanzao, 25 quil�metros a sudoeste de Macau --, na qual foram gastas 11 mil toneladas de dinamite, n�o faltou quem especulasse sobre a possibilidade de um dia realizar obras desse tipo com engenhos nucleares.
+A ex-Uni�o Sovi�tica, que levou a cabo um ambicioso programa de explos�es nucleares subterr�neas pac�ficas durante anos, parece ser a regi�o do globo onde este tipo de �know-how� foi mais desenvolvido, ainda que se desconhe�a o n�vel actual de dom�nio desta tecnologia.
+Ainda que a grande preocupa��o do p�blico se costume situar ao n�vel da radioactividade libertada, no caso das explos�es subterr�neas os especialistas dizem que ela pode ser facilmente controlada, havendo, por�m, o perigo de se contaminar len��is subterr�neos de �gua.
+Por outro lado, o que alguns receiam � que uma forte explos�o deste tipo possa provocar, ao n�vel das placas, fragilidades que venham a tornar mais graves as consequ�ncias de um futuro sismo natural.
+trata-se das Laffer Utilities (Laffer � o nome de fam�lia do her�i), apresentadas como um conjunto de programas para �perder tempo quando devia era estar a trabalhar�.
+Algures nos textos do programa l�-se que ele � capaz de �automatizar tudo o que voc� faz no escrit�rio e n�o tem que ver com trabalho�.
+Com uma base de dados de �clip-art�, um programa de impress�o de apostas, uma selec��o de anedotas tem�ticas, um criador de cartazes e algumas surpresas mais, as Laffer Utilities s�o uma boa maneira de ... perder um emprego.
+Lembrar as ra�zes foi ainda preocupa��o de Al Lowe ao preparar este disco, por isso mesmo, a par com alguns textos seus sobre os anos passados a criar Larry, uma divertida entrevista em v�deo, Al Lowe �Undubbed�, para a televis�o alem� -- e um v�deo e demo interactivo de Freddy Pharkas, um jogo de All Lowe -- h� ainda espa�o para lembrar o jogo que est� na origem do azarado her�i da s�rie: Softporn Adventure.
+O texto refere-se, nomeadamente, �s especificidades destes reclusos e constata que a comunidade prisional, intermedi�ria da san��o social atrav�s da priva��o da liberdade, passou a acolher uma popula��o dif�cil que ora se revolta, ora se adapta, ora se refugia na doen�a f�sica ou psicol�gica.
+E que efeitos se podem esperar da aplica��o da pena de pris�o �queles que cometeram delitos pela exig�ncia da toxicodepend�ncia?
+--interroga a divis�o de estudos e planeamento da Direc��o Geral dos Servi�os Prisionais.
+Bucareste acusou a Hungria de ter pretens�es territoriais na Transilv�nia -- uma regi�o atribu�da � Rom�nia em 1920, ap�s o fim do Imp�rio Austro-Hungaro -- e de apoiar a concess�o de autonomia �s regi�es dominadas pelos magiares romenos, uma vez que esta legitimaria a sua inger�ncia nos assuntos internos de outro pa�s.
+Para mais, o Governo h�ngaro recusou at� agora a assinatura de um acordo sobre a intangibilidade das fronteiras dos dois pa�ses vizinhos.
+�O di�logo pol�tico � a �nica via para melhorar as rela��es entre a Hungria e a Rom�nia�, afirmou quarta-feira Teodor Malescanu.
+Mas as palavras do ministro romeno foram recebidas pelos respons�veis da UDMR como uma jogada de �oportunismo�.
+Isto porque o problema da minoria h�ngara denegriu a imagem internacional da Rom�nia, que tenta a todo o custo fazer aprovar o seu pedido de ades�o ao Conselho da Europa (CE), que ser� votado no in�cio de Outubro.
+O objectivo do acordo � consolidar o cessar-fogo assinado o m�s passado, e proceder � troca de prisioneiros entre os dois campos.
+A guerra civil no Tadjiquist�o, que fez mais de 50 mortos, come�ou em 1992, quando as for�as do neo-comunista Rakhmonov derrubaram o governo dos islamistas, que, nas �ltimas semanas, t�m somado vit�rias e se aproximam perigosamente da capital.
+ No Dubai, as vedetas candidatas a dividir um bolo de um milh�o de d�lares s�o Goran Ivanisevic, Thomas Muster e Boris Becker.
+Mas as aten��es ser�o certamente divididas com a �armada� espanhola, composta por Carlos Moya, Albert Costa e F�lix Mantilla.
+Em Marselha (539 mil d�lares), os favoritos s�o Marcelo Rios, Thomas Enqvist e Michael Stich.
+Por c�, prossegue a segunda etapa do circuito sat�lite CESAE nos �courts� r�pidos da Associa��o Acad�mica de Coimbra.
+Dos portugueses presentes no quadro final, apenas dois actuaram ontem e ambos foram eliminados.
+O campe�o nacional Bruno Fragoso perdeu com o eslovaco Boris Borgula por 6-1, 7-6 (7-3) e Tiago Vinhas de Sousa foi derrotado pelo brasileiro M�rcio Carlsson por 6-2, 6-0.
+Contudo, as altera��es n�o se ficam por aqui.
+Para conseguir que a �originalidade� volte ao local, o arquitecto quer ainda restaurar a chamada Casa do Nuno (onde habitava Nuno, o filho de Camilo) e onde actualmente funciona a sede da junta e o posto m�dico.
+ Apesar da discuss�o que estas obras est�o j� a merecer na freguesia, o restauro da igreja paroquial �, sem d�vida, o assunto de que mais se fala.
+Desde h� j� alguns anos que chove �a eito� na centen�ria Igreja de Ceide.
+O �ltimo Inverno foi j� muito dif�cil de passar e, recordam os paroquianos, que chovia tanto dentro da igreja como c� fora.
+O padre Gabriel Pereira Lopes, p�roco da freguesia, queixa-se tamb�m do facto da igreja ser muito pequena e por isso, e para al�m do programado restauro, entrou na C�mara de Famalic�o um proposta para a amplia��o do templo.
+A Comiss�o de Obras da Igreja de Ceide quer alargar o edif�cio de uma forma que n�o ter� agradado a Siza Vieira.
+O arquitecto esteve j�, por v�rias vezes, no local e ter� manifestado a sua oposi��o � modifica��o da estrutura da igreja, defendendo antes a manuten��o do actual edif�cio.
+Mais quatro pessoas foram mortas ontem num bairro pobre de Istambul, durante os recontros entre manifestantes alevitas e a pol�cia, pouco antes de representantes de uns e outros terem chegado a um acordo pouco auspicioso.
+As �ltimas mortes de que h� not�cia ocorreram no bairro de Umraniye, na margem asi�tica da maior cidade turca, onde cerca de 1500 alevitas sa�ram para a rua para protestar contra a interven��o da pol�cia num outro bairro, Gazi Mahallesi, onde nos �ltimos tr�s dias morreram entre 17 e 23 pessoas em manifesta��es.
+E, em s�ntese, remeteu para o seu advogado, Lu�s Avides Moreira.
+A posi��o de Brand�o sustenta-se numa carta da Lacto-Lusa, de 6 de Agosto de 1993, que anula o contrato com a Socifa.
+Inten��o que foi aceite pela sociedade, conforme carta do dia 16.
+E alarga o rol dos lesados: os credores da Socifa e o pr�prio fisco, por n�o pagamento do IVA devido na presta��o de um servi�o.
+Sil�ncio sepulcral, s� os p�ssaros se ouvem.
+Mas Manoel de Oliveira continua a ouvir ru�dos.
+�� um barco que vai a passar no mar�, murmura um assistente.
+Aguarda-se que o barco se confunda ao longe com o horizonte ..
+Sust�m-se as respira��es.
+Ser� desta?
+Malkovich (Michael) e Deneuve (H�l�ne) s�o um casal -- ele americano, ela francesa -- que vem a Portugal porque Michael, historiador, julga que pode encontrar na Arr�bida documentos que lhe comprovem uma tese segundo a qual Shakespeare era um judeu de origem espanhola.
+No convento, o casal depara com alguns estranhos personagens: Baltar -- Lu�s Miguel Cintra --, o director dos arquivos; o seu assistente, Baltazar -- B�nard da Costa --, e a cozinheira -- Elo�sa Miranda.
+Para os cubanos, o Che � um semi-Deus vindo de outro lado que quis fazer deles �homens novos� inventando o trabalho gratuito, a abnega��o pol�tica, o internacionalismo.
+Mas que resta hoje destes sonhos?
+ Um monte de boas inten��es abandonadas junto dos caixotes de lixo das cafetarias em d�lares, onde jovens e velhos vasculham, ou pisadas pelos saltos-agulha das centenas de �jineteras� (prostitutas) do Malec�n e da Quinta Avenida.
+Aos 15 anos, muitas �crian�as do Che� deixam a escola prim�ria e encontram-se na rua para �buscarse la vida de qualquier forma�.
+Para os �antigos�, o Che est� prestes a morrer uma segunda vez.
+� preciso um meio dia de trabalho a um oper�rio para ganhar a famosa nota vermelha de tr�s pesos com a ef�gie do Che que lhe permitir� comprar tr�s bananas no mercado livre campon�s.
+ O d�lar e o marco fixaram respectivamente a 159.775 e 87.35 escudos face aos 156.942 e 86.871 verificados na sess�o anterior.
+Este ano vai ser aberto um concurso para projectos de investiga��o sobre a desigualdade entre homens e mulheres na sociedade portuguesa.
+O objectivo � conhecer melhor os fen�menos e as tend�ncias da desigualdade entre sexos, para que esse conhecimento possa servir de base � tomada de decis�es pol�ticas.
+Temos um exemplo logo aqui ao lado na nossa vizinha Espanha, onde os jogadores recebem bem, mas jogam futebol.
+Como � bonito ver os est�dios cheios e os espectadores verem bom futebol.
+C� em Portugal, o que vemos?
+Fraudes, mortes nos campos de futebol, �rbitros que s�o tendenciosos a favor de quem querem e outras coisas mais que s� trazem vergonha ao nosso futebol.
+A pol�cia efectuou, na manh� de ontem, nada menos do que oito deten��es no Bairro de S. Tom�, em Paranhos, na sequ�ncia de v�rias buscas domicili�rias.
+A opera��o policial, que teve in�cio cerca das 9h00, visou detectar ind�cios de tr�fico de estupefacientes.
+Num dos domic�lios, os agentes detiveram tr�s homens (um estucador de 56 anos, um padeiro de 19 e um empregado de balc�o de 24) e tr�s mulheres (uma reformada de 50 anos, uma estudante de 16 e uma tecedeira de 22), aparentemente da mesma fam�lia, tendo sido ainda apreendida uma apreci�vel quantidade de droga.
+a outra rota � por barco para portos como Southampton, Roterd�o e Antu�rpia.
+ser� correcto transaccionar f�sseis com coleccionadores particulares, enquanto os paleont�logos (envolvidos muitas vezes em descobertas importantes) est�o impedidos de fazer dinheiro com os seus achados?
+N�o poder� esta situa��o levar � corrup��o da classe?
+�Esta � a altura ideal para discutir �tica�, disse ao P�BLICO Lawrence Flynn, da Universidade de Harvard.
+O P�BLICO apurou tamb�m que as negocia��es entre a Marconi e a CN sobre a redistribui��o das participa��es estatais j� est�o bastante avan�adas.
+A reestrutura��o do quadro accionista na TMN dever� constituir um dos primeiros passos.
+A Telecom Portugal poder� vir a assumir uma participa��o significativa no operador de telem�veis -- ainda que inferior a 50 por cento -- e, em contrapartida, a Marconi ter� manifestado, mais uma vez, o seu interesse no tr�fego internacional para a Europa.
+Actualmente, o capital da TMN est� distribu�do, em tr�s partes iguais, pela Telecom, pela TLP e pela Marconi.
+A assembleia geral da Marconi deu tamb�m �luz verde� � emiss�o de papel comercial por parte da empresa -- que dever� acontecer a breve prazo --, para al�m de ter deliberado no sentido de os t�tulos da companhia passarem a revestir a forma de ac��es escriturais, de modo a permitir uma maior liquidez do papel.
+M�rio Carrascal�o desempenhou as fun��es de governador de Timor-Leste de 1982 a 1992.
+ Nessa altura, por causa do massacre do cemit�rio de Santa Cruz, descolou do regime de Jacarta, passando a manifestar algumas posi��es cr�ticas, nomeadamente em rela��o �s for�as de seguran�a.
+Foi substitu�do por um timorense bastante mais pr�-Indon�sia, o actual governador Ab�lio Os�rio Soares.
+As cr�ticas custaram-lhe o ex�lio, tendo sido nomeado embaixador da Indon�sia na Rom�nia.
+Em 1997 regressou a Timor-Leste.
+Reencontrou-se com o seu irm�o Manuel, tamb�m ele um timorense pr�-integra��o que progressivamente foi caminhando ao encontro das posi��es da resist�ncia.
+numa atitude in�dita, Robin Cook telefonou ao seu colega portugu�s antes de embarcar para Jacarta, para falar sobre Timor.
+Um dos assuntos abordados nessa ocasi�o foi a venda de armamento brit�nico ao regime de Jacarta.
+Nos meios diplom�ticos h�, ali�s, quem interprete estas manifesta��es de �sensibilidade� em rela��o � quest�o de Timor por parte do Executivo de Tony Blair com uma tentativa de compensar a m� imagem criada com as significativas exporta��es de armas �made in England� rumo � Indon�sia.
+Apesar de ter herdado do Executivo de John Major as mais pol�micas licen�as de exporta��o de armas para Jacarta, o Executivo trabalhista n�o escapou �s cr�ticas dos activistas pr�-direitos humanos, que acusam os novos governantes de n�o terem travado os neg�cios, nomeadamente a venda dos ca�as �Hawk� fabricados pela British Aerospace e alegadamente utilizados em Timor.
+No entanto, apenas tr�s meses depois da entrada em vigor do novo hor�rio de funcionamento do Mercado de Manuel Firmino, que estipulava um novo per�odo de abertura entre as 17h00 e as 20h00, foi decidido voltar a encerrar o mercado da parte da tarde.
+A reivindica��o partiu de alguns comerciantes, desagradados pelo facto de a abertura do mercado � tarde n�o ter atra�do novos consumidores.
+E nem a campanha publicit�ria desenvolvida em �rg�os de comunica��o regionais, promovendo o mercado e dando conta do alargamento do seu hor�rio de funcionamento, levou mais pessoas ao Manuel Firmino.
+O fracasso desta medida ditou que, desde o in�cio deste m�s, o mercado voltasse a funcionar apenas da parte da manh�, encerrando portas �s 14h00.
+Um dos comerciantes n�o poupa cr�ticas � C�mara, considerando que a campanha publicit�ria foi mal conduzida, que as obras est�o atrasadas e que a altera��o do hor�rio deveria ter sido feita ap�s a conclus�o das obras.
+�Nem sequer puseram ' placards ' pela cidade�, desabafa este comerciante, que considera que os an�ncios do alargamento do hor�rio da parte da tarde deveria ter sido mais publicitado.
+E acusa ainda a C�mara Municipal de n�o ter investido nos �ltimos anos no Mercado de Manuel Firmino, por pretender que os comerciantes se transfiram para o novo Mercado Municipal de Santiago.
+O facto causou a maior indigna��o junto dos estudantes daquela faculdade, cuja associa��o de estudantes se apressou, ali�s, a colocar cartazes em todos os locais da escola avisando os rec�m-chegados mais distra�dos para que �n�o passem cheques em branco�.
+Apesar do grotesco da situa��o, qualquer caloiro que procurasse saber das dilig�ncias que necessita de efectuar para se inscrever em Ci�ncias, deparava com uma longa lista de preceitos, intitulada �Aviso� e que explicava que todos os colocados na faculdade �no ano lectivo de 1994/95 (1� ano/1� vez) far�o a sua matr�cula por via postal (correio registado)�, ao que se seguia uma listagem dos documentos a enviar.
+A meio da lista, era solicitado o envio de �um cheque emitido a favor a Faculdade de Ci�ncias da Universidade de Lisboa, devidamente identificado com o nome do aluno no verso, para pagamento de propinas�.
+De montantes, nada vem referido.
+Ora, conforme explica Nuno Bio, elemento da Associa��o de Estudantes, �ainda n�o est� fixado o valor das propinas a pagar pelos estudantes, ainda n�o se sabe quem ter� direito a redu��o, o regulamento interno da faculdade ainda nem sequer foi elaborado�.
+Ali�s, aos alunos-caloiros era tamb�m pedido o envio do requerimento de redu��o de propinas �se pretender apresent�-lo�.
+Nuno Bio n�o tem d�vidas de que se trata de �uma tentativa frustrada por parte dos org�os competentes da faculdade de obrigar os caloiros, menos informados destas coisas, a pagar, para poder afirmar depois que h� muita gente a pagar propinas�.
+Um dos objectivos de Portugal nas negocia��es do Uruguay Round do Acordo Geral de Tarifas e Com�rcio (GATT) poder� ter uma solu��o positiva, dada a proposta que os Estados Unidos ontem apresentaram.
+Trata-se da aprova��o de um per�odo de 15 anos para a abertura das barreiras alfandeg�rias ao sector t�xtil.
+A Teleweb, uma nova empresa portuguesa fornecedora de acesso � Internet, anunciou estar disposta a investir um milh�o de contos de forma a atingir o seu objectivo de 50 mil clientes no ano 2000.
+ A empresa � totalmente detida pela Finantel SGPS SA, que det�m participa��es na Ensitel e na NCM-Nokia e atingiu em 1997 um volume de vendas consolidado de 9,2 milh�es de contos, com resultados l�quidos de 244 mil contos.
+ Andava pela rua de cabe�a perdida.
+Com cinquenta e tr�s anos, dizia ele de si para si, portava-se como um garoto.
+Que vergonha!
+Mas n�o via nenhuma sa�da.
+ No momento em que chegou ao jardim p�blico da rua Au-Sable viu, numa �lea deserta, uma silhueta magricela.
+Aproximou-se e o cora��o come�ou a bater com for�a.
+Era mesmo o �Pierre-Jean�, da outra vez.
+Subiram-lhe as l�grimas aos olhos, e por pouco n�o beijou o seu antigo conhecido.Subiram-lhas l�grimas aos olhos, e por pouco n�o beijou o seu antigo conhecido.
+O outro esquivou-se delicadamente, mas convidou-o a subir a bordo.
+Aparentemente impar�vel nos seus projectos de expans�o (para grande dor-de-cabe�a das marcas europeias) o an�ncio da Honda surgiu uma semana depois do acordo conseguido entre o Jap�o e a Comunidade Europeia para a abertura gradual do mercado aos carros nip�nicos a partir de 1992, e dois dias depois da morte do seu �Oyaji-san� (o grande pai), o homem que via na Europa o grande mercado a conquistar.
+Soichiro Honda foi o homem que construiu um imp�rio, que encarna o renascimento da ind�stria japonesa no p�s-guerra, desde a pequena f�brica de motocicletas, em 1947, ao grupo que factura hoje 4,5 mil milh�es de contos e tem 13 milh�es de contos de lucros.
+A par do seu envolvimento com empresas de efeitos especiais, como � o caso da ILM, alguns projectos recentes contribu�ram para dinamizar a companhia.
+� o caso da associa��o com a Time Warner Cable para a concretiza��o da TV interactiva em Orlando (na Florida), com a Nintendo para a cria��o do novo sistema de videojogos Ultra 64, ou com a Sprint of the Drums para criar uma rede de alto d�bito para a ind�stria do �entertainment� produzir filmes ou material de publicidade.
+Com a AT&T, criou a �joint-venture� Interactive Digital Solutions para o desenvolvimento e oferta de solu��es de v�deo interactivo, servi�os de informa��o ou de entretenimento, atrav�s das redes telef�nicas e de TV por cabo.
+Foi tamb�m a primeira empresa nomeada pela companhia telef�nica japonesa NTT para proporcionar servi�os interactivos na rede digital que este operador est� a instalar.
+Com sede em Mountain View, na Calif�rnia, a sua presen�a em feiras ou confer�ncias internacionais � usual e, no Siggraph deste ano (a 21� edi��o do maior certame actual de inform�tica, com 25 mil visitantes), foi o expositor com mais espa�o.
+A SGI aproveitou para anunciar a cria��o de uma nova empresa -- a Silicon Studio -- para o desenvolvimento de aplica��es para os novos mercados de ' media ' digital.
+A nova empresa trabalhar� com programadores, criadores art�sticos e distribuidores na expans�o das suas actividades correntes nos dom�nios do filme, v�deo, TV interactiva, v�deojogos ou parques de divers�es.
+�Cri�mos a nova subsidi�ria para promover o crescimento da SGI no mercado do ' entertainment '�, referiu � revista �Post Update� Mike Ramsay, presidente da Silicon Studio, que diz pensar que �a tecnologia para a �rea do ' entertainment ' comandar� o mercado nos pr�ximos cinco anos�.
+Documentos internos da empresa apontam o mercado da TV interactiva e dos videojogos como aqueles que, a longo prazo, v�o ter uma maior procura, seguindo-se ent�o o v�deo e o filme digital e, por �ltimo, os efeitos especiais.
+A Renault Portuguesa tem actualmente em curso um processo de despedimento colectivo de 144 trabalhadores.
+Os administradores nomeados pelo Estado portugu�s votaram contra esta decis�o, mas os franceses, maiorit�rios, decidiram ir em frente, o que constitui mais uma prova do confronto aberto instalado na empresa.
+Como os despedimentos ainda n�o foram consumados, a Renault continua com 765 trabalhadores e com uma produ��o di�ria de 160 autom�veis do modelo Clio.
+A administra��o da f�brica j� fez no entanto saber que � sua inten��o reduzir a produ��o di�ria para 110 unidades.
+Entretanto, os trabalhadores da f�brica de Set�bal realizaram uma greve na quinta-feira passada e marcaram nova paralisa��o para a pr�xima segunda-feira, que incluir� uma marcha sobre Lisboa.
+A administra��o francesa chegou a defender a venda da f�brica de Set�bal a outro construtor autom�vel.
+O Governo portugu�s desenvolveu alguns contactos nesse sentido, e os coreanos da Kia chegaram mesmo a deslocar-se �s instala��es.
+Mas as autoridades nacionais n�o viram grande receptividade e decidiram afastar essa hip�tese, porque n�o queriam ficar com o �nus de n�o conseguirem arranjar comprador, o que constituiria um trunfo para os franceses.
+�Na concerta��o, h� ainda muito a fazer�, constatou Maria Jos� Const�ncio, que prometeu o seu �empenhamento pessoal� para que �a falta de concerta��o entre as entidades locais e o governo central� deixe de ser um �ponto fraco� do PDI do Vale do Ave.
+�H� que coordenar a ac��o dos diversos minist�rios e munic�pios.
+Em Portugal, infelizmente, os diversos minist�rios e as autarquias t�m os seus programas, mas, por vezes, n�o t�m uma coordena��o rigorosa�, advertiu.
+Para evitar uma aplica��o desgarrada de milh�es de contos no Vale do Ave, Cravinho preconiza �um sistema de gest�o forte�, que ponha todos a trabalhar de forma articulada para uma boa aplica��o das verbas.
+Do lado portista, o dia foi calmo e sem problemas, entre Montechoro e as A�oteias, em cujo complexo desportivo a equipa fez ontem � tarde o �ltimo treino antes da partida.
+Tranquilidade absoluta, nenhuma hostilidade da parte de ningu�m, uma paz realmente sem m�cula.
+Reinaldo Teles, que chefia a comitiva � o �nico dirigente presente e n�o se espera a presen�a de mais nenhum dos altos respons�veis.
+�Como o jogo d� na televis�o julgo que n�o vem mais ningu�m�, disse-nos o homem que dirige o departamento de futebol portista.
+O deputado socialista Ferro Rodrigues mostrou-se descontente com as respostas do Governo, que considerou �vagas e evasivas�.
+O deputado disse mesmo que o �o Governo est�, deliberadamente, a querer desvalorizar politicamente um problema que � um esc�ndalo financeiro grav�ssimo que envolve muitos milhares de contos em fugas ao fisco�.
+Tudo isto porque a resposta ao Grupo Parlamentar Socialista sobre esta mat�ria foi dada pelo subsecret�rio de Estado adjunto da secret�ria de Estado do Or�amento, Vasco Valdez.
+�Quando esper�vamos a presen�a do ministro das Finan�as, Braga de Macedo, surgiu n�o o ministro, nem a secret�ria de Estado adjunta e do Or�amento, mas sim um subsecret�rio de Estado adjunto da secret�ria de Estado adjunta do ministro adjunto do primeiro-ministro�, referiu o deputado socialista.
+Para Ferro Rodrigues, n�o se trata de uma quest�o de compet�ncia, mas sim da forma como o Governo est� a tratar este assunto.
+Pal�cio Galveias.
+Campo Pequeno.
+3� a 6�, das 10h �s 19h.
+S�b., e dom., das 14h �s 19h.
+Um conjunto de grandes telas, em que a exuber�ncia crom�tica � pretexto � realiza��o de obras simp�ticas, precede a apresenta��o de um conjunto de estudos para os pain�is do Metropolitano de Lisboa -- entendidas as paredes como suporte de pintura, no sentido que possu�a antes de Maria Keil o transformar.
+n�o s�o todos os que intitulam um quadro �Um c�o, talvez�.
+Enquanto isto, foi ontem anulada a miss�o governamental angolana que deveria ter ido � Jamba, quartel-general da UNITA, recolher parte dos presos de guerra.
+Abel Chivukuvuku, membro da Comiss�o Conjunta Pol�tico-Militar (CCPM) para a fiscaliza��o do cessar-fogo, alegou � imprensa raz�es t�cnico-log�sticas para a anula��o da viagem, relacionadas quer com o n�mero reduzido de presos que desejaria voltar para Luanda quer com a �programa��o da cerim�nia�.
+O silenciamento das emiss�es da RTPi na Guin�-Bissau, aqui noticiado ontem e s�bado, vem recolocar a quest�o das complexas rela��es de Portugal com as suas antigas col�nias.
+ Um fantasma que se arrasta nestes 20 anos ainda marcados por uma descoloniza��o que foi -- ou teve de ser -- o que foi, � verdade, mas que importa exorcizar de uma vez por todas.
+Sem complexos, mas com a coragem de um projecto s�rio, consequente e respons�vel ao n�vel da coopera��o, em geral, e do audiovisual em particular.
+Foi o que n�o aconteceu, mesmo nestes �ltimos tempos de obra feita do cavaquismo.
+ R. -- O Presidente tem um prazo at� Mar�o para o assinar e apresentar ao Congresso, como um pacote que dever� ser aceite ou rejeitado.
+Caso ultrapasse esse prazo, o Congresso poder� discutir cada al�nea, o que pode adiar indefinidamente a sua aplica��o.
+� poss�vel que quem apresentou essa ideia considere que seja melhor para os Estados Unidos e para o mundo ter qualquer acordo do que n�o ter nada.
+O trabalho em fam�lia dos Hestons pai e filho (Charlton, o int�rprete, e Fraser, o realizador) revelou-se bastante interessante na �Ilha do Tesouro� que h� pouco vimos e justifica o interesse pela reincid�ncia, agora � volta de outra figura cl�ssica: Sherlock Holmes, investigando a desapari��o de um militar brit�nico, o que o leva a uma intriga de cobi�a e vingan�a.
+Legal V�deo, 1992, 101 min.
+O painel considerou que os empreiteiros teriam menos tend�ncia para relatar eventuais problemas de seguran�a e exortou a NASA a manter a sua presen�a nas f�bricas dos fornecedores.
+�A NASA n�o se deve iludir pelo aparente sucesso inicial de todos os esfor�os de transi��o�, sublinha o documento.
+O administrador da NASA, Daniel Goldin, ao fazer a sua leitura do relat�rio afirmou que este tinha considerado que o programa de vaiv�ns se encontrava de excelente sa�de.
+�O painel tamb�m aponta algumas �reas onde � preciso continuar a colocar alguma �nfase e outras onde � necess�rio melhorar�, disse Goldin.
+�A NASA concorda com essas afirma��es e j� dei instru��es para que sejam postas em pr�tica o mais depressa poss�vel�.
+S� recentemente e de uma forma restritiva, alguns tribunais come�aram a considerar que os processos contra os jornalistas devem ser suspensos at� que estejam decididos os processos contra aqueles, geralmente figuras p�blicas, de quem falaram.
+A lei de imprensa, na sua actual vers�o, nestes como em muitos outros aspectos, fere gravemente o direito de ser informado dos portugueses, pelo que se aguarda que seja brevemente discutido na Assembleia da Rep�blica um novo projecto de lei de imprensa que confirme o refor�o da liberdade de express�o, de informa��o e de imprensa.
+Orlando Miguel e Jos� Pedro partiram de bicicleta, levaram sacos-cama, roupa, comida e algum dinheiro.
+O primeiro deixou um bilhete aos pais dizendo que n�o se preocupassem e que n�o ia fugir.
+Foram cerca de 500 jovens atletas, representando escolas e clubes de v�rias regi�es do pa�s, incluindo a ilha da Madeira, que participaram no III Encontro Nacional de Andebol Feminino, j� considerado o maior da modalidade, organizado pela Associa��o de Andebol de Lisboa.
+ As expectativas dos promotores n�o ter�o sido goradas e a palavra sucesso era repetida, com insist�ncia, pelos organizadores.
+�Creio que o encontro foi muito proveitoso.
+Para al�m do conv�vio, as jovens atletas puderam praticar o seu desporto favorito e mostrarem as suas capacidades�, afirmou ao P�BLICO Isabel Cruz, da Associa��o de Andebol de Lisboa.
+3 milh�es de contos.
+Robert Koch, um Don Juan norte-americano de 51 anos, foi desmascarado e preso depois de ter burlado mais de 200 mulheres com promessas de casamento.
+Fazia-se passar por um vi�vo rico, propri�tario de f�bricas e sedento de carinho, e, depois de conseguir que as suas �presas� lhe confiassem grandes quantias em dinheiro e j�ias, desaparecia sem deixar rasto.
+Reconhecendo que �ningu�m melhor que os pr�prios sabem o que deve ser feito�, Nery pensa que deve haver uma separa��o clara entre subs�dios a equipamentos, subs�dios � produ��o e aos custos permanentes de manuten��o de uma companhia.
+Exig�ncias que obrigam os grupos a montar pe�as a correr s� para n�o serem penalizados monetariamente -- �o que n�o � saud�vel para ningu�m, muito menos para o Gil Vicente�.
+ Renitente em divulgar quais ser�o os novos crit�rios -- �embora tenha algumas ideias gerais� --, Nery espera pelos resultados dos inqu�ritos.
+Certo � que ser� um j�ri que decidir� a atribui��o dos subs�dios e que dever� incluir criadores n�o envolvidos no concurso, �cr�ticos e outros profissionais ligados ao sector�.
+Vieira Nery julga que as decis�es dever�o ser tomadas por uma comiss�o mista que envolva o Estado e �avaliadores exteriores�.
+Mas n�o acredita no convite �dos nomes pelos nomes�.
+Uma fita carregadora com 20 cartuchos de bala real para espingarda G-3, 17 balas de salva, 27 inv�lucros, uma granada de m�o defensiva M-63, uma granada ofensiva M-62, uma granada de fumo, uma muni��o para espingarda Mauser e uma muni��o de 20 milimetros, constitu�am o armamento abandonado sem qualquer tipo de sinaliza��o.
+O material, depois de inspeccionado pela GNR de Lamego, foi entregue no Centro de Instru��o de Opera��es Especiais.
+A come�ar em Lou Reed, passando por David Bowie e de Iggy Pop, h� toda uma tradi��o de estrelas anglo-sax�nicas em fase de crise existencial, que acabam por ir parar a Berlim, onde v�m a gravar discos de um pessimismo t�o cerrado quanto brilhante.
+Apesar de o Muro ter ca�do, os U2 trataram de revitalizar esta tradi��o num disco de uma negritude sem par na sua discografia pr�via que, se n�o constitui uma ruptura t�o radical com o passado quanto �The Unforgettable Fire�.
+No entanto, por esse mesmo estado de esp�rito que o anima, � um disco diferente na carreira do quarteto irland�s.
+As letras simplificam-se, para trazer � flor da pele uma amargura infinita, enquanto os arranjos denotam um frenesim recalcado, a que a produ��o de Lanois, Eno e Lillywhite vai acentuando as �nuances� dram�ticas.
+�Achtung Baby� � arte pop no seu z�nite, isto �, � beira do hospital ps�quico.
+Os adeptos da face mais ligeira da colabora��o de Kurt Weil com Bertolt Brecht devem odiar esta revis�o �hard-core� do seu report�rio, como as incurs�es dos Young Gods pela m�sica de circo e �vaudeville� n�o devem ter feito a felicidade dos adeptos da sua vertente mais industrial.
+� um disco pouco indicado para amantes de ortodoxias, o que, de modo algum, o torna inconsistente, tratando-se afinal de uma vanguarda actual, que revisita � luz dos seus pr�prios princ�pios outra vanguarda, que a antecedeu sob v�rios aspectos.
+O resultado � brutal e espectacular, baseado em vers�es radicais do report�rio mais famoso de Weil / Brecht, repletas de descargas de electricidade e avalanches de ru�dos urbanos, que da estrutura musical da famosa dupla n�o deixam mais que o esqueleto, mas lhe acentuam a sua carga teatral e inerente dramatismo, ao ponto do sufoco.
+O Sindicato dos Maquinistas dos Caminhos de Ferro Portugueses convocou mais uma greve �s horas extraordin�rias a partir de segunda-feira, informou ontem o porta-voz da CP.
+�H� algo de inacabado, amputado na nossa cultura�, especifica Ant�nio Jos� Saraiva, �uma esp�cie de inf�ncia para al�m do seu termo.
+Foi isso que nos levou � procura de outro pai al�m-Piren�us�, na CE.
+Uma segunda linha � a que engloba os tecnocratas de optimismo �standardizado� e maniqueu, com a sua subcultura do sucesso, do consumismo, do �top�, da produtividades, da efic�cia.
+No decurso da confer�ncia de imprensa, Carvalhas quis sublinhar a conjuntura em que est� a ser preparado o Congresso.
+�Decorre num quadro em que cresce o desencanto, a frustra��o e o protesto dos trabalhadores e de amplos sectores da sociedade pelas consequ�ncias de uma pol�tica que nos seus principais eixos n�o se afasta daqueles que constitu�ram a matriz pol�tica que os portugueses quiseram ver derrotada�.
+Carvalhas -- que continuar� a assegurar a lideran�a do partido -- manifestou ainda a sua convic��o de que o Congresso ser� �um importante impulso para o refor�o do PCP�.
+A ideia de instalar na frente costeira da margem direita do Douro um equipamento virado para o mar, embora longe da dimens�o que se pretende agora para o Centro de Ci�ncia e Tecnologias do Mar, tem quase 20 anos e as diversas tentativas efectuadas at� hoje nunca chegaram a bom termo, estivesse no Porto ou em Matosinhos o local escolhido.
+Tudo come�ou com a ideia, em 1980, de substituir a j� ent�o degradada Esta��o de Zoologia Mar�tima Augusto Nobre, ou Aqu�rio da Foz, como � mais conhecido, encerrado h� mais de 30 anos.
+1980 -- Por iniciativa de um bolseiro alem�o, Michael Weber (hoje docente convidado do ICBAS), os rot�rios do Porto ponderam duas ideias: reconstruir o velho aqu�rio da Foz, ou construir, de raiz, um novo aqu�rio, no Porto.
+C�mara do Porto, Universidade e Administra��o Central aderem ao projecto.
+O primeiro local apontado s�o os terrenos do futuro parque da cidade.
+No entanto, o consumidor poderia optar pelo envio de um cheque sobre o estrangeiro passado pelo banco do comprador em moeda do pa�s onde se vai fazer o pagamento -- a forma mais barata --, ou passar um cheque no pa�s da compra em moeda local.
+Cinco dias depois Pedro Concei��o deslocou-se � cidade com os documentos indispens�veis para avan�ar com o processo de matr�cula (bilhete de identidade, passaporte e n�mero de contribuinte).
+No dia seguinte, o carro podia ser levantado, tendo sido efectuado o pagamento de cerca de 1.600 contos em cheque sobre o estrangeiro.
+Com o carro, foi entregue uma guia de circula��o v�lida para 90 dias em territ�rio espanhol, o livrete, uma factura de compra e uma ap�lice respeitante a um seguro contra todos os riscos v�lido por 30 dias (custo: 38 contos).
+�Foi uma decis�o muito ponderada, muito dif�cil e muito amadurecida�, disse ao P�BLICO Anabela Moutinho, que foi lac�nica no que se refere aos motivos.
+ No cargo apenas deste Outubro, ela assegurava, entre outras coisas, a liga��o entre o IPACA e os espa�os potenciadores de novos p�blicos, como os festivais de cinema, nacionais e internacionais, as escolas e os cineclubes.
+Um projecto que ser� concretizado por meio de protocolos, mas que Anabela Moutinho n�o teve tempo de concretizar por �falta de tempo �til�.
+�Esta � uma �poca de transi��o, muito importante e dif�cil para o sector�, admitiu.
+O P�BLICO tentou saber se havia diverg�ncias de pontos de vista entre Anabela Moutinho e Costa Ramos, que ela n�o confirmou nem desmentiu.
+�A minha decis�o n�o foi pac�fica, n�o seria honesto da minha parte diz�-lo, posso dizer � que foi pac�fica e sem equ�vocos a maneira como o processo aconteceu�, declarou.
+ O autor receava, aquando da sua publica��o, que este livro fosse incompreendido.
+Raz�es?
+�Em particular, a linguagem simb�lica, certamente herm�tica� que �foge ao estilo que mais tarde escolhi�, justifica-se numa nota de contracapa.
+A aus�ncia da pretens�o did�ctica, inequivocamente pautada em livros subsequentes, que narram igualmente a saga dos angolanos, como �Mayombe� ou �As Aventuras de Ngunga�, � s� aparente, porque esta componente, embora aqui se afigure numa estrat�gia enunciativa bastante esquiva, est� flagrantemente presente.
+�Quisera acabar com a ovalidade do mundo e conseguira.
+Mas o quebrar do sonho aliou-se � impossibilidade de viver no mundo sem ovalidade.
+E ainda n�o havia m�quinas que realizassem os sonhos individuais.
+S� os de grupos.
+�(p�g. 158) Poder haver uma sugest�o maior?
+�Ela tentou olhar o lado esquerdo, mas uma montanha a separava.
+Ele fez o mesmo para o lado direito, mas a mesma montanha o impedia.
+ Cada um contemplou o seu lado, reconhecendo-se, incapaz de transpor a montanha� (p�g. 11).
+�-- Quando os corvos forem derrotados, n�o ser� s� aqui na montanha que o Sol ser� azul.
+ Por toda a parte ele dardejar� rosas sem espinhos ... -- dizia ele.
+E ela sorria �quela verdade desejada.
+ -- Os meninos brincar�o com o vento da madrugada, com ele fixando o capim � terra ...
+-- E os morcegos comer�o mel e n�o excrementos ... -- concluiu ela � (p�g. 51).
+Este assunto, ali�s, foi tamb�m discutido na CCPM, que quinta-feira efectuou a sua 27� plen�ria, ap�s um interregno de 20 dias.
+O �caso Quilengues�, em que morreram tr�s turistas brit�nicos e um neo-zeland�s, foi um dos assuntos em discuss�o, dada a contradi��o entre as partes.
+ A UNITA apresentou recentemente um suposto chefe do grupo que assassinou os turistas e que seria um oficial do ex-Minist�rio da Seguran�a de Estado.
+As declara��es do prisioneiro n�o convenceram ningu�m e o Governo rebate que a zona de Quilengues � de acantonamento de tropas da UNITA.
+V�rias pessoas citadas como tendo participado no acto desmentiram as vers�es da UNITA, e a Comiss�o Mista de Verifica��o e Fiscaliza��o (CMVF) encarregada de esclarecer o assunto n�o conseguiu apurar nada por dificuldades em chegar ao local.
+Duas organiza��es francesas, a Associa��o For�a Oper�ria Consumidores (AFOC, Altos Alpes) e a Uni�o Departamental das Associa��es Familiares (UDAF), assinaram um acordo de coopera��o com associa��es de consumidores italianas que levou � cria��o de uma estrutura franco-italiana denominada �Consumidores sem Fronteiras�.
+Esta organiza��o constituir�, segundo os seus promotores, a base de uma nova ag�ncia de informa��o europeia aos consumidores, a inaugurar no pr�ximo m�s de Junho.
+Os Consumidores sem Fronteiras prop�em-se, para j�, envolver-se na resolu��o dos lit�gios transfronteiri�os e organizar ac��es conjuntas de informa��o e esclarecimento aos consumidores dos dois pa�ses.
+Onde a �cidade do homem, n�o do lobo mas irm�o�?
+ (...) Em 1975, nos Estados Unidos, a comiss�o trilateral divulgava um relat�rio intitulado �Crise da Democracia, no qual culpava pela contesta��o � �autoridade fundada na hierarquia, na compet�ncia e no dinheiro ...� �intelectuais e grupos pr�ximos�, que estavam a tornar as democracias ocidentais, liberais, parlamentares, �ingovern�veis porque amea�adas por uma grav�ssima crise de autoridade, fomentada por intelectuais e jornalistas�.
+E conclu�a que �S� se pode sair desta crise com o restabelecimento f�rreo do princ�pio da autoridade�.
+ De ent�o para c�, o sistema tem vindo a domesticar esses sectores, seja atrav�s da sedu��o material seja pela atribui��o de honrarias, pelo que tais sectores abandonaram a contesta��o �s institui��es.
+Paralelamente, o controlo sobre os �rg�os de informa��o � de tal ordem apertada que s� vem a p�blico a verdade oficial, a do Big Brother.
+S�o tr�s os principais pontos litigiosos: a participa��o de organiza��es opostas ao processo de paz, a possibilidade dos palestinianos de Jerusal�m Oriental serem candidatos e eleitores, e ainda as compet�ncias do Conselho Aut�nomo.
+O chefe dos negociadores israelitas, Yoel Singer, real�ara anteriormente �progressos em rela��o a uma s�rie de quest�es�, salientando que as negocia��es continuar�o, dentro de duas semanas, e pela primeira vez em Jeric�.
+Na cimeira de hoje, em Erez, Arafat vai tentar convencer o primeiro-ministro israelita a fixar uma data para a retirada do Ex�rcito das cidades �rabes da Cisjord�nia e para a realiza��o de elei��es, mas Rabin provavelmente hesitar� uma vez mais, enquanto n�o obtiver garantias de seguran�a para os 130 mil colonos judeus.
+O l�der da OLP pretende tamb�m a abertura dos territ�rios, fechados desde 22 de Janeiro, para que 50 mil palestinianos possam trabalhar em Israel, embora queira tamb�m que Rabin explicite a sua ideia de uma �separa��o total� dos dois povos.
+Arafat receia que a separa��o seja sin�nimo de pris�o e n�o de independ�ncia.
+A vida de Albino Luciani, que seria o papa Jo�o Paulo I, encerra dois mist�rios: as circunst�ncias da sua morte e o teor da sua conversa de duas horas com a irm� L�cia, vidente de F�tima.
+Depois desta conversa, Luciani �nunca mais seria o mesmo�, garante a sua fam�lia.
+� por isso que, explica, n�o tem pena de Hillary Clinton.
+�Eles [ Hillary e Bill Clinton ] podem ter alguma esp�cie de acordo e quem somos n�s para dizer se � bom ou mau?
+Acho que eles t�m mais maturidade em rela��o ao sexo, como em rela��o �s drogas, do que podem mostrar�.
+�Vivo num Estado de Ironia�.
+Campos faz parte destes �rg�os, bem como Manuel Alegre que cedo se colocou ao lado do eurodeputado em nome de solidariedades antigas.
+A decis�o de Guterres foi tomada depois de segunda-feira ter reunido o n�cleo duro dos seus conselheiros pol�ticos, tendo mantido outros contactos durante todo o dia.
+Esta precipita��o dos acontecimentos vem de encontro a algumas das cr�ticas feitas � actua��o da Comiss�o Permanente, criticada por ter �deixado arrastar� durante semanas o contencioso com o eurodeputado, em vez de desde logo, como muitos defenderam, chamar Campos para este prestar esclarecimentos e colocar uma pedra sobre o assunto.
+Na busca passada � casa do suspeito, de 38 anos, que trabalha como marceneiro, foi encontrada a ca�adeira disparada tr�s dias antes contra a porta do clube, localizado naquele bairro, uma espingarda de press�o de ar, diversos cartuchos e um panfleto de hero�na.
+ Apesar de haver suspeitas de o indiv�duo estar relacionado com o tr�fico de drogas duras nas Galinheiras e tamb�m na zona de Odivelas, onde possui outra casa, nenhuma quantidade significativa foi encontrada.
+Elogiei a escolha do meu anfitri�o, enquanto discorria, a desprop�sito, acerca do papel do �consejero de cr�acion�, que tinha talvez inventado aquela esp�cie de centro de mesa, com batatas acabadas de fritar (sim, batatas fritas), certamente em fatias espantosamente finas e em azeite a ferver, tratadas a modos de �souffl�.
+Iniciei-me com uma �morcilla de Burgos�, enrolada numa massa fofa, e dediquei-me com carinho a um naco de bonito, frio e escabechado.
+A conversa seguia animada, uma dose de �marketing�, 200 gramas de imagem, levadas ao lume com �public affairs� q.b.
+Gosto de levar a s�rio o meu papel de consultor encartado.
+Eduardo Louren�o fechou, na sexta-feira, ao fim da tarde, os trabalhos da VI Reuni�o Internacional de Camonistas, que decorreu durante quatro dias na Universidade de Coimbra, com uma inspirada �li��o de encerramento� em torno das liga��es entre Cam�es e Petrarca e da luz plat�nica que sobre ambos ter�, ou n�o, incidido.
+ An�bal Pinto de Castro, organizador do encontro, obrigado pela fun��o a proferir algumas palavras finais, confessou o quanto lhe parecia �sacr�lego� falar ap�s a interven��o do rec�m-galardoado com o Pr�mio Cam�es.
+� de crer que estivesse a ser sincero.
+Eduardo Louren�o tem, de facto, esse dom, t�o contradit�rio com a sua proverbial mod�stia, de dar a tudo quanto escreve um tom definitivo.
+Provoca em quem o ouve a sensa��o de que aquilo que diz, o diz da forma mais justa, se n�o da �nica forma justa.
+Um talento que, por norma, cabe apenas aos poetas.
+A France 3, uma das principais esta��es p�blicas da televis�o francesa, abriu as hostilidades e declarou greve em nome do hor�rio laboral e da pol�tica salarial.
+A France 2 segui-lhe os passos e as emiss�es dos dois canais foram interrompidas.
+A direc��o j� mostrou boa vontade, mas a greve prossegue em todas as delega��es do pa�s.
+ Concebidos para funcionarem como instrumentos privilegiados dos gestores de sistemas de informa��o, estes Compaq Proliant viram refor�ados os seus dispositivos de seguran�a e preven��o de quebras no sistema -- refor�o esse numa perspectiva de centraliza��o f�sica deste tipo de equipamentos, sustentadores de redes cada vez mais complexas resultantes das evolu��es de �downsizing� e �rightsizing� a que muitos sistemas de informa��o estiveram ou vir�o a estar sujeitos.
+Estes sistemas exigem um acompanhamento e controlo permanentes (as 24 horas de cada um dos dias da semana), pelo que a sua centraliza��o num �nico espa�o permite racionalizar e conter os custos da sua gest�o.
+Dos dispositivos de seguran�a e preven��o faz, evidentemente, parte o Insight Manager, aplica��o de gest�o j� conhecida das linhas anteriores de servidores mas agora na sua vers�o 2.3.
+A Orquestra Cl�ssica do Porto abre hoje, em Guimar�es, a 4� edi��o dos Encontros da Primavera.
+O primeiro concerto � dirigido por Piero Bellugi, com os solistas Pedro Corostola (violoncelo), Maria do Ros�rio Ferreira e Palmira Troufa (sopranos).
+ No Pa�o dos Duques de Bragan�a, �s 21h45.
+Final do Festr�ia -- IX Festival Internacional de Cinema de Tr�ia, com a exibi��o �s 16h30 do filme premiado com o Golfinho de Ouro.
+Um industrial do Porto, Manuel Magalh�es, de 48 anos, abate a tiro dois s�cios, quatro disparos num, seis noutro, por terem passado cheques em seu nome.
+Um casal de Aveleda, povoa��o do Minho, acusa o padre da aldeia, Joaquim Carneiro, de ter rela��es sexuais com um filho seu de 12 anos.
+O povo defende o sacerdote.
+As autoridades arquivam o processo por falta de provas.
+O arcebispado, por�m, transfere o acusado para Paris.
+�� necess�rio que a constru��o [ da barragem do C�a ] seja efectivamente suspensa e que sejam reunidos os meios humanos e necess�rios a um estudo completo de toda a �rea antes que se crie uma situa��o irrevers�vel.
+ E n�o se invoquem os preju�zos imediatos de tal decis�o, pois corresponder�o, certamente, a uma parcela diminuta dos encargos com essa obra fara�nica e desajustada que � a Expo'98�.
+S�o afirma��es do reitor do Universidade do Porto, Alberto Amaral, no editorial do n� 25 do Boletim daquela Universidade, inteiramente dedicado ao tratamento da �grave situa��o do patrim�nio hist�rico-cultural do vale do rio C�a�, como igualmente refere o autor.
+Com�dia inglesa, anos 50.
+Um friso inenarr�vel de personagens marginais e/ou marginalizadas, com o p�s-guerra em pano de fundo.
+Pe�a �nica e inimit�vel no contexto do teatro ingl�s deste s�culo, oscilando entre o naturalismo e o lirismo.
+Receita que funciona lindamente, gra�as a um elenco talentoso e bem dirigido e a uma direc��o pl�stica (Jos� Carlos Barros) eficaz.
+TEATRO DA TRINDADE.
+De 3� a s�b., �s 21h30; s�b. e dom., �s 16h.
+ Cumpridos 16 dias de uma greve de 45, a paralisa��o na R�dio Televis�o da Guin�-Bissau (RTGB) foi suspensa ontem at� ao pr�ximo dia 25, devido a uma ced�ncia dos trabalhadores que t�m ouvido forte e feio da popula��o fora de Bissau que n�o tem acesso �s emiss�es da RTPi -- especialmente por causa dos jogos da selec��o portuguesa, acompanhados como se fosse a selec��o guineense.
+E foi assim que ontem -- precisamente por causa do Portugal-Turquia -- a televis�o guineense reabriu ... � hora necess�ria para transmitir o Portugal-Turquia, o que motivou grandes ajuntamentos em locais p�blicos que dispunham de televisores, tanto em Bissau como no interior do pa�s.
+ A Lisnave n�o conseguiu atingir as metas fixadas pela sua administra��o para o exerc�cio de 1994.
+No final do primeiro semestre do ano passado, a empresa previa obter um volume de vendas de 24 milh�es de contos, mas acabou por n�o ir al�m dos 21 milh�es.
+No que se refere a resultados correntes, as estimativas apontavam para um valor negativo de 2,6 milh�es de contos, mas que na realidade ultrapassou os seis milh�es.
+ Ao n�vel do Grupo, isto �, a Lisnave propriamente dita e as restantes 21 empresas em que os Estaleiros Navais de Lisboa participam no capital, os resultados saldaram-se por um preju�zo de 16,2 milh�es de contos.
+Este valor reflecte um agravamento de 6,5 por cento face aos 15,2 milh�es de contos de resultados negativos em 1993.
+As vendas consolidadas foram de 26,7 milh�es de contos em 1994, menos 1,2 milh�es de contos do que no ano anterior.
+Do ponto de vista concorrencial, o agravamento das perdas da empresa explica-se, em parte, devido � �entrada no mercado de novos estaleiros situados em pa�ses de muito baixo custo de m�o-de-obra (pa�ses do ex-bloco de Leste), e a aumentos da capacidade de docagem em �reas bem localizadas relativamente aos grandes fluxos de tr�fego mar�timo, al�m de tamb�m disporem de m�o-de-obra barata (pa�ses do M�dio e Extremo Oriente).
+ O PRESIDENTE et�ope, Mengistu Hail� Mariam, anunciou ontem a constitui��o de um novo Governo, de cuja forma��o ficou respons�vel o chefe da diplomacia, Tesfaye Dinka, agora promovido ao cargo de primeiro-ministro., revelou a r�dio de Addis Abeba.
+ A emissora anunciou igualmente que Mengistu demitira dois dos seus mais antigos ministros, o vice-Presidente Fiseha Desta e o secret�rio do Comit� Central, Legesse Asfaw.
+Tesfaye, um economista formado nos Estados Unidos, tem a miss�o de renovar o Governo e de alargar a sua base tribal, nos termos de uma decis�o adoptada pelo parlamento no in�cio da semana.
+ A PR�XIMA ronda de negocia��es sobre a paz em Mo�ambique dever� ter in�cio no dia 2 de Maio em Roma, confirmaram ontem o Minist�rio italiano dos Neg�cios Estrangeiros e a comunidade religiosa de Sant'Egidio, mediador destas conversa��es.
+Em comunicados separados, as duas entidades deram conta de que os contactos preliminares principiaram ontem em Roma.
+Segundo a ag�ncia Lusa, a delega��o da Renamoj� se encontra na capital italiana, onde s�o esperados este fim-de-semana os negociadores de Maputo.
+ �Apanhado� pela ronda do P�BLICO quando chegava �quela praia acompanhado pela mulher e pelo filho, na tarde de quinta-feira passada, o ministro dos Neg�cios Estrangeiros portugu�s escusou-se a ser fotografado, com o argumento de �estar em f�rias�, acelerando o passo (mais tarde, quando abandonava a praia, j� depois das 20 horas, acabaria por ser �apanhado� pela objectiva).
+Conhecido pelos vizinhos banhistas, l� foi acenando at� se instalar mesmo junto � �gua.
+N�o longe de Guilherme Oliveira Martins, o assessor pol�tico de M�rio Soares, que deixar� Bel�m em Setembro para assumir por inteiro a sua condi��o de futuro deputado pelo Partido Socialista.
+H� muito frequentador do local (os pais possuem uma resid�ncia em ...Boliqueime), Oliveira Martins tem visto os Tomates a transformarem-se na moda do Ver�o algarvio.
+o filho de Marcelo Rebelo de Sousa fazia anos e o ex-comiss�rio de �Lisboa 94� promovia uma festa, em local mais ou menos secreto.
+Pela Praia dos Tomates, restava apenas um discreto centrista, Fausto Quadros, al�m do porta-voz da Comiss�o Europeia, Jo�o Vale de Almeida.
+O australiano Michael Doohan (Honda) fez ontem a desfeita aos adeptos italianos, ao vencer o Grande Pr�mio de It�lia de motociclismo em 500cc, quarta prova do Mundial da modalidade, disputada no circuito de Mugello, onde os espectadores puxavam fervorosamente por Max Biaggi (Honda), que n�o foi al�m do segundo posto.
+A Honda acabaria por fazer o pleno no p�dio, com Alex Crivill� a conseguir o terceiro posto.
+ Nos arredores de Ponta Delgada uma resid�ncia do s�culo XVII, inteiramente restaurada.
+ Quatro quartos, simples por 850000 e duplos por 980000; servem refei��es quando solicitado e admitem animais.
+A sedu��o de uns olhos de longas pestanas sedosas tem agora em Cil Sublime, da Bourjois, um precioso auxiliar.
+Enriquecido com prote�nas e pr�-vitamina B5, � aplicado com uma escova especialmente criada para evitar o esborratamento do r�mel.
+ N. R. A raz�o por que se real�ou no t�tulo da not�cia em causa o benef�cio do whisky foi porque ela se refere a um estudo que real�ava esse facto, o que se compreende quando se sabe que se trata de um estudo brit�nico.
+O P�BLICO tem dedicado frequentes not�cias aos benef�cios do consumo moderado de �lcool em geral e do vinho em particular.
+Veja-se, a t�tulo de exemplo, os textos �Cora��o, vinho e legumes�, P�BLICO do passado dia 16; ��lcool faz bem ao cora��o�, 19.11.93; �Gl�rias e mis�rias do �lcool�, 15.8.93; ��lcool protege cora��o?�, 3.9.91; Vinho contra colesterol?, 12.8.91.
+O quadro de inten��es levanta uma outra quest�o.
+ Na pr�tica, privilegia-se as interven��es de fachada em detrimento do bem-estar dos habitantes das aldeias.
+A ilustr�-lo o exemplo de Pi�d�o, onde as coberturas em telha das habita��es (mesmo estando em bom estado) est�o a ser substitu�das por outras em lousa, enquanto as casas que j� t�m coberturas em lousa (ainda que chova no interior) n�o beneficiam de qualquer ajuda.
+�Pessoalmente causa-me alguma perplexidade que se trabalhe para a fotografia do visitante e n�o tanto para o bem estar de quem l� habita�, confessou Jos� Reis.
+os restantes 25 por cento s�o participa��o nacional, sustentada pelos promotores dos projectos apresentados.
+Mas tamb�m aqui h� aspectos pouco claros, j� que esta disposi��o n�o � seguida por todos.
+Quantos emigrantes mu�ulmanos residem na Europa?
+3 milh�es residem em Fran�a, 1,7 milh�es na Alemanha e 850 mil na Gr�-Bretanha.
+ Na Holanda, B�lgica, It�lia e Espanha, os n�meros oscilam entre 250 mil e 300 mil mu�ulmanos.
+A Jugosl�via tem 3 milh�es, a Alb�nia 1,7 milh�es e a Bulg�ria 800 mil.
+O sindicato dos maquinistas tamb�m n�o contabilizou os custos desta greve.
+No entanto, garante que os gastos com os autocarros fretados pela CP em Agosto para efectuar o transporte alternativo dos passageiros (principalmente emigrantes) dos comboios internacionais �chegavam para resolver os seus problemas�.
+�For�a 4�, uma placa de m�rmore disposta sobre um soclo, onde efeitos de ondula��o criam a ilus�o da passagem do vento sobre a superf�cie do mar.
+Jo�o Cutileiro tem um peixe, que ironicamente se apropria de todas as formas de peixes / fontan�rios de jardins, e que reenvia para a sua grande escultura de ninfas com um barco, mais longe, num dos lagos.
+Jos� Pedro Croft fez uma enorme caixa oca de pedra, dentro da qual disp�s bolas do mesmo material.
+� a �nica pe�a que exige algum esfor�o ao visitante, que � levado a debru�ar-se e espreitar para o interior da caixa.
+Poder� assim tomar consci�ncia do contraste entre o volume sugerido e a massa das bolas, e associar a caixa a outras formas semelhantes, feitas no mesmo material, que j� tenha visto: t�mulos, sarc�fagos ...
+Manuel Rosa e Rui Sanches, por fim, mostram-nos esculturas que t�m por referente mais ou menos expl�cito o corpo.
+mas a coluna � ela pr�pria uma met�fora do corpo que sustenta, da for�a que tudo suporta.
+e, no meio de uma estrutura de m�rmore, colocou dois vasos com �gua a diferentes n�veis, que se foi sujando -- e enchendo de moedas -- com o passar dos dias.
+A certa altura, este �corpo�, que � o corpo da escultura, foi mesmo quebrado, levando � sua substitui��o.
+Est�tua (porque � um corpo), escultura, organismo que vive, � ferido e morre?
+Todas as interpreta��es s�o poss�veis, tudo pode acontecer a este corpo.
+BRAS�LIA Pesquisa Datafolha publicada hoje revela um dado supreendente: recusando uma postura radical, a esmagadora maioria (77%) dos eleitores quer o PT participando do Governo Fernando Henrique Cardoso.
+Tem sentido -- ali�s, muit�ssimo sentido.
+Muito mais do que nos tempos na ditadura, a solidez do PT est�, agora, amea�ada.
+Nem Lula nem o partido ainda encontraram um discurso para se diferenciar.
+Eles se dizem oposi��o, mas ainda n�o informaram o que v�o combater.
+Muitas das prioridades do novo governo coincidem com as prioridades do PT.
+Desde o �ltimo dia 13, �Confiss�es de Adolescente� pode ser vista pelos teens portugueses.
+A s�rie exibida aqui pela Cultura estreou na TVI de Portugal.
+Al�m disso, a co-produ��o com o canal franc�s TCF1 para a realiza��o de mais 30 epis�dios continua sendo negociada.
+�C�mera Manchete� � o nome do novo programa jornal�stico que estr�ia quarta-feira, �s 22h30, na Rede Manchete.
+Sob o comando de Ronaldo Rosas, o programa mostrar� reportagens especiais de S�nia Pompeu.
+A dire��o do novo semanal ser� assinada por Ewaldo Ruy.
+Os jogadores se dividem pelos dez quartos do alojamento, equipados com frigobar, ar condicionado, televis�o e telefone.
+�� uma coisa do Primeiro Mundo�, afirmou o levantador Maur�cio (leia mat�ria ao lado).
+Al�m de Maur�cio, Carl�o e Paul�o, a sele��o deve contar hoje com Giovane.
+O atacante, que deveria ter se apresentado anteontem � noite, pediu mais um dia de folga ao treinador.
+Na volta de uma viagem ao exterior, vale a pena trazer uma impressora matricial.
+Free shops dos aeroportos internacionais tamb�m vendem o equipamento.
+O modelo Lx 810, da Epson, � vendido em Miami por US$ 178.
+O pre�o de lista nas revendas brasileiras � de US$ 422.
+Esse equil�brio era tido como pr�-condi��o para o sucesso do plano econ�mico.
+Parte dos recursos para a forma��o do FSE foi deslocado do or�amento da sa�de e educa��o.
+Na �poca, o ent�o ministro da Fazenda, Fernando Henrique Cardoso, fez um pronunciamento em cadeia nacional para anunciar a inten��o do governo de destinar o FSE a investimentos sociais.
+O assessor de imprensa do Minist�rio da Fazenda, S�rgio Danese, disse ontem que o ministro da Fazenda, Rubens Ric�pero n�o iria comentar o assunto porque n�o tinha informa��es suficientes.
+O projeto original do governo destinava ao TSE R$ 334,9 milh�es.
+Como n�o houve acordo entre governo e tribunal quanto ao volume de recursos, a dota��o foi inclu�da na reserva de conting�ncia sem especifica��o de despesa.
+Posteriormente, diante da amea�a do tribunal de entrar com uma a��o judicial, o governo mandou ao Congresso uma altera��o ao projeto, aumentando para R$ 452,7 milh�es a dota��o do TSE.
+Essas medidas reduziram a disponibilidade de dinheiro no sistema banc�rio e os grandes bancos passaram a n�o fornecer recursos para as pequenas institui��es.
+Quebraram aquelas que estavam �descasadas�, ou seja, que financiavam no mercado interbanc�rio de um dia (CDI) os empr�stimos e carteiras de t�tulos estaduais e municipais com prazos longos.
+Tel� -- Claro.
+Aqui s� joga quem est� bem.
+Ningu�m for�a sua escala��o porque h� quem escale o time no S�o Paulo.
+Ele s� n�o jogava porque n�o estava bem.
+Folha -- E o que o senhor acha do esc�ndalo da arbitragem carioca?
+Tel� -- Uma vergonha.
+Apenas dois �rbitros resolveram contar todos os podres, enquanto a federa��o tem mais de 70.
+O futebol precisa seguir o exemplo da CPI do or�amento e apresentar todos os podres.
+Se eu dirigisse uma federa��o, apresentaria balan�os mensais e liberaria minhas contas banc�rias.
+A Fifa e a CBF deveriam entrar de sola nesse caso e em todas as outras federa��es.
+A maioria das empresas que produzem leite das marcas interditadas n�o tinha sido comunicada ontem sobre a libera��o do produto.
+O presidente da Cooper, Benedito Vieira Pereira, 49, afirmou que pretendia distribuir leite C nos postos de venda hoje.
+Prandi disse ainda que a empresa est� elaborando �normas fact�veis de serem executadas para a solu��o ou minimiza��o dos problemas� existentes no local.
+�O Charade vai concorrer na faixa do Suzuki Swift e do Twingo, da Renault�, afirma Caparelli.
+Herbert Berger, diretor-superintendente da empresa, diz que o Charade �se aproxima do Honda Civic em tamanho e custa bem menos�.
+O Applause, um sed� quatro portas, com motor 1.6, � o carro mais caro da Daihatsu.
+O top de linha custa US$ 30 mil.
+A mudan�a do local de jogo que deve acontecer tamb�m na partida contra o Corinthians, no pr�ximo dia 17 foi determinada pela CBF, que n�o viu garantias de seguran�a no est�dio santista.
+�Na Vila, quando recebo a bola, tenho que ficar olhando sua trajet�ria, para n�o ser surpreendido.
+S� depois � que levanto a cabe�a para fazer um lan�amento�, reclama Neto.
+JFK -- A PERGUNTA QUE N�O QUER CALAR Telecine, 20h30.
+O destaque do HBO � o in�dito �Exterminador do Futuro 2 -- O Julgamento Final�, em que Schwarzenegger � um rob� com apar�ncia humana que vem do futuro para proteger um garoto.
+Para o terceiro r�u, Alexandre Cardoso, 21, o �Topeira�, o juiz determinou uma pena de 20 anos.
+Souza tamb�m negou aos r�us o direito de apelarem da seten�a em liberdade.
+Os tr�s est�o presos desde 30 de julho de 93.
+Come�ou bem antes do que se previa a batalha pela futura sucess�o na Fifa apenas seis meses depois do super-acordo que, nas v�speras da Copa, reconduziu o brasileiro Jo�o Havelange ao sexto mandato consecutivo.
+Pelo acordo, os tr�s continentes mais obstinados em cortar o reinado de Havelange, a �frica, a �sia e a Europa, aceitaram cancelar os seus movimentos de oposi��o em troca, basicamente, de dois compromissos.
+Havelange aceitaria engolir o italiano Antonio Matarrese como o seu vice-executivo e, al�m disso, esqueceria os seus modos autorit�rios, coordenando a entidade de maneira colegiada.
+O Ambulim foi um dos centros que contribu�ram para um estudo apresentado na 5� Confer�ncia Internacional sobre Transtornos Alimentares, de 29 de abril a 1� de maio em Nova York.
+Dados sobre abuso sexual em bul�micas no Brasil, �ustria e Estados Unidos foram centralizados por Harrison Pope, da Escola de Medicina de Harvard.
+Essa divis�o gera algumas distor��es terr�veis.
+Um dos injusti�ados � Alfredo Volpi, que recebe apenas um painel, com cinco telas que servem para ilustrar sua �evolu��o� de figurativo a abstrato.
+Afora historicismo, isso � menosprezar um fator interno � arte brasileira, que independe de contexto internacional.
+Volpi foi dos mais influentes pintores do pa�s para al�m da quest�o da autonomia.
+O panorama sofre preju�zos demais em favor da tese.
+Abstratos entre medianos e med�ocres, como Fukushima, P�rsio, Raimo e Douchez, t�m o mesmo ou maior destaque que Volpi e nada que se possa chamar de autonomia para oferecer como lenitivo.
+Talvez isto seja muito barulho por nada.
+Original: C�cero.
+Disse que n�o conseguia vislumbrar artif�cios fraudulentos ou pr�tica de peculato no protocolo assinado por Qu�rcia.
+Afirmou que o conjunto de fatos, em princ�pio, aponta o envolvimento de Qu�rcia.
+Recebeu a den�ncia.
+Segundo o m�dico, o caso n�o preocupa.
+Rom�rio n�o se exercitou nas cobran�as de falta e p�nalti.
+Toda a comiss�o t�cnica sabe que Rom�rio � de treinar pouco, geralmente se poupando entre dois jogos dif�ceis.
+Nem o PSB nem a coliga��o t�m compet�ncia legal para trocar o vice da chapa sem a concord�ncia de Bisol, que teve o nome aprovado em conven��o.
+Outra maneira de um partido for�ar a substitui��o seria expulsar o candidato, com base em seu estatuto.
+Neste caso, o registro da candidatura seria cancelado pela Justi�a.
+�Uma boa parte do p�blico dos Stones hoje � yuppie.
+� uma tend�ncia�, dizia o lojista Nivaldo Silva Costenaro, 34, cabeludo baterista de uma banda de blues.
+�Escuto Stones desde os 13 anos de idade.
+O f� daquela �poca vai ser f� sempre�, acrescentou Costenaro.
+Apesar de limitar a venda de quatro ingressos por pessoa, a Mesbla n�o evitava ontem que uma mesma pessoa comprasse mais de uma vez.
+A queda nas vendas teve reflexos nas negocia��es entre as confec��es e as lojas.
+�N�o est� havendo cancelamento de pedidos.
+Mas existem ind�strias que est�o sendo procuradas pelos lojistas para postergar as entregas�, conta Eduardo Costa, diretor da Abravest, que re�ne a ind�stria de vestu�rio.
+A seca que atingiu as �reas produtoras de gr�os n�o deve causar grandes estragos na safra 1994/95.
+A primeira previs�o do IBGE (Funda��o Instituto Brasileiro de Geografia e Estat�stica) indica queda de 0,62% na �rea plantada nesta safra em rela��o a anterior.
+Pocock, que publicou os resultados no �ltimo n�mero da revista �British Medical Journal�, do Reino Unido, estudou a influ�ncia do chumbo desde a gesta��o.
+Segundo ele, a exposi��o ao material durante a gravidez ou nos dois primeiros anos de vida n�o representa perigo.
+OS PAP�IS DE FLAVIO-SHIR� -- A mostra, que faz parte das comemora��es dos 50 anos de pintura do artista japon�s, re�ne 25 obras em pequenos formatos.
+Foi utilizada t�cnica mista, incluindo desenho, pastel, ceragrafia e fotografismo.
+De seg a sex das 11h �s 19h, s�b das 11h �s 14h.
+At� 30 de mar�o.
+Pre�o das obras: de US$ 2.000 a US$ 4.000.
+A aplica��o dessas observa��es ao caso americano e �s rela��es entre negros e brancos sugere uma nova maneira de conceituar os argumentos j� conhecidos do �legado da escravid�o�.
+N�o se trata simplesmente da quest�o da escravid�o certamente ter tido efeitos duradouros sobre a cultura negra, nem mesmo dela ter exercido um amplo efeito negativo sobre a auto-confian�a e auto-estima dos negros, mas mais especificamente da experi�ncia da escravatura ter desvirtuado e tolhido a evolu��o do algoritmo etnoc�ntrico que os negros americanos teriam desenvolvido no decorrer normal dos acontecimentos.
+Os brancos fizeram tudo em seu poder para invalidar ou menosprezar cada sinal de talento, virtude ou superioridade entre os negros.
+Eles tiveram que fazer isso se os escravos fossem superiores em qualidades que os pr�prios brancos valorizavam, onde estaria a justificativa moral para mant�-los escravizados?
+E, assim, tudo o que os afro-americanos faziam bem teve de ser colocado em termos que menosprezassem a qualidade em quest�o.
+Mesmo a simples tentativa de se documentar esse ponto deixa uma pessoa exposta a acusa��es de condescend�ncia e, assim, os brancos de fato conseguiram cooptar os julgamentos de valor.
+� ainda mais �bvio que � imposs�vel falar abertamente sobre o superioridade de muitos atletas negros sem ser sujeito a acusa��es de que se estar sendo anti-negro de uma maneira enviesada.
+Pela segunda vez desde quando come�ou a coordenar as a��es no Rio, h� duas semanas, o Ex�rcito mudou o nome das opera��es.
+Agora, os oficiais envolvidos se referem sempre ao comando das a��es como Centro de Coordena��o de Opera��es de Combate ao Crime Organizado (Ccocco).
+Cinco linhas paralelas, de mais de 400 km cada, foram descobertas por cientistas australianos no sul do pa�s.
+Elas est�o separadas por espa�os de 80 km a 100 km.
+As linhas, invis�veis da superf�cie, foram detectadas atrav�s de dados de sat�lites.
+Pesquisadores acham que as linhas podem ser falhas geol�gicas.
+O derramamento de �leo do petroleiro Exxon V�ldez, em mar�o de 1989, causou estragos no valor de US$ 286,8 milh�es, segundo um j�ri em Anchorage (Alaska).
+A a��o est� sendo movida por pescadores, lojistas, propiet�rios de terra e nativos.
+O valor � mais que o dobro do estimado pela Exxon, mas menor que o original pedido, de US$ 895 milh�es.
+Bombeiros e pessoal de resgate foram colocados em alerta m�ximo em Argel no final da tarde de ontem, segundo a r�dio estatal.
+Ve�culos de resgate estavam a apenas 500 metros do Aibus 300.
+Optar entre um aparelho conjugado e outro simples � outro ponto que merece aten��o.
+Para fazer uma boa compra, a t�cnica do Procon recomenda ao consumidor que verifique se j� h� dentro de casa os tradicionais equipamentos que desempenham as mesmas fun��es do conjugado, s� que separadamente.
+Caso a op��o seja pelo aparelho multiuso, o comprador deve checar se o produto tem assist�ncia t�cnica, diz ela.
+Como a id�ia de enxugar a Constitui��o enfrenta resist�ncia inclusive nos partidos que ap�iam o governo, a equipe de FHC resolveu fazer as reformas por partes.
+Primeiro aprova-se o texto enxuto e depois negocia-se a aprova��o, sem prazo definido, das leis complementares e ordin�rias.
+O Pent�gono usa a Internet, que conecta computadores a sistemas telef�nicos, para que seus funcion�rios troquem informa��es.
+Os arquivos s�o protegidos por senhas e c�digos, que acabaram mostrando-se vulner�veis.
+O conselho fiscal entende que os acionistas n�o devem mais �suportar investir dinheiro bom em companhia sem condi��es de se reerguer�.
+�Tem que torcer muito e rezar bastante, porque milagres acontecem�.
+QUADRINHISTAS DO BRASIL inteiro se re�nem entre sexta-feira e domingo em Arax� (MG).
+Haver� oficinas e um concurso.
+Informa��es pelo telefone (034) 661-2458.
+O MERCADO AMERICANO para desenhistas � o tema da oficina com David Campitti (veja trabalho dele ao lado), roteirista de hist�rias do Super-Homem e criador de v�rios personagens.
+Outros profissionais brasileiros, que atuam nos EUA, tamb�m participam.
+Informa��es pelo (011) 263-4700.
+Ontem pela manh�, os atletas que n�o atuaram contra o Vit�ria, participaram de um coletivo com a equipe principal.
+Os que jogaram, fariam treino de recupera��o � tarde no Projeto Acqua.
+Murici reclamou da postura do time do Vit�ria.
+�Eles bateram muito.
+O que fizeram com o Pereira foi um absurdo�, disse.
+Pereira levou um pis�o na cabe�a, mesmo estando ca�do no ch�o.
+Murici espera que o Inter n�o tenha a mesma atitude.
+A primeira miss�o lunar dos EUA em 21 anos teve in�cio ontem, quando um foguete Tit� 2 colocou no espa�o a astronave n�o tripulada Clementine 1, que vai passar dois meses em duas �rbitas da Lua para realizar completo mapeamento mineral�gico e topogr�fico do sat�lite da Terra.
+O nome oficial do projeto � Depse 1 (Deep Space Program Science Experiment).
+Ele � patrocinado pela Organiza��o de Defesa de M�sseis Bal�sticos e pela Nasa, em uma das primeiras opera��es espacais com fins civis e militares.
+O lan�amento ocorreu ontem na base a�rea de Vanderberg, na Calif�rnia, costa oeste do pa�s, �s 8h30 locais (14h30 em Bras�lia).
+O custo total do projeto � de U$ 55 milh�es.
+O atual perfil dos poupadores ajudar� a manter o dinheiro aplicado.
+Segundo o BC, mais da metade dos recursos da caderneta s�o de poupadores de m�dio e grande porte em tese, menos sujeitos a achar que o dinheiro perdeu rendimento com a queda da infla��o.
+A previs�o de saques reduzidos sobre a poupan�a � compartilhada por especialistas.
+�O ceticismo em rela��o a planos econ�micos faz a popula��o manter reservas�, diz o ex-diretor de Pol�tica Monet�ria do BC, Lu�s Eduardo de Assis, diretor de investimentos do Citibank.
+��s vezes peco por falta de experi�ncia, mas me considero um piloto r�pido�, afirmou ontem o brasileiro, em entrevista � Folha por telefone, da It�lia.
+Diniz come�ou sua carreira automobil�stica em 1989, no Brasileiro de F�rmula Ford, campeonato em que obteve a sexta posi��o na classifica��o final.
+A Pol�cia Civil de Ourinhos (371 km a oeste de S�o Paulo) prendeu ontem � tarde o ex-l�der religioso Jonas R�bio, 45, acusado de matar na quarta-feira a estudante Claudirene Contijo, 13, com um tiro de espingarda.
+O delegado Celso Antonio Borlina, 38, disse que R�bio confessou o crime.
+R�bio era acusado por Claudirene de t�-la estuprado no ano passado, �poca em que era o l�der da Assembl�ia de Deus na usina S�o Luiz, onde a estudante morava.
+�A m�e da estudante prestou queixa do estupro e, depois, das amea�as de morte que R�bio teria feito a Claudirene�, disse Borlina.
+Zanini diz que est� preparando a funda��o de um novo partido, que trar� expl�cito em seu programa a ressalva de que � contra o racismo.
+�Ser� um movimento nacionalista independente.
+O novo manifesto do movimento n�o ficar� pronto antes das elei��es.
+Sendo assim, n�o podemos dar nosso apoio aberto�, explicou.
+Ele afirma ter negros e judeus entre seus correligion�rios.
+Segundo a empresa, ser� o primeiro alimento produzido totalmente atrav�s de t�cnicas de engenharia gen�tica a ser vendido.
+O tomate foi projetado para manter o sabor que tem depois de colhido durante tempo prolongado na prateleira dos supermercados.
+Pa�ses produtores e consumidores de caf� entraram em um tipo de pr�-pacto internacional que n�o intervenha nos pre�os do produto, afirmou o porta-voz da Organiza��o Internacional do Caf� (OIC).
+�Temos as bases para um novo acordo.
+Pode-se agora recomendar um texto definitivo para o conselho da organiza��o�, disse o porta-voz.
+Os pa�ses integrantes do instituto enviam o texto com o novo acordo para seus governos antes da aprova��o final, que deve ocorrer na semana que vem.
+Passarinho j� descarta claramente a candidatura ao Planalto.
+�Meu compromisso com o Par� � inarred�vel�, afirmou ontem depois de conversar com Amin.
+O senador catarinense ainda hesita.
+Mas faz quest�o de lembrar que, aos 46 anos, ainda ter� v�rias outras oportunidades para disputar a presid�ncia.
+A maior facilidade criada � a supress�o das barreiras alfandeg�rias para a importa��o de produtos de um pa�s pelo outro.
+Isso vai facilitar as atividades de franqueadores que precisam importar produtos para distribui-los a seus franqueados que operam em outro deles.
+Por exemplo: o master-franqueado da Arby's no M�xico enfrentar� menos problemas e menos burocracia para importar o rosbife (que, ao contr�rio do que ocorre com a Arby's Brasil, vem dos EUA) e as batatinhas (canadenses) que serve em seus restaurantes pr�prios e franqueados.
+O maior impacto do Nafta a curto prazo ser� sobre as franquias do setor automotivo.
+Pelo acordo, s�o reduzidas de imediato todas as barreiras tribut�rias e alfandeg�rias relacionadas � importa��o por qualquer pa�s do acordo, de ve�culos automotores, autope�as e acess�rios fabricados nos demais pa�ses membros.
+Roberto Capuano, presidente do Creci-SP (Conselho Regional de Corretores de Im�veis), diz que o mercado de usados sofre com a car�ncia de financiamentos desde 70.
+�Em 79 houve uma leve reabertura de cr�dito, mas nada significativo�, afirma.
+Segundo ele, hoje as concess�es de financiamento banc�rio s�o restritas e elitistas.
+A Benetton volta a provocar pol�mica com um cartaz da s�rie �United Collors�.
+Desta vez, com uma camiseta branca furada a bala e empapada de sangue acima de cal�as militares.
+A empresa informou que a roupa era de Marinko Gagro, soldado croata-b�snio morto em julho do ano passado.
+�L'Osservatore Romano�, o jornal do Vaticano, qualificou a campanha de US$ 15 milh�es como �terrorismo de imagem�.
+Os franceses �Le Monde� e �Le Figaro� e o alem�o �Frankfurter Allgemeine� recusaram o an�ncio.
+Ilmar Galv�o e Moreira Alves votaram pela concess�o do mandato de seguran�a.
+Eles sustentaram que a ren�ncia interrompeu o processo de impeachment.
+Moreira Alves tamb�m votou a favor de Collor em outros dois mandatos de seguran�a, mas foi vencido nos dois julgamentos.
+Aquele pensamento provocou-me um arrepio estranho e delicioso.
+N�o falei nada, mas visualizei a Sra. Oke, sentada no sal�o amarelo, o mesmo sal�o onde nenhum Oke de Okehurst exceto ela ousava permanecer sozinho, envergando o vestido de sua antepassada e confrontando-se, por assim dizer, com aquela coisa vaga, plangente, que parecia permear o aposento ... aquela vaga presen�a -- assim me parecia do galante poeta assassinado.
+O filme �Barfly� (traduzido no Brasil como �Condenados pelo V�cio�) foi inspirado na vida de Bukowski.
+O escritor, identificado com a cultura beatnik, admitia somente ter sido inspirado por um outro autor da Calif�rnia, John Fante (�Pergunte ao P�, entre outros), que Bukowski descobriu nas suas leituras em bibliotecas p�blicas quando jovem.
+Michael Schumacher culpou um degrau existente nas zebras do circuito de Adelaide por seu acidente ontem no final da primeira sess�o de treinos oficiais.
+�Quando analisei a pista na quinta-feira, pedi � FIA que melhorasse as condi��es daquele ponto�, disse o alem�o.
+Na sua opini�o, o Estado n�o pode continuar em setores onde a iniciativa privada j� comprovou sua efici�ncia.
+Em entrevista � Folha, Angarita elogiou a iniciativa do Sindicato dos Eletricit�rios de S�o Paulo, que articulou uma a��o judicial para tentar suspender a venda de a��es da Eletropaulo.
+Cutler admitiu que as conversas entre funcion�rios da Casa Branca e do Tesouro sobre investiga��es sobre o banco de McDougal n�o eram �recomend�veis�, mas n�o constitu�ram crime.
+Essas conversas levaram em mar�o � demiss�o de Bernard Nussbaum, amigo da primeira-dama, do cargo de assessor jur�dico da Casa Branca.
+Fiske come�ou seu trabalho em 20 de janeiro.
+Comiss�o Parlamentar de Inqu�rito sobre o caso Whitewater come�a a funcionar dia 29.
+O relat�rio n�o diz se as reuni�es entre funcion�rios da Casa Branca e do Tesouro violaram a �tica.
+O secret�rio do Tesouro, Lloyd Bentsen, disse que a quest�o �tica vai ser examinada agora.
+S� o procurou tempos depois, quando o secret�rio de Assuntos Especiais, Eliezer Baptista, resolveu preparar o macroplanejamento do pa�s uma id�ia esplendorosa que, infelizmente, n�o resistiu ao impeachment de Collor e convocou Mour�o para sua equipe.
+Com medo de perder poderes, Modiano chamou Mour�o e, numa atitude mesquinha, comunicou-lhe que, como o financiamento do projeto seria bancado pelo BNDES, ele iria para l�, mas na qualidade de representante do banco.
+A tela de abertura do programa � simp�tica.
+Uma sala tem como �cones uma pilha de discos, um aparelho de som, uma enciclop�dia e uma janela que mostra a encruzilhada onde Robert Johnson teria feito um pacto com o diabo, trocando a alma pelo sucesso.
+�Peppermint Tea House -- The Best of Shoukichi Kina�.
+Lan�ado dentro do projeto �Asia Classics�, um subselo do Luaka Bop (o mesmo que lan�ou Tom Z� no exterior), essa colet�nea � mais um passo de Byrne na sua busca do sublime na m�sica.
+No primeiro volume de �Esses Byrne� mostrou soberbas trilhas sonoras de filmes indianos.
+Com Kina, ele chegou bem perto da perfei��o.
+Em todo o shopping, que est� decorado com lanternas, leques e at� um portal japon�s, h� exposi��es de iquebana (arranjos florais), bonecas, cer�micas, armaduras, m�scaras, quimonos e espadas.
+Em outro espa�o, h� exibi��o de v�deos que demonstram, por exemplo, a cerim�nia do ch� e o teatro kabuki, al�m de pontos tur�sticos do Jap�o.
+Pr�ximo � pra�a da alimenta��o, um sushiman estar� preparando pratos t�picos, como sashimis e o sushis, que estar�o � venda.
+Para isso, ele afirma pretender inaugurar uma c�mara setorial para discutir as quest�es espec�ficas da economia no Estado.
+Para atender � pol�tica desejada pelo governador eleito, Barelli vai comandar uma revis�o da antiga pasta (Secretaria das Rela��es do Trabalho).
+O economista diz que, para gera��o de empregos, v�rias propostas ter�o que ser analisadas.
+Segundo ele, �nem os grandes pa�ses souberam resolver o problema do desemprego�.
+Segundo ele, italianos e belgas j� descobriram que comprar im�veis em Natal � um �timo neg�cio.
+�Um diplomata da B�lgica comprou apartamento para passar as f�rias de julho e janeiro, mas fora desse per�odo ele aluga o im�vel por US$ 1.000�, diz Bezerra.
+Apesar de morarem aqui do lado em Miami, Teresa e Emerson Fittipaldi n�o d�o nem tchuns para a Copa.
+Pegam os dois pimpolhos Joana e Luca e se mandam para Lake Powell, Arizona.
+Para tanto � imprescind�vel priorizar algumas a��es como a recupera��o de unidades armazenadoras de cereais e das rodovias, al�m do incentivo � utiliza��o das ferrovias.
+Tamb�m � necess�rio alertar toda a sociedade para a import�ncia de se reduzir as perdas de alimentos.
+Aos horticultores � preciso oferecer informa��es de mercado seguras e atuais para a programa��o da produ��o e desenvolver novas alternativas para o acondicionamento e embalagem dos produtos.
+Helmet � brutal.
+� metal punk.
+Tem tamanho, peso intr�nseco que dispensa volume alto.
+A banda foi cortejada por v�rios selos multinacionais, que viam nela um poss�vel novo Nirvana.
+Acabou assinando com o Interscope, ligado � Warner, por US$ 1 milh�o por tr�s discos.
+Como notou o jornalista Jim Greer, da revista americana �Spin�, � uma boa grana, mas nada muito absurdo.
+�Meantime�, disco de estr�ia da banda no Interscope, vendeu 500 mil c�pias.
+As duas guitarras do Helmet atacam sempre juntas, em pequenas c�lulas separadas por um sil�ncio nunca menos que perturbador.
+Algo como um samba de breque transposto para o dom�nio eletrificado do rock.
+As letras ir�nicas, eruditas, denunciando que os m�sicos talvez tenham estudado um pouco demais s�o gritadas.
+N�o h� nenhuma �nfase na melodia.
+�O Helmet � uma grande se��o r�tmica�, definiu o baixista da banda, Henry Bogdan, em uma entrevista � �Spin�.
+M�todo japon�s exige calhama�os de exerc�cios; opositores dizem que 't�cnica peca pelo mecanicismo'.
+A matem�tica ainda � o �bicho de sete cabe�as� para grande parte dos estudantes do 1�. e 2�. graus.
+Na semana passada, a Secretaria Nacional de Direito Econ�mico baixou uma portaria, que aplica uma multa de 250 Ufirs (R$ 170) aos seis col�gios por cada matr�cula recusada.
+�A Justi�a foi aplicada porque o Estatuto da Crian�a e do Adolescente garante o direito de escolas a todas as crian�as�, disse M�rcia Mendes, presidente da Associa��os dos Pais de Alunos do Pit�goras, um dos seis col�gios acusados de recusar as matr�culas.
+O mesmo n�o se pode dizer da pintura de Cy Twombly.
+Da segunda gera��o de expressionistas abstratos, Twombly � tamb�m do segundo time.
+� um grande pintor, e sua �ltima fase � o que mais sustenta a afirma��o; mas � irregular, tem fracassos evidentes.
+Ainda que Robert Hughes, da revista �Time�, ache que o tema de Twombly � mesmo o fracasso, um fracasso � sempre um fracasso.
+�Os brasileiros escolhem Cardoso� foi o t�tulo na primeira p�gina.
+O jornal compara a elei��o presidencial brasileira � francesa, afirmando que o premi� Edouard Balladur, ex-favorito, tamb�m pode perder, em abril de 95.
+�A explica��o da reviravolta n�o reside em uma pretensa especificidade brasileira, que seria um coquetel de leviandade, inconst�ncia e pusilanimidade, tudo ao som de salsa�, ironiza o editorial.
+Os empres�rios abriram m�o de posi��es hist�ricas, eventualmente visando sua prote��o, para construir e defender id�ias exclusivamente de interesse coletivo para o desenvolvimento global.
+Folha -- Quais s�o as implica��es que o encerramento da revis�o traz para a economia?
+Resposta -- J� pedimos a cria��o de um conselho para produzir essa defini��o.
+� medida que o grupo de servi�os que entram nos lares e nas empresas evolui, a defini��o de servi�o universal vai evoluir tamb�m.
+Deve haver, no m�nimo, uma linha digital chegando a cada casa, mas acho que servi�o universal deveria significar muito mais do que isso, mais do que o fio.
+Amato -- O cen�rio ideal para o Plano Real seria a vit�ria inequ�voca de um dos candidatos j� no 1� turno das elei��es.
+Folha -- Quem vence?
+�Eu tenho uma origem pol�tica.
+Meu av� materno foi prefeito de Limeira, meu pai foi duas vezes prefeito de Itapira, onde passei toda minha inf�ncia.
+Desde cedo fui orador da turma, sempre gostei de declamar.
+Da� a alguns anos, os Gershwins foram para Hollywood, a fim de fazer o que intimamente desprezavam: escrever can��es para filmes.
+Mas, mesmo que quisessem n�o conseguiriam fazer nada �menor�.
+Num �nico ano, 1937, eles compuseram �They Can't Take That Away From Me�, �Let's Call the Whole Thing Off�, �A Foggy Day�, �Nice Work if You Can Get it�, �They All Laughed�, �Love Walked In�' e �Love Is Here to Stay�', e essas s�o apenas as que ficaram universalmente conhecidas.
+Infelizmente, 1937 seria tamb�m o �ltimo ano de George, porque ele morreria em julho, de um tumor cerebral, aos 38 anos.
+A desgra�a � que, cientificamente, como demonstra a moderna criminologia, quase nada dessa pol�tica criminal �simb�lica� serve para atenuar o grav�ssimo problema da criminalidade.
+A interven��o militar �localizada�, que pode imediatamente oferecer algum al�vio, tem como efeito principal o seguinte: o crime s� muda de lugar.
+Com a interven��o no Rio, que se cuide o resto do pa�s!
+Quarenta por cento dos gerentes japoneses est�o insatisfeitos com os sal�rios recebidos, segundo pesquisa divulgada por um instituto privado em T�quio.
+Cerca de 72 % dos empres�rios da constru��o querem que o pr�prio setor negocie a convers�o dos contratos para a URV, enquanto 28 % desejam que o governo estabele�a as regras.
+Folha Qual a sa�da?
+Broz N�o tenho a receita exata, tenho a certeza de que algo precisa ser feito diante do confronto entre nacionalismo alban�s e s�rvio.
+O candidato deu r�pida entrevista no sagu�o do aeroporto e dirigiu-se ao Hotel Delmonicos, na Park Avenue, em Manhattan, onde a di�ria � de US$ 195 (CR$ 253 mil) no fim de semana e US$ 250 (CR$ 325 mil) de segunda a sexta-feira.
+Lula e outros 20 convidados almo�aram a convite do c�nsul-geral do Brasil em NY, Marco Cesar Meira Naslausky, no restaurante Peter Luger Steak House, tradicional no bairro do Brooklin.
+A conta, paga pelo embaixador, ultrapassou US$ 1,4 mil (CR$ 1,8 milh�o).
+A li��o parece clara em ambos os casos: n�o ser� um artif�cio legal nem a boa vontade dos governos que proteger�o a riqueza dos poupadores ou o poder de compra dos assalariados, mas sim um ambiente de estabilidade econ�mica consistente.
+Pelas raz�es alinhadas, que n�o refletem apenas conjeturas te�ricas, mas a experi�ncia duramente vivida pela na��o nas �ltimas duas d�cadas, se imp�e dar um basta definitivo a todos os remanescentes de indexa��o formal, sejam eles nas taxas de juros nominais (TR), no recolhimento dos tributos (Ufir e correlatos) e nos sal�rios (IPC-r).
+O castelo de cartas ruiu logo.
+H� uma esquerda reformista no PSDB.
+Mas ela parece ser mera exce��o.
+Agora, o partido retirou a m�scara.
+Mostrou que a maioria dos seus pol�ticos profissionais cuja respeitabilidade n�o pode ser posta em d�vida oscilava entre o centro nacionalista e os radicais do PMDB.
+Considerados globalmente, n�o faziam diferen�a.
+Eram o retrato do cora��o e do c�rebro deste partido.
+Nada mais!
+O tancredismo concedeu ao seu principal te�rico, professor e senador Fernando Henrique Cardoso, a oportunidade de esbo�ar a filosofia pol�tica do mudancismo.
+Na pr�tica, os que iriam constituir o cerne e a base do partido iam mais longe: condenavam o autoritarismo, o oportunismo, o clientelismo e o fisiologismo.
+Ostentavam um patamar aparentemente s�lido, que evidenciava a radicalidade dos estratos m�dios e dos intelectuais envolvidos na rejei��o do �status quo�.
+Na largada, Ayrton Senna, que havia feito a pole position, manteve a primeira coloca��o.
+Por causa de um acidente envolvendo o finland�s J.J. Letho, da Benetton, e o portugu�s Pedro Lamy, da Lotus, o �safety car� entrou na pista.
+Durante seis voltas, os pilotos tiveram que manter suas posi��es por raz�es de seguran�a e aproveitavam para aquecer os pneus enquanto os carros envolvidos no acidente eram retirados.
+O �caso Paubrasil� j� rendeu � Receita Federal, segundo o coordenador-geral de fiscaliza��o do �rg�o, Luiz Henrique Barros de Arruda, cerca de US$ 1,5 milh�o em cr�dito tribut�rio -- a soma do imposto devido, multas, corre��o monet�ria e juros de mora -- de cerca de 20 empresas envolvidas no esc�ndalo que procuraram a Receita espontaneamente.
+Essas empresas, diz Arruda, reconheceram ter emitido �notas frias�, sem o recebimento de nenhum servi�o da empresa do pianista Jo�o Carlos Martins e de seu s�cio Ettore Gagliardi.
+Os jogadores do Palmeiras treinam �s 9h, no Centro de Treinamento.
+O treino deve ser s� recreativo.
+Depois do treino, os jogadores voltam ao Lord Hotel, onde est�o concentrados desde a noite de ontem.
+Interessado em organizar um livro de arte sobre viajantes, Ronaldo Gra�a Couto, da produtora Metav�deo/Metalivros, tomou conhecimento da pesquisa e procurou Belluzzo.
+Couto procurou a Odebrecht.
+Entre 40 outras id�ias, �Brasil dos Viajantes� foi selecionado para comemorar os 75 anos de atividade do grupo empresarial.
+Como naquela �poca ainda n�o existia a fotografia, eram os pintores que faziam os retratos das fam�lias.
+Rembrandt era o preferido das pessoas mais importantes.
+Ele era t�o bom artista, que muitos pintores quiseram aprender a pintar com ele.
+O livro �Apresentando Rembrandt -- Pintor, Gravador, Professor� (editora Klick), de Alexander Sturgis, conta muitas outras coisas sobre a vida e as obras do pintor.
+O governo do Estado realizar� um concurso p�blico para preencher 200 vagas de agente fiscal.
+As inscri��es come�am no pr�ximo dia 12, nas ag�ncias dos Correios.
+A taxa � de CR$ 4.300,00.
+Cerca de 20 mil pessoas fizeram cursos proficionalizantes no Senac (Servi�o Nacional de Aprendizagem Comercial) no ano passado, na Para�ba.
+Segundo o diretor regional da entidade, Glauco Pereira Chaves, a crise no mercado de trabalho estaria dificultando o aproveitamento dos formandos.
+Tudo indica que os telejornais da Globo v�o ganhar novo rep�rter especial.
+Antes de estrear como editor senior da �Playboy�, Nirlando Beir�o fez uma s�rie de testes de v�deo.
+Cientistas dos EUA confirmaram ontem em Quito (Equador) a contamina��o causada por atividades petrol�feras na Amaz�nia equatoriana.
+Segundo eles, casos de doen�as de pele em �ndios estariam relacionados � contamina��o.
+O Centro para Direitos Econ�micos e Sociais e m�dicos da Universidade Harvard informaram que amostras de �gua t�m n�veis entre 10 e 1.000 vezes maiores que os recomendados pela Ag�ncia de Prote��o Ambiental dos EUA.
+Para tentar resolver o problema do tr�fico de drogas e da viol�ncia no Rio de Janeiro, o governador do Rio e o Presidente da Rep�blica chamaram o Ex�rcito.
+Agora, os soldados e a pol�cia est�o trabalhando juntos para prender os traficantes.
+Eles est�o fazendo uma �guerra� contra o tr�fico.
+S� quando h� problemas muito graves, como o do Rio, o Ex�rcito � chamado para ajudar.
+Quando Jos� Sarney come�ou a escrever semanalmente para a Folha, chegaram protestos � reda��o.
+Como aceitar que, depois de uma passagem t�o ruim pela Presid�ncia, Sarney dispusesse de espa�o privilegiado no jornal?
+O tom familiar, coloquial, benigno de suas cr�nicas (agora reunidas em �Sexta-feira, Folha�, ed. Siciliano) foi pouco a pouco vencendo as resist�ncias do p�blico.
+Como cronista, Sarney toma o cuidado de n�o indignar ningu�m, e � dif�cil antipatizar com ele.
+Como a separa��o das marcas � tida como uma tend�ncia irrevers�vel, caberia � Volkswagen construir sua pr�pria f�brica.
+A TV-A atribui ao Plano Real a conquista de 17 mil assinantes em outubro, contra a m�dia de 10 mil assinaturas/m�s no in�cio do ano.
+Segundo o ministro, os juros desses empr�stimos ser�o compat�veis com os praticados no mercado internacional (8 % a 12 % ao ano).
+Ciro n�o quis confirmar se a TR (Taxa Referencial) deixar� de corrigir esses financiamentos.
+Ciro minimizou a estimativa de infla��o de 3% em outubro, de acordo com a Fipe.
+�A pr�pria Fipe prev� que em novembro isso tudo passa e a infla��o cai�, disse.
+Na segunda seguinte, o dia 4 ao dia 8, os bancos ficar�o abertos at� uma hora ap�s o expediente normal, apenas dez delegacias regionais do Banco Central e em ag�ncias do Banco do Brasil encarregadas de entregar o real aos demais bancos.
+A distribui��o acontecer� at� o final do m�s, partindo das capitais para o interior do pa�s.
+Cada banco buscar� no BC ou no BB o volume de dinheiro compat�vel com suas opera��es.
+Apenas 5 milh�es de contribuintes haviam entregue sua declara��o de Imposto de Renda at� ontem pela manh�.
+A estimativa da Receita Federal era receber 7 milh�es de declara��es.
+O n�mero final do total de declara��es entregues s� deve ser divulgado na pr�xima semana.
+Os 12 computadores da Future Kids oferecem jogos educativos que treinam o racioc�nio l�gico.
+No estande da Tec Toy h� tr�s consoles Mega Drive, al�m de um Master System e quinze cartuchos de jogos.
+As crian�as podem ficar 15 minutos em cada um dos jogos.
+Caso n�o haja fila, o per�odo de uso pode ser maior.
+A partir do dia 17, a Playtronic tamb�m ter� um estande no Iguatemi.
+Quatro meses depois do atentado, o governo argentino ainda n�o conseguiu avan�ar nas investiga��es sobre o atentado.
+No in�cio do m�s, o presidente Carlos Menem criou um fundo especial de US$ 1 milh�o para recompensar quem fornecesse informa��es que ajudem nas investiga��es.
+Quando os brit�nicos retomaram o Egito em 1801 a pedra foi parar no Museu Brit�nico, mas os franceses fizeram c�pias dela.
+O resultado foi que um franc�s, Jean-Fran�ois Champollion, que s� visitou o Egito em 1828, foi o primeiro a decifrar os hieroglifos.
+Uma carta de Champollion de 1822 � Academia de Inscri��es e Belas Artes, mostrando seus resultados, � o nascimento da egiptologia, segundo Vercoutter.
+Hoje, a praia ganhou nova infra-estrutura com bares, quiosques, sistema de ilumina��o, banheiros e quadras de esportes.
+Os jovens t�m se encontrado em Atalaia, principalmente entre quarta-feira e domingo, quando s�o realizados jogos de voleibol.
+A verdade � simples.
+Lula cresceu e prosperou enquanto tudo dava errado no pa�s.
+F�cil subir nos palanques, defendendo que � necess�rio �mudar tudo o que est� a�.
+Seu sucesso estava, essencialmente, sustentado no fracasso.
+No clima de sinistrose, as fragilidades do PT n�o ficaram t�o vis�veis � opini�o p�blica: a dificuldade de se fazerem alian�as capazes de sustentar Lula quando virasse presidente; os aloprados que conquistaram posi��es estrat�gicas no partido; a falta de generalizada convic��o sobre o valor da democracia.
+No final da reuni�o, o premi� italiano, Silvio Berlusconi, leu um documento de 45 par�grafos, pelo qual 140 pa�ses consideram que o crime organizado � o maior inimigo das democracias.
+Segundo n�meros da ONU, os cart�is do crime como a M�fia, a Yakuza japonesa e os cart�is de Cali e Medell�n, na Col�mbia, faturam anualmente US$ 750 bilh�es em atividades il�citas.
+Os �contratos de gaveta�, vistos pela CPI como mais um ind�cio de que as empreiteiras atuavam em cartel, s�o compromissos sigilosos entre as empresas.
+O sigilo � uma das exig�ncias contratuais.
+Al�m do contrato com a Servaz, h� outros do mesmo tipo, envolvendo mais empreiteiras.
+Os documentos foram encontrados em papel ou retirados de disquetes apreendidos na casa de Ailton Reis, diretor da Odebrecht.
+Com a Servaz, segundo o documento obtido pela Folha a Odebrecht acerta o pagamento de US$ 110 mil mais uma porcentagem referente a obras de abastecimento de �gua em Roraima, ainda a serem licitadas.
+Al�m disso, tem que se alterar o aparato jur�dico para favorecer a��es de negocia��o.
+Todo esfor�o deve ser no sentido de transformar o local do trabalho em primeira inst�ncia do processo trabalhista, com o poder de compor conflitos.
+S� assim se evitar� a estratifica��o da legisla��o, permitindo que o sistema acompanhe de maneira din�mica as transforma��es na economia.
+A Justi�a do Trabalho passaria a tratar apenas dos conflitos individuais, que n�o sejam resolvidos no pr�prio ambiente de trabalho.
+H� a necessidade de uma representa��o permanente do trabalhador que converse com o empregador e signifique garantia de que n�o haver� persegui��o aos negociadores, diz Siqueira.
+Foi enterrado ontem de manh� no cemit�rio Chora Menino, em Santana (zona norte de S�o Paulo), o ex-chefe do Departamento de Fotografia da Folha Waldemar Cordeiro, 63.
+Cordeiro morreu devido a complica��es decorrentes de um enfisema pulmonar (diminui��o da �rea dispon�vel para troca de ar nos pulm�es).
+A patinadora Tonya Harding, banida das competi��es por ter planejado atentado contra a rival Nancy Kerrigan, ser� o destaque de setembro da revista �Penthouse�.
+Segundo o jornal �USA Today�, a revista publicar� fotos nas quais Harding aparece mantendo rela��es sexuais com seu ex-marido, Jeff Gillooly.
+A legisla��o prev� multa de 300% sobre o valor da venda para quem se recusar a emitir a nota.
+Depois, haver� campanha de fiscaliza��o e aplica��o de multas para quem for pego em flagrante.
+O objetivo da campanha � aumentar a arrecada��o dos impostos.
+� gratificante exibir opini�es progressistas, dessas que fazem chover cartas de apoio no �Painel do Leitor�.
+A verdade, por�m, n�o d� cartas nem votos.
+A verdade ofende �somos uns bo�ais�.
+O fato � que o caos sangrento de nossas ruas � o resultado conjunto de nossas pr�prias a��es, embora ningu�m individualmente tenha a inten��o de produz�-lo ou sequer tenha poder para isso.
+A responsabilidade � a um s� tempo de todos e de ningu�m.
+N�s, motoristas, pais e jovens brasileiros, somos os piores inimigos de n�s mesmos.
+� um equ�voco reduzir a liberaliza��o comercial ao neoliberalismo.
+Afinal, a abertura come�ou na economia brasileira movida por considera��es pragm�ticas e objetivas.
+Simplesmente tornara-se imposs�vel, por raz�es tecnol�gicas, comerciais e financeiras, estimular o desenvolvimento com base na substitui��o de importa��es.
+Tal pol�tica convertera-se, com o avan�o da globaliza��o econ�mica, em sin�nimo de prote��o ao atraso, ao desperd�cio e � inefici�ncia.
+O governo agora apenas antecipou uma rodada de redu��o de tarifas prevista para janeiro de 95.
+E, com essa medida, pode matar dois coelhos.
+De um lado refor�a a estabiliza��o, pois qualquer eventual aumento irrespons�vel de pre�os depara-se agora mais rapidamente com a concorr�ncia de similares importados; de outro, ajuda a aliviar a supervaloriza��o do real, pois quanto mais importa��es houver, maior ser� a procura por d�lares.
+Fazer avan�ar uma reforma estrutural (como a abertura da economia) e, ao mesmo tempo, calibrar um processo de estabiliza��o n�o � por�m uma tarefa f�cil.
+Resta portanto saber se as medidas anunciadas, em tese adequadas, ter�o na pr�tica o efeito que se espera.
+Com o d�lar comercial j� a R$ 0,85 e o governo dizendo que n�o h� limite para baixo, a necess�ria abertura pode involuntariamente desaguar num arriscado escancaramento.
+O livro de Marilena Ansaldi tamb�m oferece detalhes sobre uma das mais bem-sucedidas experi�ncias culturais de S�o Paulo: o Teatro da Dan�a, que funcionou a partir de 1975 na Sala Gil Vicente do Teatro Ruth Escobar.
+Esse palco, al�m de revelar in�meros talentos que depois se �refugiaram� na Europa, como Sonia Motta, serviu para Marilena inaugurar a fase mais f�rtil de sua carreira.
+BRIZOLA � O 3� NO RIO, com 13 % .
+Estava vivo o sentimento �tnico e nativista.
+Seu instrumento de ser ouvido era a for�a da viol�ncia.
+Em dois pa�ses esses �ndios ainda n�o esqueceram o trauma da conquista: Cort�z, no pr�prio M�xico, e Pizarro, no Peru.
+� o del�rio que alimenta o Sendero Luminoso.
+� a vigorosa hist�ria mexicana que surpreende em Chiapas.
+O acordo n�o se tornou p�blico, mas, agora, se sabe que foram feitas muitas concess�es pol�ticas.
+Nas grandes cidades, multid�es imensas sa�ram �s ruas em apoio aos rebeldes.
+A rebeldia n�o � a revolu��o, como bem acentua Octavio Paz.
+A rebeldia � um ato pessoal; a revolu��o � uma manifesta��o coletiva.
+Mesmo asim, Van Himst pretende surpreender a defesa saudita, colocando dois atacantes velozes.
+Al�m do zagueiro Grun e do centroavante Weber, o lateral-esquerdo Vital Borkelmans encontra-se tamb�m com um cart�o amarelo e pode ficar de fora hoje.
+Eles se intitulavam filiais da MIT Dealing Management Inc., dos Estados Unidos.
+A empresa norte-americana informou � PF que n�o tem filiais ou representantes no Brasil.
+A MIT brasileira n�o aplicava o dinheiro.
+O golpe foi denunciado por funcion�rios da empresa.
+A a��o da PF come�ou �s 16h30 de anteontem e s� terminou ontem �s 15h30.
+A MIT operava no mercado desde junho passado.
+A PF ainda n�o levantou os n�meros exatos do golpe, mas acredita que pelo menos cem pessoas tenham perdido seu dinheiro.
+Em n�vel internacional, h� um processo de conglomeriza��o de empresas que, ao permitir a integra��o de softwares, conferiu aos grandes grupos vantagens expressivas sobre os independentes.
+Tem que se pensar, para o Brasil, em uma empresa com modelo acion�rio flex�vel, que permita incorporar, em uma �nica marca, os esfor�os individuais desses criadores e dispor da sinergia necess�ria para investir no mercado internacional.
+O jogador disse que antes de voltar � Espanha visitar� o Estado onde nasceu, a Bahia.
+Ontem levou, no Rio, os dois filhos a uma visita de rotina ao pediatra.
+�Se sentir que h� um desejo coletivo, ele vai encaminhar as emendas constitucionais�, disse Luiz Henrique ap�s uma hora de conversa com Itamar, ontem, no Planalto.
+Luiz Henrique disse que o PMDB vai procurar os presidentes e l�deres dos outros partidos em busca do consenso sobre a vota��o da reforma fiscal e tribut�ria.
+A aprova��o das emendas � considerada essencial para o Plano Real.
+O corredor come�ar� na marginal Tiet�, na altura do Cebol�o, passar� pelo Ceagesp, pra�a Panamericana, Faria Lima, Lu�s Carlos Berrini e terminar�, por enquanto, no shopping Morumbi.
+O escrit�rio de J�lio Neves j� come�a a preparar novos estudos para o prolongamento deste corredor al�m do shopping Morumbi, em dire��o � ponte do Socorro.
+N�o sei se fui claro: seria mais f�cil um pinguim cruzar tr�s Saaras do que FHC comer carne de bode espremido a um magote de sertanejos suados.
+Compreendeste?
+N�o fosse a elei��o, d. Luciano Mendes teria de negar Cristo tr�s vezes para que nosso senador empuleirasse no lombo de um cavalo, envergando aquele chap�u de cangaceiro.
+O caso de Lula � semelhante.
+O candidato do PT bateu na trave em 89.
+Agora imagina que � chegado o momento.
+Sente o bafejar da sorte a acariciar-lhe a cara.
+Os metal�rgicos do Estado de S�o Paulo da base da For�a Sindical (data-base em novembro) romperam negocia��o com os setores de autope�as, forjaria e parafusos e preparam greve.
+Os empres�rios ofereceram 10,4 % em novembro para S�o Paulo, Osasco e Guarulhos e 6 % mais 6,27 % em mar�o para o interior.
+A categoria pede 69,69%.
+O Sindicato dos Metal�rgicos do Rio e do Grande Rio (data-base em outubro) se re�ne hoje com representantes da Firjan (federa��o das ind�strias do Rio) pela primeira vez desde o in�cio da greve da categoria, na ter�a-feira.
+O sindicato pede 99% e os empres�rios oferecem 13,56%.
+A greve �por empresa� atingia ontem sete empresas e 12 mil empregados.
+O governador da Fl�rida, Lawton Chiles, 63, resolveu processar o governo dos EUA para recuperar os US$ 739 milh�es anuais que ele diz estar gastando em despesas com imigrantes ilegais em seu Estado.
+Ele acusa o governo federal de n�o cumprir bem sua miss�o de impedir a entrada no pa�s de estrangeiros e de n�o repor o dinheiro gasto pelos Estados com eles.
+A decis�o tem evidentes objetivos eleitorais.
+Chiles, que pertence ao Partido Democrata e ao mesmo grupo neoconservador que ajudou Bill Clinton a chegar � Casa Branca, vai tentar se reeleger em novembro deste ano.
+Sua popularidade est� mais baixa do que nunca e um de seus poss�veis oponentes � filho de George Bush.
+O governo se comprometeu a n�o �garfar� a poupan�a, nas palavras do presidente do BC.
+Estabeleceu que as tarifas p�blicas v�o todas marchar ao compasso da URV at� meados de maio.
+Depois ser� a vez dos financiamentos rurais e, no in�cio de junho, acenar� os rumos que poder�o tomar os pontos fundamentais do plano a pol�tica monet�ria e a pol�tica cambial.
+Ao justificar sua necessidade, o presidente da Rep�blica abriu uma fresta para a realidade na propaganda sobre as virtudes da URV.
+Mesmo indexado, �o sal�rio convertido em cruzeiros reais continua a ser corro�do pela infla��o�.
+Mais de 110 mil pessoas assistiram � partida, no est�dio Santiago Bernabeu.
+O atacante Rom�rio n�o atuou bem.
+Afastado de seu companheiro de ataque, Stoichkov, pegou pouco na bola.
+Para Itamar, Stepanenko 'est� perdido'.
+O presidente Itamar Franco disse ontem que o ministro Alexis Stepanenko (Minas e Energia) autor de v�rios bilhetes pedindo apoio ao candidato Fernando Henrique Cardoso �est� perdido�.
+�Ele est� perdido na China�, disse Itamar, com bom humor.
+Stepanenko viajou para a China h� uma semana e deve voltar na sexta-feira.
+A demiss�o do ministro � dada como praticamente certa.
+Ir a uma feira livre serve de aula para os principiantes aprenderem a comprar frutas, legumes e verduras mais baratos e de melhor qualidade.
+Mas � preciso disposi��o para andar a feira toda, com caderno e l�pis na m�o, comparando os pre�os dos produtos.
+O advogado Mariano Gon�alves Neto, autor da a��o popular que se arrasta h� 12 anos, acusa os ex-ministros de aprovarem uma superavalia��o dos terrenos entregues pelo grupo Delfin como pagamento de uma d�vida que, em 1982, chegava a Cr 70 milh�es.
+O ex-ministro da Fazenda Ernane Galv�as (governo Figueiredo) disse que sua participa��o no epis�dio limitou-se a um �de acordo� no processo que teria vindo pronto do Banco Nacional de Habita��o, subordinado na �poca ao Minist�rio do Interior.
+O epis�dio � emblem�tico do tipo de pol�tica externa exercitada pelo governo Clinton e ajuda a explicar por que ela � t�o mal avaliada.
+At� agora, o Bradesco Visa financiava em at� quatro parcelas, com os juros cobrados sobre os cruzeiros reais.
+Para o lojista, a nova modalidade n�o traz mudan�a alguma.
+Ele receber� o valor � vista ap�s 30 dias.
+O candidato � Presid�ncia pelo PT (Partido dos Trabalhadores), Luis In�cio Lula da Silva, � o convidado de Mar�lia Gabriela no �Cara Cara�.
+CARA CARA Bandeirantes, 0h.
+Marco Antonio Nahum, advogado dos canadenses, nega que o premi� venha ao Brasil tratar da expuls�o.
+�Isso ainda n�o ficou acertado pelo governo canadense�, disse.
+A fam�lia de Christiane j� gastou aproximadamente US$ 1 milh�o para tentar a sua liberta��o.
+Contratou a empresa Winfreys, a maior especialista em atividades de lobby, em todo o Canad�.
+Terminado o Mundial, todas as aten��es das sele��es j� se voltam para a Olimp�ada de Atlanta.
+Um exemplo � o t�cnico norte-americano, Taras Liskevich, que n�o esconde seu objetivo: conquistar a medalha de ouro ol�mpica.
+Para isso, a comiss�o t�cnica n�o se cansa de estudar os outros times.
+Nesta temporada, um dos mais visados foi o Brasil.
+Ele filmaram todos os jogos das brasileiras.
+A d�cada de 90 n�o tem sido boa para a China.
+O time, que nos anos 80 dominou o v�lei feminino, voltou a decepcionar no Mundial.
+Novamente, n�o conseguiu ficar entre os quatro melhores.
+Em 92, na Olimp�ada de Barcelona, as chinesas terminaram em s�timo lugar, coloca��o que provocou grandes mudan�as na equipe.
+O Santo Andr� perdeu por 2 a O para o Rio Branco, de Americana, ontem � tarde no est�dio Bruno Jos� Daniel, em Santo Andr�.
+Para o Rio Branco, bastou o primeiro tempo do jogo para determinar o resultado.
+Souza, ponta-esquerda do time de Americana, abriu o marcador aos 25min.
+Paulo Cesar, cobrando falta na cabe�a da �rea, fechou o marcador aos 43min, sem chances de defesa para o goleiro S�lvio, do Santo Andr�.
+Para se ter uma id�ia, no Brasil a melhor marca � de Hilma, com 3,12 m..
+N�o por acaso, o t�cnico Bernardinho diz que o time cubano ataca do terceiro andar.
+Parar esse ataque no bloqueio � uma miss�o quase imposs�vel.
+O presidente Itamar Franco tem, como qualquer cidad�o brasileiro, o direito de apoiar a candidatura para a Presid�ncia da Rep�blica de quem bem entender.
+Pode tamb�m cobrar dos membros de sua administra��o que ap�iem o seu candidato ou deixem o governo.
+Ainda assim, os planos do Planalto de dedicar apoio total ao ex-ministro Fernando Henrique Cardoso, como esta Folha revelou na edi��o de ontem, esbarram em problemas.
+Embora n�o ocorra no Brasil h� muito, � absolutamente normal, nos pa�ses de maior tradi��o democr�tica, que o governo tenha um candidato e o ap�ie.
+Ronald Reagan n�o poupou esfor�os para eleger seu vice, George Bush, presidente dos Estados Unidos, em 1988.
+A defesa apresentar� o �recurso extraordin�rio� no �ltimo dia do prazo: tr�s dias �teis ap�s a publica��o no Di�rio da Justi�a da decis�o do TSE sobre o �embargo declarat�rio� pedido pelos advogados de Lucena.
+Os advogados de Lucena calculam que isso n�o deve ocorrer antes de ter�a ou quarta, com Lucena j� eleito, segundo acreditam.
+O embargo � uma medida para que o TSE admita a constitucionalidade da quest�o o que permite que o assunto seja levado ao STF.
+FHC -- N�o h� temor de recess�o.
+N�o estamos fazendo pol�tica recessiva.
+N�o estamos diminuindo investimentos do setor privado e, no setor p�blico, o BNDES est� mudando a orienta��o de investimentos do Nordeste.
+Estamos num ciclo de expans�o.
+Folha -- Mas no momento em que a nova moeda for criada, se os pre�os forem convertidos no pico, n�o ser� necess�rio um aperto monet�rio, uma recess�o tempor�ria?
+A ag�ncia de comunica��o �Free Press�, do comit� do candidato Fernando Henrique Cardoso (PSDB-PFL-PTB), reconheceu ter errado na divulga��o de uma entrevista do diretor para Assuntos Internacionais do Banco Central, Gustavo Franco, concedida a ela como pe�a de campanha.
+Na sua defesa encaminhada ontem ao TSE, ela admitiu ter cometido um �erro de interpreta��o� na produ��o da reportagem na qual Franco afirma que o PT, caso ganhe as elei��es presidenciais, pode dar �um calote na d�vida interna�.
+Promotor p�blico, procurador da Justi�a e um dos criadores da Comiss�o Justi�a e Paz da Arquidiocese de S�o Paulo, Bicudo lista como outras prioridades temas de sua especialidade: a reforma da Justi�a e do sistema penitenci�rio.
+Folha -- Quais s�o suas linhas priorit�rias de atua��o na C�mara dos Deputados?
+Para tentar se recuperar do jet lag e da correria da exposi��o do marido no Yerba Buena, Regina Cas� fez ontem sa�da pela tangente.
+Junto com seu guapo Luiz Zerbini, foi at� Muir Woods um bosque nas imedia��es da cidade, onde s� queria era abra�ar �rvores de prefer�ncia, enormes sequ�ias.
+O grupo P�o-de-a��car, 2� maior rede de supermercados do pa�s, iniciou negocia��es para a aquisi��o das 27 lojas G.Aronson, todas localizadas em S�o Paulo.
+A Folha confirmou os entendimentos junto a empres�rios do setor de eletrodom�sticos e a funcion�rios do P�o-de-a��car.
+Oficialmente, a rede nega.
+Folha -- Em torno dele n�o estavam pessoas que hoje est�o na sua candidatura?
+FHC -- Certamente, e da�?
+O candidato do PT apresentou o vice de seu principal advers�rio como algu�m �identificado com o autoritarismo�.
+At� aqui, os tiros de Lula parecem n�o ter abalado o alvo.
+�Meu passado muito me orgulha�, d� de ombros Maciel.
+O governador se convenceu ainda de que, se abandonassse a candidatura, poderia se tornar ref�m de Qu�rcia e perderia o poder de influ�ncia na escolha do candidato do PMDB.
+A declara��o de Fleury de que continua candidato n�o convenceu alguns prefeitos que o acompanhavam.
+�Ele j� est� abra�ando a candidatura Qu�rcia�, disse Itamar Borges (PMDB), de Santa F� do Sul.
+Fleury aproveitou a viagem para responder ao ex-secret�rio Jos� Machado de Campos Filho, que o chamara de traidor por n�o apoiar Qu�rcia.
+�Para quem quer ser candidato ao Senado, faltou bom senso e ju�zo�, disse.
+Ao visitar a ponte rodoferrovi�ria de Rubin�ia, criticou o deputado quercista Alberto Goldman (PMDB-SP), ex-ministro dos Transportes.
+�Ele veio aqui e liberou US$ 5 milh�es, 1% do valor da obra.�, declarou.
+Colaborou Aur�lio Alonso, da Folha Norte.
+Apesar de todas as restri��es, a esmagadora maioria dos consultados (86%) acredita que Lula tomar� posse sem resist�ncia de qualquer setor da sociedade.
+H� uma pequena parcela, no entanto, que � catastrofista ao ponto de afirmar que Lula sequer conseguir� tomar posse.
+S�o apenas 2%.
+Para outros 9%, haver� resist�ncias mas n�o a ponto de inviabilizar a posse.
+Burle Filho disse que combinou a ida de seu auxiliar ap�s telefonar para seu colega carioca, Ant�nio Carlos Biscaia.
+Segundo Burle Filho, Biscaia n�o lhe adiantou nada por telefone sobre o assunto.
+�Queremos saber se o bicho de S�o Paulo e o do Rio s�o ramifica��es de uma grande organiza��o criminosa.
+Ao mesmo tempo, estamos esperando den�ncias sobre a atua��o dos bicheiros paulistas�, disse Burle Filho.
+Mas o deixaram partir.
+Infante -- E pagaram as passagens.
+Me deram uma licen�a de dois anos, acreditavam que o ex�lio ia me neutralizar.
+Tinha uma fam�lia para sustentar, e pensavam que isso me impediria de fazer qualquer coisa.
+Pensavam que meu livro �Tr�s Tristes Tigres� seria recebido sem grande estardalha�o.
+Outro lado de Lara: ela estudou canto l�rico, adora m�sica e costuma cantar em shows e jam sessions com amigos.
+Como fazia em Madri.
+A id�ia para o livro surgiu ap�s uma entrevista que Andr�a fez em meados de 93 para o �Tokyo Journal� com o jogador Zico, � �poca atuando no Kashima.
+�O editor da revista me telefonou e deu a id�ia.
+Fizemos um projeto e comecei a elaborar a estrutura do livro�, diz Andr�a.
+Nos fundos de commodities h� liquidez di�ria ap�s 30 dias.
+Na transi��o para o real, s�o mais recomendados os fundos de renda fixa DI (Dep�sito Interfinanceiro).
+RIO DE JANEIRO Aos trancos e barrancos, eis que mais uma vez o pa�s tenta dar um passo em dire��o a um futuro mais feliz.
+O panorama � assustador: amea�a de greve, possibilidade de hiperinfla��o e descontrole de pre�os, perda salarial, nuvens negras para todos os lados.
+Exercendo o direito de n�o estar a favor ou contra ningu�m, gostaria aqui de remar contra a mar� e pasmem manifestar meu otimismo.
+Por que n�o?
+Se h� um artigo em falta no mercado hoje todo tipo de mercado, diga-se � o otimismo.
+Por que n�o ousar ainda que ingenuamente esperar que as coisas desta vez se ajeitem?
+Olha o Carnaval de sal�o!
+S� bela!
+S� piranha!
+S� lulite!
+�Eu n�o s� lulite, s� gostosa�.
+Rarar�!
+E bomba!
+Bomba!
+Sim�o Emerg�ncia!
+Um camel� cearense inventou o kit Aids: um par de joelheiras e 45 camisinhas!
+Rarar�!
+Come�ou a esculhamba��o?!
+� Carnaval!
+E os nomes dos blocos?
+Balan�a Rolha, Sacode a Rosquinha e Peru Esperto!
+�Bloco da Galinha Azul Solta a Franga�.
+�Alpinista Sexual s� Scala Monte L�bano�.
+�N�o, eu vou pescar�.
+�Ah bem, pensei que voce ia pescar�.
+� mole?
+� mole mas sobe!
+Principalmente no Carnaval!
+T� no ar mais uma cal�nia do Macaco Sim�o!
+Pior � o Itamar que ficou intoxicado com o pr�prio pastel�o!
+Nervosas!
+Atacadas!
+Inadimplentes do sexo em geral!
+Tem pelad�o no sal�o!
+Os Piranhos do Kung Fu invadem o sal�o!
+Essa � a grande novidade do Carnaval 94: os pelad�es.
+Antes o Scala e o Monte L�bano s� contratavam peladas.
+Depois do Clube das Mulheres (em Portugal, Clube das Bigodudas) os pelad�es invadem os sal�es.
+Gente, � cada piranh�o!
+Parece remador de porta avi�o!
+E bem Clube das Mulheres: sunga, punho e gravatinha.
+E eu sei que � tudo bofe mas parece tudo gay.
+Capa da revista �Honcho�.
+Est�tica gay!
+Um rinoceronte de sunga � mais sexy!
+Pelo menos agora na televis�o tem pra todo mundo.
+No Brasil n�s j� estamos com oito tipos de op��o sexual: perua, bofe, bofeca, boneca, traveca, hermafrodita, drag queen e drag king.
+E tem gente que ainda fica na m�o?
+Rarar�!
+Carnaval devia ser o ano todo.
+Cinco dias � pra amador!
+A fazenda de renas Napapiiri, nas proximidades (tel. 38-4048) promove jantares com carne de rena e faz exibi��es folcl�ricas.
+Pre�o: cerca de US$ 43.
+Dos museus, o melhor � o Arktikum (leia reportagem ao lado).
+Mas h� dois muito instrutivos, a c�u aberto por isso, as visitas v�o s� de junho a setembro.
+�Achei a id�ia muito interessante e educativa.
+A gente precisa de reciclagem em geografia, pois tudo muda muito r�pido.
+L� em Portugal, eu costumo comprar a Folha�, afirma Barra.
+Em S�o Jos� do Rio Preto, as duas bancas mais tradicionais da cidade tiveram filas na distribui��o do terceiro fasc�culo do Atlas Folha/�The New York Times�.
+Esses n�cleos coloniais, entre eles o do Vale do Ribeira, foram ocupados a partir de terras vendidas pelo governo.
+Para quem vai fazer a pesquisa no computador a separa��o de dados n�o faz nenhuma diferen�a: tudo funciona como se fizesse parte de um s� arquivo.
+Segundo Milton Julio Zanluqui, da Paraklin P�ra-raios, entre os tipos mais comuns (veja exemplos ao lado) est�o o sistema de gaiola (Gaiola de Faraday), composto de seis partes principais: captador do tipo terminal a�reo, cabo de cobre, suportes isoladores, tubo de prote��o, malha de aterramento e conector de medi��o.
+Esse equipamento �envolve� o pr�dio e � indicado para constru��es que ocupem �reas horizontais, como galp�es e ind�strias.
+� preciso que tenhamos a consci�ncia de que os nossos tratados, que envolvem quest�es transcendentais que buscam uma circula��o de bens e servi�os sem barreiras entre nossos pa�ses, exigem uma vis�o da hist�ria.
+N�o podemos deixar morrer o esp�rito de Igua�u.
+Ele fechou um tempo de diverg�ncias para abrir uma era de converg�ncias.
+Jos� Sarney escreve �s sextas-feiras nesta coluna.
+�O Rio sempre foi um lugar que deu exemplo para o Brasil�, disse o corregedor eleitoral, tentando desfazer o car�ter de �interven��o branca� atribu�do � sua viagem ao Rio de Janeiro.
+�N�s n�o vamos acreditar que um povo t�o culto como o da Guanabara possa ter problemas como andam dizendo por a�, afirmou, referindo-se � popula��o do antigo Estado da Guanabara, que desapareceu na fus�o com o Estado do Rio de Janeiro.
+Um, o convencional, � o que o Plano Real contempla e tem como instrumentos essenciais a importa��o � taxa cambial fixa e baixa e quase sem tarifa aduaneira de mercadorias, tornando imposs�vel pela competi��o os produtores e comerciantes brasileiros aumentarem seus pre�os; e a redu��o da demanda por bens de consumo e de investimento, mediante juros altos e corte do gasto p�blico, impedindo as remarca��es de pre�os pelo excesso de oferta suscitado.
+O outro, heterodoxo e inconvencional, consiste em controlar as press�es inflacion�rias em sua origem, nas cadeias produtivas, mediante a fixa��o negociada entre todos os setores participantes de tetos para os aumentos de valores, sejam estes pre�os ou sal�rios.
+A agita��o ontem foi grande tamb�m na Bolsa de Mercadorias com frequentes boatos de dificuldades de institui��es financeiras em cumprirem seus compromissos no mercado futuro do �ndice Bovespa.
+A proje��o de juros acumulados para este m�s no mercado futuro caiu de 3,76% no dia anterior para 3,71% ontem.
+O impacto provocado pela jazz�stica trilha de �A Marca da Maldade�, com temas influenciados por Stan Kenton e at� por pianistas de bordel, chamou a aten��o de Edwards, que em 1958 procurava um m�sico novo, talentoso e barato para compor o prefixo da teless�rie �Peter Gunn�.
+O jovem (34 anos), talentoso e barato Mancini entrou rachando.
+A fanfarra que a seguir comp�s para �Dizem que � Amor� (High Time) s� n�o ficou tamb�m gravada na mem�ria de todo mundo porque, daquela vez, Edwards nem sequer chutou na trave.
+De todo modo, uma s�lida e longa parceria se configurava, alternando com�dias e dramas cuja qualidade cinematogr�fica variava infinitamente mais que as virtudes de suas trilhas sonoras.
+Quando os dois acertavam juntos (�Bonequinha de Luxo�, �V�cio Maldito�, �A Pantera Cor-de-Rosa�), era uma festa para todos os sentidos.
+E faturamento em dobro para o compositor, que n�o perdia uma chance de estourar no �hit parade� com uma can��o.
+Por duas delas, �Moonriver� e �Days of Wine and Roses�, ambas com letra de Johnny Mercer, ganhou o Oscar.
+O baiano Dias Gomes, 70, est� se preparando para ressuscitar um cl�ssico da teledramaturgia brasileira.
+Semana que vem, come�a a reescrever �Irm�os Coragem�, novela de Janete Clair que a Globo exibiu com enorme sucesso entre junho de 1970 e julho de 1971.
+A emissora pretende lan�ar o �remake� em janeiro de 95, �s 18h, no lugar de �Tropicaliente�.
+No centro de todos esses nossos esfor�os de qualidade est� o nosso cliente.
+� tudo pelo cliente.
+A principal meta da melhoria da qualidade da Degussa no Brasil � a satisfa��o total do cliente.
+� o cliente quem define os requisitos que teremos de cumprir e, at� mesmo, superar, praticando qualidade em sua plenitude e obtendo, assim, seus benef�cios, que s�o competitividade, lucratividade e satisfa��o.
+Entendemos que palavras como respeito, aten��o e �tica, incluindo a efici�ncia na presta��o dos servi�os, devem mais do que nunca, nortear nossa conduta nas rela��es com nossos clientes.
+PT, PDT, PC do B e os outros �contras� (partidos contr�rios � revis�o) exigiram que as vota��es do Congresso revisor come�assem pela quest�o tribut�ria.
+A nova reivindica��o irritou os partidos favor�veis � revis�o, que criticaram a proposta.
+No dia 1� de mar�o, a funda��o inaugurou mais uma escola.
+Ela foi constru�da em Mar�lia (cidade onde foi fundada o banco) com capacidade para mil alunos.
+Como nas outras escolas, ali os alunos recebem gratuitamente o ensino, o material did�tico, o uniforme, a alimenta��o e tratamento m�dico e odontol�gico.
+A exclus�o de Maradona, por ter jogado dopado contra a Nig�ria, abalou os jogadores.
+O time, antes bastante aguerrido e solid�rio, passou a discutir em campo.
+A Argentina fez uma op��o bastante ofensiva.
+Variava entre um 4-3-3 e um 4-2-4.
+O t�cnico Alfio Basile tentou aproveitar seus melhores jogadores.
+Ainda h� muito o que fazer para desgravar completamente as exporta��es de impostos indiretos, conforme permitem as regras do Gatt e imp�e a realidade de um ambiente internacional competitivo.
+Mas a Uni�o foi ao limite em termos de desgrava��o das exporta��es.
+Antes da urgente reforma constitucional, a extens�o da desonera��o tribut�ria das exporta��es para todas as etapas da cadeia produtiva enfrenta dificuldades operacionais insuper�veis no �mbito federal.
+Seria invi�vel estimar, por exemplo, com a acuidade necess�ria para a concess�o de cr�dito fiscal, qual a incid�ncia efetiva de PIS e Cofins em cada etapa do processo produtivo anterior � produ��o dos insumos usados diretamente na produ��o para exporta��es.
+Reginaldo Duarte pertence a uma fam�lia de pol�ticos de Juazeiro do Norte (CE), mas nunca havia disputado elei��o.
+�Sempre fugia da raia.
+Achava que n�o tinha voca��o pol�tica�, explica.
+Em 90 ele n�o p�de fugir.
+�O Tasso Jereissati quis homenagear a minha regi�o e pediu um nome.
+Fui escolhido e n�o queria aceitar.
+Mas aceitei porque foi uma gentileza do governador�, diz.
+J�nice Trist�o afirma que acompanha a vida pol�tica de Elcio �lvares h� duas d�cadas.
+Nunca havia disputado mandatos eletivos.
+Acha que foi escolhido como suplente pela sua imagem empresarial formada em mais de 50 anos.
+O t�cnico Sacchi elogiou ontem a �bravura� de seus jogadores, especialmente de Baggio, �com seu toque m�gico para ganhar partidas�.
+Ainda assim, foi a m�dia que amplificou as acusa��es, ao divulg�-las sem uma checagem mais aprofundada.
+Culpa da m�dia, ent�o?
+� primeira vista, sim, mas � a� que se chega ao verdadeiro n� da quest�o.
+Por partes: a lista de nomes mais completa foi ao ar durante o Jornal Nacional, da Rede Globo, por volta de 20h.
+De que adiantaria os jornais do dia seguinte omitirem os nomes, se eles j� haviam sido arrastados � lama por um notici�rio que atinge cerca de 60 milh�es de pessoas, certamente mais do que a tiragem conjunta de todos os jornais brasileiros?
+O treinador Levir Culpi n�o quis adiantar o substituto de �der Aleixo.
+Segundo ele, a sa�da do meia-ponta representa uma �perda importante� para o time.
+�Vamos com calma achar o substituto do �der�, disse.
+Culpi, no entanto, acha que o time pode suportar a press�o e �confirmar a vantagem� em S�o Paulo.
+O time pode empatar a segunda partida para se classificar.
+O assessor de comunica��o da Reduc, Fernando Fortes, disse que os grevistas est�o desrespeitando a determina��o do TST.
+�O que nos preocupa � o cumprimento da ordem judicial.
+Os grevistas est�o irredut�veis.
+O TST fala em manter a produ��o de combust�veis e g�s.
+Se eles n�o substitu�rem o pessoal, v�o estar praticando desobedi�ncia judicial�, disse Fortes.
+�Nenhum eleitor evang�lico deve se sentir culpado por ter opini�o diferente de seu pastor ou l�der espiritual�, diz a cartilha.
+O texto defende o voto para a Presid�ncia baseado em programas de governo.
+E condena a dissemina��o de boatos, como os que atingiram Luiz In�cio Lula da Silva na elei��o de 89.
+A IRLF s� aprova o aborto quando a gravidez coloca em risco a vida da m�e.
+Mesmo quando a gravidez decorre de estupro ou quando o feto n�o tem chances de sobreviv�ncia, o aborto � visto como crime.
+A IRLF condena ainda a eutan�sia e os m�todos contraceptivos n�o naturais, inclusive a p�lula.
+A t�cnica tradiciona de tomografia n�o conseguia �ver� o abcesso no dente.
+Com o uso do m�todo tridimencional, foi poss�vel localiz�-lo e descobrir a causa da morte.
+O artista, cujo nome de batismo � Lu�s Carlos dos Santos, pagou uma fian�a de CR$ 100 mil e foi liberado.
+A Folha procurou falar com Melodia sobre o incidente.
+O telefone da casa do m�sico permaneceu conectado a uma secret�ria eletr�nica.
+�Foi barra pesada, pelo som pareciam armas superpotentes, n�o foi um tiroteio qualquer�, afirmou Tatiana Marques Pedrosa, 24, estudante de psicologia.
+Tatiana mora na rua S�o Clemente, Botafogo, em frente � principal via de acesso ao morro.
+Morando h� seis anos ali, Tatiana disse que nunca viu um per�odo t�o violento.
+�H� dois meses que a coisa est� pior.
+Com tanta viol�ncia, sinto uma ang�stia muito grande�, afirmou.
+A expectativa do Sebrae era que o volume de neg�cios chegasse a US$ 7 milh�es na Fenit.
+Em 95, no entanto, a participa��o dos pequenos subsidiados pelo Sebrae na Fenit vai diminuir.
+�Vamos reduzir o n�mero de expositores para cem.
+A id�ia � privilegiar os de maior qualidade e as feiras estaduais�, diz Souza.
+Ao lado da Fenit foi realizada a Fenatec (Feira Internacional de Tecelagem), com representantes da Argentina, Alemanha, Fran�a e Estados Unidos.
+O IPMF padece de males profundos.
+Mas est� correto o conselheiro Paulo Planet Buarque.
+Analisado isoladamente, � um imposto democr�tico e que, se levado � sua consequ�ncia l�gica, o Imposto �nico, poderia consumar a revolu��o tribut�ria que o pa�s deseja e precisa.
+MARCOS CINTRA CAVALCANTI DE ALBUQUERQUE, 48, doutor em economia pela Universidade de Harvard (EUA), � vereador da cidade de S�o Paulo pelo PL e professor da Funda��o Get�lio Vargas (SP).
+Foi secret�rio do Planejamento e de Privatiza��o e Parceria do munic�pio de S�o Paulo (administra��o Paulo Maluf).
+Tamb�m nisto B�scoli estava em seu elemento: poucos podiam ser mais hilariantemente viperinos desde que a piada n�o fosse dirigida contra voc�.
+E havia outro motivo para que ele atra�sse a ira de tantos: seu sucesso com as mulheres.
+Cite uma cantora da bossa nova ou estrela da televis�o dos anos 60 e h� poucas chances de que ele n�o a tenha namorado sempre dando a impress�o de que fazia isto por tarefa.
+�Pela bossa nova, namoro at� o Trio Irakitan�, dizia.
+De acordo com Maluf, a equipe de elabora��o do programa ter� 30 dias para trabalhar e o nome mais cotado para coorden�-la � o do presidente nacional do PFL, Jorge Bornhausen.
+Segundo ele, a parte econ�mica do programa poder� ser elaborada pelo ex-ministro da Fazenda Gustavo Krause (PFL-PE).
+Segundo Maluf, o programa de governo seria �o ponto de converg�ncia� para a discuss�o uma futura alian�a eleitoral na sucess�o presidencial.
+�Ser� um programa capaz de unir todas for�as que querem levar o Brasil para o Primeiro Mundo.
+Um programa capaz de combater a infla��o, o desemprego e que abra uma corrente de investimentos para o pa�s�, disse Maluf.
+Se eu fosse voc�, passava a andar de t�xi especial e apresentava a conta � concession�ria.
+Ligue imediatamente para o Procon (tel. 829-3055) e solte os cachorros.
+O livro est� em toda parte e eu sou daqueles que n�o conseguem ver a imagem em outra dimens�o.
+O que fa�o?
+O Brasil � um pa�s do Terceiro Mundo, ainda sem cara e sem personalidade, mas metade do seu territ�rio � a Hil�ia, que tem fascinado os iluminados, de Humboldt a Euclides da Cunha; acontece que, no passo de Jeca Tatu em que caminha, o Brasil amea�a transformar a Hil�ia, no s�culo 21, que j� nos olha com cara feia por cima do muro, num gigantesco cinzeiro, cheio da cinza das �rvores e da cinza dos �ndios.
+O Brasil s� sair� da sua mediocridade se se deixar incorporar � Amaz�nia.
+E jamais chegar� a isto se n�o tiver a total dedica��o de algu�m como Ricupero, um excepcional �cadre�, que em pouco tempo convenceu os brasileiros da sua intelig�ncia e compet�ncia na vida p�blica.
+Quando Ricupero foi pin�ado da sua miss�o amaz�nica, lembrei Osvaldo Aranha, que dizia que o Brasil � um deserto de homens e de id�ias.
+A TVs Bandeirantes e TVA/ESPN abrem espa�o para o futebol americano hoje.
+A Bandeirantes mostra o videoteipe de Kansas e Atlanta, �s 17h45.
+A TVA apresenta em VT a partida entre Detroit e Dallas, �s 7h.
+Promotores da Sic�lia pediram � Justi�a local o julgamento de 23 pessoas, acusadas de conspirarem para assassinar Salvo Lima, pol�tico democrata-crist�o morto pela M�fia pouco antes das elei��es de 1992.
+A pol�cia deu ordem para prender mais sete pessoas suspeitas de envolvimento.
+O caso Lima coincide com a pris�o de 54 pessoas, como parte de uma opera��o anti-M�fia no oeste da ilha.
+Ignacio Perea, 32, foi condenado a cinco penas de morte em Miami.
+Ele sequestrou e violentou tr�s meninos com a inten��o de lhes transmitir o v�rus da Aids de que se sabia portador.
+As penas foram para sequestro, viola��o e tentativa de homic�dio.
+� a primeira vez nos EUA que um tribunal considera o v�rus HIV como uma arma letal.
+O modelo de privatiza��o previsto na MP foi organizado pelo presidente do BNDES, P�rsio Arida.
+No Planalto, n�o existe consenso sobre a MP.
+Inicialmente, a equipe econ�mica chegou a estudar a privatiza��o das empresas do grupo Nuclebr�s, al�m de Angra 1 e 2, mas n�o chegou ao fim.
+N�o � apenas Marluce Dias da Silva a poderosa superintendente da Globo que � considerada g�nio em casa.
+T�o tchans quanto ela, seu marido Eurico � o dono de uma cole��o de restaurantes estrelados no Rio.
+O PDT prop�e uma auditoria para a d�vida interna.
+Os pap�is da d�vida seriam obrigatoriamente trocados por outros de vencimento a longo prazo (a partir de 15 anos, segundo o documento).
+O programa prop�e a monetiza��o da d�vida interna (pagamento com dinheiro impresso) acompanhada da cria��o de uma nova moeda (�assegurada a convers�o da poupan�a em termos favor�veis�) e imposi��o de r�gido controle monet�rio.
+Essa proposta permitiria ao governo negociar em posi��o favor�vel o resgate dos t�tulos p�blicos, j� que o programa admite o �n�o-pagamento parcial da d�vida, assegurada a poupan�a�.
+O PDT pretende reduzir os impostos federais a quatro.
+Sobre reforma agr�ria, o projeto prev� terra para 20 milh�es de pessoas com assentamento de comunidades em �reas vizinhas a estradas vicinais.
+O grupo su��o baseia sua m�sica na combina��o de barulhentas guitarras sampleadas, vocais guturais e pesadas batidas dan�antes.
+Ao vivo, sua m�sica n�o difere muito do pop agressivo de Ministry, Young Gods a quem substituiu no BHRIF, Nine Inch Nails e KMFDM.
+Os alem�es, por outro lado, fazem um tecnopop assobi�vel e nost�lgico no seu uso de sintetizadores anal�gicos anteriores ao surgimento da tecnologia dos samplers digitais.
+Segundo o COE, eram remotas as chances de Lauro e Alex serem encontrados com vida porque eles haviam levado comida suficiente para apenas um dia.
+�S� sobrevivemos porque encontramos duas cabanas de ca�adores com mantimentos e cobertores�, disse Nascimento.
+O PPR tentou derrubar a reten��o das verbas de habita��o e educa��o no primeiro turno, mas uma manobra do governo barrou a mudan�a.
+Na ocasi�o, o PMDB apoiou o governo, mas agora n�o estaria totalmente unido para repetir a dose.
+A equipe econ�mica considera que cedeu tudo o que podia durante a vota��o da emenda no primeiro turno.
+O governo concordou em incluir no Or�amento de 94 investimentos em educa��o nos mesmos n�veis do ano passado e destinar para habita��o tr�s vezes mais do que o registrado em 1992.
+Os n�meros destes investimentos ainda est�o sendo fechados pelo Minist�rio do Planejamento.
+O presidente Itamar Franco disse que �ser�o mais de 50 demiss�es� nas estatais do Minist�rio dos Transportes por causa da desobedi�ncia � MP de convers�o � URV.
+V�rias empresas ligadas a este Minist�rio converteram os sal�rios pelo �pico� e n�o pela m�dia dos �ltimos quatro meses.
+Itamar disse que at� agora poucas demiss�es foram efetivadas porque �h� sempre um processo legal e burocr�tico que precisar ser cumprido�.
+�Mas todos ser�o demitidos�, acrescentou.
+O presidente disse que pediu ao ministro das Minas e Energia, Alexis Stepanenko, que cheque tamb�m as estatais ligadas a ele para saber se tamb�m houve desobedi�ncia.
+Participa de reuni�o partid�ria em Maravilha (Santa Catarina).
+� tarde, vai a Belo Horizonte e visita o jornal �O Estado de Minas� em companhia de H�lio Costa, candidato do PP ao governo do Estado.
+� noite vem a S�o Paulo para participar do programa �Fogo Cruzado�.
+Segundo Marcondes, o valor do patrim�nio do hospital � de US$ 50 milh�es (R$ 42,7 milh�es) e o total de d�vidas, de R$ 50 milh�es.
+Ap�s a negocia��o das d�vidas, a reabertura do hospital est� condicionada ao destombamento peloo Condephaat (�rg�o que cuida do patrim�nio hist�rico da cidade) de uma �rea de 10 mil m2, que dever� ser comercializada.
+As f�rias nos EUA podem terminar no balc�o do consulado, onde o passaporte � devolvido �s 16h.
+O da advogada Zilda (n�o quis dar o sobrenome), 49, veio junto com uma carta dizendo que seu visto foi recusado.
+A Divis�o de Vigil�ncia Sanit�ria informou que foram registrados 402 casos de c�lera, com quatro mortes, este ano no Estado.
+Os munic�pios com maiores incid�ncias s�o Nossa Senhora do Socorro, com 85 casos, Aracaju, 66, e Laranjeiras, 46.
+O Banco de Leite Humano da Secretaria Estadual de Sa�de montou um estande na pra�a Fausto Cardoso, centro de Aracaju.
+O objetivo � informar sobre a import�ncia da amamenta��o.
+A atriz C�ssia Kiss e a primeira-dama do Estado, Maria do Carmo Nascimento Alves, visitaram ontem o local.
+Iniciado em 1958, o estudo de �O Capital� por jovens intelectuais da USP s� terminaria cinco anos mais tarde.
+Entre eles estavam figuras hoje conhecidas, como o fil�sofo Jos� Arthur Giannotti, o cr�tico liter�rio Roberto Schwarz e o economista Paul Singer.
+FHC foi o primeiro a tirar consequ�ncias de tal estudo.
+Em 1962, publica �Capitalismo e Escravid�o no Brasil Meridional�, primeira obra de sua autoria a se tornar refer�ncia nas ci�ncias sociais do pa�s.
+Para melhorar a ventila��o, podem ser criadas janelas nos telhados ou pequenos v�os.com telas para evitar a entrada de insetos.
+Os v�os permitem a sa�da do ar quente, que fica na parte superior do ambiente.
+Vidros e janelas d�o transpar�ncia � casa e facilitam a entrada do ar.
+Mesmo com os ventos do litoral, � suficiente o vidro de seis mil�metros, n�o temperado.
+�� uma solu��o econ�mica e segura�, diz Vinograd.
+No piso, carpetes est�o vetados.
+Deve-se usar pedras (pedra goi�s e ard�sias), madeiras n�o-brilhantes (que riscam facilmente) e cer�micas n�o-esmaltadas.
+Outra solu��o, utilizada pelos decoradores Oscar Mikail e Fernando Rodrigues Alves na casa de praia do apresentador de TV Gugu Liberato, � o piso com p� de m�rmore e cimento queimado branco.
+�Uma casa de praia tem que ser pr�tica, ao mesmo tempo que confort�vel�, diz Mikail.
+Rubens Barrichello promete um an�ncio nos pr�ximos dias.
+Fica com o pacote Jordan-Peugeot.
+Por enquanto, se dedica � prepara��o f�sica.
+Est� com quilos de sobra.
+Christian Fittipaldi, por sua vez, negocia com a Tyrrell.
+Seu Sauber pode acabar trocando o certo pelo incerto ao esperar por Karl Wendlinger.
+A Divis�o de Vigil�ncia Sanit�ria de Sergipe registrou nos primeiros 115 dias deste ano 87 casos de c�lera com uma morte.
+A chefe da Vigil�ncia, No�lia Soares, disse que, apesar da exist�ncia de um surto da doen�a, o controle da qualidade das �guas dos rios, po�os e lagos tem evitado a propaga��o do vibri�o.
+Aristides est� esperando a publica��o do texto do FSE aprovado pelo Congresso para concluir se a desvincula��o dessas verbas atinge ou n�o direitos individuais.
+Pela desvincula��o, o governo fica desobrigado de cumprir os artigos da Constitui��o que mandam aplicar percentuais fixos do Or�amento em educa��o e habita��o.
+Aristides tem defendido junto ao governo a aplica��o do art. 212 da Constitui��o, que prev� que Uni�o deve destinar pelo menos 18 % da receita de impostos � educa��o.
+Os Estados e munic�pios t�m que destinar 25 % desta receita.
+O papa Jo�o Paulo 2� escolheu a irm� Em�lia Ehrlich para ocupar uma das tr�s secretarias no s�nodo que acontecer� em outubro.
+A reuni�o de bispos discutir� a vida de padres e freiras.
+� a primeira vez que o papa indica uma mulher para um cargo desse n�vel na Igreja Cat�lica.
+As bibliotecas do Vaticano poder�o ser consultadas em breve atrav�s da Internet, rede de computadores com milh�es de usu�rios.
+O material ser� transferido para a mem�ria de computadores.
+O projeto � das empresas Xerox e Ernst & Rubican.
+Foram analisadas apenas a primeira amostra dos produtos e ainda deve ser feita contraprova.
+A Otker e a Somel v�o pedir contraprova.
+A Mocotex vai sofrer an�lise para saber onde ocorre a contamina��o.
+Uma das poucas surpresas na elei��o anticlim�tica por que passou o pa�s foi a alta incid�ncia de votos brancos e nulos nas elei��es legislativas e para os governos estaduais.
+Se abordo este tema aqui n�o � para arriscar-me na an�lise pol�tica, coisa que outros far�o melhor do que eu.
+Meu intuito � tentar entender que papel teve a imprensa nessa hist�ria.
+S�o os garotos do Nabuco que v�o administrar o neg�cio.
+Decis�o do Supremo Tribunal Federal permite aos micropartidos disputar a Presid�ncia.
+O STF derrubou dispositivos da lei eleitoral que restringiam a participa��o dos partidos de acordo com sua representa��o na C�mara.
+15 de maio -- Sarney desiste de disputar a pr�via do PMDB contra Requi�o e Qu�rcia.
+Ela � vencida por Qu�rcia, indicado candidato � Presid�ncia.
+A absten��o � de 52 % dos peemedebistas credenciados a votar.
+A primeira etapa do M2.000 Summer Concerts amanh� em Santos e domingo no Rio, inaugura a temporada 94 de ca�a aos shows.
+Espalhados pelos 7.408km do litoral brasileiro, os shows deste in�cio de ano tem l� suas atra��es internacionais.
+Nada compar�vel aos shows de Madonna, Michael Jackson e Paul McCartney no final de 93.
+Saem os megas e entram os micros, minis e m�dis.
+A dupla planeja agora temporada de dolce far niente.
+Em meio a tantos scuds por causa dos imbroglios de seu vice Jos� Paulo Bisol, Luiz In�cio Lula da Silva teve anteontem um providencial refresco.
+Preocupado com a facilidade de comunica��o de seu advers�rio, Francisco Rossi, M�rio Covas acatou as recomenda��es de assessores do presidente eleito, Fernando Henrique Cardoso, e se submeteu �s t�cnicas de marketing.
+Avesso a um tratamento mais t�cnico de candidatos durante as campanhas eleitorais, Covas protagonizou a mudan�a mais significativa no hor�rio eleitoral gratuito.
+A vida de alunos como Z n�o � f�cil.
+Eles t�m que comparecer no Juquery de segunda a sexta-feira, das 8h �s 17h.
+Em um dia da semana, Z faz plant�o � noite.
+Fica mais de 36 horas ininterruptas no pronto-socorro.
+Uma vez por m�s, d� plant�o no fim-de-semana.
+Segundo ele, os traficantes tamb�m teriam interesse em negociar uma redu��o em suas penas.
+�Eles t�m medo de ser mortos, porque a coisa est� ficando violenta demais.
+Entregar-se, com a garantia de redu��o da pena, seria um bom neg�cio�, disse Maciel.
+A di�ria do aluguel de um Uno Mille sai por R$ 63, com quilometragem livre, na Avis.
+Na locadora Clean Car a di�ria do mesmo carro sai por R$ 72 com limite de 200 quil�metros por dia.
+A Localiza tem di�ria promocional de R$ 41 sendo que cada quil�metro rodado custa mais R$ 0,26.
+Na promo��o o cliente ganha uma di�ria gr�tis a cada tr�s.
+Um advogado nova-iorquino transformou seu apartamento no Brooklyn num para�so tropical, com �rvores, um sol artificial e pelo menos cinco jacar�s na foto, os policiais no momento da retirada de um deles.
+Os animais foram entregues � cust�dia do zool�gico do Bronx.
+Um s�nodo da Igreja Cat�lica concluiu ontem que as freiras devem ter maior poder e responsabilidade.
+Mas postos de comando da igreja permanecer�o exclusivamente masculinos, disse o cardeal Eduardo Martinez Somalo em Roma.
+Par�grafo 3� -- Da aplica��o do disposto neste artigo n�o poder� resultar pagamento de benef�cio inferior ao efetivamente pago, em cruzeiros reais, na compet�ncia de fevereiro de 1994.
+Par�grafo 4� -- As contribui��es para a Seguridade Social, de que tratam os arts. 20, 21, 22 e 24 da Lei n� 8.212, de 1991, ser�o convertidas em URV e convertidas em UFIR nos termos do art. 53 da Lei n� 8.383, de 30 de dezembro de 1991, ou em cruzeiros reais na data do recolhimento, caso este ocorra antes do primeiro dia �til do m�s subsequente ao de compet�ncia.
+Par�grafo 5� -- Os valores das parcelas referentes a benef�cios pagos com atraso pela Previd�ncia Social, por sua responsabilidade, ser�o atualizados monetariamente pelos �ndices previstos no art. 41, Par�grafo 7�, da Lei n� 8.213, de 1991, com as altera��es da Lei n� 8.542, de 1992, at� o m�s de fevereiro de 1994, e convertidas em URV, pelo valor em cruzeiros reais do equivalente em URV no dia 28 de fevereiro de 1994.
+A pol�cia do Mato Grosso do Sul vai combater a pesca predat�ria durante a piracema (�poca de reprodu��o dos peixes), que termina no dia 31 de janeiro.
+At� esta data, a pesca est� proibida.
+Os pescadores detidos em flagrante usando tarrafas (redes de pesca) ser�o acusados de crime contra fauna, que prev� de um a tr�s anos de pris�o.
+A pol�cia de Arealva (386 km a noroeste de S�o Paulo) prendeu tr�s rapazes acusados de estuprar a funcion�ria p�blica A.P.S., 30, no �ltimo domingo.
+Renato Batista Pedroso, 21, Douglas Jos� da Silva, 21, e A.L.F., 16, est�o incomunic�veis.
+Segundo o delegado Roberto Ilhesca, Silva teria confessado do crime.
+Outros tr�s suspeitos est�o foragidos.
+�Ele confirmou as id�ias que eu j� tinha, um candidato s�rio, respons�vel.
+Sobre minhas quest�es em particular, na �rea de educa��o, ele me impressionou muito.
+�Tem muitos conhecimentos gerais e faz digress�es um pouco longas.
+Daria a mesma nota para ele e para o Rossi.
+A Nig�ria conquistou ontem o t�tulo da 19� Copa Africana de Na��es, ao derrotar Z�mbia por 2 a 1, em T�nis (Tun�sia).
+O zagueiro Elijah Litana abriu o placar para Z�mbia aos 3min de jogo.
+O atacante Emmanuel Amunike empatou dois minutos depois e marcou o gol da vit�ria aos 2min do segundo tempo.
+Uma proposta plaus�vel � a de se fixar um prazo determinado, tr�s ou quatro meses, para que a sociedade se adapte voluntariamente � URV, estabelecendo regras privadas de convers�o.
+Findo esse prazo, a convers�o para URV seria obrigat�ria, conforme regras previamente fixadas pelo governo.
+N�o h� hip�tese do Plano ser lan�ado antes do Congresso votar, primeiro, a emenda constitucional que cria o Fundo Social de Emerg�ncia e, em seguida, o Or�amento federal para 1994 com deficit zero.
+E com o Carnaval no meio.
+Isso significa que, tudo indo bem, a cria��o da URV fica para mar�o.
+A atriz e apresentadora norte-americana de TV Ricki Lake e outras 14 pessoas passaram a noite de anteontem na cadeia em Nova York, depois de haverem realizado um protesto nos escrit�rios do estilista Karl Lagerfeld em Manhattan.
+As 15 pessoas s�o membros da Peta (People for Ethical Treatment of Animals), entidade norte-americana que luta pelos direitos dos animais.
+Elas se algemaram entre si e nos m�veis do escrit�rio em protesto contra a utiliza��o de peles animais em cria��es do estilista.
+O cantor Stevie Wonder anunciou anteontem que sua turn� norte-americana a primeira desde 1989 reverter� fundos para campanhas contra a fome.
+O dinheiro arrecadado com a venda do single �Take the Time Out� ir� para uma campnaha organizada pelo American Express.
+A falha ocorreu no momento de conferir, reembalar e repassar os pacotes de moedas para a tesouraria do banco.
+Calliari explicou que a posi��o das embalagens permitiu a queda de dois sacos, que depois foram parar na lata do lixo.
+Segundo Calliari, o barulho na ag�ncia causado pela reforma do pr�dio abafaram o ru�do da queda.
+O diretor-geral da PF, coronel Wilson Rom�o, disse que recebeu �recomenda��es expressas para investigar tudo a fundo�.
+A vit�ria, por�m, n�o foi suficiente para acalmar o ambiente no clube santista.
+O presidente Miguel Kodja Neto ignorou a diplomacia que normalmente marca o futebol para acusar o ex-jogador e empres�rio Pel�.
+�Ele foi um �timo jogador, mas � um p�ssimo dirigente�, disse Kodja, que quer o afastamento de Pel�, assessor de assuntos internacionais, e do vice Samir Abdul Hack.
+Pelechian -- Sim, e gosto.
+Folha -- O sr. acredita ter influenciado estes filmes?
+Jo�o Paulo Bordon, vice-presidente da Swift-Armour, alega n�o ter sido comunicado sobre a realiza��o da assembl�ia da empresa para ratificar a concordata.
+O an�ncio legal foi publicado recentemente.
+O grupo Bordon enfrenta briga de fam�lia.
+O balan�o do Eldorado S/A informa que as irm�s Ver�ssimo, filhas do fundador, ocupam cargos na vice-presid�ncia e diretoria do grupo.
+Sem acesso �s contas do grupo, elas informam que n�o assinaram o balan�o.
+Elas n�o apenas restringem a liberdade e o futuro dos que tentam deixar Cuba como tamb�m oferecem expectativa de �reformas democr�ticas for�adas� aos que ficam na ilha.
+O principal objetivo norte-americano � evitar a repeti��o da fuga de mais de 125 mil cubanos em dire��o � Fl�rida ocorrida em 1980 a partir do porto de Mariel.
+Na ocasi�o, Fidel Castro permitiu a sa�da de milhares de pessoas insatisfeitas com seu regime e abriu um novo espa�o para se manter no poder.
+Profissionais com longa experi�ncia em cargos gerenciais e administrativos devem se preocupar, ao fazer o curr�culo, em mostrar a ess�ncia da carreira, resumindo suas conquistas e realiza��es.
+Muitos, no entanto, preferem descrever em detalhes toda sua trajet�ria educacional e profissional o que acaba deixando o curr�culo extenso e de dif�cil leitura.
+O filme � bom, divertido.
+As mulheres jornalistas que est�o na tela t�m in�meros problemas.
+Alguns at� se parecem com os meus.
+Mas nenhuma delas tem medo de n�o encontrar o carro na sa�da do cinema ou de ser agredida pelo guardador, ou de parar no sinal de tr�nsito na volta para casa.
+Amanh� e por mais 15 dias tem mais filme.
+O de meia-noite � �timo, mas � em Botafogo e acaba tarde.
+O das cinco tamb�m � bom, mas � no shopping mais concorrido da cidade e n�o vai ter ingresso.
+Nem vaga para estacionar, embora seja seguro.
+� a Companhia de Dan�a de Deborah Colker que faz a abertura.
+Com figurinos de Tufi Duek e Yam� Reis e festa doppo com a turma do ValDemente.
+Em discurso feito na semana passada em Johannesburgo, Arafat conclamou os mu�ulmanos a uma �jihad� pela cidade de Jerusal�m.
+O discurso, transmitido posteriormente por uma r�dio israelense, colocou o di�logo em perigo.
+Nenhum filme brasileiro foi escolhido, quebrando assim a expectativa criada em torno de �Alma Cors�ria� de Carlos Reichenbach e �Mil e Uma� de Suzana de Moraes.
+O Brasil est� assim fora das sele��es de Cannes-94.
+Tampouco nenhum filme latino-americano passou pelo crivo da �Quinzena dos Realizadores�.
+Ainda assim, � expressiva a presen�a nas outras se��es do festival.
+Na competi��o estar� o mexicano �La Reina de La Noche� de Arturo Ripstein.
+A paralela �Um Certo Olhar� selecionou �Los Naufragos� do chileno Miguel Littin e �Sin Compassion� do peruano Francisco Lombardi.
+�Adeus America� de Jan Schutte, �O Casamento de Muriel� de P.J. Hogan, �71 Fragmentos da Cronologia de Um Acaso�, de Michael Haneke, �Fresh� de Boaz Yakin, �Venus de Neve� de Sotiris Gortisas, �Costas �s Costas , Cara a Cara� de Huang Jianxin, �Rainha Bandida� de Shekkar Kapur, �Sem Pele� de Alessandro d'Allatri, �Wrony� de Dorota Kedzierzawska, �Tr�s Palmeiras� de Jo�o Botelho, �Katia Ismailova� de Valeri Todorovski, �Os Sil�ncios do Pal�cio� de Moufida Tiati e o filme coletivo �Homem, [...]
+O advers�rio que os romenos mais temem n�o � nenhuma sele��o, mas o cansa�o.
+O fuso hor�rio ser� um duro obst�culo.
+A tabela da primeira fase prev� uma viagem Los AngelesDetroitLos Angeles para a sele��o romena, em um espa�o de oito dias, na primeira fase.
+O pr�prio agredido informou � pol�cia de que n�o se tratava dos agressores.
+No gramado, antes do jogo, representantes das torcidas dos grande times de S�o Paulo (Corinthians, S�o Paulo, Palmeiras e Santos) fizeram uma manifesta��o pela paz.
+Como exemplo no combate � viol�ncia, o prefeito carioca cita a cria��o da Guarda Municipal, um grupo de atividades especiais que tem atuado mais na libera��o de �reas tradicionalmente ocupadas por camel�s.
+�Foram reabilitados 65 pontos da cidades, usados como pontos de venda de drogas e que viraram quiosques de flores�, conta Maia.
+A princ�pio, a d�vida era sobre a necessidade de se jogar com dois volantes meias que tamb�m auxiliam na defesa, Dunga e Mauro Silva.
+Esta arma��o d� � equipe um car�ter excessivamente defensivo, o que desagrada �queles que esperam da sele��o brasileira um futebol alegre, sempre visando ao gol.
+O funcionamento � bastante simples.
+O cliente passar� seu cart�o de d�bito ou de cr�dito Bradesco Visa no terminal da empresa conveniada e fixar� as datas de pagamento de suas compras.
+Na data acordada, os computadores do Bradesco debitar�o (saque) a conta do cliente e creditar�o (depcsito) a empresa vendedora.
+O objetivo � facilitar a vida dos clientes e reduzir os custos da institui��o financeira.
+O custo do processamento dos cheques � de US$ 0,70 por folha.
+O do pr�-datado eletr�nico, US$ 0,15.
+Sem educa��o, reforma ficar� 'capenga'.
+O per�odo da tarde do Semin�rio Internacional foi dominado pelo conceito de Qualidade Total.
+N�o no �mbito da empresa, mas de um pa�s inteiro.
+Tema: mercado, rela��es de trabalho e educa��o.
+As duas palavras n�o chegaram a ser ditas, mas resumem o consenso: investir em treinamento nas empresas � bom, mas n�o basta.
+No domingo, em Campos (280 km ao norte do Rio), Brizola concentrou as cr�ticas em Cardoso e Lula.
+Disse que FHC � incompetente administrativamente e que Lula � inexperiente e que n�o gosta de trabalhar.
+Segundo os organizadores, havia 6.000 pessoas no com�cio �s 21h, quando Brizola come�ou a falar.
+Quando ele terminou, �s 22h10, metade das pessoas havia abandonado o local.
+A aposentadoria �, em tese, o tempo de realizar planos antes abandonados pela necessidade de trabalhar, mas nem sempre � isso que acontece.
+Muitas vezes as mudan�as na vida do rec�m-aposentado levam a uma sensa��o de perda de capacidade.
+Essas mudan�as geralmente coincidem com a �crise da meia idade�, que � definida n�o pela faixa et�ria, mas como um certo momento em que as pessoas reavaliam as escolhas feitas na sua vida.
+Que a organiza��o investiu muito tempo e dinheiro na sensibliza��o para o programa e que, agora, � com eles.
+Que n�o faria nada, a n�o ser cobrar os resultados do investimento.
+Zagalo assistiu ao jogo entre Flamengo e Corinthians, domingo, no Maracan�.
+Elogiou dois jogadores flamenguistas e um corintiano.
+Ele afirmou que os atacantes Magno e S�vio, do Fla, podem disputar o torneio Pr�-Ol�mpico, em mar�o de 1996 na Argentina.
+Primog�nito de Tsunezaemon Maeda, Takayuki tinha tr�s anos quando o pai, a m�e e seis irm�os chegaram a Ituverava, procedente de Sagaken, no Jap�o, para trabalhar como colonos na fazenda Santa Tereza.
+Tsunezaemon, que havia sido barbeiro em Lima, no Peru, onde morou por 14 anos, acumulara dinheiro com o qual comprou casas e pr�dios ao retornar ao Jap�o.
+Segundo Arnaldo Leite, coordenador do Programa de Capacita��o Gerencial do Sebrae (Servi�o Brasileiro de Apoio �s Micro e Pequenas Empresas), o programa se distingue de similares de outros Estados por ter aval pr�vio do Tribunal de Contas.
+De acordo com Leite, o Tc considerou desnecess�rias as licita��es nas opera��es do programa porque o Sebrae � uma entidade civil sem fins lucrativos.
+Na tarde de ontem, diante de dezenas de jornalistas, Pertence �zerou� os computadores da Justi�a Eleitoral.
+A opera��o � uma garantia de que todo o sistema estar� vazio para receber os n�meros oficiais da apura��o.
+Ao todo, ser�o usados cerca de 5.800 computadores.
+Os de maior porte est�o nos TREs (Tribunais Regionais Eleitorais) e no TSE.
+O Contru (Departamento de Controle e Uso de Im�veis) desinterditou ontem dez salas de cinema.
+Tr�s do shopping Ibirapuera, seis do Belas Artes e o Cinearte, no Conjunto Nacional.
+Continuam interditados o Cine West PLaza 3, o Comodoro e o Iguatemi, que s� deve voltar a funcionar em dezembro.
+�A produtividade das f�bricas de n�vel internacional foi o dobro da verificada nas demais, enquanto a qualidade foi cem vezes melhor.
+Elas registraram uma produ��o por hora de trabalho 43% superior �s demais�, afirma o relat�rio.
+A frequ�ncia na entrega de mercadorias para os clientes tamb�m revela grandes diferen�as entre as empresas �world class� e as concorrentes.
+O ex-auxiliar de produ��o em cervejarias Joilson da Silva, 23, casado com um filho, pedia mais informa��es sobre o abrigo.
+�Eles d�o comida pra gente nesse lugar?
+Podemos dormir?
+O motivo seria a indefini��o das regras de convers�o dos alugu�is.
+O advogado Waldir de Arruda Miranda Carneiro, especializado em loca��o diz que a convers�o espont�nea pode ser interessante dependendo da negocia��o entre as partes.
+�Se o inquilino conseguir um bom desconto na hora da mudan�a do aluguel de cruzeiros reais para URV � neg�cio para ele�, diz.
+Isso porque ningu�m sabe quais ser�o os crit�rios da convers�o autom�tica, explica.
+Uma parada no Boathouse Cafe (East 72 com a Park Drive North) pode ser uma boa id�ia para quem est� indo ao Metropolitan pelo Central Park.
+A lanchonete � simples, mas faz um saboroso cachorro-quente por US$ 3.
+Quem gosta de comida italiana, deve experimentar o Christo's (Lexington, quase esquina com a rua 49).
+O card�pio varia entre pratos de massa al dente (cerca de US$ 15) e peixes grelhados (US$ 20).
+H� algo de novo nestes quatro filmes que Cac� Diegues fez para a TV Cultura, em cima de quatro can��es de Gil, Caetano, Chico e Jorge Ben Jor.
+N�o � maestria de virtuose, n�o � rigor formal racional, n�o � mensagem revolucion�ria, n�o � filia��o �s artes conceituais, n�o � achado de marketing.
+�Queremos tamb�m negociar com as ind�strias rentabilidade m�nima de 15%�, diz Ara�jo.
+Outra sa�da que ser� buscada � o aumento da venda de carne �in natura�, que hoje � de apenas 30 % do total produzido.
+A meta � chegar a 50 % , diz Ferreira Jr.
+Outro dia mesmo, durante a Copa, um jornalista ingl�s que escreve sobre o futebol para os principais jornais do mundo, reclamava dos poucos autores de express�o que escreveram sobre o fut.
+No caso do autor de �O Amanuense Belmiro� (recomendo a leitura de um dos inaugurais romances urbanos brasileiros) n�o havia nada especial a demandar a pergunta feita atrav�s do Alcino.
+Os agentes de turismo do Jap�o est�o dando tratos � bola para novo tour em solo brasileiro.
+Querem organizar pacotes para visitas ao t�mulo do tricampe�o Ayrton Senna eternamente favorito de 10 entre 10 japoneses.
+O n�mero � a soma dos US$ 8,6 bilh�es arrecadados mais US$ 3,3 bilh�es de d�vidas das empresas (transferidas para os novos controladores) mais US$ 3 bilh�es que o governo deixou de investir.
+Montoro disse que sobre esses US$ 15 bilh�es o governo deixa de pagar US$ 1,5 bilh�o/ano de juros anuais, mais US$ 1,5 bilh�o/ano em amortiza��es de d�vidas e mais US$ 2 bilh�es/ano em aportes de capitais �s empresas.
+A produ��o de petr�leo na Argentina aumentou 30 % entre 1991 e 1993.
+O salto na produ��o � atribu�do ao sucesso da privatiza��o da ind�stria petrol�fera.
+De 1991 a 1993, a produ��o de petr�leo local pulou de 20 milh�es para 34,60 milh�es de metros c�bicos.
+�ltima dica: n�o espere subir ao cesto do bal�o para iniciar suas fotos.
+A prepara��o para o v�o, por si s�, j� � um espet�culo � parte.
+CIRO COELHO � editor-assistente de Fotografia.
+A ex-prefeita Luiza Erundina participa hoje da divulga��o de um balan�o comparativo entre sua administra��o 89 a 92 e o primeiro ano da gest�o Paulo Maluf � frente da Prefeitura de S�o Paulo.
+O balan�o aponta para uma queda nos investimentos nas �reas de sa�de, educa��o e habita��o e redu��o nos sal�rios da prefeitura.
+Uma pesquisa da Associa��o Nacional de Ag�ncias de Viagens (Abav) revelou que o setor foi incrementado 29 % em m�dia desde a implanta��o do Plano Real.
+Para o presidente da Abav, S�rgio Nogueira, a expectativa � que o mercado de viagens e turismo recupere este ano os mesmos �ndices favor�veis de 1990, ano em que foi registrado o maior volume de vendas em passagens a�reas.
+As ag�ncias movimentaram cerca de US$ 2,5 bilh�es em 90.
+Na sonolenta Eindhoven, cidade do Brabante (regi�o do sul da Holanda), seriedade e pontualidade s�o apreciadas.
+�Gostamos de gente que cala a boca e faz seu trabalho, como Ronaldo.
+Rom�rio era querido, mas em certos bares ele n�o entrava�, diz um taxista.
+O tricolor, ap�s tantas decep��es na temporada, vai �s finais da Conmebol, ca�a-n�queis t�o inexpressivo que o pr�prio S�o Paulo inscreveu apenas jogadores jovens, que se denominaram Le�es, ao inv�s do tradicional Expressinho.
+Apesar disso ou exatamente por causa disso, conseguiu a classifica��o diante do Corinthians, que ficou no meio do caminho: nem escalou sua for�a m�xima, nem optou por um time de juniores com alguns reservas de refor�o.
+Assim, o tricolor conseguiu impor seu melhor conjunto.
+Mas ter� sido apenas isso?
+Vamos confrontar os titulares que tudo perderam com esses meninos que s�o a derradeira esperan�a de um t�tulo neste ano: enquanto os titulares de Tel� privilegiam o meio-campo, com tr�s, quatro, �s vezes cinco volantes congestionando o setor, o que, na pr�tica, n�o confere nem poder ofensivo ao time, tampouco prote��o infal�vel � defesa, os garotos jogam com Mona, o �nico volante t�pico.
+O resto � composto de meias-armadores e atacantes.
+Ser� mera coincid�ncia?
+Em respeito � hist�ria, diga-se que o mal que a UDN fez ao pa�s n�o decorreu de seu moralismo, mas de sua amoralidade.
+O mal residia, enfim, no fato de que a pol�tica real da UDN se traduzia no que conspirava entre quatro paredes, sem uma antena parab�lica que lhe pusesse termo e sem uma imprensa independente que revelasse, livre de paix�es partid�rias, sua voca��o para renunciar ao Estado de Direito ao toque da primeira corneta.
+A imprensa independente n�o quer cassar de Itamar Franco o direito de fazer seu sucessor.
+Tamb�m n�o lhe pede que cometa o maior pecado de um governante, que � n�o governar e n�o servir ao p�blico.
+Que o fa�a, no entanto, todos os dias do ano, n�o apenas quando o pa�s est� � boca da urna, e nos limites da lei.
+Embora j� estejam com as chaves do im�vel novo, s� devem ir para l� em fevereiro.
+�Nossa vontade era mudar antes.
+O casal quer trocar o piso do banheiro, revestir o piso das �reas sociais com madeira e instalar arm�rios.
+Metade dos Estados e munic�pios do pa�s � federativa e economicamente invi�vel, segundo Asp�sia Camargo.
+A soci�loga e cientista pol�tica falou sobre o tema na �ltima confer�ncia do encontro da Associa��o Nacional de P�s-gradua��o em Ci�ncias Sociais (Anpocs).
+Na noite de ontem, havia cerca de 3.000 pessoas retidas no aeroporto de Ezeiza, o principal de Buenos Aires.
+Os v�os da tarde de ontem entre Buenos Aires e S�o Paulo ou Rio foram retidos em Ezeiza.
+�Sei disso.
+�Procure saber se voc� se entrega, ou se voc� foge de suas emo��es.
+Mas n�o fa�a perguntas como esta, porque o amor n�o � grande nem pequeno.
+� apenas o amor.
+N�o se pode medir um sentimento como se mede uma estrada.
+Se voc� tentar medi-lo, estar� enxergando apenas seu reflexo, como a da lua em um lago; n�o estar� percorrendo seu caminho�.
+�Eu estou decepcionado�, afirmou Coulthard ap�s o treino.
+O escoc�s atribuiu o seu desempenho ruim ao fato de desconhecer o carro sob as novas regras.
+Pela primeira vez neste s�culo, vastos setores das classes m�dias ocidentais n�o conseguir�o efetivar a proeza social -- cimento da democracia e do capitalismo -- realizada pelas gera��es precedentes: garantir a seus filhos um n�vel de vida igual ou superior ao que seus pais haviam desfrutado.
+Ensino universit�rio, contas de poupan�a nos bancos, investimentos em a��es e im�veis n�o asseguram a transmiss�o do patrim�nio familiar dos pais para os filhos.
+Esse quadro de inseguran�as e de incertezas que j� dura h� duas d�cadas contribui para transformar o integrismo, o racismo, a intoler�ncia pol�tica num formid�vel desafio ao sistema democr�tico neste nosso fim de s�culo.
+LUIZ FELIPE DE ALENCASTRO, 48, historiador, � pesquisador do Cebrap (Centro Brasileiro de An�lise e Planejamento) e professor do Instituto de Economia da Unicamp (Universidade Estadual de Campinas).
+Em�lia conta a Adelaide, tamb�m, sobre o livro que est� escrevendo, que trata da fam�lias mais importantes de S�o Paulo.
+Adelaide critica Em�lia por esta sair pouco de casa e, em seguida, deixa a sala irritada pela pouca aten��o qu lhe � dispensada pela m�e.
+Cada curso dura dois s�bados, em um total de 14 horas, e custa US$ 250.
+As pr�ximas datas dos cursos s�o dias 23 e 30 de julho.
+Santana era morador antigo de Lambari d'Oeste.
+Os tr�s cortadores de cana eram de Alagoas e estavam na cidade havia 15 dias.
+Lambari d'Oeste (5.000 habitantes) virou munic�pio h� dois anos.
+Tal cobertura, assistem�tica e voltada para buscar esc�ndalos, leva-os, tamb�m, a trabalhar sobre fatos consumados.
+N�o estamos concluindo que este comportamento � intencional, mas sim que h� uma forte predisposi��o cultural (preconceito) em criticar o Legislativo antes mesmo de conhecer tudo que cerca algumas decis�es.
+Os agentes pol�ticos, sociais e econ�micos tamb�m t�m de se reciclar, pois sua a��o sobre o Legislativo paulistano s� ocorre em fun��o de interesses espec�ficos.
+Que a democracia contempor�nea � fortemente corporativa o sabemos, mas esperamos que n�o continue a ser t�o cheia de particularismos!
+RUI TAVARES MALUF, 35, mestre em ci�ncias pol�ticas pela Unicamp (Universidade Estadual de Campinas), � analista da Superintend�ncia de Experi�ncias Metropolitanas Internacionais da Emplasa (Empresa de Planejamento Metropolitano da Grande S�o Paulo).
+Vinte marcas criaram t-shirts especialmente para o evento, sob o tema �Atitude� Forum, Zoomp, Zapping, Reinaldo Louren�o, Walter Rodrigues, Der Haten, Viva Viva, Armadilha, Ellus, I�dice, Sucumbe a C�lera, Cia. do Linho, Special K, Lorenzo Merlino e Herchcovitch.
+Do movimento underground est�o presentes Anderson Rubbo, Cac� di Guglielmo, Divas e Alessandro Tierni.
+O dire��o geral � de Paulo Borges.
+No galp�o ser�o mostradas fotos dos bastidores dos lan�amentos de moda em registros de Claudia Guimar�es, da Folha Imagem.
+O mercado acion�rio foi bastante agitado ontem.
+A Bovespa (Bolsa de Valores de S�o Paulo) movimentou R$ 450,1 milh�es.
+� o maior volume desde 1� de novembro �ltimo (R$ 763,6 milh�es).
+O mercado trabalhou com a expectativa de aprova��o, pelo Conselho Monet�rio Nacional, de medidas de socorro a bancos em dificuldades.
+O �ndice Bovespa operou em alta durante toda a manh� e in�cio da tarde, mas fechou em baixa de 0,34%, considerando as medidas do CMN como restritivas ao funcionamento do sistema financeiro.
+Seria uma repeti��o do que ocorreu em 1989, durante o Plano Ver�o, quando �desapareceram� 51,87 % de infla��o do �ndice oficial de corre��o monet�ria.
+Muitas empresas entraram na Justi�a para obter o direito de contabilizar essa corros�o inflacion�ria em seus balan�os, e ganharam as a��es em primeira inst�ncia.
+Os processos continuam tramitando.
+O d�lar futuro na BM&F foi cotado a R$ 0,977425 para os neg�cios com vencimento no pr�ximo dia 29.
+O �ndice Bovespa futuro fechou a 39.600 pontos, com expectativa de valoriza��o de 7,10 % ao m�s.
+Aposto uma boneca Barbie como os pais de crian�as entre cinco e doze anos ainda v�o perder v�rios fios de cabelo procurando por um tal de Mighty Max debaixo do sof�, no cinzeiro do carro ou atr�s da geladeira.
+Mighty Max � um boneco de pl�stico, que convive com monstros e morcegos em um universo miniaturizado.
+O brinquedo � a mais recente sensa��o da apetitosa loja Fao Schwarz, da Quinta avenida, em Nova York.
+Juan Manuel Fangio, sobrinho do argentino pentacampe�o de F�rmula 1, est� testando na Calif�rnia um novo prot�tipo que vai dar o que falar na F�rmula Indy.
+Trata-se do Eagle uma reedi��o com motor Toyota e chassis Lola do legend�rio carro feito por Dan Gurney.
+Assinaram o acordo Luxemburgo, B�lgica, Holanda, It�lia, Fran�a, Alemanha, Espanha, Gr�cia e Portugal.
+N�o s�o signat�rios Inglaterra, Irlanda e Dinamarca.
+Os pa�ses visitados poder�o exigir a comprova��o de �meios de subsist�ncia� para a perman�ncia do turista.
+Essa permiss�o de entrada de brasileiros, sem o visto, partiu de uma negocia��o entre Portugal e a Uni�o Europ�ia.
+Na segunda-feira, a m�e de Christian, Suzy Fittipaldi, dissera que gostaria de pedir ao filho para que abandonasse as pistas.
+Ontem, ela foi ao aeroporto e se disse contente por t�-lo �de volta ao colo�.
+�s 11h30 de ontem, no escrit�rio da Fittipaldi Promo��es, empresa da fam�lia, Christian afirmou que, enquanto estiver motivado e achar boas as condi��es de seguran�a, continuar� na F�rmula 1.
+Folha O Banco Central est� se espelhando na experi�ncia de algum outro pa�s, em opera��o semelhante?
+Tavares N�o usamos nenhum modelo porque temos uma caracter�sitica muito nossa, que � o tamanho do Brasil.
+� um continente e � um pa�s de Terceiro Mundo.
+Vai ter real chegando de barco, caminh�o, avi�o, todos os meios de transporte.
+�J� tinha feito dieta, gin�stica localizada, exerc�cios com aparelhos e n�o conseguia eliminar essas gordurinhas.
+Achei que a 'lipo' seria uma boa alternativa�.
+A cirurgia durou duas horas.
+Silvia tomou anestesia peridural (aplicada na regi�o da coluna) e p�de retornar para sua casa no mesmo dia.
+No Tatuap�, onde a falta de atendimento � comum nos fins-de-semana, havia ontem cl�nicos e cirurgi�es trabalhando.
+Ningu�m foi encontrado ontem nas secretarias municipais e estaduais da Sa�de para comentar o assunto.
+Na semana passada, o mercado de gado leiteiro foi marcado por uma surpresa: o gado girolando atingiu pre�o superior ao holand�s puro-sangue nos leil�es.
+Dia 19 �ltimo, o 2� Special Girolando atingiu a m�dia de R$ 3.600 por 44 f�meas.
+Dois dias depois, o Salute Milk, baixou a m�dia para R$ 3.000 na venda de metade das 30 vacas holandesas que foram apresentadas para neg�cio.
+Segundo o Instituto M�dico Legal de Tabo�o da Serra, todas as v�timas levaram mais de um tiro, a maioria na cabe�a.
+A arma utilizada foi uma pistola autom�tica 380.
+Os crimes no Itaim Paulista ocorreram na rua Enseada de Itapacor�ia, 327, por volta das 20h de s�bado.
+Muller chega �s 5h25 de hoje a S�o Paulo, vindo de Londres.
+O jogador contou o motivo que o obrigou a n�o acertar com o Everton.
+�N�o garantiram casa, carro, assist�ncia m�dica e passe livre depois de quatro anos�, disse o diretor Kalef Jo�o Francisco, que falou ontem com Muller.
+O contrato, diz ele, expirou ano passado e s� foi renovado em fevereiro.
+Ele disse que a receita da ferrovia n�o cobre as despesas.
+Gazeta Filho confirmou que os dormentes se deterioraram muito antes do previsto.
+Segundo ele, os cerca de mil dormentes podres ser�o trocados em curto prazo.
+O percurso pelas avenidas Consola��o, Rebou�as e Francisco Morato durou exatos 20 minutos, sem tr�nsito moroso.
+Foi poss�vel cumprir os 13,4 quil�metros do percurso em uma velocidade m�dia de 40 km/h, incluindo-se a espera nos sem�foros.
+O que corresponde a uma rentabilidade de 13,03 % sobre o patrim�nio l�quido.
+O grupo Votorantim recebeu o ISO-9002 para o cimento Votoran, produzido em Volta Redonda.
+Os servi�os de intelig�ncia da Argentina que trabalham com a ajuda do Mossad (servi�o secreto de Israel) e do FBI (Pol�cia Federal dos EUA) atribuem o atentado ao Hizbollah, grupo mu�ulmano xiita pr�-Ir�.
+O grupo e o governo iraniano negaram envolvimento.
+Ontem, o juiz Juan Jose Galeano, respons�vel pelo caso, viajou para a Venezuela.
+A corrida sucess�ria come�a esta semana com um quadro mais claro e definido do que o da semana passada.
+Embora ainda n�o possa ser definitivamente descartada, a hip�tese da candidatura do ex-presidente Jos� Sarney encontra-se pelo menos momentaneamente bastante afastada, abrindo espa�o para que a campanha comece a deslanchar com base no atual grid de largada.
+A �nica alternativa que resta a Sarney de fato � que o Superior Tribunal de Justi�a acate a den�ncia de estelionato apresentada peloo Minist�rio P�blico Federal contra Orestes Qu�rcia e que, neste caso, o ex-governador de S�o Paulo renuncie � candidatura do PMDB.
+� uma possibilidade remota.
+N�o s� a expectativa � de uma tramita��o demorada at� a decis�o do STJ o que pode inviabilizar uma eventual candidatura Sarney por falta de tempo at� as elei��es, como tamb�m h� informa��es dando conta de que o ex-presidente poder� em breve anunciar seu apoio a Fernando Henrique Cardoso, do PSDB.
+Estudo da Associa��o dos Engenheiros da Petrobr�s sobre a estrutura de pre�os dos combust�veis mostra que a Petrobr�s tem uma margem de at� 29% no pre�o do combust�vel que poderia ser usada para reduzir o valor da gasolina.
+O c�lculo baseia-se na compara��o dos pre�os brasileiros com o americano.
+A Folha procurou a Petrobr�s mas n�o obteve retorno.
+Os Bulletin Board Systems (BBS) est�o oferecendo um novo servi�o aos seus usu�rios: a conex�o com a rede das redes, a InterNet.
+Essa rede, criada h� 20 anos pelo Departamento de Defesa dos Estados Unidos para conectar os seus v�rios centros de pesquisa, � considerada a maior rede de comunica��o entre computadores, contando com milh�es de usu�rios espalhados por v�rias cidades do mundo.
+Os servi�os mais procurados na InterNet s�o correio eletr�nico e transfer�ncia de arquivos, conhecida como FTP (File Transfer Protocol).
+Para se ter uma id�ia da import�ncia da InterNet, a Casa Branca est� conectada a essa rede, disponibilizando publica��es e documentos para qualquer pessoa do mundo.
+O acantonamento Itere� oferece futebol, v�lei, caminhadas, brincadeiras, e est� com vagas para as temporadas de 17 a 23 e de 24 a 30 de julho.
+Fica na fazenda Itere�, em Juquitiba, a 120 quil�metros de S�o Paulo.
+Steven quer aproveitar a viagem ao Brasil para completar sua discografia dos Mutantes e conhecer Arnaldo Baptista.
+A advogada diz que o Mais! adulterou a obra �para satisfazer a vontade de neonazistas brasileiros�.
+Vera Lucia Vassouras, que � negra e se diz militante do PDT, sustenta que os autores do livro �em nenhum momento afirmaram que os negros s�o geneticamente inferiores ao brancos�.
+No texto publicado no Mais!, os autores do livro afirmam que �as pontua��es de QI aumentam com o status econ�mico, em ambas as ra�as.
+Mas a magnitude da diferen�a entre negros e brancos nos desvios padr�es n�o se reduz.
+Na realidade, ela aumenta � medida que as pessoas ascendem a escada socioecon�mica�.
+�H� exce��es.
+Uma blusa preta pode ser usada com um suti� branco se a saia for branca�, explica.
+A an�gua deve ser usada sob roupa transparente.
+Nunca use an�gua curta com uma saia comprida.
+CONCERTOS INTERNACIONAIS GLOBO, 0h.
+A com�dia rom�ntica �Suburbano Cora��o� conta a hist�ria de Lovemar (Andr�a Beltr�o), uma sonhadora costureira do sub�rbio que idolatra o locutor de r�dio Rog�rio (Marco Nanini).
+Adaptado da pe�a hom�nima de Naum Alves de Souza, o especial tem ainda no elenco Marisa Orth, Pedro Paulo Rangel e Diogo Vilela.
+A dire��o � de Guel Arraes.
+Al�m de Bras�lia, devem atrasar o rel�gio os Estados de S�o Paulo, Rio Grande do Sul, Santa Catarina, Paran�, Rio de Janeiro, Minas Gerais, Esp�rito Santo, Goi�s, Mato Grosso do Sul, Mato Grosso, Amazonas e Bahia.
+A altera��o deve ser feita de acordo com a hora local, respeitando portanto as diferen�as normais de fusos hor�rios.
+Quem est� duas horas mais cedo que Bras�lia mudar� o hor�rio duas horas antes, portanto.
+Com a mudan�a, o metr� de S�o Paulo vai funcionar hoje uma hora a mais, como se tivesse duas meias-noites.
+Os rel�gios ser�o atrasados � meia-noite para 23h, e os trens continuar�o a correr at� a �segunda meia-noite�.
+Isso vale para os trens das linhas Norte-Sul e Leste-Oeste.
+O metr� Paulista continua fechando �s 20h30 e, portanto, n�o ser� afetado pela mudan�a; as outras linhas do metr� funciona das 5h00 �s 24h00.
+A prolifera��o nessa �rea � maior pelo fato dos ind�genas n�o pagarem impostos com a atividade, mas apenas taxas.
+Muitos dos cassinos t�m grandes grupos por tr�s, que usam incentivos aos �ndios como fachada.
+A emo��o cresce quando atravesso o sal�o de festas e chego �s salas do museu, do Memorial de Menininha do Gantois.
+Aqui era o quarto pobre, simples, limpo e acolhedor.
+A cama n�o era um leito de enferma, era um trono de rainha.
+Apoiada nos travesseiros, o busto levantado na anima��o da conversa, o rosto concentrado no jogo dos b�zios, no instante da adivinha��o, Menininha do Gantois personificava a verdade do Brasil, de um Brasil mais profundo e mais belo, situado al�m da corrup��o, da injusti�a, da viol�ncia, da mentira, das pequenezes, da dela��o transformada pelos pobres homens da baixa pol�tica em suprema virtude nacional.
+Ai, m�e Menininha, acode-nos nesta hora de quase desespero, d�-nos o alimento da confian�a e do sonho.
+Aqui, neste espa�o onde se reverencia sua mem�ria, eu a recordo amiga de toda uma vida, nossas longas vidas vividas na intensidade da paix�o: com ela aprendi a bondade e o povo.
+Me ensinou que s� o povo constr�i grandeza e o faz desinteressadamente, no dia-a-dia da generosidade.
+O Instituto Brasileiro do Patrim�nio Cultural reinaugura dia 18 o Museu Victor Meirelles, em Florian�polis.
+O museu funciona na casa em que nasceu Meirelles, autor do quadro �A Primeira Missa no Brasil�, sua obra mais famosa.
+O pr�dio foi restaurado.
+A Cartilha da Seguran�a Escolar foi lan�ada anteontem, em Porto Alegre.
+O autor do projeto, vereador Isaac Ainhorn (PDT), disse que o objetivo � orientar os estudantes sobre seguran�a pessoal, no seguran�a no tr�nsito e preven��o contra as drogas.
+O patrim�nio era de CR$ 26,3 trilh�es, equivalentes a US$ 16,3 bilh�es, contra US$ 6,9 bilh�es dos fund�es tradicionais, que v�m encolhendo.
+A grande vantagem dos fundos de commodities � a possibilidade de saques di�rios, sem perda do rendimento, ap�s 30 dias.
+Um grupo de miul policiais militares dos EUA sob o comando do ex-chefe de pol�cia de Nova York, Raymond Kelly, comeu a trabalhar ontem em Porto Pr�ncipe com o objetivo de ajudar a manter a ordem p�blica.
+Trezentos soldados de cinco pa�ses da regi�o do Caribe tamb�m chegaram ontem ao Haiti com a mesma miss�o.
+O coordenador da Intelig�ncia Fiscal da Receita, Marcos Vin�cius Lima, disse que foram encontrados em 14 estabelecimentos de um conceituado col�gio privado de S�o Paulo os equipamentos importados por uma funda��o educacional ligada ao mesmo grupo.
+Esse col�gio foi multado semana passada em 2 milh�es de Ufir quase R$ 1 milh�o, valor que corresponde ao II (Imposto de Importa��o) e IPI (Imposto sobre Produtos Industrializados) sonegados por meio da fraude.
+Manoel Carlos Marques Beato, do restaurante Fasano e Jos� Sebasti�o Figueiredo, sommelier e propriet�rio do La Bicocca, garantiram suas vagas no Concurso Brasileiro de Sommeliers.
+No segundo semestre, o concurso escolher� um destes profissionais (respons�veis pelo servi�o de vinhos num restaurante) para representar o Brasil no concurso mundial, no Jap�o, em 95.
+Jack �Girafa� Charlton, t�cnico da Irlanda, usou seu pesco�o comprido e seus quase dois metros de altura para vislumbrar a passagem da equipe para as quartas, depois de um estr�ia vitoriosa contra a It�lia na primeira fase.
+Acabou saindo nas oitavas.
+Com a desclassifica��o, Jack Girafa n�o teve o comportamento pacato que se espera da gigante das savanas.
+Criticou a Fifa at� pelas diretrizes sobre suprimento de �gua durante as partidas.
+�Embora seja concorrente respeitado, a Nielsen n�o representa uma amea�a real�, diz Fl�vio Ferrari, diretor da Ibope M�dia.
+Acrescenta que o cinquenten�rio instituto que, a exemplo da Nielsen, tem atua��o internacional, se disp�e a oferecer eventuais relat�rios diferenciados do concorrente.
+No final do �ltimo ano, Coelho dividiu a cena com o tenor espanhol Pl�cido Domingo.
+H� um m�s, estrelou �Salom�, de Richard Strauss.
+O jornalista LU�S ANT�NIO GIRON viaja � �ustria a convite da ag�ncia Austr�aca de Turismo e da Lufthansa.
+No s�bado, dia 2, a CET (Companhia de Engenharia de Tr�fego) restabeleceu o sentido normal de circula��o da rua Visconde da Luz, no Itaim-Bibi (zona sul), da Jo�o Cachoeira para a Clodomiro Amazonas.
+A mudan�a foi determinada como op��o de retorno ou acesso � avenida Santo Amaro.
+Parte do crescimento de Fernando Henrique na �ltima pesquisa Datafolha deve-se � evolu��o do tucano entre os eleitores que declaram simpatia pelo PMDB de Orestes Qu�rcia e pelo PPR de Esperidi�o Amin.
+FHC foi de 33% para 41% entre os peemdebistas (oito pontos percentuais de crescimento) e de 3% para 42 % no eleitorado do PPR (11 pontos).
+24 de novembro de 92 -- Um Boeing 737, da China Southern Airlines, bate em um morro pouco antes de aterrissar na cidade tur�stica de Guilin, causando a morte de 141 pessoas.
+23 de julho de 93 -- Um jato brit�nico Aerospace 146, da China Northwest Airlines caiu na regi�o aut�noma de Ningxia Hui.
+Dos 133 passageiros e tripulantes que estavam na aeronave, 55 morreram.
+Pergunta Quais suas opini�es sobre a atua��o do Minist�rio P�blico na investiga��o da contraven��o?
+Resposta Injusta principalmente quando nos acusa de tr�fico de drogas, de participar de quadrilha e sequestro.
+A �R�scal Pizza & Cozinha� criou um card�pio especial para o almo�o da P�scoa de amanh�, com uma massa com um recheio e um molho � base de peixes.
+Pelo pre�o de CR$ 6.200, os clientes poder�o se servir � vontade do ravi�li com recheio de salm�o.
+Os que preferirem carne, podem optar por um nhoque preparado � base de vitela.
+A pizzaria foi inugurada pelo Grupo Viena h� dois meses e oferece 21 tipos de pizzas no forno � lenha, al�m de massas especiais, grelhados e saladas.
+Presidente arruma cargos para integrantes do 'Grupo de Juiz de Fora'.
+Na reta final de seu mandato, o presidente Itamar Franco decidiu arrumar empregos, a partir de janeiro pr�ximo, para seus amigos que comp�em o chamado �Grupo de Juiz de Fora�.
+Parecia concurso de perguntas idiotas a cobertura do carnaval nos sal�es, ter�a.
+Manchete, Gazeta e Bandeirantes empataram.
+Voc� se acha louca?
+Os cargos oferecidos s�o, por exemplo, ger�ncia de produ��o industrial, engenharia de projeto ou desenvolvimento de produtos.
+Pesquisa da Laerte Cordeiro & Associados, com base em an�ncios de emprego, indica que a �rea voltou a ocupar a segunda posi��o na demanda.
+Pa�ses ex-comunistas como Pol�nia e Hungria t�m interesse em entrar na Otan.
+Eles temem, por�m, que um relacionamento especial entre Moscou e a alian�a os deixe sem garantias de seguran�a.
+De seu lado, a R�ssia manifestava preocupa��es semelhantes diante da aproxima��o da Otan em rela��o aos antigos membros do Pacto de Vars�via (a alian�a militar pr�-URSS, extinta em 1990).
+S�o tr�s as certezas entre as partes que negociam o aumento de sal�rio para o funcionalismo: a decis�o n�o pode ser adiada, a apar�ncia de equil�brio fiscal deve ser preservada e Ricupero ficar� no cargo em quaisquer circunst�ncias.
+Comandantes militares voltaram a receber comunicado esta semana, destinado � tropa, dizendo que o reajuste do soldo sair� este m�s.
+Os maestros candidatos ao pr�mio de melhor regente s�o Claudio Abbado, Simon Rattle, Valery Gergiev, Mariss Jansons e John Eliot Gardiner.
+Para melhor cantor, figuram Jos� Carreras, Thomas Hampson, Ben Heppner, Sergei Leiferkus e Bryn Terfei.
+Entre as mulheres, est�o Cecilia Bartoli, Jessye Norman, Anne Sofie von Otter, Dawn Upshaw e Galina Gorchakova.
+O Classical Music Award � promovido pela �BBC Music Magazine� e pelo jornal �The Independent�, e tem o patroc�nio da empresa Kenwood.
+Outros pr�mios incluem personalidade do ano, melhor produ��o de �pera, melhor transmiss�o de TV, melhor grupo de m�sica antiga, entre outros.
+Antonio Jorge, diretor t�cnico da Amil, diz que a empresa ainda estuda a convers�o, mas que os carn�s de junho j� ser�o emitidos em URV.
+Ele garante que o aumento real nas mensalidades n�o vai superar os 22 % propostos pela Abramge.
+Outras empresas, entretanto, v�o praticar um reajuste maior na convers�o.
+P�ssimo come�o de temporada para os dois melhores tenistas do mundo.
+Pete Sampras e Michael Stich perderam no Torneio de Doha, no Qatar, para jogadores mais acostumados ao calor forte o marroquino Karim Alami e o haitiano Ronald Agenor.
+O �nico favorito a vencer foi Stefan Edberg, que arrasou o tamb�m sueco Anders Jarryd por 6/1 e 6/1.
+Sampras esteve irreconhec�vel.
+Alami, de 20 anos e n�mero 204 do ranking, teve a maior vit�ria de sua carreira.
+�Tinha v�o marcado para esta noite (ontem), porque n�o imaginava ser capaz de vencer Sampras�, disse Alami.
+S�O PAULO -- Se o leitor se deu ao trabalho de ler todas as repercuss�es sobre o real publicadas ontem pela Folha, ter� verificado que, como sempre ocorre nessas ocasi�es, h� opini�es contr�rias, a favor e mais ou menos.
+Se separar as opini�es conforme a caracter�stica do entrevistado, ter� verificado igualmente que todos os empres�rios ouvidos s�o a favor.
+Seria bom grifar a� o todos.
+Os 160 expositores da mostra da ind�stria de cosm�ticos estimam vendas de US$ 150 milh�es at� a pr�xima segunda-feira.
+S�o esperados cerca de 60 mil visitantes.
+A Cosm�tica'94 acontece no Parque Anhembi (zona norte de SP) e � aberta ao p�blico.
+A Secretaria das Finan�as da Prefeitura de S�o Paulo informou ontem que a partir de segunda-feira, dia 8, a UFM (Unidade Fiscal do Munic�pio) valer� R$ 28,15.
+O reajuste � de 6,08 %, equivalente ao IPC-r de julho.
+Os contribuintes que pagarem seus tributos (IPTU, ISS etc.) hoje sair�o ganhando, pois o c�lculo ser� feito pela UFM de R$ 26,54.
+Ser� lan�ado no pr�ximo dia 9, no audit�rio da Folha, a partir das 19h30, o livro �Tributa��o no Brasil e o Imposto �nico�, organizado pelo economista Marcos Cintra.
+Autor da proposta do Imposto �nico, que pretende reduzir a enorme gama de tributos a um, Marcos Cintra far� uma palestra sobre o assunto.
+Ainda em setembro, no final do m�s, encerram-se as inscri��es nos col�gios Santa Cruz (para a 1� s�rie do 1� grau) e Santo Am�rico.
+As escolas limitam os vestibulinhos a algumas s�ries que t�m maior procura por vagas.
+MAIORIA DOS PROGRAMAS EST� LIGADA � REDE DE ESGOTOS*.
+A modelo Monique Evans, um dos destaques do Carnaval carioca deste ano, vai submeter-se a uma tomografia computadorizada na pr�xima semana para verificar a origem de �um pontinho� que apareceu em seu rim.
+�N�o acho que seja c�ncer, pode ser algo relacionado aos meus medos e ang�stias�, afirmou.
+Monique, 37, disse que descobriu a �marquinha, que n�o � pedra no rim� quando se separou do marido, em junho passado.
+�Senti dores e, como tenho uns casos de c�ncer na fam�lia, decidi fazer o exame�, contou a modelo.
+Antonio Delfim Netto, deputado federal pelo PPR-SP, ontem no �Jornal do Brasil�.
+Itamar e as autoridades que o acompanhavam chegaram ao cemit�rio � 1h50 de ontem.
+Ele permaneceu no vel�rio por aproximadamente uma hora e seguiu para descansar no hotel Gl�ria at� o hor�rio do enterro.
+Segundo amigos do presidente, ele repetiu v�rias vezes que Ariosto era o filho homem que nunca teve.
+Ele voltou a ter crises de choro.
+Na dianteira, o freio � a disco ventilado de 320 mm, com pist�o duplo.
+Na traseira tamb�m h� disco ventilado de 230 mm.
+Ambos t�m acionamento hidr�ulico.
+A suspens�o frontal � por garfo telesc�pico invertido, marca Marzochi, com curso de 200 mm.
+A suspens�o traseira � progressiva com um s� amortecedor Boge.
+O TJM (Tribunal de Justi�a Militar) vai come�ar a ouvir neste m�s as testemunhas de acusa��o do processo contra os 120 Policiais Militares denunciados pelo massacre de 111 presos no pavilh�o 9 da Casa de Deten��o, no Carandiru (zona norte de S�o Paulo).
+O massacre ocorreu quando a tropa de choque da PM invadiu o pavilh�o para acabar com uma rebeli�o, em 2 de outubro de 92.
+Mal, pelo menos por enquanto, na elei��o presidencial, o PMDB mostra na corrida dos senadores que ainda tem cacife.
+S�o peemedebistas os l�deres no Rio Grande do Sul (Jos� Foga�a), Paran� (Roberto Requi�o) e Cear� (Mauro Benevides).
+No espectro pol�tico, Foga�a e Requi�o s�o rotulados como membros da centro-esquerda.
+O Conselho Municipal de Defesa dos Direitos das Mulheres e das Minorias promove hoje em Natal, a partir das 14h, no cal�ad�o da rua Jo�o Pessoa (centro), um ato p�blico.
+A Coordenadoria Especial da Mulher de Londrina (PR) promove debate, �s 20h30, no Catua� Shopping Center, com a sex�loga Marta Suplicy.
+Em Curitiba, as comemora��es v�o incluir envio de fax, plantio de �rvores e passeatas.
+Em Salvador, haver� hoje a inaugura��o na esta��o da Lapa (centro) de um balc�o de informa��es sobre as quest�es da mulher.
+Em plant�o especial, funcion�rios da Delegacia Regional do Trabalho e do Instituro Pedro Melo expedir�o carteiras de trabalho e de identidade para mulheres.
+A excurs�o est� sendo organizada pelo clube More Fun, que promove a troca de correspond�ncias entre gays que n�o querem expor publicamente suas prefer�ncias sexuais.
+At� sexta-feira, 38 pessoas j� haviam reservado suas passagens.
+�Queremos lotar dois �nibus com a 80 pessoas e fechar o hotel�, disse o administrador de empresas E., 31, um dos organizadores.
+Curiosamente, quanto mais os novos programas e redes �facilitam� a vida dos informatizados, maior a tend�ncia dles de ficarem grudados no computador.
+Hoje em dia, qualquer americano medianamente equipado pode fazer diante do teclado tarefas que antes exigiam que ele tirasse o bumbum da cadeira: mandar e receber fax e mensagens eletr�nicas, reservar passagens a�reas, controlar a conta banc�ria, acessar bancos de dados e dezenas de outros servi�os.
+Para marcar a doa��o da cole��o de Takeo Hirata para a biblioteca do Masp, dois designers japoneses fizeram ontem uma palestra na Fiesp.
+Takuo Hirano e Tetsuyuki Hirano, pai e filho, vieram falar sobre o novo conceito de design que est�o desenvolvendo.
+Atrav�s da sua empresa, a Hirano Design International Inc., que tem sede no Jap�o e filial em Chicago, eles promovem o casamento entre o design e a administra��o de empresas.
+�Eu estou apaixonado�, declarou o presidente Itamar Franco, 63, no hall do Teatro Nacional de Bras�lia, onde assistiu anteontem o bal� �Quebra Nozes� com a pedagoga June Drummond, 31.
+June n�o respondeu se tamb�m estava apaixonada.
+Apenas sorriu.
+Itamar disse que o ass�dio da imprensa continua �atrapalhando muito� seu namoro com June.
+Com cinco livros publicados, e um sexto em prepara��o, o escritor aguarda apenas, para a definitiva consagra��o, que Hollywood consiga adaptar alguma de suas hist�rias malucas para as telas.
+�Strip-tease�, que a Companhia das Letras lan�a hoje no Brasil, � o �ltimo romance de Hiaasen.
+A mesma editora publicou, em 93, o livro de estr�ia do escritor, �Ca�a aos Turistas�.
+O ciclista espanhol, 48, se suicidou em Caupenne d'Armagnac, no sul da Fran�a com um tiro.
+Em 1973, Oca�a venceu a Volta da Fran�a, a maior competi��o do g�nero no mundo.
+Piotr Wator, 21, do Gajowianka, da 3� divis�o, morreu domingo durante partida, em Cracovia.
+Wator se chocou com um companheiro de equipe.
+Cerca de 80 % da �rea do munic�pio � considerada de preserva��o ambiental.
+�O aumento na arrecada��o possibilitou que, depois de 30 anos, a prefeitura pudesse comprar tr�s caminh�es, um �nibus e um trator�, disse o prefeito Fausto Camper, 43 (PMDB).
+E a Regina Cas� me disse que churrasco da Brahma, vulgo churrasquinho de Los Gatos, � assim: voc� enfia o dedo na picanha e fica rodando.
+Rarar�.
+Dedo na brasa!
+E sabe quem eu ainda n�o vi?
+O Chato Mesquita do �Pernil�.
+O Jabaury Jr. a gente encontra toda hora.
+Esse t� trabalhando.
+T� cobrindo at� vaca no pasto!
+Rarar�!
+No caso do cart�o Unidas, o possuidor tem desconto de at� 5 % na loca��o, possui tratamento preferencial e melhoria na categoria de carro.
+O cart�o TVA vai permitir a seus portadores, a partir deste m�s, a participa��o nas primeiras exibi��es realizadas em espa�os culturais do Banco Nacional.
+A American Express estuda tamb�m lan�ar cart�es de afinidade.
+Os cart�es Sollo j� realizaram contratos, entre outros, com o Jockey Club de S�o Paulo, a Funda��o Get�lio Vargas, a Associa��o Paulista de Propaganda, a Birello e a Jog�.
+Moradora da Mooca, nos �ltimos cinco anos Regina virou o ano com Hilton, o namorado, no �Recanto da Cachoeira�, uma est�ncia no munic�pio de Socorro (SP).
+Brigada com o namorado, Regina optou por estrear na S�o Silvestre.
+� Foi �timo, voc� fez um minuto melhor do que o esperado�, disse a Medeiros ao cruzar a linha de chegada.
+Ganhou um abra�o, um beijo no rosto, e foi para a Mooca estourar uma champanhe � meia-noite.
+O sindicalista Luiz Ant�nio de Medeiros correu a S�o Silvestre em busca de resultados.
+E chegou l�.
+Com uma ponte de safena e uma mam�ria implantadas h� um ano e meio, fez o trajeto em 1h41 58 minutos atr�s do vencedor.
+Na chegada do ano em que disputar� uma vaga no Senado pelo PP-SP, deu mais de duas dezenas de entrevistas, posou para fotos, e deixou 1993 nas emissoras de r�dio e TV.
+As chibatadas rompem a pele e as cicatrizes ficam por toda a vida.
+A dor faz os golpeados desmaiarem.
+Ao ouvir a negativa a seu recurso, Fay n�o demonstrou rea��o.
+Em Cingapura, a pena de chibatadas � obrigat�ria para tentativa de homic�dio, roubo, estupro, tr�fico de drogas e vandalismo.
+Na discuss�o com a bancada, FHC deve tratar da revis�o constitucional e da delega��o que recebeu do partido para negociar as alian�as eleitorais.
+A reuni�o da Executiva do PMDB deve ser palco de mais um embate entre os adeptos da candidatura do ex-governador Orestes Qu�rcia e os antiquercistas.
+No PPR, a discuss�o ser� sobre a pol�tica de alian�as e se ela inclui o PSDB, al�m dos novos candidatos do partido a presidente.
+O mais cotado � o senador Esperidi�o Amin (SC).
+Apesar da entrada da frente fria no Estado, s�o poucas as chances de chuvas no interior, regi�o mais afetada pela falta de chuvas.
+Mas o abrandamento da invers�o t�rmica (fen�meno causado por massas de ar quente que impedem a dispers�o dos poluentes) e a volta dos ventos dever�o contribuir para melhora da qualidade do ar.
+FHC ressalvou que n�o conhecia a proposta com detalhes e que o governo est� aberto a negocia��es.
+Entretanto, disse n�o saber �que perdas s�o essas�, e que �at� agora todo mundo est� ganhando com a URV�.
+Pelo que a Folha apurou, a equipe econ�mica n�o est� disposta a aceitar modifica��es nos dispositivos sobre sal�rios da medida provis�ria que instituiu a URV.
+�N�o podemos aceitar medidas que contrariem o combate � infla��o�, disse FHC, ao responder sobre o assunto.
+Falando aos membros da Comiss�o de Assuntos Econ�micos do Senado, o ministro disse estar feliz por n�o haver, at� agora, senten�as judiciais contr�rias �s medidas do plano econ�mico.
+Ele disse que a Justi�a do Trabalho em S�o Paulo havia decidido que n�o havia perdas salariais para os metal�rgicos do Estado.
+Ao contr�rio do que tem acontecido em suas �ltimas declara��es p�blicas, FHC evitou ontem fazer ataques aos chamados �aumentos abusivos de pre�os� por parte dos oligop�lios grandes empresas que dominam determinados segmentos da economia.
+O ministro limitou-se a dizer que havia conversado por telefone com assessores do presidente Itamar Franco sobre a proposta de uma nova legisla��o contra abusos do poder econ�mico.
+Esse prazo, no Brasil, � muito flex�vel.
+O Opala durou 23 anos, o Chevette, 20, a Kombi segue firme desde 1957.
+As redes de revendedores das duas marcas festejam a dissolu��o.
+Os distribuidores Ford est�o exultantes.
+Foram recebidos na semana passada em Detroit por Alex Trotman, principal executivo da Ford norte-americana.
+Assim como a China, aceleramos a reforma no campo, come�amos por a�.
+Todos sabem que as reformas na agricultura foram a locomotiva que levou a China a seu estado atual.
+Por isso, j� em 1991, come�amos a introduzir a propriedade privada da terra, desmontando o sistema socialista.
+Hoje n�s temos o maior setor agr�cola privado entre os pa�ses da Comunidade de Estados Independentes (alian�a que substituiu a URSS).
+Por enquanto, s� est�o definidos investimentos da ordem de US$ 600 milh�es nos pr�ximos tr�s anos na Argentina e mais US$ 500 milh�es no M�xico, anunciou Douglas Ivester.
+Nos �ltimos cinco anos, s� na Argentina, a Coca-Cola investiu US$ 800 milh�es na aquisi��o de novos equipamentos e desenvolvimento de novas tecnologias, lembrou Ivester ap�s encontro com o presidente argentino Carlos Menem.
+Uma das consequ�ncias disso � outra aberra��o na sa�de brasileira: em 1991, os hospitais psiqui�tricos cadastrados � rede federal consumiam 7,5% das despesas com interna��es realizadas pelo SUS e representavam o equivalente a 18% dos leitos.
+Esses n�meros v�o de encontro com a tend�ncia mundial, segundo a qual o paciente deve ficar o menor tempo poss�vel internado, e os esfor�os devem ser para reintegr�-lo na sociedade.
+Apesar de seu discurso duro contra eventuais aumentos de pre�os, Ciro admitiu, pela primeira vez, rever a estrat�gia de redu��o das al�quotas de importa��o.
+Ele disse aos empres�rios -- que mantiveram suas cr�ticas � pol�tica de abertura comercial indiscriminada -- que o governo pode aumentar de novo as al�quotas de importa��o em todos os setores.
+Os juros est�o em alta.
+O rendimento projetado das cadernetas saiu de um patamar de 42% nos dep�sitos na semana passada para 46% nas aplica��es nos pr�ximos dias.
+Francisco Lafayette, administrador do Banco Banorte, diz as cadernetas est�o com rendimento atraente, mas que os fundos de commodities d�o a vantagem adicional do resgate di�rio com rentabilidade a partir da car�ncia de 30 dias.
+Cada modelo de fac-s�mile, com suas especifica��es t�cnicas peculiares, recebe uma patente do tipo MI (modelo industrial).
+N�o � um PI porque s� funciona com algo que j� existia (o telefone).
+O registro do tipo DI (desenho industrial) diz respeito � configura��o e cor do produto.
+O Coprotest � uma vers�o revista e melhorada das anti-higi�nicas e pouco seguras latinhas para coleta de fezes para exames parasitol�gicos.
+�O produto n�o tem similares em todo o mundo e est� sendo muito bem aceito nos Estados Unidos�, afirma Adolfo Moruzzi, 49, diretor da NL.
+A pra�a deveria ser reconstru�da ap�s a constru��o do �Piscin�o�.
+A pra�a, por ser tombada, deveria ficar como era antes da obra.
+Segundo o sindicato dos arquitetos, o projeto da prefeitura modifica o desenho original.
+Entre as modifica��es, est� a constru��o de um anfiteatro ao ar livre.
+O sindicato fundamenta o pedido de embargo da obra com a omiss�o da prefeitura, que n�o submeteu o projeto de reurbaniza��o da pra�a ao Condephaat.
+A assessoria de imprensa da Secretaria de Vias P�blicas disse ontem que a prefeitura s� vai se manifestar depois que for notificada pela Justi�a.
+Jason � o maior assassino de todos, com 126 v�timas.
+Myers vem a seguir, com 46.
+Freddy est� na rabeira, com 37.
+Coincidentemente, Jason tamb�m levou o maior n�mero de balas (132), contra 27 de Michael Myers e apenas seis de Freddy Krueger.
+Os avais s�o respeit�veis, mas n�o bastam.
+Tem que se demonstrar atrav�s de contas e de racioc�nios que o expurgo significar� perda.
+Como n�o se pretende o monop�lio da verdade, apresento o racioc�nio (e as contas) que fundamentam a tese de que n�o haver� perdas.
+Quem pensa o contr�rio, que trate de comprovar que o racioc�nio est� furado.
+Ao desembarcar no Canad�, troque a moeda norte-americana por d�lares canadenses.
+N�o esque�a dos �culos escuros e uma capa de chuva.
+Existem outras formas de burlar o reajuste anual.
+Alguns propriet�rios fazem um contrato para o apartamento e um acordo, muitas vezes verbal, para a loca��o da garagem e/ou do telefone.
+Nesse caso, embora o valor do aluguel do im�vel fique fixo por 12 meses, os da garagem e do telefone podem ser reajustados at� mesmo mensalmente.
+H� ainda aqueles que optam pela loca��o de temporada.
+Ou seja, alugam o im�vel por um prazo m�ximo de 90 dias e ainda podem receber o valor adiantado.
+A equipe do ministro da Fazenda, Rubens Ricupero, avalia que o processo de crescimento do volume de dinheiro em circula��o, decorrente da queda da infla��o, est� praticamente encerrado.
+Segundo o diretor de Assuntos Internacionais do Banco Central, Gustavo Franco, as emiss�es de reais cresceram bastante na primeira semana de julho, mas est�o est�veis desde ent�o.
+Blatter disse que o Comit� n�o levou em considera��o o fato do jogador ter se desculpado e comparecido ao hospital para visitar o norte-americano Tab Ramos.
+�Tamb�m n�o nos interessa se o jogador estava internado ou n�o.
+O desenlace, por�m, n�o veio na forma do �golpe de tim�o� com reforma radical, mas foi precipitado pelas duas balas que tiraram a vida de Colosio.
+Mas o resultado, surpreendentemente, foi o mesmo no que se refere � reconstitui��o do princ�pio da autoridade.
+Uma hora ap�s a morte de Colosio -- na realidade minutos depois do p�blico receber a tr�gica not�cia, Octavio Paz pedia um �basta aos excessos verbais e ideol�gicos de alguns intelectuais e jornalistas� e �s �numerosas e irrespons�veis apologias da viol�ncia�.
+Cantora rejeita som 'enlatado'.
+A cantora e violinista Meca Vargas n�o faz coro com os contentes com o computador.
+Para ela, a m�quina produz �m�sica enlatada, desprovida de for�a viva�.
+Os paulistanos est�o pagando menos tamb�m pelos produtos da cesta b�sica.
+Outra pesquisa do Datafolha mostra que estes alimentos recuaram 0,47% na �ltima semana de novembro.
+Mesma tend�ncia teve o custo da cesta b�sica pesquisado pelo Procon, em conv�nio com o Dieese.
+Este levantamento de pre�os que inclui tamb�m produtos de higiene e de limpeza mostra recuo de 1,58% em rela��o � sexta-feira anterior.
+Estamos na transi��o.
+Desculpe-se.
+Agosto ser� outro mundo.
+Mais perto da elei��o.
+O governo n�o est� errado em formular regras que possibilitem o alongamento.
+No Brasil, coisa esquisita, j� se negociou CDB de seis meses uma eternidade, hoje em dia.
+A droga �salmeterol� � mais eficaz para tratar asma a longo prazo do que a comumente usada, segundo estudo na �ltima edi��o da revista da Associa��o M�dica Norte-americana (�Jama�).
+A droga pode ser inalada atrav�s de bombinhas de aerossol.
+Ela teve a��o prolongada como dilatadora dos br�nquios (tubo por onde o ar passa aos pulm�es).
+O alem�o n�o teve muita chance de lutar pela pole.
+Ap�s fazer uma �nica volta r�pida, Schumacher emprestou seu carro para o companheiro Jos Verstappen, que havia rodado nos treinos da manh�.
+O piloto holand�s, em sua primeira volta, perdeu o controle do Benetton e foi parar na brita.
+Final de treino para ele e Schumacher.
+Em rela��o aos rem�dios, houve um aumento abusivo, perto dos 60% em janeiro.
+Dallari, ap�s negocia��es com os laborat�rios, candidamente anunciou que �para janeiro, n�o se pode fazer mais nada, mas em fevereiro os reajustes ser�o iguais � infla��o�.
+Sequer lhe ocorreu que em fevereiro o �excesso� de aumento poderia ser compensado.
+Distra��o, claro.
+Mas, em fevereiro, os laborat�rios ultrapassaram a infla��o outra vez.
+Em mar�o, esbo�a-se a mesma tend�ncia.
+A Fenasoft deixou de ser um reduto de especialistas em inform�tica para atrair tamb�m quem pretendia comprar o primeiro computador.
+�Estou comprando o micro -- o port�til Aero, da Compaq -- para acelerar as pesquisas da minha tese de mestrado�, disse Deborah Caldas.
+Enquanto isso (ou entrementes, como nas antigas hist�rias em quadrinho), o ex-governador ACM descola suntuoso empr�stimo para socorrer os cacaueiros da Bahia, quatro anos de car�ncia, juros de 2% subsidiados, um empr�stimo de pai (governo) para filho (fazendeiros).
+No fundo, uma indecente doa��o do dinheiro p�blico.
+Enquanto isso (outra vez a vontade de escrever entrementes), a Receita Federal divulga a lista dos devedores do Imposto de Renda que nada pagar�o.
+Mais da metade do rombo na Previd�ncia tem como causa a sonega��o de empresas.
+Entrementes (resistir tr�s vezes quem h� de? ), o governo de FHC amea�a reduzir as aposentadorias para cinco m�nimos.
+O Depav tamb�m est� zoneando o parque.
+O trabalho, coordenado pela diretora Vera Bononi, 49, prop�e solu��es para o aproveitamento do espa�o constru�do, em especial a marquise.
+A Asuapi doou 10 bicicletas para a guarda municipal fazer o policiamento do parque.
+Para entrar em a��o, a tropa de ciclistas est� esperando apenas que aconte�a o cerimonial de inaugura��o, diz Santini.
+A guarda usar� uniformes especiais.
+Segundo Santini, com as bicicletas, o trabalho ser� mais r�pido e eficiente.
+Ele afirma que n�o tem previs�o para o in�cio do trabalho dos ciclistas.
+Na disputa pelo mercado de impressoras quem sai ganhando � o consumidor.
+Vale a pena conferir os produtos e comparar a qualidade de impress�o dos modelos jato de tinta.
+A Epson mostra na Comdex quatro modelos com tecnologia jato de tinta, que dever�o substituir os modelos matriciais no dia-a-dia dos usu�rios e empresas.
+O Departamento de Estado dos Estados Unidos ordenou ontem refor�os na seguran�a das representa��es diplom�ticas de Israel no pa�s para prevenir eventuais ataques terroristas.
+Em Nova York, o FBI (Federal Bureau Investigation, a pol�cia federal dos Estados Unidos) recebeu uma liga��o an�nima no final da noite de anteontem informando que o Consulado de Israel sofreria um atentado a bomba.
+O secret�rio disse que aguardava apenas uma defini��o de Paulo Maluf quanto a seu futuro pol�tico para anunciar medidas de base.
+�As den�ncias que v�m sendo feitas procedem�, disse Raia.
+J� os quatro leil�es realizados durante a Expomilk venderam 124 cabe�as de ra�as holandes, jersei, pardo-su��o e girolanda.
+O faturamento bruto atingiu R$ 615,7 mil, equivalentes a US$ 721,8 mil.
+Uma vaca pardo-su��o conseguiu a maior cota��o da Expomilk, R$ 36 mil (US$ 42,2 mil).
+Foi apresentada pela Agropecu�ria Am�rica e arrematada pela Citrovita Agropecu�ria.
+Folha -- O sr. j� fez algum discurso contra a monarquia no Parlamento?
+Skinner -- N�o � permitido falar muito contra a monarquia no Parlamento porque supostamente voc� deve jurar fidelidade � rainha.
+A carga tribut�ria das institui��es financeiras poder� aumentar em 73% caso o Congresso Nacional aprove o aumento da contribui��o social sobre o lucro dos bancos e a inclus�o da obrigatoriedade de recolhimento do PIS (Programa de Integra��o Social).
+Mesmo assim os bancos continuam entre os setores da economia que recolhem menos tributos.
+O ganho de arrecada��o com o aumento da carga de impostos dos bancos vai ajudar a fechar em US$ 16,1 bilh�es o FSE (Fundo Social de Emerg�ncia).
+O governo teve que elevar de 15% para 20% o percentual dos recursos dos tributos que v�o fazer parte do FSE para compensar a perda de recursos do Fundo de Participa��o dos Estados e munic�pios.
+Essa decis�o dever� dar mais de US$ 12 bilh�es ao FSE, s� com tributos que pertencem � Uni�o.
+O pa�s vive, neste momento, a parte mais crucial da grande luta pela implanta��o da cidadania.
+� quando o rastreamento das pistas da contraven��o come�a a bater em personalidades acima de qualquer suspeita, dentro do pequeno c�rculo �ntimo da elite cosmopolita brasileira um am�lgama que re�ne intelectuais de esquerda e direita, eleitores de Lula a Maluf, passando por todo o espectro intermedi�rio, empres�rios, profissionais liberais, jornalistas e membros do judici�rio.
+A corrup��o institucional brasileira vai muito al�m de ratos gordos como Jo�o Alves.
+Caio Gorentzvaig, cotista da Petroplastic, disse que o arresto cair� automaticamente quando seu pai, Boris, �se der por citado e oferecer bens im�veis � penhora�.
+Prev� que isso aconte�a em dez dias, quando seu pai voltar a S�o Paulo.
+O tucano tem passado a maior parte do tempo em seu quarto e, ao contr�rio dos 60 h�spedes, n�o tem feito exerc�cios.
+Mesmo assim, o governador eleito tem seguido a dieta hipocal�rica da cl�nica, onde a ingest�o di�ria � de 350 calorias.
+�� boa, mas � pouca�.
+Apesar de ter dito h� dias que �entre o justo e leg�timo e o legal�, opta pelo justo e leg�timo, Lula disse que a reforma agr�ria, em sua eventual governo, ser� feito dentro da lei.
+Lula fica na �frica do Sul at� quarta-feira � noite, quando viaja ent�o para a Alemanha, onde permanece at� sexta-feira � noite.
+�Disaster Movie�, de 1979, foi uma revela��o: apesar da produ��o apertada, esse curta-metragem revelava um talento mais que incontest�vel.
+� um pouco em fun��o disso que �Anjos da Noite� decepciona.
+A incurs�o de Wilson Barros ao universo dos seres noturnos paulistanos revela um tra�o tipicamente brasileiro: uma esp�cie de vergonha de ser brasileiro.
+Talvez por isso a S�o Paulo de �Anjos da Noite� pare�a com qualquer lugar do mundo.
+O talento de Barros continua l�, apesar de Marilia P�ra se empenhar em tomar conta do filme com um histrionismo fora de lugar.
+Mas de alguma forma representa a tend�ncia formalista que marcou o cinema brasileiro na segunda metade dos anos 80.
+Al�m de Serpa e Teixeira, j� estavam sob investiga��o judicial eleitoral os candidatos M�rcia Cibilis Viana (federal, PDT), Paulo de Almeida (federal, PSD) e Aluizio de Castro (estadual, PPR).
+Ontem, o plen�rio do TRE do Rio determinou a recontagem dos votos da 81� se��o da 77� zona.
+O tribunal anulou 108 votos de Nilton Cerqueira (federal, PP) e 93 de Emir Larangeira (estadual, PFL) na 113� zona (Niter�i).
+Os votos foram preenchidos com a mesma letra.
+Tamb�m foram anuladas duas urnas em Barra Mansa.
+A Reebock teve resultado melhor do que o esperado no segundo trimestre.
+O lucro cresceu 24% para US$ 51 milh�es e as vendas, 18%, para US$ 776 milh�es.
+A Nike tamb�m teve bons resultados, pondo fim a especula��es de que teria passado o auge das famosas fabricantes de t�nis dos EUA.
+O hotel Deville, perto do aeroporto de Guarulhos, diz que, al�m de executivos, tem hospedado muitos turistas, principalmente com destino ao Caribe e � Disney.
+Afirma que, na primeira quinzena de julho, o �ndice m�dio de ocupa��o foi de 80%.
+Para elaborar a lista, o grupo mede o custo de vida em quase 200 lugares.
+S�o analisados os pre�os de 155 produtos e servi�os.
+Atr�s de Buenos Aires, est�o duas cidades asi�ticas (Hong Kong e Taip�) e duas europ�ias (Zurique e Genebra, ambas na Su��a).
+As declara��es foram recebidas por Maradona como �uma tacada de beisebol na cabe�a�, como o pr�prio meia definiu.
+Maradona negou veementemente as cr�ticas da m�e de Franco.
+�N�o sei o que dizer.
+Estou surpreso�, foi o que Franco disse a sobre as declara��es da m�e.
+A resolu��o normativa de 4 de junho de 1987 do Instituto Brasileiro de Turismo (Embratur) afirma que �os meios de hospedagem de turismo dos tipos hotel, hotel de lazer e hotel-resid�ncia, de qualquer categoria, ficam obrigados a dispor de local espec�fico, devidamente sinalizado e apropriado ao embarque e desembarque de usu�rios portadores de defici�ncias, al�m de estacionamentos para seus ve�culos�.
+A lei diz ainda que 2% dos apartamentos t�m que ser adaptados e devem haver facilidades para acesso e circula��o em todas as depend�ncias sociais do hotel.
+As rodas de liga de alum�nio ou magn�sio t�m manuten��o menos frequente.
+A Secretaria da Fazenda do Estado de S�o Paulo determinou que, quando um contribuinte realizar venda de mercadorias em feiras, exposi��es ou em locais chamados de �outlets� ou �feira de promo��es� e permanecer na �rea determinada por mais de 60 dias, ser� obrigat�ria a inscri��o do referido local no cadastro de contribuintes do ICMS (Fund. portaria CAT 116/93).
+A restitui��o do IRPF poder� ser feita a terceiros, desde que: se de valor at� 80 Ufir, mediante simples autoriza��o por escrito do benefici�rio, acompanhada de c�dula de identidade e CPF do representante e do representado, para verifica��o das assinaturas; e se de valor acima de 80 Ufir s� poder� ser paga a procurador (Fund. instru��o normativa DRF n�. 38/92).
+Segundo ele, o que retardou o trabalho da equipe foram as contus�es de alguns jogadores importantes, como Marcelo Negr�o e Giovane.
+J� pensando no Mundial da Gr�cia, que ser� disputado em setembro em Atenas, o t�cnico brasileiro retoma o comando dos treinos da equipe a partir da pr�xima segunda-feira.
+�Corre��o: hoje trazemos as respostas das palavras cruzadas de s�bado.
+A resposta para as palavras cruzadas de hoje foram publicadas ontem.
+Segundo Rosa, nutricionista e professora da PUC de Campinas, junto com os alimentos se �ingerem e digerem afetos�.
+Isto �: os diversos tipos de refei��es s�o investidos de s�mbolos que alteram o valor que as pessoas atribuem ao que comem.
+Para chegar a suas conclus�es, Rosa entrevistou um grupo de funcion�rios p�blicos (t�cnicos administrativos) que trabalham e comem no centro velho da cidade de S�o Paulo (ruas L�bero Badar�, S�o Bento e adjac�ncias).
+Observou tamb�m os lugares onde essas pessoas almo�am.
+VIT�RIA Josias de Souza ensinou sexta-feira, nesta Folha, que �pesquisa n�o � urna�.
+Eu tamb�m acho, mas � preciso avisar o pessoal do PT.
+Nunca vi um partido render-se t�o r�pida e incondicionalmente como o PT est� fazendo.
+� not�vel o baixo astral da milit�ncia, de quadros dirigentes e de simpatizantes em geral.
+De acordo com o amassado, sem que seja preciso interferir na pintura.
+Agora, as negocia��es s�o internacionais: Am�lia vai conversar com a Krupp alem� e a Japan Steel.
+Cada obra consome 1,5 tonelada de a�o (US$ 6 mil) mais US$ 10 mil de m�o-de-obra.
+Multiplique isso por 15, some o valor do transporte e se tem US$ 400 mil.
+�Sete Ondas� n�o est� partindo do marco zero.
+Am�lia j� tem o OK inicial de tr�s pa�ses: Alemanha, Fran�a e EUA.
+Na Alemanha, a obra ficaria na Casa da Cultura dos Povos em Berlim.
+O Centro Internacional de Arte e Escultura, de Vaison-la-Romaine, sul da Fran�a, tamb�m tem interesse pela obra.
+Nos EUA, o Museu da OEA (Organiza��o dos Estados Americanos) j� tem agendada uma exposi��o de Am�lia, mas ela n�o sabe se instala l� suas �Sete Ondas�.
+A negocia��o com patrocinadores � o processo mais complexo, segundo a artista.
+As esculturas, ela diz que faz em seis meses.
+Nos per�odos de choques, 41% dos contratos do setor de confec��es precisaram ser renegociados entre compradores e vendedores.
+Nas �pocas de normalidade da economia inflacion�ria brasileira, esse percentual ainda chega a 7,5%, contra 2,3% do Chile, pa�s que j� alcan�ou sua estabiliza��o.
+Duas pessoas morreram em um acidente ocorrido na noite de ontem no km 2 da rodovia Jo�o do Amaral Gurgel, que liga Ca�apava a Jambeiro, no interior de S�o Paulo.
+Um Gol e um Voyage se chocaram.
+Eles viajavam em sentido contr�rio.
+O motorista do Gol e uma passageira do Voyage morreram na hora.
+A pol�cia ainda n�o sabe qual foi a causa do acidente.
+O diretor de Assuntos Internacionais do Banco Central, Gustavo Franco, disse que o BC vai comprar d�lares por uma cota��o �bem abaixo� de R$ 1,00, pre�o pelo qual vende a moeda norte-americana.
+Ontem, no mercado privado, sem interfer�ncia do Banco Central, o d�lar foi negociado a R$ 0,90.
+Segundo Franco, o BC vai comprar d�lares quando a cota��o cair a um ponto determinado, �n�o conhecido pelo mercado�.
+O povo, assim, � mais destinat�rio de um discurso pol�tico, que ele mesmo n�o profere, quando muito escuta, que seu emissor ou senhor.
+Por isso, estando o �demos� na recep��o e n�o na produ��o do discurso pol�tico, � melhor dizer regime dem�tico que democr�tico.
+O �kratos� n�o � do povo; � seu, se tanto, o ouvido, o olhar.
+Mas, nesta sociedade, cabe papel relevante ao conforto.
+Os discursos da frugalidade, da austeridade ficaram para as rep�blicas gregas e romanas, aquelas que Montesquieu ainda celebrava, em meados do s�culo 18, quando dizia que o regime republicano se caracteriza pela disposi��o de seus cidad�os a renunciar ao bem privado em favor do bem comum e por isso, acrescentava ele, trata-se de regime imposs�vel na modernidade, na qual se busca a vantagem pessoal.
+Segundo o jogador, o Palmeiras s� ter� dificuldades para garantir o t�tulo na partida contra o Corinthians pois deve enfrentar antes advers�rios teoricamente f�ceis (Mogi Mirim, Ituano e Santo Andr�).
+�Em o �ltimo jogo, n�s teremos um prazer, ou de sermos campe�es ou de dar o t�tulo ao S�o Paulo�, garantiu Moacir.
+�Nosso objetivo no campeonato, portanto, ser� definido depois do domingo, quando enfrentamos o S�o Paulo�, disse o volante.
+A entrada de capitais, para ser esterilizada monetariamente, teve de ser absorvida pelo crescimento da d�vida p�blica, o que por sua vez requereu um super�vit fiscal prim�rio crescente.
+h� mais de um ano, o M�xico encontra-se numa situa��o de desequil�brio potencial permanente no balan�o de transa��es correntes e no or�amento fiscal, que foi compensado mais recentemente por um endividamento crescente do setor p�blico nas pra�as financeiras internacionais, em particular na Bolsa de Nova York.
+A quinta rodada da Superliga Nacional Masculina de V�lei da temporada 94/95 prossegue hoje com mais quatro jogos.
+O Palmeiras/Parmalat que est� em terceiro lugar na classifica��o geral, tenta defender sua invencibilidade no torneio diante do Flamengo, no Rio.
+Na tese, despreza o papel do povo num governo.
+Mas, nos anos 90, os yuppies desenvolvem consci�ncia.
+Joe Pesci vai ser a consci�ncia em carne-e-osso.
+O ator vive um sem-teto, que o acaso coloca diante do estudante.
+A TV por assinatura no Brasil est� nascendo concentrada nas m�os de quatro grupos: Organiza��es Globo, Multicanal (subsidi�ria da Companhia de Minera��o do Amap�), Rede RBS e Abril.
+E abre s� para almo�o e drinks ao cair da tarde.
+Apesar de ningu�m l� ser mais amigo do rei, a turma continua na ativa.
+Os policiais foram a uma casa, indicada pelas duas mulheres, onde estariam outros integrantes da quadrilha, mas nada foi encontrado ali.
+O Parque S�o Carlos � habitado por moradores de renda baixa.
+A casa em que Faria ficou era de dois quartos.
+O quarto em que ficou era pequeno e sua porta permaneceu sempre aberta.
+O conjunto fica pr�ximo � estrada de Madureira, uma das principais vias de Nova Igua�u.
+Segundo a pol�cia, a estrada � procurada por grupos de exterm�nio para desova de cad�veres.
+Esta � a simplicidade que mata, quando mal traduzida em simploriedade.
+Mas Michael Tilson Thomas, regendo tudo com uma seguran�a natural e nenhuma pompa, parece o regente por excel�ncia desse nosso tempo, que ao menos na m�sica � um tempo de refinamento e intelig�ncia, al�m de comunica��o.
+Simp�tico, fluente (com as palavras tamb�m), � vontade no papel de regente jovem (embora beirando os 50 anos), n�o h� um nome melhor para dirigir a New World Symphony, uma orquestra de bolsistas.
+Nem simples, nem muito menos beethoveniano � o �Concerto para Violino� de Tchaikovski.
+� uma obra infelizmente popular.
+Est� longe de ser o melhor de Tchaikovski, mas o diabolismo da parte solista n�o deixa nunca de impressionar.
+Diabolismo n�o falta ao solista Robert McDuffie: � um violinista de mil dedos, afinad�ssimo e sem medo da m�sica.
+Mas toca tamb�m com certa brutalidade, impaci�ncia ou agressividade com a m�sica.
+� um m�sico atl�tico, no limite extremo da escola americana de interpreta��o.
+Os juros despencaram ontem com a expectativa otimista para a infla��o de dezembro.
+As taxas de juros equivalentes dos CDBs diminu�ram 2,31 pontos percentuais em rela��o � m�dia da �ltima segunda-feira.
+Em 1920, a sujei��o do sult�o Mohamed 6� ao Tratado de S�vres desagradou os nacionalistas liderados por Mustaf� Kemal, militar que lutou nas guerras balc�nicas e na Primeira Guerra Mundial.
+J� em Ancara que se tornaria capital Kemal presidiu a grande Assembl�ia Nacional.
+Eleito presidente e primeiro-ministro, ele cuidou pessoalmente das opera��es militares durante as guerras grego-turcas (1920-22).
+Depois do tratado de paz de 1923, aboliu o califato, a poligamia, substituiu pr�ticas legais isl�micas por outras de inspira��o europ�ia, adotou o alfabeto romano e o calend�rio gregoriano e concedeu �s mulheres direito de voto.
+O assalto foi impedido pela seguran�a do carro blindado.
+Houve tiroteio.
+O carro-forte do Banespa estava estacionado no p�tio interno quando um caminh�o basculante. com homens armados na traseira. bloqueou o port�o.
+O turismo de pesca ainda atrai 75 % dos visitantes que v�o ao Mato Grosso do Sul.
+Muitos hot�is, no entanto, perceberam o crescente interesse pelo ecoturismo e diversificaram suas atra��es.
+Promovem saf�ris fotogr�ficos, focagem noturna de jacar�s e outros passeios.
+Cobram di�rias a partir de US$ 50 (CR$ 22,9 mil) por pessoa, com pens�o completa.
+H� op��es mais econ�micas, como os campings, por US$ 5 (CR$ 2.200,00) por pessoa.
+Aguap� (067-241-2889) -- A 59 km de Aquidauana, a fazenda tem 7.000 hectares, campo de pouso e sete apartamentos.
+Os pacotes, com pens�o completa, incluindo passeios, v�o de US$ 150 (CR$ 68,8 mil), dois dias, a US$ 300 (CR$ 137,7 mil), cinco dias.
+O pacote de pescaria, com barco, motor, piloteiro e pens�o completa, � 20% mais caro.
+No camping, a di�ria � de US$ 5 (CR$ 2.200,00).
+Nessa obra literalmente seminal, Miller estabeleceu o padr�o do que seria dali para diante a sua literatura: mem�rias pessoais vistas com a min�cia de um estudo biol�gico e narradas no tom exaltado de um profeta b�bado.
+A mat�ria-prima desse primeiro livro � o per�odo que o autor passou em Paris nos anos 20, perambulando de prost�bulo em bar, de quarto emprestado a banco de esta��o, enquanto tentava escrever.
+O otimismo de Van Himst se justifica, em parte, pela entrada do centroavante belgo-croata Josip Weber no time.
+Ele marcou cinco gols na vit�ria sobre a Z�mbia, no come�o do m�s.
+Foi o mais expressivo resultado j� atingido por uma sele��o belga na hist�ria.
+�Weber deu maior profundidade � equipe.
+Encontramos um homem-gol de que sent�amos falta h� muitos anos�, disse Van Himst, ele pr�prio um dos maiores jogadores que a B�lgica j� teve.
+Carmelita Aralse Tebele, 89, moradora do oitavo andar, passou mal com a fuma�a e teve de ser carregada pelas escadas at� uma ambul�ncia.
+Beth Tebele, 62, filha de Carmelita, foi medicada junto com a m�e.
+O comerciante Albano Miguel Ferreira, 37, disse que estava na �rea de servi�o de seu apartamento, no primeiro andar, e viu o fogo come�ar em um barrac�o nos fundos da loja.
+No caso dos microcomputadores, o aumento foi de 200 unidades para 500 unidades mensais, no mesmo per�odo.
+�H� uma busca fren�tica por eletr�nicos que facilitem o dia-a-dia do cidad�o em casa e no escrit�rio�, afirma Quintana.
+Andrade afirmou que a limpeza � feita diariamente.
+No largo Santa Cec�lia, funcion�rios da Administra��o Regional recolheram material acumulado pelos indigentes e desinfetaram bueiros.
+Nos primeiros 20 dias deste m�s foram assassinados no Rio 223 pessoas m�dia de 11 por dia.
+O �ndice � o mesmo registrado em novembro do ano passado.
+O diretor da DRFVAT, Leonilson Ribeiro, disse que a presen�a de militares e policiais nas ruas causou a diminui��o do n�mero de carros roubados e furtados.
+Muitas vezes em v�os quase sem escalas, algumas esp�cies voam mais de 10 mil km, do �rtico at� a lagoa.
+� o caso de um ma�arico-de-papo-vermelho, anilhado na lagoa do Peixe e observado 13 dias depois no Estado de Nova Jersey (nordeste dos EUA).
+Apesar de ser a primeira �rea no Brasil a ser inclu�da, em 1991, � rede hemisf�rica de reservas para aves migrat�rias na categoria de reserva internacional, o Parque Nacional da Lagoa do Peixe, criado em 1986, n�o existe de fato, pois ainda n�o h� um plano de manejo e de desenvolvimento.
+O aumento da cesta b�sica na cidade de S�o Paulo na terceira semana de junho foi de 10,27% em cruzeiros reais, segundo pesquisa do Procon em conv�nio com a Fipe.
+No acumulado do m�s atinge 43,85% , superando as varia��es de mar�o (40,91%), abril (40,23%) e maio (40,23%).
+Em 94 acumula alta de 701,5%.
+Entre as maiores varia��es da semana est�o a sardinha em lata (30,52%) e papel higi�nico (28,18%).
+J� as maiores quedas, dos pre�os m�dios, ficaram com a cenoura (- 13,82), batata comum (5,14%) e p�o franc�s (- 1,25%).
+O grupo alimenta��o, por�m, vem tendo as maiores altas do m�s.
+J� o Regency St. John Resorts ter� amplida sua �rea esportiva e aprimorado seu servi�o de h�spedes.
+O Hyatt Regency Aruba tamb�m ter� melhorias em seu cassino e a instala��o de duas piscinas de hidromassagem.
+Essas dificuldades, que surgiram desde o lan�amento da id�ia, estimularam economistas e governo a buscar caminhos mais r�pidos.
+� o caso da segunda op��o, uma vers�o mais apressada da primeira hip�tese.
+O governo cria a URV e estabelece um prazo de tr�s a quatro meses para que a sociedade negocie convers�es volunt�rias.
+E anuncia, no momento mesmo da cria��o do indexador, que a partir de determinada data a convers�o ser� obrigat�ria, conforme regras predeterminadas.
+Dando um tempo para os acertos, FHC cumpre a promessa de que ser� tudo volunt�rio.
+Mas na segunda fase, a convers�o obrigat�ria exige regras de convers�o, tablita e possivelmente um congelamento para impedir especula��o na nova moeda.
+E isso � um choque com data marcada.
+Mas � uma vantagem deixar todos os passos definidos desde o momento de lan�amento do plano.
+O Banco Mercantil de S�o Paulo realizou lucro l�quido de CR$ 52,3 bilh�es no primeiro semestre.
+O que corresponde a 3,65% sobre o patrim�nio l�quido.
+O lucro l�quido do Banco Bandeirantes no primeiro semestre foi de CR$ 44 bilh�es, com um retorno de 12,6% sobre o patrim�nio l�quido.
+Teve direito at� � rara presen�a de Maria Beth�nia na plat�ia a �ltima performance carioca de Gilberto Gil e Caetano Veloso.
+Junto com sua Magda Collares, Fausto Silva decola dia 13 para temporada de trivial variado cultura e showbizz em Nova York.
+Na quarta-feira, o S�o Paulo tomou um vareio do Palmeiras.
+Se o jogo terminasse 6 a 0 no primeiro tempo n�o seria mais que o fiel espelho da partida.
+Parte porque o Palmeiras jogou com extremo empenho e raro talento.
+Parte porque o tricolor entrou em campo mal escalado por Tel�, que tentou consertar logo ap�s, com a entrada de V�tor na lateral e a passagem de Cafu para o lugar de Jamelli.
+Por fim, pela inexpressiva atua��o das duas maiores estrelas tricolores Leonardo e Cafu.
+Esta tarde, por for�a da suspens�o de Axel, j� come�a melhor no papel, com V�tor na lateral e Cafu no meio.
+Mas � hora de Tel� dar um toque de classe nesse meio-campo com V�lber e voltar com Palhinha mais � frente, que esse jogo vale o campeonato.
+Em algumas cidades, onde os pais t�m poder de decis�o no sistema educacional, os projetos de novas escolas incorporam computadores, CD-ROM, modem e linhas telef�nicas para permitir que os estudantes se comuniquem on line.
+Americano � exagerado.
+Costuma encarar a tecnologia como a solu��o milagrosa para todos os problemas.
+Computador na sala de aula n�o salva ningu�m da ignor�ncia.
+Mas pode facilitar a vida dos estudantes.
+Traz dicas de produ��o gr�fica, como alinhamento.
+Tem v�rios ap�ndices de software para DOS ou �Windows�.
+A venda em bancas de jornal ou pelo telefone (0800) 11-5353.
+Voltado para usu�rios leigos e avan�ados, os 13 cap�tulos d�o dicas para aproveitar melhor os recursos do sistema operacional da Microsoft.
+Inclui disquete de atualiza��o do 6.0 para 6.2.
+Pre�o: CR$ 16,9 mil.
+Tome-se o caso de Ventersdorp, 200 km a oeste de Johannesburgo, uma das 14 cidades declaradas pelo governo ��reas de turbul�ncia�, est�gio suave do estado de emerg�ncia.
+Em Ventersdorp, a nova e a velha �frica do Sul s�o vizinhas.
+Na delegacia de pol�cia, a nova bandeira de seis cores est� hasteada no mastro e o juiz David Postman acabou de concluir a contagem dos votos.
+Creio que essa voca��o de poeta determinou um dos maiores problemas (h� v�rios) na realiza��o da obra de Murilo Mendes.
+H�, com efeito, um �poetismo� em seus poemas.
+Murilo sempre est� partindo do pressuposto de que ele � poeta.
+Afirma-o, de maneira expl�cita, em sua obra.
+O resultado � muitas vezes ing�nuo, infantil.
+A CompUSA de Los Angeles vendia, na semana passada, a impressora jato de tinta Epson Stylus 800 por US$ 199.
+Na loja de Nova York, o Compaq Aero est� por US$ 996,97 (se chegar �s 9h da manh�, ganha o cupom de desconto de US$ 100).
+N�o aceita cart�o de cr�dito internacional.
+A J&R Computer World, apesar de meio fora de m�o, tem pre�os competitivos.
+A equipe de Maluf n�o � a primeira a deixar de lado o problema de enchentes.
+A prefeita Luiza Erundina priorizou o transporte p�blico.
+J�nio Quadros tamb�m gastou em obras vi�rias e Mario Covas marcou sua gest�o pela constru��o de corredores de �nibus exclusivos, como o da avenida Nove de Julho.
+Ontem a cidade conferiu os estragos causados pela chuva de quinta feira, a maior chuva de mar�o desde 1930.
+Reynaldo de Barros, secret�rio de Vias P�blicas, participou de uma cerim�nia pela manh� e depois n�o p�de mais ser encontrado em seu gabinete.
+O Banco Mundial, o FMI e o Gatt s�o institui��es p�blicas que deveriam ter suas pol�ticas submetidas a esferas p�blicas de decis�o e ter a transpar�ncia necess�ria para serem avaliadas e monitoradas pelos cidad�os.
+Recentemente, o Banco Mundial constituiu um grupo independente de inspe��o para analisar queixas sobre seus projetos, iniciou uma nova pol�tica de informa��es e o Gatt organizou o primeiro semin�rio com a participa��o de ONGs.
+A Secretaria da Sa�de de Goi�nia recebeu ontem CR$ 163,7 milh�es, atrav�s de conv�nio assinado com a Funda��o Nacional de Sa�de.
+Desse total, CR$ 127 milh�es ser�o destinados � campanha de combate � dengue e ao pagamento dos 200 agentes de sa�de contratados para a campanha.
+Goi�s ter� dentro de dois anos o seu Zoneamento Ecol�gico-Econ�mico.
+O superintendente de Meio Ambiente, Clarismino J�nior, disse que um dos objetivos � identificar as macrozonas existentes no Estado.
+Segundo ele, a proposta inicial � zonear no primeiro ano a Bacia do rio Meia Ponte.
+As faixas, listras e c�rculos desenhados nos gramados norte-americanos da Copa n�o s�o apenas um recurso est�tico.
+Eles servem para camuflar o improviso dos est�dios da Copa.
+Os nove est�dios do Mundial s�o normalmente usados para o futebol americano, que tem marcas pr�prias na grama.
+Comentei com minha amiga espanhola, Dolores de Pan�a, a profus�o de beijos melados que massacraram minhas aveludadas bochechas na Bienal.
+Ela subiu nos tamancos flamencos.
+�� por essas e outras que a gente acaba pegando uma micose na chica e depois n�o sabe explicar por qu�, bradou a espanholita.
+E prop�s que eu desencadeasse nesta coluna uma campanha nacional contra sauda��es lambuzadas.
+Ta�, gostei da id�ia!
+Al�, v�timas do abuso oscular do Brasil!
+Fa�am como eu!
+Em sinal de protesto, de hoje em diante, quem me cumprimentar com beijo suado leva uma muqueta no focinho.
+Viva Howard Hughes!
+O IBGE divulgou tamb�m ontem a infla��o registrada no Rio e S�o Paulo entre os dias 9 de mar�o e 7 de abril.
+O IPC-amplo (cesta de consumo de fam�lias com renda de at� quarenta sal�rios m�nimos) registrou 42,65% em S�o Paulo e 41,96% no Rio.
+Esses �ndices revelam queda de infla��o de 0,23 ponto percentual em S�o Paulo e 0,59 ponto percentual no Rio, em compara��o com o per�odo anterior.
+O candidato do PMDB chegou a Bauru �s 10h de ontem, num jatinho fretado pelo partido.
+Foi recebido no aeroporto por militantes, prefeitos e vereadores.
+Qu�rcia disse que tem sido v�tima de uma �campanha s�rdida� com o objetivo de �denegrir� sua campanha.
+O modo de escapar disse � simples: aplica-se um reajuste de, digamos, 40% ; em seguida, avalia-se qual a infla��o di�ria, digamos de 1,5% , e a� se aplica ao pre�o cheio um desconto decrescente.
+No primeiro dia ap�s o reajuste de 40% , o desconto � de 38,5% ; no segundo de 37% e assim por diante.
+Assim, um carro de CR$ 10 milh�es, sairia no primeiro dia ap�s o reajuste por CR$ 6,15 milh�es.
+No segundo dia, por CR$ 6,3 milh�es e assim sucessivamente.
+� uma URV �s avessas.
+A generaliza��o dessa indexa��o di�ria n�o s� acostuma a economia, como passa a exercer press�o para que o governo oficialize e universalize a pr�tica.
+O assunto veio � tona durante conversa do advogado Osmar (Nuno Leal Maia) com o jardineiro Kennedy (Alexandre Moreno), jovem negro que sofrera insultos racistas do patr�o, o empres�rio Raul Pelegrini (Tarc�sio Meira).
+Uma semana depois, tr�s organiza��es paulistas resolveram contestar a informa��o na Justi�a Federal.
+Os termos da reforma foram assinados quarta-feira pelo prefeito Paulo Maluf, pelo abade Dom Isidoro Oliveira Preto (mosteiro) e por representantes do Banco de Boston, Philips e Akzo/Tintas Ypiranga, empresas que patrocinam as obras.
+As reformas no mosteiro e a coloca��o de um novo cal�amento no largo de S�o Bento devem consumir US$ 200 mil.
+Esperava aparecer como o �nico respons�vel pelo plano de estabiliza��o.
+O fracasso das medidas antiinflacion�rias poderia lhe custar o acesso ao Planalto.
+Desta forma, para se proteger de uma reviravolta inoportuna dos �ndices econ�micos, FHC e seus amigos do governo tiveram o cuidado de fixar um calend�rio vantajoso.
+Foram detidas 25 pessoas no escrit�rio e em outros cinco endere�os ligados a Castor de Andrade.
+As listas apontam quase todas as divis�es e departamentos da Pol�cia Civil do Rio como recebedoras de quantias mensais.
+Minha experi�ncia, nos dois �ltimos anos, supervisionando programas de qualidade total dentro do grupo Rh�ne-Poulenc tem mostrado isso com frequ�ncia.
+Cabe ao empres�rio, ao empreendedor, seja pequeno, m�dio ou grande, criar o ambiente adequado para a valoriza��o de um recurso humano com essas qualifica��es.
+Do contr�rio, todo o esfor�o feito estar� destinado ao fracasso.
+Jovens estudantes na Belo Horizonte dos anos 20, o futuro m�dico Juscelino Kubitschek e o futuro advogado Jos� Maria Alkimin costumavam sair juntos em busca de mo�as namoradeiras.
+Iam aos bailes no Autom�vel Clube e reparavam nas poucas donzelas que ousavam frequentar os caf�s.
+Um dia Juscelino comunicou a Alkimin que estava namorando a mo�a Sara Lemos, filha do pol�tico Jayme Lemos.
+Alkimin achou que tamb�m era hora de firmar namoro e pediu a JK que o apresentasse �s amigas da namorada.
+Para ele, os clubes precisam encontrar uma f�rmula que concilie os dois aspectos.
+Segundo Nujud, as contas do Corinthians ainda n�o foram fechadas, para se saber se o clube teve lucro ou preju�zo.
+Qual o maior vexame que j� deu?
+�Sempre os mais recentes s�o os piores.
+Ao parar de fumar, acabei perdendo a estribeira algumas vezes.
+�Nunca tive muito interesse, apesar de j� ter experimentado.
+Boa not�cia: o Parreira j� comprou sua Mitsubishi!
+Claro, � a �nica que lhe d� garantia de ir at� a Copa de 98!
+Porque n�o escala aquele goleiro j�nior, o Pitarelli.
+Que defendeu os tr�s p�naltis do S�o Paulo.
+E os s�o-paulinos podem se conformar com o Zico.
+Que perdeu apenas 4 p�naltis em sua vida.
+Rarar�!
+P�ssima Not�cia: o Brizola lan�ou sua biografia em v�deo.
+Campanha no ar!
+Pelo pre�o simb�lico de CR$ 1.000.
+Simb�lico?!
+Em se tratando do v�deo, pra mim � uma fortuna.
+Simb�lico seria CR$ 1,00.
+E olhe l�!
+A� a gente tirava a fita e gravava em cima!
+Ent�o eu vou lan�ar o Pr�mio Simb�lico!
+Pago CR$ 1.000 pra quem assistir o v�deo at� o fim.
+O Roberto D�vila n�o vale!
+Porque foi ele que fez!
+Acho que o �nico que viu at� o fim!
+As apar�ncias s�o as principais v�timas do sarcasmo de Altman, desde as �peruas� que discutem a cor de seus cabelos at� a noiva, que usa aparelho, escovando os dentes alegre diante do espelho.
+O humor de Altman vai na dire��o das conven��es de uma sociedade que, tendo que aceitar tudo sob as apar�ncias, acaba perpetuando uma vasta rede de hipocrisia.
+�Pertencemos a um grupo de caridade.
+Fazemos um n�mero de dan�a.
+Dan�amos em hospitais e manic�mios�, diz uma das tias do noivo, uma senhora, para as colegas �peruas�.
+H� uma boa dose de realismo tamb�m.
+Os personagens de �Cerim�nia de Casamento� parecem sa�dos das p�ginas de �rec�m-casados� do �The New York Times�, uma das se��es mais provincianas da imprensa mundial e de maior leitura num jornal que vende a imagem de um dos mais cosmopolitas do mundo.
+O diretor de cr�dito agr�cola do Banco do Brasil, Said Miguel, disse ontem na Comiss�o de Agricultura da C�mara dos Deputados que os agricultores j� renegociaram R$ 1,7 bilh�o das d�vidas em atraso.
+Faltam serem renegociados mais R$ 700 milh�es.
+Os acordos j� feitos abrangem 33 mil agricultores.
+O financiamento p�blico seria baseado em fundos sociais (FGTS, FAT), na atra��o de recursos de pens�o e nas parcerias com a iniciativa privada.
+A taxa��o de grandes fortunas e o dinheiro de privatiza��o -- de estatais n�o-estrat�gicas (exclui min�rios, petr�leo e telecomunica��es), complementaria esses fundos.
+O ex-presidente Fernando Collor, sua mulher Rosane e o secret�rio Lu�s Carlos Chaves jantaram anteontem � noite no restaurante Golden Horn, em Aspen, no Colorado (EUA).
+Durante o jantar, pessoas que estavam em uma mesa pr�xima � de Collor e Rosane tiraram fotos.
+Embora n�o fosse o alvo, o ex-presidente assustou-se com os flashes.
+Na sua avalia��o, a oposi��o sabe que o plano � bom, �mas n�o quer aprov�-lo agora, pois seria bater palma para o advers�rio�.
+Votar contra seria perigoso porque o plano tem apoio popular.
+Segundo o presidente da Matis-Paris Brasil, Louren�o Lopes, 44, a empresa francesa foi escolhida por ter mais estrutura e n�mero de produtos para cl�nicas de beleza, medicina est�tica e cirurgia pl�stica.
+Al�m dos produtos, a Matis oferece infra-estrutura a seus franqueados.
+A empresa possui laborat�rio pr�prio na Fran�a e fabrica os equipamentos usados em suas cl�nicas.
+Os produtos s�o profissionais (para institutos) e para o consumidor.
+S�o Paulo foi literalmente invadida pela cozinha japonesa.
+E o tempo vem mostrando que o fen�meno n�o � mero modismo.
+A cidade tem uma larga popula��o de origem japonesa e isso sustenta a prolifera��o e a qualidade de seus restaurantes.
+Mas se em muitas casas japonesas da cidade o sashimi e os sushis s�o os carros-chefes em outros se destacam por iguarias nem t�o comuns.
+� o caso do Iti Fuji, restaurante fundado 12 anos atr�s pelo japon�s Hashiro Yawata, 74 (propriet�rio de um dos mais antigos restaurantes japoneses da cidade, o Kokeshi, aberto h� 33 anos).
+Trinta empres�rios argentinos e 25 brasileiros se re�nem na pr�xima sexta-feira em semin�rio da FGV para debater perspectivas do setor privado nos dois pa�ses com o Mercosul.
+Os empres�rios brasileiros acreditam que os setores mais prop�cios a investimentos na Argentina s�o: cerveja, celulose, alimentos, petroqu�mica, telecomunica��es e g�s.
+Segundo Luiz Estev�o, Collor, vestido com uma cal�a marinho e camisa de algod�o branca, sem mangas, estava �confiante no trabalho dos advogados�.
+Tamb�m esteve na Casa da Dinda Eun�cia Guimar�es, amiga de Rosane.
+Collor recebia telefonemas o tempo inteiro, dos advogados, de pol�ticos amigos e de pessoas que prestavam solidariedade.
+Um amigo da fam�lia ligou para informar sobre o estado de sa�de do irm�o do ex-presidente, Pedro Collor, que est� sob tratamento nos Estados Unidos.
+Subiu ao palco com o candidato a vice, Aloizio Mercadante, e os candidatos ao governo do Estado, Jos� Dirceu (PT), e ao Senado, Luiza Erundina (PT) e Jo�o Hermann (PPS).
+Lula respondeu perguntas do p�blico sobre sa�de e cultura.
+Mercadante foi escalado para responder quando o tema foi aposentadoria.
+Apartamento na rambla Armenia, Montevid�u.
+Valor: US$ 69 mil.
+50% de apartamento na avenida Atl�ntica, Copacabana, zona sul do Rio.
+Valor da fra��o: US$ 100 mil.
+A Reuters Holdings, baseada em Londres, pretende lan�ar um servi�o financeiro por TV em meados do ano.
+Inicialmente restrito � Europa, a empresa o tornaria mais tarde dispon�vel em todo o mundo.
+Entre os teens, os tri�ngulos est�o em alta com direito a forma��o de casais �a tr�s�.
+No filme, uma garota (Alex) � colocada por engano no mesmo quarto de dois rapazes e se apaixona por Eddy.
+Mas Eddy ama Stuart e eles acabam se desencontrando.
+A Eufrasio Ve�culos, revendedora Ford, aposta no showroom sazonal e se prepara para come�ar a atuar em Campos do Jord�o por dois meses.
+Pesquisa Update, da C�mara Americana, revela que os empres�rios esperam queda na infla��o com o real.
+O �ndice mensal de expectativa de infla��o teve este m�s o resultado mais baixo desde que foi criado h� tr�s anos.
+� verdade.
+Sei que agora tenho um longo caminho a percorrer.
+Tenho que estudar muito.
+Este foi meu primeiro filme, mas sei que posso ser uma grande atriz.
+Sinto isso.
+Quero ter mais experi�ncia, estudar e me dedicar 100% a isto.
+Quero ser uma das melhores atrizes do mundo.
+Este � meu objetivo.
+E como voc� se sente como �estrela de cinema�?
+Meu Deus!
+Tenho muito trabalho a fazer antes de me sentir assim.
+Mas as coisas que t�m acontecido com mim s�o espantosas.
+Eu venho de um bairro muito pobre, muito pobre de Nova York.
+Voc� n�o pode imaginar que qu�o longe eu venho.
+Eu me tirei das ruas e comecei a estudar.
+Aprendi a falar direito e tentei fazer minha vida melhor para mim.
+Quando se est� num ambiente como o que eu vivia, voc� n�o sabe que existe todo um mundo diferente fora dali.
+Tive muita sorte mesmo.
+Eu podia ser uma drogada hoje, podia viver na rua.
+Espero que possa voltar l� e dizer �s crian�as que elas n�o precisam ficar presas l�, que existe todo um outro mundo fora dali.
+Hoje as coisas est�o melhores para a comunidade latina nos EUA.
+At� em Hollywood, h� pessoas como Sonia Braga e Andy Garcia.
+Quando eu era crian�a, tinha vergonha de ser latina.
+Alec Eu quero viver, eu quero n�o fazer nada.
+Ali, no torneio que encerrava a temporada 93, Steffi aguardava mais um trof�u, olhando para o ch�o, sem ligar para as 12 mil pessoas em volta da quadra.
+Minutos antes, tinha pulverizado a espanhola Arantxa Sanchez.
+Nos discursos de agradecimento em Wimbledon e Us Open, meses antes, a alem� lembrara o p�blico de que n�o ficava feliz em vencer torneios na aus�ncia de Monica Seles, afastada das quadras depois de ser esfaqueada na quadra.
+�A proposta da Anfavea � uma indexa��o disfar�ada�.
+Do assessor especial da Fazenda para pre�os, Jos� Milton Dallari.
+Chegaram famintos e esfarrapados �s cabeceiras do Purus ap�s quatro meses de viagem.
+Mas desvendaram o mist�rio da sua liga��o com outros rios, feita atrav�s de canais abertos pelo homem para o com�rcio e o contrabando.
+De volta ao Rio de Janeiro, Euclides preparou alguns dos mapas que serviram de base para o tratado de fronteiras entre o Brasil e o Peru.
+Foi assassinado em agosto de 1909, um m�s antes do acordo entre os dois pa�ses.
+Escrevia � �poca �Um Para�so Perdido�, livro sobre a Amaz�nia, interrompido com a morte.
+A tarefa foi transferida a Falc�o.
+�Trata-se de um candidato que estreou em com�cio sob vaias e quer, pela via do preconceito, enfraquecer quem lidera as pesquisas�, disse Falc�o sobre FHC.
+�N�s n�o vamos dar corda ao debate rasteiro�, acrescentou o presidente do PT, que acha improv�vel o STF permitir a candidatura de Jos� Sarney por um partido que n�o seja o PMDB.
+Os motoristas da EMTU (Empresa Metropolitana de Transportes Urbanos), empresa do governo do Estado, t�m o mesmo piso salarial das empresas privadas.
+Segundo a EMTU, cerca de 215 mil pessoas utilizam diariamente o corredor do tr�lebus.
+A fam�lia do bicheiro Jos� Scafura, o �Piru�nha�, espera para qualquer momento a liberta��o de Andr�, 15, sequestrado na noite de ter�a-feira.
+Neto de �Piru�nha�, Andr� foi capturado perto de sua casa, na Aboli��o (zona norte).
+A fam�lia diz estar reunindo os US$ 100 mil do resgate.
+Segundo o pai de Andr�, Luis Carlos Scafura, os sequestradores provaram que ele est� vivo, ao responder corretamente algumas perguntas sobre a vida do rapaz.
+Scafura afirmou que os sequestradores j� ligaram tr�s vezes.
+Na sequ�ncia, em 1498, Vasco da Gama chegou �s �ndias e inaugurou uma nova era.
+Um cabo, ensinam os dicion�rios, � uma faixa de terra que entra pelo mar o oposto de fiorde, que � uma por��o de mar que avan�a pelo continente, como acontece na Noruega e Su�cia.
+� o mesmo que capo, cabedelo ou promont�rio, extens�o de terra maior que a ponta e menor que a pen�nsula.
+As vendas brutas ca�ram de US$ 3,78 bilh�es para US$ 3,57 bilh�es, consequ�ncia de redu��es tanto no mercado interno como no externo.
+A receita l�quida, exclu�da a carga tribut�ria, foi de US$ 1,15 bilh�o em 93, contra US$ 1,27 bilh�o em 92.
+O mercado brasileiro de cigarros apresentou uma queda de 6,5% com o total de unidades vendidas passando de 127,8 bilh�es em 92 para 119,5 bilh�es no ano passado.
+A Souza Cruz, que lidera o mercado, teve sua participa��o reduzida de 83% em 92 para 79,6% em 93, consequ�ncia de uma retra��o de 10,5% nas suas vendas.
+Durante a apresenta��o do balan�o, M�ton Cabral disse que a empresa reagiu no final do ano, voltando a deter mais de 80% do mercado, gra�as ao sucesso da marca Derby.
+A Ensec, brasileira, foi uma das cinco empresas pr�-qualificadas para participar de concorr�ncia de US$ 11 milh�es para reconstruir o sistema de seguran�a do estacionamento do World Trade Center, em Nova York, destru�do com a explos�o de um carro-bomba.
+O Sinicesp impugnou concorr�ncia da Prefeitura de Santana de Parna�ba argumentando que o edital para obra de infra-estrutura contraria a Lei 8.666.
+O hotel ter� cinco alas com motivos de cada um dos cinco esportes mais populares dos Estados Unidos futebol americano, basquete, beisebol, t�nis e surfe.
+A segunda fase do projeto prev� a constru��o (para 1995) do All-Star Music Resort, no mesmo estilo e com mesma capacidade, que homenagear� os estilos musicais do country, jazz, rock, calypso e musicais da Broadway.
+Entre os cat�licos que se intitulam �n�o-praticantes�, 94% n�o tomam decis�o baseado nos conselhos religiosos.
+� mais alto do que os que se dizem ateus ou sem religi�o (93%).
+Indagados sobre se concordavam com a posi��o da Igreja com rela��o ao uso de camisinha, 74% discordaram totalmente e 13% discordaram em parte.
+Convencido de que Munhoz tem uma boa ret�rica, o publicit�rio pretende deixar o candidato �falar o que quiser� para, antes da grava��o final, decidir o que ir� ao ar.
+Centrado no slogan �Fome e desemprego . Agricultura � a solu��o�, Munhoz vai falar durante o tempo a que tem direito sobre a necessidade de aprimoramento do setor.
+A Folha realizou um teste em um dos �nibus que integram a nova linha Penha-Lapa, acompanhando a d�cima viagem de ontem.
+A principal diferen�a verificada � a sensa��o de �frio na barriga� quando se passa pelas curvas do elevado Costa e Silva (Minhoc�o), mesmo em uma velocidade m�dia de 60 km/h.
+Quem ainda n�o escolheu sua carreira ou sonha dar uma guinada na vida profissional pode recorrer aos servi�os de orienta��o vocacional oferecidos por algumas institui��es em S�o Paulo.
+Geralmente procurados por estudantes adolescentes, esses servi�os s�o abertos a qualquer interessado e t�m a vantagem de ser gratuitos ou cobrar pre�os baixos se comparados aos de empresas de consultoria.
+O objetivo do projeto, segundo Miro, � facilitar a investiga��o policial e a obten��o de provas contra o crime organizado (delitos praticados por atividades de quadrilha ou bando).
+Uma das novidades � a permiss�o para a pol�cia se infiltrar em quadrilhas para obter provas.
+Atualmente, esse tipo de prova n�o � aceito na Justi�a.
+O projeto tamb�m permite � pol�cia impedir, interromper e interceptar a escuta e a grava��o das conversas telef�nicas.
+Mas isso s� teria validade com a aprova��o de um projeto de lei complementar � Constitui��o, regulamentando a escuta telef�nica.
+Esse projeto tamb�m � do deputado do PDT e j� foi aprovado pela C�mara.
+Quem queria comprar, por exemplo, meia d�zia de p�es e um litro de leite, mas s� tinha cruzeiros reais no bolso, teve que enfrentar fila de at� duas horas.
+Um consumidor que tivesse apenas uma nota de CR$ 50.000,00 para fazer uma compra de valor menor tinha de solicitar um cheque de convers�o no valor aproximado da mercadoria e receber o troco em cruzeiros reais.
+As vitrinas sempre guardam surpresas, variando desde o caro refinamento da avenida Montaigne (que concentra a alta costura, no Champs Elys�es) at� as pechinchas nas pequenas lojas espalhadas pela cidade e nos magazines.
+Paris �... o encanto dos sentidos.
+Mesmo abstraindo-se tudo o que a cidade representa como patrim�nio cultural da humanidade, ainda assim ela resulta �parisdis�aca�, adornada de todo o sortil�gio.
+Com essas defini��es, a equipe teria condi��es de estabelecer as regras o real.
+Amaral disse que as reservas internacionais ser�o mesmo a garantia da nova moeda.
+�Existem algumas op��es de como fazer isso�, disse S�rgio Amaral, chefe de gabinete do ministro Rubens Ricupero.
+O real ter� que ter uma refer�ncia em rela��o ao d�lar, segundo Amaral, que n�o quis dizer se o c�mbio ser� fixo ou vari�vel.
+Primeiramente, gostaria de dizer que o vosso caderno � �D+�!!!
+Adoraria se voc�s �engordassem� um pouco mais o Folhateen.
+E eu gostaria de saber como fazer para adquirir a carteirinha de estudante.
+No interior, o jeito mais f�cil � procurar o gr�mio estudantil ou a dire��o de sua escola.
+Eles devem ter informa��es para orientar voc�.
+O ex-deputado Ibsen Pinheiro (PMDB-RS) tamb�m n�o quis falar sobre a absolvi��o do ex-presidente.
+Ex-presidente da C�mara, Ibsen conduziu a vota��o autorizando o impeachment de Collor, em 29 de setembro de 1992.
+O prefeito de S�o Paulo, Paulo Maluf (PPR), e o candidato derrotado do PMDB � Presid�ncia, Orestes Qu�rcia, tamb�m n�o quiseram comentar o resultado.
+O meia Mazinho deve substituir o meia defensivo Mauro Silva segunda-feira contra os Estados Unidos pela segunda fase (oitavas-de-final) da Copa do Mundo.
+O t�cnico Carlos Alberto Parreira afirmou ontem que a sele��o brasileira precisar� tocar mais a bola contra um advers�rio que ele prev� muito defensivo.
+O n�vel de detalhamento do atlas Folha/�The New York Times� n�o tem similar no mercado brasileiro.
+Os mapas de pa�ses, como os do terceiro fasc�culo, mostram 16 tipos de vias de comunica��o (de estradas a aeroportos de tr�fego local e internacional), rios, lagos, cachoeiras, montanhas (acompanhas da altitude), parques nacionais, reservas de g�s natural etc.
+�Senti que eu j� conhecia o Brasil antes de ter vindo aqui�.
+Saar �descobriu� no Brasil o bambu verde.
+Sua id�ia � fazer na 22� Bienal uma instala��o com bambus no segundo andar que seja vista por cima, do terceiro, �com toda sua cor e padr�o�.
+A partir de hoje a �rea de conviv�ncia do Sesc Pomp�ia abriga instala��o in�dita do artista pl�stico Nuno Ramos, �Montes�.
+O trabalho faz parte do projeto �Exerc�cios Visuais� do Sesc Pomp�ia, que visa a ocupa��o inusual de espa�os da ex-f�brica.
+Os integrantes dos movimentos Pinheiros Vivo e Vila Ol�mpia Viva receberam com alegria a not�cia.
+�Acho que o Consema foi sens�vel ao que est� acontecendo.
+A obra vai causar um impacto t�o grande na regi�o que deve ser analisada pelo Estado�, disse Hor�cio Galvanese, do Pinheiros Vivo.
+Para o arquiteto Siegbert Zanettini, do Vila Ol�mpia Viva, �agora um �rg�o independente est� analisando o impacto da obra sobre o meio ambiente�.
+Segundo Zanettini, a an�lise do Consema foi resultado do compromisso assumido, na sexta-feira passada, pelo governador Fleury.
+O �ndice de Pre�os ao Consumidor (IPCA) de Belo Horizonte fechou a �ltima quadrissemana de novembro com varia��o de 3,12%, contra os 2,54% registrados em igual per�odo de outubro.
+Em compara��o � quadrissemana anterior, quando o IPCA ficou em 3,28%, houve redu��o.
+O dado � do Ipead (Instituto de Pesquisas Econ�micas e Administrativas/MG) e mede os gastos das fam�lias com renda de 1 a 40 sal�rios m�nimos em Belo Horizonte.
+O IPCR, para fam�lias com renda de 1 a 8 m�nimos, ficou em 3,06%, contra os 2,85% do mesmo per�odo de outubro.
+Teremos, ent�o, um meio-campo ainda mais s�lido na prote��o � zaga e mais inteligente, veloz e agressivo, na combina��o com o ataque.
+Teremos o balan�o entre defesa e ataque que tanto Zagalo e Parreira defendem, mas que o time n�o revela em campo.
+Alberto Helena Jr., 52, � colunista da Folha.
+Sua coluna na Copa � di�ria.
+�F�ria Metal� entrevista os integrantes do Ramones, banda surgida em Nova York na d�cada de 70.
+Johnny (guitarra), Joey (vocal), CJ (baixo) e Mark (bateria) falam sobre o surgimento do grupo, a influ�ncia de Iggy Pop, processo de cria��o e o revival do punk considerado rid�culo e ileg�timo por CJ.
+Entre os clips que o programa exibe est�o �Blitzkrieg Bop�, �I Wanna Live� e �Poison Heart�.
+Furia Metal MTV, 21h30.
+Entendo que a minha miss�o n�o � torcer.
+� o dilaceramento permanente desta profiss�o possessiva, exclusivista, desconfiada e ciumenta.
+Isto n�o impede que, no �ntimo, eu deseje alguma esp�cie de uma complexa, delicada e dif�cil soberania para n�s.
+Na hora da invas�o da delegacia, s� havia um motorista no plant�o.
+Os invasores quebraram portas, janelas, grades e cadeados.
+Os outros 13 presos n�o tentaram fugir.
+O delegado disse que t�m se tornado frequentes os linchamentos na regi�o.
+Citou outros ocorridos recentemente em Carinhanha, Santa Maria da Vit�ria, Paratinga e Guanambi.
+Com esse caso em Caetit�, chega a 14 o n�mero de linchamentos registrados na Bahia desde o in�cio deste ano.
+O aquecimento das vendas nos �ltimos dois meses fez com que o empres�rio carioca Arthur Sendas desengavetasse antigo projeto de constru��o de shopping center, hipermercado e conjunto residencial em �rea do grupo localizada na Via Dutra, no final da Linha Vermelha.
+O grupo Sendas entra com o terreno de 154 mil metros quadrados, e o Nacional Iguatemi Empreendimentos, dono de shoppings centers no Nordeste e s�cio de Tasso Jereissati no Iguatemi de S�o Paulo, com os recursos para bancar o empreendimento.
+No s�bado, Goldman relatou a conversa a alguns quercistas no Pal�cio Bandeirantes.
+Ele disse que a conversa foi �preliminar�.
+Goldman defendeu a necessidade de novas negocia��es.
+Em sua avalia��o, a conversa com o deputado fluminense mostrou que ambos desejam a aproxima��o.
+O m�sico e humorista Juca Chaves inaugura no dia 5 de outubro, a partir das 22h, um novo teatro em S�o Paulo.
+O Jucabar�-Theatro Inteligente vai funcionar na rua Major Sert�rio, 661, no centro de S�o Paulo.
+A renda da noite de inaugura��o ser� doada ao Fundo Social de Solidariedade do Estado.
+33... dias ser�o completados na pr�xima segunda-feira, dia de elei��o, sem que os vereadores da C�mara de S�o Paulo votem algum projeto importante.
+Dos 55 vereadores, 24 s�o candidatos � Assembl�ia Legislativa, C�mara dos Deputados ou Senado.
+Se todos conseguirem ser eleitos, quase metade (43%) dos vereadores paulistanos v�o trocar de Casa Legislativa.
+Yoshiaki Nakano -- economista, ex-secret�rio especial de assuntos econ�micos do Minist�rio da Fazenda e ex-secret�rio nacional de pol�tica econ�mica (gest�o Bresser Pereira).
+Pasta indefinida.
+Vladimir Rioli -- Pode ficar com a presid�ncia do Banespa.
+� economista e ex-vice-presidente do Banespa (governo Fleury).
+Como num grandioso �mea culpa�, os �ltimos filmes que Eastwood dirigiu �Os Imperdo�veis� e �Um Mundo Perfeito� apontam justamente para uma crise do her�i, de sua autoridade restauradora, de sua integridade monol�tica.
+Em consequ�ncia, p�e a nu uma crise dos pr�prios g�neros com que trabalha.
+Em �Um Mundo Perfeito�, o tema e seu desenvolvimento s�o de uma clareza exemplar.
+No Texas, em 1963, um menino sem pai, Philip Perry (T.J. Lowther), � tomado como ref�m por um criminoso fugitivo, Butch Haynes (Kevin Costner).
+Ambos partem numa fuga para o norte (rumo ao pai de Haynes, que mora no Alasca).
+O programa sobre a 2� Guerra Mundial traz informa��es sobre 2.100 eventos, tem 900 fotos e 140 mapas de batalhas, permitindo estudar as t�ticas usadas na guerra, al�m de textos e v�deos de grava��es originais.
+Quem prefere as miss�es espaciais tem � disposi��o cerca de oito horas de informa��es sobre 1.600 miss�es, incluindo o programa Apollo e o passeio do astronauta Neil Armstrong na Lua.
+A organiza��o tamb�m iniciou um inqu�rito para averiguar o massacre de cerca de 500 mil pessoas.
+Encarregado do caso, o marfinense Rene Degni-Segui se disse favor�vel a um tribunal internacional para julgar os acusados.
+�Pessoalmente prefiro um tribunal internacional a dependermos dos tribunais nacionais (de Ruanda), que ter�o dificuldade em ser imparciais�, disse o professor de direito da Costa do Marfim.
+A surpresa descrita por Pedro Collor atingiu todos os partidos que integram a chamada �Nova For�a� da pol�tica local, liderada por Lessa.
+At� o final da tarde de ontem, nenhum dos partidos da �Nova For�a� havia se manifestado sobre o assunto.
+Lessa disse que est� disposto a ouvir uma proposta formal do governador.
+O Congresso Internacional de Direitos Autorais enviou, na �ltima sexta, uma recomenda��o ao ministro da Cultura, Luiz Roberto Nascimento Silva, para reativar o �rg�o governamental que cuidava da �rea da propriedade intelectual.
+O CNDA (Conselho Nacional de Direito Autoral), que respondia pela fiscaliza��o, consulta e assist�ncia nas mat�rias relativas a direitos autorais, foi desativado pelo governo Collor.
+FHC e Malan n�o esclareceram, at� o momento, quanto o governo gastou e quando foram comprados os pap�is.
+Ainda assim, s� o an�ncio de que as garantias ser�o adquiridas junto ao mercado poder� elevar os gastos com as futuras compras, raciocinam senadores como Espiridi�o Amin (PPR-SC), Ronan Tito (PMDB-MG) e Gilberto Miranda (PMDB-AM).
+Assessores da equipe econ�mica j� t�m pronta parte da argumenta��o.
+FHC e Malan dever�o defender que o uso das reservas foi necess�rio para salvar o acordo da d�vida, sem o qual o pa�s n�o normalizar� suas rela��es financeiras internacionais.
+Para explicar as compras em sigilo, o argumento � justamente a necessidade de evitar o encarecimento dos pap�is.
+O acordo da d�vida implica a troca dos pap�is atuais por uma combina��o de cinco tipos diferentes de novos pap�is.
+A opera��o dar� um desconto inicial de US$ 4,3 bilh�es sobre o valor do d�bito.
+Essas condi��es, para renegociar US$ 35 bilh�es, foram aprovadas pelo Senado no final de 93.
+O mesmo n�o ocorre com a mistura de tinta para a parede.
+Segundo a consultora de cores Renata Louren��o, o trabalho pode ser feito por qualquer pessoa seguindo as instru��es da lata, sem a necessidade de um pintor profissional.
+Mas deve-se tomar o cuidado de acabar com infiltra��es na parede para que a tinta possa ser bem aproveitada e dure mais tempo.
+�lvaro Vidigal, da Bovespa, registra a �prud�ncia� dos candidatos �s elei��es de hoje, �que est�o sabendo preservar a institui��o acima dos interesses pessoais�.
+Preocupa��o de Vidigal: �Manter junto �s autoridades a imagem de seriedade cuidadosamente cultivada no decorrer desses anos�.
+A Secretaria Estadual da Sa�de investiu R$ 11 milh�es na campanha, de acordo com o diretor do Centro de Vigil�ncia Epidemiol�gica, Wagner Costa.
+Na primeira fase, no dia 11 de junho, foram vacinadas cerca de 3,2 milh�es de crian�as em S�o Paulo.
+Neukirchen disse que a proposta apresentada aos bancos inclui uma inje��o de 2,7 bilh�es de marcos (US$ 1,55 bilh�es) para aumento do capital acion�rio.
+Al�m disso, a companhia pede que os bancos aumentem suas linhas de cr�dito em 500 milh�es de marcos (US$ 287,1 milh�es) e concedam uma morat�ria de tr�s meses no pagamento das d�vidas.
+No mercado, estima-se que as d�vidas do grupo superam os nove bilh�es de marcos.
+Os bancos devem responder � proposta do grupo at� o dia 12.
+A crise no MG aprofundou-se no in�cio de dezembro com o decl�nio nos pre�os internacionais do petr�leo.
+A pesquisa, divulgada no �ltimo s�bado, aponta o candidato do PSDB em primeiro lugar, com 48% das inten��es de voto.
+A defini��o do tempo no hor�rio eleitoral gratuito foi encaminhada pela corregedoria a todos os partidos que disputam a sucess�o do governador Luiz Antonio Fleury Filho.
+A escolha dos t�cnicos Johan Cruyff, do Barcelona da Espanha, e Tel� Santana, bicampe�o mundial pelo S�o Paulo, como colunistas da Folha durante a Copa dos EUA recebeu elogios de ex-jogadores da sele��o brasileira.
+Todos lembraram a import�ncia dos dois treinadores na hist�ria do futebol e no cen�rio atual mundial.
+Folha -- Como vai atuar no jogo contra o Palmeiras?
+Sandoval -- Vou entrar com determina��o, sabendo que vai ser dif�cil.
+Confio na nossa equipe e espero sair do Parque Antarctica com a vit�ria.
+Uma nova escalada obscurantista na China afastou ontem de Cannes o cineasta Zhang Yimou, diretor do c�lebre �Lanternas Vermelhas�.
+Os neg�cios com ouro na BM&F (Bolsa de Mercadorias e Futuros) somaram 54 toneladas em maio, o que representa uma queda de 2,70% em compara��o ao m�s imediatamente anterior, quando foram movimentadas 55,5 toneladas, de acordo com informa��es da entidade.
+A m�dia di�ria de neg�cios com o metal em maio foi de 2,43 toneladas.
+Em rela��o ao mesmo m�s do ano passado, quando os neg�cios atingiram 139,8 toneladas de ouro, a redu��o � de 61,37%.
+A m�dia di�ria naquele m�s foi de 6,6 toneladas, segundo dados da Bolsa de Mercadorias & Futuros.
+�Se n�o fiz�ssemos isso, os jogadores sofreriam mais.
+Os hor�rios das refei��es foram adaptados ao fuso local�, disse Sant'Anna.
+Apenas para estadas mais breves, de um a dois dias, o preparador teria optado por manter o fuso de Los Gatos.
+O Conia, por exemplo, � a parte do congresso que discute o uso da inform�tica pelos profissionais liberais.
+O Conia ter� 18 palestras dirigidas �s �reas de medicina, odontologia, veterin�ria, arquitetura, urbanismo, engenharia e direito.
+�� �nico no mundo.
+Felicito.
+� uma coisa a se estudar�.
+Mas deixa escapar o que pensa.
+Na cabe�a do treinador ou no papel, o esquema t�tico pode ser ofensivo.
+Mas n�o o ser�, na pr�tica, �se n�o houver determina��o para vencer�.
+J� Vail oferece entre outras coisas espet�culos ao longo do ver�o realizados num anfiteatro aberto.
+O destaque, sem d�vida, � o bal� Bolshoi da R�ssia, que h� pelo menos cinco anos consecutivos se apresenta na cidade.
+O Flamengo ficou na espera do Gr�mio, procurando apenas aproveitar o contra-ataque.
+Mesmo assim, n�o criou muitas chances.
+O gol da vit�ria flamenguista foi marcado aos 20min do segundo tempo.
+Magno desceu pela direita e cruzou para a �rea.
+N�lio teve apenas o trabalho de escorar para o gol vazio.
+Emissora: S�rie em seis epis�dios, exibidos de segunda a quinta, na Cultura.
+�Propaganda - O Poder da Imagem� � o tipo de s�rie que poderia ser adotada com proveito nos curr�culos escolares.
+Os respons�veis pelo programa vasculharam uma multid�o de arquivos de filmes (russos, norte-americanos, franceses, ingleses, alem�es) e selecionaram imagens produzidas pelas ag�ncias oficiais desses pa�ses que comp�em uma formid�vel ilustra��o da hist�ria do s�culo 20, tal como vista por seus protagonistas.
+O governo do Estado diz que a greve � ilegal.
+Os professores da UFPB (Universidade Federal da Para�ba) tamb�m entraram em greve geral ontem.
+Eles reivindicam 105% de reposi��o de perdas salariais.
+Os funcion�rios da universidade est�o em greve desde o �ltimo dia 19.
+Patr�cia -- Minha literatura vem da imagem, da TV, do cinema.
+Eu tento usar o processo de edi��o, como se trabalha em uma mesa de edi��o.
+Folha -- Como � teu m�todo?
+O ingl�s John Ritchings vai escrever um livro sobre os meses que foi torturado pelo galo da vizinha, diz o jornal.
+O galo, Corky, cantava das 3h �s 7h diariamente e Ritchings n�o conseguia dormir.
+A estrela s�mbolo do PT vai emoldurar com destaque o cen�rio dos programas do candidato Luiz In�cio Lula da Silva.
+Seus programas de r�dio ter�o reportagens de rua, vetadas na TV.
+Segundo Markus Sokol, coordenador de comunica��o, a estrat�gia de comunica��o foi delineada em um semin�rio realizado em dezembro passado.
+Executivos em cargos de lideran�a assumem seus cargos com mais uma pesada fun��o: implementar cada passo da estrat�gia que eles pr�prios esbo�aram.
+Em junho, Christensen esteve no Brasil a convite da Funda��o Dom Cabral de Belo Horizonte, para acompanhar a segunda etapa do curso STC-Executivo.
+As constru��es lembram vagamente uma verdadeira vila de pescadores.
+Perto de qualquer uma, o cen�rio de �Tropicaliente� parece uma grande aldeia de milion�rios.
+�� bonito isto daqui�, dizia admirado um pescador nativo, que participava de uma grava��o como figurante no �ltimo dia 30.
+�Licen�a po�tica� � parte, as praias e as dunas de Fortaleza foram escolhidas como cen�rio de �Tropicaliente� por Paulo Ubiratan, diretor-art�stico da Globo, por causa da beleza natural do lugar.
+Crian�as mais velhas que nunca seguiram a dieta podem ter retardo mental, desenvolvimento motor prejudicado e altera��es significativas do comportamento (isolamento e agressividade).
+Pesquisas t�m mostrado que o fenilceton�rico deve seguir a dieta durante toda a vida.
+Os adultos que voltam a comer prote�nas podem ter piora da concentra��o e da mem�ria e uma maior agita��o.
+Quando astronautas embarcarem numa esta��o orbital ou, eventualmente, estiverem a caminho de Marte, ter�o como companhia necess�ria alguns bilh�es ou trilh�es de micr�bios comil�es.
+A Nasa (ag�ncia espacial dos EUA) encomendou � empresa Micro-Bac o desenvolvimento de um sistema de purifica��o de esgoto utilizando bact�rias conhecidas por devorarem material org�nico.
+O estudo custou US$ 400 mil.
+Os testes de laborat�rio deram certo, e agora a empresa se prepara para comercializar essa �mini-usina tratadora de esgotos�.
+Em um dos subprodutos mais ir�nicos da pesquisa, a tecnologia criada para uma �casa� orbital de bilh�es de d�lares, a esta��o Freedom, poder� facilitar a constru��o de casas populares na Terra, segundo Ercole Am�rico Carpentieri Jr., diretor da filial brasileira da Micro-bac.
+A tecnologia pode reduzir a necessidade de rede de esgotos em determinados locais.
+O que deve ser destacado de forma resumida em um item sobre qualifica��es profissionais s�o os pontos fortes da experi�ncia e da forma��o acad�mica e os atributos que podem diferenciar o candidato no mercado.
+Segundo Laerte Leite Cordeiro, consultor especializado em recoloca��o, o curr�culo deve ser dividido em tr�s partes: �rea geral de interesse (administrativa, cont�bil, financeira etc.), qualifica��es profissionais e descri��o da trajet�ria de empregos (empresas, cargos, per�odos).
+Comenta o contrabando de plut�nio, dizendo que �� s�ria advert�ncia para que governos controlem os excessos deste material que o fim da Guerra Fria criou�.
+O jornal kuaitiano diz que o governo do pa�s fechou um sal�o de beleza por uma semana por infringir a lei que pro�be mulheres de trabalhar em sal�es para homens e vice-versa.
+O Houston Rockets conquistou sua 27.� vit�ria na temporada da NBA (contra apenas quatro derrotas), em casa, s�bado, ao vencer o Philadelphia 76ers por 100 a 93.
+Mais uma vez o piv� Hakeem Olajuwon foi o fator de desequil�brio a favor do Houston, um dos mais destacados times do campeonato.
+Ele converteu 23 pontos e ganhou 17 rebotes.
+O Philadelphia tem apenas 12 vit�rias e j� perdeu 19 vezes.
+Contra o Houston, o cestinha da equipe foi Jeff Honaceck, com 20 pontos, seguido por Clarence Whiterspoon, com 16.
+Lamentavelmente, mais uma vez, um pa�s do subcontinente sul-americano suspende as garantias do Estado de Direito.
+Desta feita, por�m, n�o foi uma aventura castrista ou uma injun��o pol�tica qualquer que levou o governo venezuelano a este ato extremo, mas sim a conjuntura econ�mica.
+E, de fato, os ventos n�o sopram em favor dos venezuelanos.
+Eleito em 1988, o ex-presidente Carlos Andr�s P�rez (social-democrata) encetou uma pol�tica econ�mica neoliberal que imp�s sacrif�cios � popula��o dando in�cio a uma s�rie de violentos protestos.
+P�rez enfrentou duas tentativas de golpe militar e, a exemplo de Fernando Collor, acabou sofrendo impeachment por corrup��o e, hoje, encontra-se numa cela � espera de julgamento.
+Aprofundando as d�vidas, Leila se deixou cair na tenta��o do duplo.
+Se n�o h� mais bossa nova realmente nova, o melhor � mimetizar o criador do estilo, o cantor Jo�o Gilberto.
+O resultado dessa tentativa de metempsicose est� no rec�m-lan�ado CD �Isso � Bossa Nova� (EMI-Odeon), s�timo disco em 12 Anos de sua carreira.
+A lei tamb�m prev� que at� o final de junho o governo enviar� ao Congresso um projeto sobre a eleva��o do valor real do sal�rio m�nimo.
+Se o m�nimo sofrer altera��o, a Constitui��o determina que as aposentadorias tamb�m sejam revistas.
+O Mogi Mirim venceu ontem o Uni�o S�o Jo�o de Araras por 2 a 0 em Mogi e deu mais um passo para conquistar uma vaga na Copa Bandeirantes.
+Esse torneio reunir� os seis primeiros colocados da S�rie A-1 e o campe�o da A-2 do Campeonato Paulista.
+Em S�o Paulo, n�o � necess�rio deslocar-se muito � procura de artigos de segunda m�o.
+H� pontos de concentra��o de lojas que facilitam a pesquisa de pre�os.
+Entre uma loja e outra podem ocorrer grandes diferen�as.
+�Nas aulas, Candelori usa reportagens de jornais para introduzir seus temas e colocar toda a classe para falar.
+Na lista dos assuntos est�o a campanha contra a fome, racismo, imigra��o nordestina, aborto, corrup��o pol�tica e aborto.
+�Tem gente que acha a aula in�til.
+S�o as mesmas pessoas que acham que sua responsabilidade social se limita a pagar impostos�, diz Let�cia Zioni Gomes, 16.
+�� preciso cobrar o governo para as coisas funcionarem�, completa Bruno Fanizzi, 15.
+Fanizzi conta que antes das discuss�es em aula era uma pessoa de �extrema direita�, que n�o pensava nas suas opini�es.
+�A aula � boa porque o professor jamais imp�e a opini�o dele.
+Ele levanta a discuss�o e deixa o conflito de id�ias rolar�, diz Manoela Nicoletti, 15.
+Segundo Candelori, seu curso � de �sensibiliza��o� para temas �ticos.
+�Eles n�o suportariam a aridez da teoria, mas discuto temas como espa�o p�blico e privado atrav�s de fatos como o comportamento do Itamar no Carnaval�, diz.
+Sobre o controle antidoping, a Fifa estabelecer� uma �fax hotline�, em opera��o 24 horas por dia, para que os m�dicos das delega��es consultem a organiza��o da Copa antes de fornecer qualquer medicamento a seus jogadores.
+As consultas ser�o feitas por escrito para evitar contesta��o.
+O brasileiro Jo�o Havelange consolidou ontem, em Nova York, a sua candidatura para o sexto mandato consecutivo como presidente da Fifa.
+O presidente da Uefa, Lennart Johansson, negou que a entidade europ�ia v� apresentar oposi��o a Havelange.
+�O fonoaudi�logo tem uma �rea vasta de atua��o, apesar de estar muito concorrida.
+A gente pode trabalhar com atores de teatro, crian�as, na preven��o de doen�as ou no exame de trabalhadores expostos a ru�dos.
+O atacante do Barcelona obteve o �Onze de Ouro�, distin��o outorgada por leitores da revista francesa �Onze Mundial�.
+Recebeu 13.576 votos (33,94%) em 40 mil respostas.
+O time anunciou ontem a contrata��o do craque argentino.
+Naquele ano, Fleury j� aparecia com 20% das inten��es de voto em setembro, um m�s antes do primeiro turno.
+A menos de uma semana da elei��o deste ano, Munhoz n�o passou dos 12%.
+Fleury foi para o segundo turno e venceu o atual prefeito de S�o Paulo, Paulo Maluf (PPR).
+Na pesquisa Datafolha divulgada no �ltimo s�bado, o primeiro colocado, M�rio Covas, aparece com 47%, nove pontos percentuais acima da soma dos �ndices dos demais candidatos.
+ACM disse que �n�o conseguiu entender nada� do novo plano econ�mico elaborado pela equipe do ministro Fernando Henrique Cardoso.
+�Eu estou igualzinho ao presidente Itamar, igualzinho ao minist�rio, igualzinho a 150 milh�es de brasileiros, que n�o sabem coisa alguma sobre este plano�.
+Segundo ACM, �quem disser que conhece o plano, de um modo geral, est� mentindo�.
+O governador acrescentou que pedir� ao Nosso Senhor do Bonfim para que o plano d� certo.
+O acordo inclui votar, sem novas emendas, o projeto de Or�amento para 95.
+Os l�deres acertaram que, havendo quorum, o Or�amento de 95 s� n�o ser� votado hoje se n�o ficar pronta a publica��o do parecer aprovado na Comiss�o Mista de Or�amento, acolhendo cerca de R$ 2,5 bilh�es em emendas.
+O STJ decide que Halpern e Decaro, indiciados pelo MPF sob a acusa��o de fraude processual, devem ser julgados pela Justi�a Federal em S�o Paulo.
+Os deputados petistas Luiz Gushiken (federal) e Luiz Azevedo (estadual) pedem que o Minist�rio P�blico estadual reabra processo sobre as importa��es.
+O partido atua no movimento sindical atrav�s da �Corrente Sindical Classista�, filiada � CUT.
+Controlam ou s�o influentes em sindicatos importantes em S�o Paulo, como o dos metrovi�rios, o da Sabesp e banc�rios.
+Os sindicatos dos metal�rgicos de Betim (MG), de Ribeir�o Preto (SP) e da Bahia tamb�m s�o da tend�ncia.
+Com o resultado, o time perdeu a chance de garantir antecipadamente a vaga nas quartas-de-final.
+A equipe entra na quadra amanh� �s 20h30 (15h30 de Bras�lia) para disputar com o Canad�, no chamado �grupo da morte�, um lugar na pr�xima fase do Mundial (leia mat�ria nesta p�gina).
+Entre os exemplares destes vinhos que chegaram �s nossas praias, todos safra 92, o �Bourgueil L�chellerie�, milita decididamente na primeira fila.
+Ele � um vinho ligeiro, de aroma simples de frutas vermelhas e boa acidez, ideal para ser bebido refrescado.
+O �Chinon Les Granges� e o Saumur-Champigny �Domaine de la Martini�re� s�o pouco mais encorpados.
+O �Les Granges� tem aroma e sabor marcado por morangos e evolui bem na boca, enquanto o Saumur, tamb�m frutado, � mais s�rio e (levemente) t�nico.
+O sushiman Oscar Tadaioshi Izumi baixa com hashi e raiz forte hoje nos dom�nios de Wilma Kovesi para entregar os segredos da cozinha japonesa.
+Ningu�m ignora os terr�veis problemas que o pa�s atravessa, dos quais a corrup��o policial e os n�veis de viol�ncia s�o apenas uma faceta.
+Ainda assim, � de se supor que existam bons agentes de pol�cia.
+� particularmente a estes bem como a toda a sociedade que interessa a apura��o completa e rigorosa dos massacres.
+Toda vez que um crime cometido por representantes do poder p�blico fica sem puni��o, o �nus recai sobre a corpora��o como um todo, gerando um turbilh�o que indifer�ncia os bons dos maus policiais.
+� evidente que quem perde s�o os primeiros.
+O pior, por�m, � que nesse processo o pr�prio Estado cai em descr�dito e acaba cedendo mais espa�o �s for�as do caos.
+Folha -- O Mercosul foi objeto de declara��es meio euf�ricas nos �ltimos dias.
+Que pontos, no entanto, o deixaram frustrados?
+Amorim -- N�o posso dizer que algo me frustre, levando em conta que h� tr�s anos a id�ia do Mercosul surgia no Tratado de Assun��o.
+Talvez prefer�ssemos que o com�rcio estivesse totalmente liberalizado, sem as listas de produto em regime de adequa��o (sobre os quais s�o mantidos os impostos alfandeg�rios) que afetam de 5 % a 8 % do com�rcio regional.
+E como ningu�m � de ferro e Orlando fica perto, duas horas e meia de avi�o, a turma depois d� uma passadinha para um milk shake com Mickey e Pateta.
+Aberto desde 1948 no Beverly Boulevard quando os clientes eram Errol Flynn, Frank Sinatra e Billy Wilder, o restaurante Dominick's continua na moita.
+�Tudo leva a crer que estamos num processo de endemia�, sustenta o patologista Bruce Mendes Campos, respons�vel pela an�lise do sangue.
+De 205 pacientes com suspeitas da doen�a, 47 revelaram-se positivas neste ano.
+�Estamos assustados�, afirmou o diretor do hospital, Aloysio Campos da Paz, professor visitante das universidades de Oxford (Inglaterra), Stanford, Harvard e Nova York (EUA).
+Para viabilizar a viagem de pelo menos parte dos 20 integrantes do coral advogados, banc�rios, secret�rias, estudantes, aposentados e apenas um padre, a diretora diz que se uma empresa se dispuser a cobrir os custos da viagem, o coral oferece em troca a grava��o de um disco promocional.
+�Qualquer doa��o seria muito bem recebida�, diz madre Maria.
+Os interessados em viabilizar a participa��o do coral no festival belga podem entrar em contato com a dire��o do coral nos telefones (011) 231-5346 e 288-3514.
+Olhando com aten��o os jogos da Copa � poss�vel chegar a uma conclus�o oposta � de Parreira.
+Foi justamente gra�as � arte de alguns de seus jogadores que equipes como a Bulg�ria, a Rom�nia e a Nig�ria chegaram muito mais longe do que jamais tinham conseguido.
+Num campeonato em que predominaram as defesas fechadas, o que fez a diferen�a foi a imagina��o ouso dizer, para horror de Parreira, a �magia� de um punhado de craques: Rom�rio, Baggio, Stoichkov, Hagi, Brolin...
+No final � feita uma auditoria para ver se a auto-avalia��o feita pela f�brica corresponde � realidade.
+Segundo Migues, algumas f�bricas da Autolatina ainda n�o atingiram o padr�o de excel�ncia Q1.
+As f�bricas que atingiram receberam uma placa e, em alguns casos, um ve�culo � sorteado para seus funcion�rios.
+A disciplina de cirurgia pl�stica da Faculdade de Medicina da USP est� lan�ando um programa de cirurgia pl�stica em crian�as com a colabora��o do Banco Real.
+O objetivo � divulgar as t�cnicas mais modernas de tratamento das deformidades cong�nitas nas diversas regi�es do pa�s.
+A primeira etapa do programa come�a hoje e vai at� o dia 2 em Jo�o Pessoa (PB).
+Um novo protocolo desenvolvido por um grupo de m�dicos de quatro institui��es p�blicas de S�o Paulo est� trazendo para o Brasil o Interferon Beta, para o tratamento de pacientes com esclerose m�ltipla (doen�a degenerativa que causa enrijecimento muscular, debilidade das fun��es motoras e perda parcial da vis�o).
+A droga foi aprovada em julho passado pela FDA �rg�o de fiscaliza��o de medicamentos dos Estados Unidos para o tratamento da doen�a.
+Informa��es pelo tel. (011) 887-8740.
+O ex-ministro e candidato presidencial Fernando Henrique Cardoso disse � Folha que frequentava no passado os sebos do centro, perto da Faculdade de Direito da USP.
+Quando vai aos EUA, ele n�o se limita a comprar blazers caros.
+Tamb�m vai � livraria Barnes and Nobles, em Nova York.
+Nem todos t�m tempo e paci�ncia para sebos.
+O fil�sofo Jos� Arthur Giannotti diz que se sente angustiado em livrarias e prefere comprar seus livros por cat�logos.
+�Tantos livros, t�o pouco tempo para ler�.
+O �nibus era da Via��o Boa Vista e transportava cerca de 40 passageiros na hora do acidente, segundo a Pol�cia Rodovi�ria.
+Com a colis�o, cinco passageiros ficaram feridos e foram atendidos no Hospital M�rio Gatti.
+Todos passam bem.
+De um lado, acirrou-se a rea��o marcadamente militante, mais diretamente identificada com as propostas pol�tico-ideol�gicas das esquerdas organizadas, cujas manifesta��es procuravam real�ar o que se entendia na �poca por �cultura nacional e popular�, recha�ando a influ�ncia �imperialista� e suas armas culturais entre as quais incluiam-se a televis�o voltada para o consumo e para a �aliena��o�, as formas art�sticas �americanizadas�, a cultura pop e at�... a guitarra el�trica.
+Esta vertente, francamente conteudista, derivava das experi�ncias realizadas no per�odo pr�-64 pelos Centros Populares de Cultura (CPCs), ligados � Uni�o Nacional dos Estudantes, que privilegiavam a �mensagem� e procuravam falar uma idealizada linguagem do �povo�.
+Ele teve importante papel ao manter a pol�tica externa e muitos assuntos internos fora da esfera decis�ria da UE, ganhando a simpatia brit�nica.
+J� assessores do federalista Delors, que em dez anos na presid�ncia da CE aumentou os poderes do cargo e entrou em conflito com Margaret Thatcher (antecessora de Major), temem que a escolha de Santer sirva para enfraquecer os poderes do presidente, o que abriria caminho para o eixo Alemanha-Fran�a consolidar o seu dom�nio.
+O fato de haver razo�vel consenso acerca desses temas n�o significa, por�m, que seja f�cil ou r�pido aprovar tais reformas no Congresso.
+Elas dependem em boa parte de emendas constitucionais, o que est� longe de ser simples de fazer.
+Ora, se o presidente chegar � posse sem que estejam no m�nimo delineados, divulgados e negociados com os setores organizados da sociedade os projetos que consubstanciem tais reformas, torna-se muito mais dif�cil faz�-los aprovar no Congresso no prazo de um semestre definido pelo pr�prio FHC.
+Por tudo isso, n�o cabe formalismo em excesso.
+Cabe, � l�gico, todo o respeito �s prerrogativas do atual governante.
+Mas, at� por ser tamb�m o candidato do presidente, FHC deve come�ar a trabalhar j�.
+As paulistas Mancha Verde (Palmeiras) e Gavi�es da Fiel (Corinthians) aceitaram, mas as cariocas se recusaram.
+A carta de Stepanenko engrossa desde ontem o inqu�rito contra o ministro no TSE.
+Trata-se de outra evid�ncia irrefut�vel.
+Itamar n�o precisa da ajuda de Vicentinho.
+Tem diante de si um portentoso exemplo de desvio de conduta.
+Ali�s, j� poderia ter agido.
+Seu sil�ncio soa a cumplicidade.
+Mas os pontos mais importantes n�o foram tocados.
+N�o ouvi falar nada sobre arbitragens.
+N�o se fala na cobran�a dos �rbitros que apitam maldosamente, para prejudicar uma equipe.
+Isso aconteceu muitas vezes este ano.
+Outro ponto � que as bolas que v�o ser usadas no campeonato n�o devem ser escolhidas por interesses comerciais.
+Cerca de 300 cubano-americanos protestaram na frente do pr�dio da miss�o cubana na ONU, onde as negocia��es ocorreram.
+Funcion�rios do escrit�rio cubano tentaram abafar os protestos na rua colocando caixas de som nas janelas do pr�dio com m�sica alta.
+Em Miami, duas bombas caseiras (coquet�is Molotov) foram encontradas ontem em frente � revista cubana �Replica�, que ap�ia as negocia��es entre Cuba e EUA para tentar resolver a crise.
+As bombas n�o chegaram a ser acionadas.
+Se podemos nos orgulhar de nosso futebol, n�o podemos dizer o mesmo dos nossos �rbitros, aqui representados exatamente por Marsiglia.
+Falando em futebol, em bom futebol, devemos saudar a chegada de Ronald�o aos Estados Unidos.
+As empresas Tejofran e Power informaram ontem, em nota enviada ao TRE (Tribunal Regional Eleitoral), que prestam servi�os aos comit�s do PSDB e PMDB.
+A Tejofran e a Power s�o acusadas por Rui Ramos, ex-integrante da campanha tucana, de financiar a estrutura pol�tica do candidato do PSDB ao governo, M�rio Covas.
+Na cheia, entre outubro e abril, quando a m�dia mensal de chuvas supera 290 mil�metros, a irriga��o do arroz � feita por inunda��o.
+No per�odo mais seco, quando as chuvas n�o passam de dez mil�metros/m�s, os produtores plantam soja, que exige menos �gua.
+Na sua primeira declara��o, Queiroz omitiu US$ 1,7 milh�o em bens.
+Entre eles estavam a empresa �ticas Trevo, terrenos e pontos comerciais em Salvador.
+O presidente do partido no Rio, S� Freire, disse que o pedido de cassa��o foi feito porque Walter Queiroz n�o teria comparecido �s reuni�es marcadas para que ele explicasse sua nova declara��o de renda.
+J� o sistema de tratamento de esgotos residenciais, com a esta��o do ABC, dever� atingir 150 toneladas/dia.
+Somando-se os dois n�meros, chega-se a 500 toneladas/dia, aproximadamente 50% das 1.100 toneladas de polui��o org�nica di�ria jogadas no rio.
+Antes da derrota por 2 a 0 na primeira partida entre os dois clubes, quinta-feira no Mineir�o, o t�cnico botafoguense Renato Trindade desenhou a t�tica advers�ria num quadro negro do vesti�rio.
+Ele circundou com giz os cinco meias.
+�O segredo do Atl�tico � o quinteto.
+Eles v�m de tr�s com a bola e nos dificultam�, disse.
+Ele foi o respons�vel pela inclus�o de peixes amaz�nicos no livro dos recordes da International Game Fish Association.
+A Pol�cia Militar do Amazonas est� desenvolvendo uma campanha educativa de tr�nsito.
+Ela est� alertando os motoristas que saem de Manaus pela rodovia AM-10.
+Policiais orientar�o os motoristas com placas educativas que indicar�o os cuidados b�sicos contra acidentes.
+Agora suponha que a economia toda vire um bando de gente que ganha mesada e de gente que d� mesada.
+Para quem recebe em URV, � �timo.
+Para quem paga, pode n�o ser um grande neg�cio ter de pagar mais dinheiro todos os dias.
+� mais ou menos o que est� acontecendo na discuss�o de quem tem im�vel com quem paga o aluguel, por exemplo.
+Ou do dono da escola com seu pai, que paga a mensalidade.
+Se voc� tem um pai p�o duro, n�o tenha d�vidas pe�a a URV na sua mesada.
+Entre os itens importados, devem chamar aten��o na Feicon os revestimentos de cer�mica para piso trazidos da It�lia.
+Importados pela C�ramus, os pisos de uso comercial (para hot�is e shopping centers) destacam-se pela grande resist�ncia.
+Algumas ainda est�o sendo calculadas pela Caixa.
+Eles ter�o ainda de pagar multa equivalente a R$ 647,90.
+Al�m dos dois, tamb�m foram condenados outros 12 diretores da CEF.
+A necessidade de reparar os equipamentos durante a viagem teria alterado os planos de trabalho.
+Damascene viaja raramente de navio, mas n�o � um estranho ao mar.
+Quando jovem, esteve engajado na Marinha grega.
+Conhecia bem grandes embarca��es e saberia como agir numa emerg�ncia.
+A situa��o tende a se agravar, uma vez que nenhuma das partes parece mostrar disposi��o de recuar.
+O governo recebeu ontem outra m� not�cia: empres�rios se declararam contr�rios ao CIP.
+O motivo � que ele obriga a empresa que contratar jovens a manter um tutor para ensinar o trabalho aos novos empregados.
+Com isso, dizem, aumentam os custos de contrata��o dos jovens.
+Ao contr�rio do que poderia esperar a direita norte-americana na vig�ncia da Guerra Fria, a vinda de Soljenitsin n�o p�de ser capitalizada em discursos e apari��es p�blicas.
+Por dois motivos: primeiro, a indisponibilidade do escritor, que simplesmente n�o estava interessado em percorrer os Estados Unidos ou debater suas id�ias; segundo, seus coment�rios sobre a �decad�ncia moral� do Ocidente logo o tornaram inconveniente.
+Sua �ltima apari��o foi numa formatura em Harvard, em 1978.
+Depois disso, caiu no ostracismo.
+Sua partida dever� ser igualmente discreta.
+Seu editor, Claude Durand, disse � Folha por telefone, de Paris, que Soljenitsin n�o tem inten��o de fazer qualquer despedida p�blica dos Estados Unidos.
+�Sei que jamais terei uma chance de trabalhar t�o calmamente de novo.
+Na R�ssia, serei dilacerado pelos acontecimentos e pelas trag�dias das pessoas�.
+Est� visto que nem o conjunto, nem qualquer outra frase dele, d� � frase negada por Fernando Henrique o sentido que ela tem por si mesma.
+A leitura de todo o trecho leva, ali�s, a outra mentira, esta sobre os tempos de TV do PSDB e do PT.
+N�o � a primeira vez, e espero n�o ser a �ltima, que Fernando Henrique e esta coluna se atritam pelo mesmo motivo.
+A primeira fez dez anos h� pouco.
+Foi quando noticiei que Fernando Henrique estava em contatos sigilosos com o governo Figueiredo, oferecendo um plano de concilia��o entre o regime que se exauria e o que nasceria.
+Para isso, a presid�ncia n�o poderia ficar com o candidato �bvio na �poca, Ulysses Guimar�es, que supostamente representava o risco de confronto com os militares.
+Tancredo tivera com o governo conflitos impeditivos, aqueles que o levaram a dissolver o PP de ent�o.
+N�o preciso repetir, agora, quem era o homem que Fernando Henrique apresentava como adequado ao plano, sob o argumento de que tinha tr�nsito nas esquerdas para encabe�ar um pacto conciliador.
+O professor Leit�o de Abreu, que era ministro do Gabinete Civil, n�o vive para confirmar as visitas ofertantes de Fernando Henrique.
+Mas h� testemunhas da veracidade da not�cia, inclusive o ex-presidente Figueiredo, que repeliu a proposta.
+Apesar da veracidade ainda hoje comprov�vel, na ocasi�o Fernando Henrique me fez graves acusa��es.
+Com a mesma hombridade que exibe mais uma vez.
+Pois experimente ir a um show de m�sica, por exemplo, ofender Tim Maia ou Jo�o Gilberto na boca do palco, e voc� ver� que a atitude de Edmundo foi, no m�nimo, compreens�vel.
+Pois �, Palmeiras e Corinthians fazem hoje a final�ssima do Brasileiro.
+Provavelmente ser� um bel�ssimo jogo.
+O Corinthians ter� de atacar os 90 minutos para reverter a enorme vantagem palmeirense.
+Como Collor, o novo presidente falou em: �reforma do Estado�, �abertura da economia�, �avan�o da privatiza��o� e �flexibiliza��o dos monop�lios estatais�.
+�Isto come�ou a acontecer no governo anterior ao do presidente Itamar Franco�.
+Morreu no domingo � noite aos 80 anos o cientista Roger Sperry, ganhador do Pr�mio Nobel de Fisiologia e Medicina de 1981.
+O an�ncio foi feito segunda-feira pelo Instituto da Calif�rnia para Tecnologia Avan�ada (Pasadena), onde Sperry trabalhou at� 1984.
+Na rela��o divulgada ontem em Washington, o Brasil � citado com outros 34 pa�ses.
+O Jap�o � o principal alvo da lista por barreiras aos produtos, servi�os e investimentos norte-americanos, que segundo o USTR, colaboraram para que o d�ficit dos EUA no com�rcio com os japoneses atingisse US$ 60 bilh�es no ano passado.
+O relat�rio dedica 42 p�ginas ao Jap�o, �cujas barreiras � importa��o de produtos e servi�os s�o muito maiores do que as dos demais membros do Grupo dos Sete (pa�ses mais industrializados) e representam um entrave inaceit�vel ao sistema de com�rcio global�.
+Tendo em mente o Jap�o, os EUA reinstalaram recentemente a cl�usula conhecida como Super-301, mecanismo legal que amplia os poderes de retalia��o comercial da Casa Branca.
+A Super-301, que j� foi aplicada contra o Brasil quando esteve em vigor em 1989 e 1990, estabelece um prazo de 18 meses para que as negocia��es bilaterais levem � elimina��o do que os EUA consideram barreiras ao com�rcio.
+Depois disto, os EUA podem impor tarifas punitivas de at� 100% �s importa��es do pa�s ofensor.
+F� do t�cnico s�o-paulino Tel� Santana (�� muito modesto�), Bianchi v� o time paulista t�o forte como em 1992 e 93.
+O principal jogador da equipe � o meia-direita Jos� Basualdo.
+Ele atuou pela sele��o argentina na Copa do Estados Unidos.
+Entre a defesa e o ataque, a bola quase sempre passa por seus p�s.
+�A L�lian sem calcinha fez mais estrago do que um bando do Comando Vermelho com rev�lver.
+Ela � uma profissional e conseguiu o objetivo dela.
+Mas o presidente tem de ter postura.
+O bom � que o Fernando Henrique n�o estava presente e n�o se comportou como o Maur�cio Corr�a.
+�S� tenho um sentimento: estupefa��o.
+Todo presidente � uma autoridade p�blica e deve cuidar da imagem daquilo que ele representa.
+O fato do presidente ter posado ao lado de aquela atriz foi um vexame para toda a na��o brasileira.
+As letras do Digable Planets n�o refletem este radicalismo.
+S�o mais cool, falam de amor, crescimento e amizade sempre com um vi�s �engajado�, por�m.
+O pr�prio nome da banda, segundo Ann Marie, vem de �leituras sistem�ticas de Sartre, Marx, Kafka�.
+Significa �planetas bacanas�.
+O nome seria uma tentativa de passar �s pessoas o conceito de auto-sufici�ncia cada ser � um �planeta� mas tamb�m de inter-rela��o entre os �planetas�.
+�Da� os nomes de insetos para n�s�, continua Ann Marie.
+O atlas que a Folha vai distribuir aos leitores em 19 fasc�culos tem tr�s patrocinadores: o Banco Itamarati, a Companhia Brasileira de Alum�nio (do grupo Votorantim) e a OAS Participa��es.
+N�o foi dif�cil encontrar patrocinadores para o projeto, segundo Antonio Carlos de Moura, 35, diretor comercial da empresa Folha da Manh� S/A, que edita a Folha.
+Uma mesa de encostar, em jacarand� claro, feita � m�o no Rio no s�culo 18, vai a leil�o no pr�ximo m�s em S�o Paulo por US$ 15 mil, pre�o equivalente a dois carros populares zero quil�metro, como o Fusca.
+O leil�o ser� na casa Renato Magalh�es Gouv�a Escrit�rio de Arte, nos Jardins (zona oeste).
+Colosio, candidato do Partido Revolucion�rio Institucional (PRI, governista) foi morto a tiros em Tijuana durante um com�cio.
+Ele era o favorito para as elei��es programadas para 21 de agosto.
+Foi substitu�do como candidato por Ernesto Zedillo, que chefiava sua campanha.
+O percurso, que ser� divulgado momentos antes da prova, deve incluir a avenida Bandeirantes, via Anchieta, represa Billings e chegada na av. Nove de Julho, em frente ao Banana Banana.
+Ao final, todos os corredores comemoram a prova com uma feijoada na casa noturna.
+Ali estar� exposto o Lancer Evolution, carro de competi��o da Mitsubishi.
+Ele � acusado pela morte de pelo menos 83 pessoas e de ter ferido outras 200.
+O Chacal chegou ontem �s 10h30 (5h30 no Brasil) ao Pal�cio da Justi�a, usando algemas e cercado por um forte esquema de seguran�a.
+S�O PAULO A elei��o presidencial est� na sua pior fase, que � aquela das articula��es pol�ticas de bastidores, da costura sigilosa de alian�as, das rasteiras e baixarias entre advers�rios e correligion�rios.
+� a fase do vale-tudo que antecede a escolha dos candidatos pelos partidos.
+� uma fase tr�gica para n�s, eleitores.
+Imagino que para a grande maioria dos pol�ticos este seja o momento de gl�ria porque exercita o que considera uma virtude: a arte da esperteza, a pol�tica segundo os c�nones herdados da velha tradi��o das raposas pol�ticas.
+H� quem se orgulhe dos golpes que consegue imaginar e confunda artimanhas com estrat�gia.
+� que �os dirigentes petistas acreditavam tanto na vit�ria no primeiro turno que dispensaram os pol�ticos�, quer dizer, a direita.
+Agora ela est� de volta.
+Jos� Genoino, entrevistado, j� falava como novo centro do poder.
+�N�s vamos colocar a campanha na rua�, dizia, sorrindo.
+A secess�o est� s� come�ando.
+Maria Mole: mistura de bebidas, conhaque com martini e/ou vodka.
+Muito N: coisa ou conversa negativa demais.
+Futebol e pol�tica n�o se misturam.
+Essa � a postura que prevalece para a maioria dos jogadores da sele��o brasileira de futebol.
+Apenas o goleiro Gilmar (Flamengo) revelou o seu voto para as elei��es presidenciais deste ano.
+Gilmar pretende votar no senador Esperidi�o Amin (PPR-SC).
+�No quadro de hoje, voto nele�, disse o goleiro do Flamengo.
+Para governador ele ainda est� indeciso.
+�s 21h15, o tucano Jos� Serra, rec�m-eleito senador, juntou-se ao casal presidencial.
+�s 22h43, o governador eleito de S�o Paulo, M�rio Covas, e sua mulher Lila chegaram para o jantar.
+Covas disse que �Z� Milion�rio� � um velho amigo seu.
+Jos� Carlos Ara�jo foi s�cio de Oct�vio Cavalcanti Lacombe (morto em 1992), que fundou h� mais de 30 anos a Paranapanema, companhia de explora��o de min�rios principalmente estanho e cassiterita na regi�o amaz�nica.
+O ladr�o verificou qual era o saldo da conta de Maria Estela e, em seguida, retirou R$ 225 do caixa.
+Antes de ir embora, o ladr�o pegou o t�tulo de eleitor e o CIC da assistente de secretaria e os jogou atr�s do caixa.
+�Ele disse que era para eu apanh�-los e para n�o tentar reagir e impedir sua fuga�, afirmou Maria Estela.
+Mas, quando o ladr�o ia deixar o caixa, dois outros clientes do banco chegaram.
+Arrelia foi o primeiro palha�o brasileiro a ter um circo na televis�o.
+Era o �Circo do Arrelia�, na TV Record, em 1953.
+Ele ficou famoso no todo o pa�s.
+Arrelia se chama Valdemar e tem 87 anos.
+Quando menino, gostava de estudar e n�o queria ser de circo, como seus pais.
+Para Serra, se n�o for aprovada a emenda constitucional que desvincula dos fundos de participa��o dos Estados e munic�pios as receitas dos novos tributos, de nada adiantar� o aumento de impostos.
+�Ainda n�o d� para ficar tranquilo.
+O prazo � curto e os advers�rios s�o muitos�, disse Covas.
+Para ele, a pr�xima semana ser� decisiva para o governo, quando as MPs que tratam do aumento de impostos ter�o de ser necessariamente votadas.
+Elas n�o podem ser reeditadas porque a cria��o ou altera��o de impostos s� podem ser feitas no ano anterior ao de sua cobran�a.
+O Banespa ainda n�o esclareceu as supostas dificuldades de uma ag�ncia no exterior por haver pago parte das importa��es sem licita��o de equipamentos israelenses no governo Qu�rcia.
+O BC n�o teria aprovado a opera��o.
+Se o Minist�rio P�blico Estadual comprovar danos ao patrim�nio p�blico com as importa��es de Israel, os respons�veis estar�o sujeitos a a��o de ressarcimento.
+Mas o �Concerto para Harpa, Obo� e Orquestra de C�mara� (1980) e sobretudo sua �Cha�ne 2� para violino e orquestra (1985) apontam para uma escuta mais livre, hedonista.
+Resumindo: Lutoslawski radicalizou Bart�k naquilo que este tinha de noturno, de dram�tico e �moderno�.
+Os acasos e interrup��es misteriosas da m�sica noturna bartokiana se fizeram, em Lutoslawski, sil�ncios e vibra��es aleat�rias da orquestra.
+Jogos.
+Os folhetos ter�o uma rede de distribui��o in�dita at� agora em qualquer campanha institucional do governo: carteiros da ECT (Empresa de Correios e Tel�grafos), ag�ncias e postos de atendimentos do Banco do Brasil, Caixa Econ�mica Federal, al�m de postos lot�ricos.
+�Durante 15 dias, os bancos v�o lhe dar reais em troca de cruzeiros reais.
+Se for necess�rio, esse prazo ser� prorrogado.
+Por isso, voc� n�o precisa correr para trocar o seu dinheiro�.
+Essa � a inscri��o obrigat�ria nos folhetos e cartazes.
+O atacante Casagrande aponta a falta de entrosamento como uma das causas para o seu baixo desempenho.
+�Sei que n�o estou rendendo tudo que posso, mas estou em evolu��o�, disse.
+Folha -- O que voc� acha das cr�ticas que vem recebendo da torcida e da imprensa?
+Segundo o m�dico Fl�vio Pozzuto, as fun��es neurol�gicas do torcedor se mant�m sem altera��o ele respira com ajuda de aparelhos.
+O m�dico diz que o edema neste caso � irrevers�vel.
+�A les�o foi grande e vai deixar sequelas�.
+A pol�cia pretende localizar todos os envolvidos na briga at� o dia 21 de outubro.
+O Conselho Monet�rio Nacional aprovou ontem uma nova linha de empr�stimos do Banco Central para socorrer bancos de qualquer porte.
+Os empr�stimos poder�o ser pagos em 90 dias e prorrogados pelo mesmo prazo.
+� a terceira linha especial de socorro criada pelo BC desde o lan�amento do real, todas com o objetivo de atender bancos que tiveram sua sa�de financeira comprometida.
+A frase do ex-presidente � uma refer�ncia a uma das promessas feitas por Cafeteira, um ex-sarneyzista que insinuava, durante a sua campanha, mandar Sarney para fora do seu Estado.
+Se derrotar o candidato do PPR, Roseana se tornar� a primeira governadora eleita na hist�ria do pa�s.
+No caso de impedimento do presidente da Rep�blica ou de vac�ncia do cargo nos primeiros dois anos de mandato, a substitui��o ser� feita atrav�s de elei��o direta em 90 dias.
+Se a vac�ncia ocorresse nos �ltimos dois anos de mandato, a elei��o seria feita pelo Congresso, em 30 dias.
+Nos dois casos, at� a elei��o a Presid�ncia seria exercida pelo presidente da C�mara dos Deputados.
+Ficando novamente vago o cargo, o presidente do Senado e, sucessivamente, do Supremo Tribunal Federal (STF) assumem a incumb�ncia.
+O n�mero de a��es ordin�rias em junho apresentou queda de 10,36% comparado com o m�s de maio.
+Foram requeridas 606 a��es em junho contra 676 no m�s anterior.
+Em rela��o ao mesmo m�s do ano passado a queda � ainda maior 34,98%.
+A pesquisa � feita pela Hubert Im�veis e Administra��o.
+Em junho foram pedidas 145 a��es revisionais de aluguel.
+Comparadas com o m�s anterior, as revisionais tiveram queda de 9,94%.
+Por�m, se o total de junho for comparado com o mesmo m�s de 93, verifica-se aumento de 28,32%.
+No 1� semestre de 94 foram feitos 662 pedidos no total.
+Segundo o secret�rio, US$ 43,5 milh�es foram liberados pelo prefeito quatro meses atr�s para amplia��o ou reforma em dez hospitais e unidades de sa�de.
+Entre elas, est�o as reformas e amplia��es dos hospitais do Tatuap� (US$ 7,4 milh�es), Vila Nova Cachoeirinha (US$ 17 milh�es) e Tide Setubal (US$ 9,4 milh�es).
+Raia disse ainda que sua equipe levantou as necessidades de material de consumo da rede e que US$ 3 milh�es est�o sendo liberados agora para gastos do primeiro trimestre.
+Ao saber que os militares pretendiam a democracia e o fim da guerra colonial, os portugueses come�aram a dar cravos aos soldados, que os colocavam na ponta dos seus fuzis -- da� o nome Revolu��o dos Cravos.
+No processo que seguiu � derrubada da ditadura, o poder caiu na rua.
+Decis�es governamentais eram ultrapassadas pela for�a de manifesta��es.
+A Secretaria Municipal da Administra��o afirmou ontem que poder� rever o decreto do prefeito Paulo Maluf que pro�be a contrata��o de portadores de doen�as com possibilidade de se tornarem incapazes no futuro.
+O decreto foi publicado no �Di�rio Oficial� do dia 29 de julho.
+No par�grafo �nico do seu primeiro artigo, ele afirma que os candidatos n�o podem apresentar �patologia com perspectiva presente, de incapacidade no futuro�.
+Os fatos est�o a� e s�o estarrecedores.
+J� passou da hora de governo, prestadores de servi�os e usu�rios se empenharem para n�o s� aprofundar as discuss�es sobre a inefic�cia do sistema de sa�de, como tamb�m para buscar alternativas e solu��es vi�veis, a fim de proporcionar um atendimento mais digno � popula��o em geral.
+A crise da sa�de tem solu��o, basta vacin�-la contra as inger�ncias pol�tico-partid�rias, as omiss�es e desacertos dos respons�veis pelo setor.
+O Supremo interpreta, em �ltima inst�ncia, a Constitui��o e as leis, mesmo que elas digam respeito a contratos privados.
+Como poder� faz�-lo, por�m, respeitando ele pr�prio a lei, quando existem legalmente duas moedas?
+Como julgar em consci�ncia contratos escritos ou impl�citos que dependem de uma moeda como meio de pagamento e de outra como unidade de conta?
+A primeira varia tanto, dia-a-dia, que uma semana � suficiente para produzir um aumento de mais de 10% nos vencimentos antecipados dos congressistas e dos magistrados.
+Cabe perguntar: Se o Supremo n�o est� de acordo com a suposta falta de regra das duas moedas, que existem de fato desde a emiss�o da �ltima medida provis�ria, por que n�o a proclamou imediatamente inconstitucional e se serviu dela privadamente?
+O Supremo Tribunal � por acaso uma Justi�a privada ou � o baluarte da Justi�a P�blica?
+Como se v�, trata-se de quest�es altamente perturbadoras.
+Assim, se voc� est� usando uma planilha, por exemplo, pode selecionar parte (ou todo) e levar o que foi selecionado para outra planilha ou qualquer programa que estiver sendo usado.
+Os arquivos das pastas tamb�m podem ser abertos a partir do menu, facilitando o uso de arquivos e aplica��es.
+O oper�rio da constru��o civil negro e desempregado Rodney King, 28, vai receber US$ 3,8 milh�es da Prefeitura de Los Angeles como compensa��o pelos efeitos da surra que levou de quatro policiais brancos em mar�o de 1991.
+A absolvi��o em abril de 1992 dos policiais provocou 48 horas de conflitos raciais em LA que resultaram em 58 mortes, 4.000 feridos e US$ 1 bilh�o em preju�zos.
+A contracultura, o movimento pelos direitos civis, a terceira onda gay e o politicamente correto nasceram aqui.
+Ningu�m fica parado com o vento que sopra do Pac�fico.
+O politicamente correto institucionalizou-se, mas nem os americanos o suportam, exceto, � claro, os eternos paroquiais.
+� uma nova religi�o.
+A pr�pria Receita Federal pode ter, indiretamente, inspirado este golpe.
+Quando o secret�rio da Receita era Osires Lopes Filho, v�rias cartas foram expedidas a profissionais liberais e empresas �convidado-os� a acertar as contas com o le�o.
+O delegado informa que foi aberta sindic�ncia interna no minist�rio para tentar apurar o caso e que a Pol�cia Federal dever� tamb�m abrir um inqu�rito.
+Para avaliar as perspectivas deste ano, a Folha ouviu economistas e consultores que t�m entre seus clientes muitas das principais empresas brasileiras.
+Todos trabalham com tr�s cen�rios: o prov�vel, o otimista e o pessimista.
+De uma ponta a outra, as varia��es s�o imensas e refletem a instabilidade de uma economia em infla��o cr�nica e alta.
+Para a taxa de infla��o de dezembro de 1994, por exemplo, as previs�es variam de civilizados 2% para explosivos 150%.
+Uma resposta est� numa palavra: tranquilidade.
+Num Mundial, esta palavra tem um significado determinante.
+A tranquilidade pode levar uma equipe ao t�tulo ou pode destro��-la no caminho.
+Num grupo de 30 pessoas, entre jogadores, t�cnicos, m�dicos e massagistas, que devem conviver durante tanto tempo, surgem problemas diariamente.
+O McDonald's proibiu o cigarro nos 1.400 restaurantes de sua propriedade nos Estados Unidos.
+Mais 3.600 concession�rios da marca aderiram � proibi��o.
+William Rhodes, vice-presidente mundial do Citibank, veio ao Brasil para reuni�o com executivos do banco.
+Hoje deve se encontrar com FHC.
+�O problema � grave.
+Se n�o fizermos nada, pode haver uma trag�dia.
+O nosso jogo aqui � preven��o�, disse o secret�rio de Seguran�a P�blica do Estado de S�o Paulo, Ant�nio Correia Meyer.
+O secret�rio comandou ontem uma reuni�o sobre seguran�a nos est�dios, na sede da Federa��o Paulista de Futebol.
+Estiveram presentes dirigentes do futebol, e autoridades policiais civis e militares.
+Corn�lio Pires -- Folclorista e contador de causos (1884-1958), foi o primeiro produtor independente de discos do Brasil.
+Raul Torres & Flor�ncio -- Dupla que atuou dos anos 40 aos 60 e criou um estilo rom�ntico de interpreta��o.
+Reis n�o quis dizer se tamb�m abandonar� Esperidi�o Amin.
+Um acidente envolvendo um caminh�o e um �nibus, na �ltima sexta-feira, matou 16 estudantes, tr�s motoristas e deixou nove feridos no km 5,4 da estrada, que � chamada de �rodovia da morte�.
+De acordo com a assessoria, o secret�rio dos Transportes, M�rcio Ribeiro, determinou que os engenheiros do Departamento de Estradas de Rodagem (DER) apressassem os estudos para a duplica��o.
+Crian�as norte-americanas encontram em maio mais dois programas educativos em CD-ROM.
+�Through the Woods� (Pelo Bosque) e �At the Seashore� (Na Praia) foram desenvolvidos pela IBM.
+�Through the Woods� � voltado para crian�as que est�o no primeiro ano do 1� grau.
+Cada um dos produtos deve custar US$ 329.
+Incluem ainda um outro CD, que serve como guia para uso em sala de aula.
+A maior parte da produ��o (25O t) j� est� comprometida com uma ind�stria de processamento da polpa.
+O restante ser� comercializado em supermercados, feiras e ind�strias de sorvetes do Paran�.
+O fazendeiro, que cultiva tamb�m 1 hectare de rosas, diz que optou pelo morango ap�s experi�ncias �nem sempre vantajosas� com outras culturas.
+Eu tenho um projeto, que vou revelar aqui pela primeira vez, que tem a assinatura de um dos homens mais inteligentes e criativos do mundo, Eliezer Batista.
+Ele est� me municiando e me dando o instrumental de que care�o para imaginar a solu��o dos problemas do Estado.
+Schumacher era a esperan�a de conferir alguma emo��o a um Mundial que j� parecia ter dono.
+E emo��o significa p�blico, o qual atrai patrocinadores.
+Depois, com a morte de Senna, vieram as exig�ncias por medidas de seguran�a na F-1.
+Mas a bordo de um 'cadillac' preto convers�vel, o presidente Menem atravessou a pista do parque da Sociedade Rural Argentina sob os aplausos da plat�ia de cerca de 10 mil agricultores.
+�N�o chegamos ao para�so, mas j� conseguimos sair do inferno�, disse Carlos Menem da tribuna de Palermo.
+Ontem, os bancos j� se adequaram �s novas regras.
+�Todas as linhas foram travadas em tr�s meses�, disse M�rio Luna, gerente de Departamento de Cr�dito e Financiamento do Bradesco.
+Luna afirmou que � cedo para fazer uma avalia��o completa do impacto das novas medidas.
+O t�cnico Jair Pereira deve escalar um �time de reservas� amanh�, �s 17h, contra o Flamengo, em Caio Martins.
+�Quero todos totalmente recuperados para as quartas-de-final, afirmou o treinador corintiano.
+Ele considerou surpreendente o desempenho do russo Alexander Popov.
+O ex-jogador defendeu tamb�m a ado��o de penas para menores de 18 anos que cometam crimes.
+�Ningu�m, por mais crian�a que seja, pode ir para o est�dio com uma arma.
+A participa��o aumentaria 3% at� meados de 95.
+O trabalho da publica��o inglesa, divulgado pelo �Financial Times�, indica que h� menos preocupa��o com riscos pol�ticos e econ�micos do que com a falta de liquidez e diversifica��o.
+No caso de Alvarez, a intimidade com os EUA levou-o a conhecer a mentalidade norte-americana, seus gostos, seus costumes.
+De certa forma, afiou-o para a luta em que se engajaria a partir dos anos 60.
+� desse combate, da maneira como ele chegou a criar as formas que geraram uma obra original, que �O Olho da Revolu��o� tira seu interesse.
+O livro chega num momento em que a id�ia de cinema engajado est� meio por baixo.
+Cuba tamb�m est� com a imagem um tanto abalada (crise, �xodo por mar etc).
+Mas justamente por isso �O Olho da Revolu��o� chega num momento apropriado.
+�� perto de casa e tranquilo�, diz Rossi, em meio a goles de refrigerante em lata, dentro de seu Opala estacionado atr�s de uma igreja.
+Folha -- O estado de s�tio seria um golpe?
+Serra -- Seria e o foi apresentado assim, numa reuni�o da qual participei.
+Nela, o presidente confidenciou n�o acreditar mais que terminaria o seu mandato.
+Foi seis meses antes do golpe.
+As dificuldades que bancos federais e estaduais provavelmente enfrentar�o refor�am as d�vidas quanto � conveni�ncia do setor p�blico ser propriet�rio de tantas institui��es financeiras.
+O Brasil disp�e de um dos mais modernos sistemas banc�rios privados do mundo.
+O fato de que os bancos p�blicos venham a ser, recorrentemente, fontes de preju�zo vem s� refor�ar as desvantagens da atua��o do Estado nesse setor.
+Na B�blia, h� duas hist�rias que tentam explicar a origem de tantas l�nguas.
+Essas hist�rias s�o chamadas de mitos.
+A hist�ria da Torre de Babel (G�nesis, cap�tulo 11, no Antigo Testamento) diz que a �Terra tinha uma s� l�ngua e um s� modo de falar�.
+Ent�o, os homens encontraram um lugar plano e resolveram construir uma torre que chegasse at� o c�u.
+Trata-se de uma estrat�gia de infla��o reprimida, pronta para rebelar-se pela via da explos�o cambial no momento em que os agentes econ�micos se convencem de que n�o � sustent�vel, e em que os capitais come�am a fugir.
+Ela dever� inviabilizar a atual oportunidade de estabilizar afetivamente a economia brasileira e, de quebra, dever� fazer uma s�rie de estragos irrevers�veis, que em muito debilitar�o seu potencial de desenvolvimento a longo prazo.
+O PRN n�o tem mais candidato � Presid�ncia da Rep�blica.
+O partido expulsou ontem Walter Queiroz por considerar que ele �faltou com a verdade� no epis�dio da dupla declara��o de renda.
+Depois de registrado, Queiroz apresentou uma segunda declara��o ao TSE (Tribunal Superior Eleitoral), acrescentando bens que alegou ter �esquecido� nos primeiros pap�is que foram apresentados.
+N�o h� diferen�a essencial entre as armas individuais do tr�fico e as do Ex�rcito.
+Elas pertencem basicamente � categoria �fuzil autom�tico�, isto �, um h�brido surgido na Segunda Guerra entre os tradicionais fuzis e metralhadoras de m�o ou submetralhadoras.
+Fuzis disparam muni��o mais poderosa e a um alcance maior, de at� um quil�metro.
+J� as submetralhadoras disparam balas semelhantes �s dos rev�lveres, mas podem faz�-lo em rajadas, a um alcance curto, de dezenas de metros.
+Nesta ala merece men��o o seu Bruto Ros�, um espumante elaborado com uva Baga (100%) que conserva um pouco da cor e muito de frutado desta variedade t�pica da regi�o.
+Este � um vinho bem seco e encorpado capaz de escoltar uma refei��o completa.
+O de 91, tem bolhas finas, abundantes e persistentes, que lhe d�o um paladar cremoso marcado como o aroma por um toque atraente de cascas c�tricas e framboesa.
+J� no cap�tulo dos goles decididamente rubros, desponta o Jo�o Pato (75% Baga, 25% Cabernet Sauvignon).
+Quem experimenta a �ltima vers�o (safra 91) descobrir� um vinho novo ainda, adstringente, mas de aroma e sabor intensos.
+Um vinho marcado pelo belo contraponto entre a especiaria da Baga e as frutas vermelhas da Cabernet, quase uma marca registrada na cria��o deste vinicultor portugu�s.
+A atacante Ana Moser (26 anos, 1,85 m e 70 kg), revelou ontem que, depois do v�lei, sua segunda paix�o � o futebol.
+�Quando era menina gostava muito de jogar futebol com meu irm�o�, disse a jogadora, que, ao contr�rio do v�lei, onde atua como atacante, prefere jogar como quarto-zagueiro no futebol.
+Anteontem, o Banerj, que administra a d�vida p�blica estadual, n�o conseguiu que o Banco Central negociasse R$ 84 milh�es em t�tulos estaduais, trocando-os por t�tulos federais.
+� o que se chama de �alavancagem� com um empr�stimo de US$ 10, por exemplo, pode-se entrar num jogo de US$ 100 ou mais.
+Se a aposta for errada, o risco de bancarrota � uma forte possibilidade.
+A expans�o dos �derivativos� n�o foi gratuita.
+Os economistas alinham v�rios motivos b�sicos, entre eles a possibilidade de invers�o aberta com a queda dos regimes socialistas, a onda liberalizante que varre a Am�rica Latina, a forte expans�o econ�mica dos pa�ses do Sudeste Asi�tico e tamb�m a baixa e est�vel remunera��o de tradicionais aplica��es.
+Nos EUA, os CDBs, at� antes da alta dos juros, rendiam 3% ao ano para uma infla��o de 2,5%.
+O IRA (Ex�rcito Republicano Irland�s, separatista) admitiu o envolvimento de seus militantes no assassinato de um funcion�rio do correio durante um roubo na cidade de Newry (Irlanda do Norte).
+No �ltimo dia 11, Frank Kerr, 53, foi morto com um tiro na cabe�a quando tr�s homens assaltaram um posto do correio.
+Os tr�s levaram cerca de US$ 210 mil.
+�Eles s�o a raz�o de tudo o que fa�o, de tudo aquilo em que acredito�, disse, com grandes espa�os entre as palavras.
+Tentava, a custo, firmar a voz, sob flashes dos fot�grafos.
+Ricupero, na verdade, foi o �ltimo do grupo que o acompanhava familiares e assessores a sucumbir � emo��o, pelo menos publicamente.
+Vice de FHC, Maciel nunca usou tanto a express�o �ternurar�.
+Ele a emprega quando tem que dizer n�o a uma pessoa sem desagrad�-la no caso, a seus colegas de PFL que querem repartir os cargos com o PSDB antes do 1� turno.
+�Quem adere depois da vit�ria � fisiol�gico�.
+Como melhorar o ensino superior sem transformar as universidades em �escol�es�, onde os professores s� repetem e se afastam da pesquisa?
+Essa foi a principal quest�o do 1� dia de debate, do qual participaram os professores Jos� Arthur Giannotti (Departamento de Filosofia da USP), Jos� Augusto Guilhon Albuquerque, (Departamento de Ci�ncia Pol�tica da USP), e Luiz Pinguelli Rosa (diretor da Coordena��o de Programas de P�s-Gradua��o da Universidade Federal do Rio de Janeiro).
+O bom uso dos modems e fax/modem depende dos programas para gerenciamento e transmiss�o de dados que acompanham os equipamentos.
+Todos os fax/modem saem de f�brica com dois programas um para gerenciamento das transmiss�es dfa e outro para gerenciamento das transmiss�es de dados.
+Na delegacia, que fica no Jardim Mutinga trabalham seis policiais com dois carros.
+A Caravan foi queimada.
+Resta um Gol.
+Os outros carros incendiados s�o tr�s Passats, um Chevette, dois Fuscas e um Escort.
+�Somos quercistas no atual momento.
+Ele � desenvolvimentista, defende a na��o soberana contra o imperialismo, representa as for�as populares.
+�Temos v�rtebra pol�tica, n�o somos gel�ia.
+O PT n�o tem estatura ideol�gica para enfrentar o candidato conservador.
+Por isso est� como cego em tiroteio.
+Pesquisa da Cia. City de Desenvolvimento mostra que os empres�rios procuram locais com infra-estrutura, facilidade de acesso, boa oferta de m�o-de-obra, pr�ximos de S�o Paulo e das rodovias e longe de movimentos sindicais.
+�Temos problemas com clientes e fornecedores que n�o conseguem encontrar a empresa�, afirma S�rgio Ueta, gerente administrativo da Ueta Ind�stria e Com�rcio de Aparelhos Eletr�nicos.
+A empresa, com sede em Caieiras (35 km a norte de SP), pretende se mudar para o Empresarial Jaragu�, onde adquiriu 6.000 m2.
+Ao mesmo tempo o atual papado esmagou a teologia da liberta��o, teorias de concorr�ncia feroz s�o implantadas em toda parte, o fascismo est� de volta, Fidel Castro se tornou uma esp�cie de Somoza e at� Martin Luther King, ficou provado, era plagi�rio.
+No plano dos costumes, a reviravolta se apresenta sob aspecto viral, epid�mico.
+Os outrora odiosos valores de fam�lia, sucesso e consumo renascem, sobretudo entre a popula��o jovem.
+Sexo livre, fumo e droga est�o condenados.
+O narcisismo assume uma fei��o cada vez mais fr�vola.
+O substitutivo d� aos minist�rios da Fazenda e Justi�a poder de pedir interven��o judicial nas empresas acusadas de praticar aumento injustificado de pre�os.
+Os senadores que faltarem �s sess�es do Congresso revisor, a partir de hoje, ter�o cortes no valor de um dia de trabalho por falta.
+O desconto seria de CR$ 83 mil por dia, com base nos vencimentos de fevereiro.
+A decis�o foi anunciada na tarde de ontem pelo presidente da revis�o constitucional, senador Humberto Lucena (PMDB-PB), que tomou a decis�o mesmo tendo em m�os parecer contr�rio da sua assessoria jur�dica.
+Lucena se declarou �pressionado� pela imprensa e pela necessidade de buscar o qu�rum nas sess�es da revis�o.
+Segundo o senador, a decis�o quanto ao corte dos sal�rios dos deputados a decis�o cabe ao presidente da C�mara, deputado Inoc�ncio de Oliveira (PMDB-PE).
+Inoc�ncio j� havia anunciado a ado��o do corte de um 30 avos dos sal�rios dos deputados por falta em sess�o da C�mara.
+Os primeiros descontos aconteceriam na folha de pagamento deste m�s, que se encerra dia 15.
+At� as 20h, ele n�o havia se pronunciado sobre a extens�o da medida tamb�m aos trabalhos da revis�o.
+O vice-presidente da C�mara, Adylson Motta, disse que o regimento da C�mara impede o corte.
+Favor�veis ao pressuposto da integra��o, devemos ser criteriosos, n�o descartando e nem acatando, de pronto, tudo o que nos oferece o governo federal.
+Os exemplos de Simon e Brizola revelam que a precipita��o e a busca de resultados pol�ticos imediatos constr�em o pior caminho para o desenvolvimento e s�o v�cios que o curso da hist�ria n�o tarda a desnudar.
+RICARDO ANT�NIO SILVA SEITENFUS, 45, doutor em Rela��es Internacionais pelo Instituto de Altos Estudos Internacionais em Genebra, � coordenador do curso de mestrado em Integra��o Latino-americana da Universidade Federal de Santa Maria (RS).
+Foi secret�rio especial para Assuntos Internacionais do Rio Grande do Sul (governo Pedro Simon).
+O avi�o em que o presidente da Argentina, Carlos Menem, viajava no �ltimo dia 30 de dezembro para Anillaco, sua cidade natal, iria ser atacado por um m�ssil, disse ontem o jornal de Buenos Aires �Ambito Financiero�.
+A seguran�a teria sido avisada por dilomatas e desviado a rota do avi�o.
+O governo nega a suposta tentativa de atentado.
+Come�aram ontem pela manh� em Nova York os desfiles de pr�t-�-porter outono-inverno.
+A primeira estilista foi a Donna Karan, com a linha DKNY.
+Os desfiles se realizam pelo segundo ano em tendas ou pavilh�es localizados no Bryant Park, que fica atr�s da Biblioteca Nacional, na rua 40.
+Deputados ligados a ACM diziam ontem, em conversas com colegas, que Benito Gama (PFL-BA) foi vetado para ser vice de FHC porque o grupo avalia que s�o cada vez mais reduzidas as chances de vit�ria do tucano.
+Ontem, v�rios operadores da Bovespa creditavam a queda do �ndice da bolsa � percep��o do mercado financeiro de que o PFL leia-se ACM est� desembarcando da candidatura de FHC.
+As constantes desaven�as com seu pai, John Paul Getty, fizeram John Paul Getty 2� entrar no �affaire�.
+�As Tr�s Gra�as�.
+A c�lebre escultura de Antonio Canova, atualmente na Gr�-Bretanha, est� sendo cobi�ada pelo museu Getty, de Malibu, na Calif�rnia.
+Hoje a m�dia brasileira � de cerca de 1.800 quilos por hectare.
+Segundo os produtores, o incentivo � pesquisa elevar� este desempenho para 2.400 kg/ha.
+As possibilidades do trigo irrigado tamb�m s�o animadoras.
+O plantio no Cerrado atingiu 5.000 kg/ha nas lavouras comerciais e mais de 8.000 kg/ha em �reas experimentais.
+Os quatro grandes jornais deram, na segunda-feira, a mesma manchete para informar como acabou o Grande Pr�mio do Brasil, que aconteceu domingo passado.
+Era �Senna erra e Schumacher vence� (em �O Globo�, a frase ficou pouco maior que isso).
+Com essa constata��o enviesada de que o alem�o s� chegou em primeiro porque o brasileiro cometeu uma bobagem, a imprensa coroou o festival �sennista� da semana anterior, quando todas -- sem exce��o, todas -- as coberturas apontavam a vit�ria de Ayrton Senna em Interlagos.
+A Folha chegou a escrever que o melhor piloto do mundo (Senna), na dire��o do melhor carro do mundo (a Williams) teria sua consagra��o no aut�dromo.
+E que s� Schumacher acreditava em sua possibilidade de vencer.
+Senna, todo mundo viu, rodou quando estava em segundo lugar e com poucas chances de recuperar o primeiro.
+Enterrou as previs�es mais do que otimistas, e deixou ver quanto � viciado o notici�rio esportivo.
+O compromisso com a precis�o (e, por extens�o, com o leitor) vale menos do que a torcida da imprensa nessas horas.
+A imprensa j� se esqueceu de que os oligop�lios s�o o vil�o da URV e do plano FHC.
+O pa�s convive com o novo indexador h� um m�s, o ministro virou candidato e os pre�os chamados de �abusivos� continuam em vigor.
+O assunto saiu da pauta, ainda que esteja dentro do bolso do leitor.
+JUNIA NOGUEIRA DE S� � a ombudsman da Folha.
+A ombudsman tem mandato de um ano, renov�vel por mais um ano.
+Ela n�o pode ser demitida durante o exerc�cio do cargo e tem estabilidade por um ano ap�s o exerc�cio da fun��o.
+Suas atribui��es s�o criticar o jornal sob a perspectiva do leitor recebendo e checando as reclama��es que ele encaminha � Reda��o e comentar, aos domingos, o notici�rio dos meios de comunica��o.
+Cartas devem ser enviadas para a al. Bar�o de Limeira, 425, 8� andar, S�o Paulo (SP), CEP 01202-001, a. c. Junia Nogueira de S�/Ombudsman.
+Para contatos telef�nicos, ligue (011) 224-3896 entre 14h e 18h, de segunda a sexta-feira.
+No ranking dedicado aos homens aparecem o ator Sean Connery e o cantor Sting.
+Entre os corpos mais feios est� o da top-model Kate Moss, que segundo o jornal tem ombros ca�dos e o peito liso.
+Os deputados que negociam o plano de estabiliza��o econ�mica do governo criticaram o ministro Fernando Henrique Cardoso, que admitiu ontem a possibilidade de disputar a Presid�ncia da Rep�blica.
+A declara��o do ministro repercutiu mal tamb�m entre os l�deres pol�ticos no Congresso.
+A frase do ministro desagradou at� mesmo ao PSDB.
+�Para o plano econ�mico, n�o ter dito essa frase seria melhor�, afirmou o senador Beni Veras (PSDB-CE).
+O S�o Paulo come�ou o primeiro tempo ap�tico, mas acordou aos 10min com uma falta n�o marcada do lateral Mac Allister em Euller, quando este ia entrar na �rea.
+Em seguida, Euller fez mais duas jogadas.
+A partir dos 22min, o S�o Paulo passou a marcar sob press�o e anulou o advers�rio.
+O Boca n�o conseguia sair jogando e perdia bolas em seu campo.
+O autor do ataque seria Nagi Mohamed Mustafa e o l�der do grupo seria Basseem Khalil, morto num caf� do Cairo (capital) ap�s tiroteio contra a pol�cia.
+O atentado teria sido planejado pelo grupo radical isl�mico Gama'a al-Islamiya, que lan�ou campanha contra o governo eg�pcio em 1992 para a cria��o de um Estado Isl�mico fundamentalista.
+Khalil e Mustafa, disfar�ados, tentaram matar Mahfouz um dia antes, mas ele n�o estava em casa.
+Muitos neg�cios foram iniciados ou fechados durante o Siaf.
+Produtores da Para�ba, por exemplo, venderam abacaxi a um grupo de empres�rios espanh�is, no valor de US$ 323 mil.
+A cooperativa de Cura�� (BA), no Vale do S�o Francisco, praticamente acertou uma joint venture com um empres�rio portugu�s para produzir mel�o e uva.
+Empres�rios alem�es avan�aram nas negocia��es para instalar unidades de beneficiamento de polpas de frutas.
+Pode-se dizer, em outras palavras, e a �grosso modo�, que o aluguel do primeiro m�s de reajuste sempre embute (por estimativa) uma infla��o que reduzir� o aluguel nominal ao seu valor real na metade do per�odo de reajuste subsequente.
+Portanto, na primeira metade, o locador sai ganhando, pois recebe mais aluguel do que vale a loca��o.
+Na segunda metade, a vantagem � do inquilino que papa menos aluguel do que vale o uso do im�vel.
+A m�dia no per�odo de reajuste equilibra as vantagens de cada contratante durante todo per�odo de uniformidade nominal do aluguel.
+Tudo isso deve ser considerado para que se entenda que uma exata transposi��o do aluguel praticado em cruzeiros reais para o sistema da URV s� � poss�vel se os contratantes procederem a uma extra��o da m�dia dos alugu�is nos �ltimos meses que corresponderem a periodicidade de reajustamento do contrato, para, s� ent�o, transformar tal m�dia em URVs.
+Folha -- No caso de �Vereda da Salva��o�, como foi esse trabalho conjunto?
+Serroni -- A pe�a previa tr�s casas numa clareira de uma floresta.
+Depois de conversar com Antunes, sempre fa�o uma maquete.
+Como a pe�a trata de religi�o, de misticismo, achamos que dev�amos usar muitos troncos, dar uma verticalidade, o que tem a ver com a ascens�o etc.
+Decidimos ent�o, por uma quest�o de economia c�nica, abolir uma das casas e s� insinuar as outras duas, fazendo apenas suas entradas, que parecem as de um templo.
+�Isto foi fundamental, pois conseguimos aumentar nossos criat�rios e agora usamos como garanh�es selecionadores apenas aqueles realmente de ponta�, diz.
+O resultado positivo da estrat�gia pode ser comprovado agora, dizem os criadores, uma vez que hoje s�o os EUA que est�o buscando cavalo �rabe no Brasil.
+Os franqueados brasileiros come�am a utilizar uma nova �arma� para aumentar seu poder de influ�ncia junto � c�pula das empresas franqueadoras.
+Trata-se do conselho de franqueados, que come�a a surgir em algumas redes como McDonald's, �gua de Cheiro, Multicoisas e Localiza.
+J� bastante difundidos nos EUA, os conselhos t�m atenuado problemas e melhorado o tradicionalmente dif�cil relacionamento franqueado X franqueador.
+Atuando em conjunto, os franqueados ganham for�a e passam a ter voz ativa em quest�es vitais para o neg�cio, como definic�o de produtos, prazos de pagamento e estrat�gias de propaganda.
+A liminar, ainda n�o cumprida pelo governo do Esp�rito Santo, determina que o delegado respons�vel pelo caso, Francisco Badenes, seja reconduzido ao cargo.
+Em maio �ltimo, Badenes foi afastado e transferido para o interior.
+O juiz determina ainda o restabelecimento dos meios necess�rios ao funcionamento da Comiss�o de Processos Administrativos Especiais, respons�vel pelo inqu�rito.
+O l�der Tasso Jereissati (PSDB) passou de 61% para 58%, perdendo tr�s pontos percentuais em rela��o � �ltima pesquisa.
+N�o deve haver segundo turno no Estado.
+Wilson Barbosa (PMDB) subiu seis pontos, atingindo 55%.
+O candidato tamb�m deve vencer a elei��o no primeiro turno.
+A margem de erro da pesquisa Datafolha � de 3,0 pontos percentuais, para mais ou para menos, exceto no Distrito Federal, que � de 4,0 pontos.
+A dire��o do datafolha � dos soci�logos Antonio Manuel Teixeira Mendes e Gustavo Venturi.
+Nada do que eu havia lido ou ouvido antes de vir para c� me preparara para as emo��es que eu iria viver e v�rias vezes me senti numa montanha-russa emocional.
+Em v�rias ocasi�es eu me vi, como anotei em meu di�rio, � beira das l�grimas.
+Mas em muitos outros momentos eu me esfor�ava para conter risadas.
+Ou ent�o lutava contra uma depress�o repentina, tentava controlar minha raiva, fazia piadas com algu�m ou fazia for�a para n�o sentir saudades de casa.
+Art. 1� Dispensar a obrigatoriedade da express�o de valores em cruzeiros reais constante dos incisos II e III, do art. 8�, da medida provis�ria n� 482, de 29 de abril de 1994, desde que, no caso de fixa��o dos pre�os em Unidade Real de Valor (URV) seja exposto, em lugar vis�vel e de f�cil leitura, o valor da URV do dia.
+Art. 2� � obrigat�ria a express�o dos valores em cruzeiros reais nas notas fiscais.
+As negocia��es com os petebistas se prolongaram at� o in�cio da madrugada de ontem.
+Al�m do PSDB e do PFL, coligou-se em torno da candidatura Covas o pequeno PV.
+Mas � at� poss�vel encontrar pelo menos um ponto interessante no disco: algumas de suas letras, compostas por Michael Callahan e Marc English.
+Na faixa mais interessante do disco, �Wooden Nails�, a primeira, o tema abordado � a supera��o da depend�ncia de drogas e da depress�o.
+Uma melodia melanc�lica emoldura a can��o.
+O fato � que a sombra do R.E.M. � uma presen�a muito forte nesse �Building Our House�.
+Os vocais de Callahan e English s�o calcados ao extremo nos de Michael Stipe e Mike Mills.
+O beat de certas can��es acaba tornando-as filhotes das compostas pela banda de Athens, como �Losing My Religion�.
+Al�m disso, deve lan�ar mais cinco novidades at� o final do ano, al�m de aumentar sua distribui��o.
+Apesar dos esfor�os, estima fechar o ano vendendo 75 milh�es de litros de sorvetes (US$ 120 milh�es).
+Em 93 vendeu 83 milh�es de litros (US$ 128 milh�es).
+Ou na hip�tese remota de sua vota��o antes de 30 de agosto.
+Para Holanda, �n�o � poss�vel a mudan�a pretendida pelos bancos�.
+Por dois motivos: a eventual queda na arrecada��o e problemas jur�dicos que impedem mexer na base de c�lculo do imposto por meio de MP, j� que ela foi fixada em emenda constitucional.
+O governador do Paran� � ligado politicamente ao ex-governador Roberto Requi�o que � candidato ao Senado pelo PMDB.
+Requi�o � um dos mais ferrenhos advers�rios de Orestes Qu�rcia, que disputa a Presid�ncia da Rep�blica pelo partido.
+Enfim, nem tudo est� perdido neste futebol automatizado e excessivamnete defensivo que a Copa est� consagrando.
+Afinal, por mais que tentem matar o craque, ele sempre sobrevive, aqui ou ali, onde menos se espera.
+Alberto Helena Jr., 52, � colunista da Folha.
+Sua coluna na Copa � di�ria.
+O presidente argentino, Carlos Menem, disse que lan�ar� sua candidatura para a elei��o de 2003 se for eleito em 1995.
+Depois de dois mandatos, Menem n�o poderia ser reeleito em 1999.
+�Espero quatro anos e volto a me apresentar�, disse.
+Duas bombas mataram duas pessoas em Teer� ontem, segundo a ag�ncia oficial de not�cias �Irna�.
+N�o foram apontados suspeitos pelo atentado embora tenha sido dito que a pol�cia achou pistas dos terroristas.
+A e�sfora incidia sobre o valor das propriedades, as quais estavam devidamente registradas nos �diagramas� p�blicos.
+Tratava-se de um imposto sobre o capital.
+Outro tributo importante, e tamb�m sob a forma de contribui��o volunt�ria, foram as trierarquias.
+Tratavam-se de contribui��es para construir os �geis barcos trirremes.
+Tanto a constru��o quanto a manuten��o da equipagem corria por conta dos que tinham mais recursos.
+O Movimento Zona Sul foi criado no ano passado, durante a pol�mica a respeito da realiza��o das obras de prolongamento da avenida Faria Lima (zona sul).
+Comandado pelo advogado Luiz Antonio Siqueira Dias, o grupo defendeu as obras, que foram iniciadas recentemente.
+O Folha Informa��es atendeu no �ltimo fim-de-semana 3.978 pessoas, que acionaram o servi�o para saber o resultado de Guarani e Palmeiras.
+O pico de audi�ncia aconteceu entre as 20h e 21h de s�bado, quando o sistema atendeu 309 liga��es.
+O Folha Informa��es � um servi�o do Banco de Dados da Folha de S. Paulo (tecnologia da Telesis Sistemas em Telecomunica��es).
+Nenhum dirigente do PMDB compareceu ontem, em S�o Paulo, � cerim�nia de instala��o do Instituto Ulysses Guimar�es de Estudos Pol�ticos e Sociais.
+Ulysses morreu em 1992 como presidente nacional do partido.
+Orestes Qu�rcia e Luiz Antonio Fleury Filho n�o estiveram na sede da entidade.
+Foi tamb�m o caso de candidatos a cargos majorit�rios ou dirigentes regionais.
+Os carros ganharam ainda prote��o sob o c�rter e sob o tanque de gasolina, para suportar com menor risco os choques contra as pedras.
+A altura do solo foi ampliada em alguns cent�metros para facilitar as investidas fora de estrada.
+A crise no M�xico deve fazer com que todos os pa�ses emergentes recebam um menor volume de capitais externos de renda fixa (aplicados, por exemplo, em eurob�nus).
+Ele acredita, no entanto, que o mercado acion�rio brasileiro deve crescer em 1995.
+A institui��o � respons�vel pela guarda (cust�dia) do equivalente a US$ 3,7 bilh�es de investimentos estrangeiros nas Bolsas.
+Nos �ltimos dias, houve uma sa�da de US$ 100 milh�es da cust�dia da institui��o, por causa da crise do M�xico.
+Meirelles afirma que esses recursos n�o sa�ram de neg�cios tradicionais de Bolsas, mas sim de opera��es de �box� (financiamentos tendo como lastro a��es).
+Ele prev�, contudo, que o mercado acion�rio crescer� e a cust�dia de a��es de investidores estrangeiros na institui��o chegar� a US$ 5,5 bilh�es ao final de 1995.
+Apesar de evitar dar um n�o definitivo, Marise deixou claro que deve recusar o convite de Brizola.
+�Fiquei muito honrada com a proposta, mas o governador Brizola tem nomes muito bons para compor sua chapa�, afirmou.
+O governo do Estado informou, atrav�s de nota de sua assessoria de imprensa, que as declara��es do presidente Itamar Franco e do presidente eleito Fernando Henrique Cardoso sobre medidas federais de combate ao tr�fico de drogas �configuram posi��es an�logas �s que o governador do Estado vem defendendo�.
+Segundo a nota, �as provid�ncias capazes de alterar o quadro atual de combate aos traficantes dizem respeito � Pol�cia Federal e �s For�as Armadas, com o objetivo de interromper o fluxo de armas proibidas e coca�na no Rio�.
+Confiante numa eventual vit�ria do candidato Fernando Henrique Cardoso, a c�pula do PSDB j� articula a forma��o de um superpartido para n�o depender do PFL no Congresso.
+Os tucanos consideram perigoso deixar um poss�vel governo de FHC sujeito �s manobras dos deputados e senadores pefelistas.
+Grava disse que os dois sofreram rompimentos de feixes musculares em m�sculos da coxa direita.
+O problema de Viola � num m�sculo da parte de tr�s da coxa e de C�lio est� no lado interno.
+O m�dico negou que esteja tratando mais algu�m.
+O Cobra privilegiava os atos impulsivos e as cores fortes e seus principais nomes foram Appel e Alechinsky.
+Doucet se inspirava muito em suas viagens e refletia isso no t�tulo de v�rios de seus trabalhos, como �Guatemala Blues� e �Mostar Sarajevo�.
+Tamb�m s�o obras suas �Labyrinthe de la Lumi�re� e �Turbulences d'Abysses�.
+O candidato do PDT disse que quer fazer o mesmo.
+Os grandes com�cios ser�o realizados apenas em setembro.
+Ao contr�rio do que fez em campanhas anteriores, Brizola quer promover com�cios r�pidos, com tr�s oradores.
+O filme passa como vento pelas quest�es. nada pequenas. dos tratamentos psiqui�tricos contempor�neos e se fixa no que existe de mais �bvio na hist�ria.
+Isso termina por arrast�-la destestavelmente, sem que se chegue a parte alguma.
+Paradoxalmente, � por a� que se podem ver as virtudes de Figgis.
+A ONU admitiu que os s�rvios n�o haviam cumprido as exig�ncias, mas recusou autoriza��o para bombardeio.
+Yasuhi Akashi, reprsentante da organiza��o, argumentou que os s�rvios estavam em meio ao processo de retirada.
+J� nas primeiras palavras a Medida Provis�ria que institui a Unidade Real de Valor incorre em erro e abre brecha para contesta��es na Justi�a.
+A constata��o � do jurista Saulo Ramos, ex-consultor Geral da Rep�blica e ex-ministro da Justi�a, que foi convidado a analisar o rascunho das medidas e na ocasi�o alertou para dois problemas: o artigo primeiro e o par�grafo primeiro da MP 433 praticamente instituem duas moedas no pa�s, o que � proibido.
+Al�m disto, a manuten��o da Ufir para corre��o de tributos � ilegal, segundo o jurista.
+Outros especialistas apontam mais defeitos t�cnicos na MP.
+Diz o artigo primeiro que fica institu�da a URV, dotada de �curso legal� para servir �exclusivamente� como padr�o de valor monet�rio.
+Isto restringe � URV todos os poderes da moeda do pa�s.
+�� um erro t�cnico grave e que pode resultar num conflito judici�rio muito grande�, segundo Saulo Ramos.
+As negocia��es para a volta de Mansell � F-1 tiveram in�cio logo ap�s as 500 Milhas de Indian�polis, realizada no m�s passado, e continuaram durante as 200 Milhas de Milwaukee, disputada no �ltimo dia 5, quando recebeu a visita de alguns engenheiros da Williams.
+Quinto colocado no Mundial de F-Indy, com 46 pontos, Mansell disse que pretende continuar a correr na categoria at� o final da temporada, condi��o que ele imp�s a Frank Williams para voltar � equipe pela qual ganhou o t�tulo mundial de 1992.
+Nos �ltimos dias, aumentaram as especula��es sobre quem ocuparia o lugar de Mansell na Newmann/Hass.
+O mais cotado na bolsa de apostas � o brasileiro Raul Boesel, que ontem n�o consegui terminar a corrida em Detroit.
+Eles estranharam o fato dele n�o sair do barraco de manh� e chamaram a pol�cia.
+Jair morava sozinho no barraco e, segundo seus vizinhos, era alco�latra.
+O segundo caso aconteceu no Pari (centro).
+Um indigente conhecido como Daniel foi encontrado ca�do na rua Canind� na madrugada de ontem por policiais que estavam fazendo patrulhamento.
+Quem � que vai, ent�o?
+S� intelectual quatro-olhos?
+S� aquela turma do Cebrap?
+S� soci�logo e antrop�logo?
+Socorro!
+Ent�o � bom a Joyce Pascowitch ir se preparando.
+O Fernando Collor, evidentemente, n�o foi convidado para a posse.
+�So what�, n�o � mesmo?
+Ele compensou em Aspen, dando um malho naquela flor de nome Rosane, casualmente, na frente de fot�grafos.
+Na alquimia medieval, a �rvore simbolizava a transforma��o, sempre cercada de globos reluzentes que corresponderiam aos planetas.
+Para o psicanalista Carl Jung, esta seria a origem dos globos pendurados nas �rvores contempor�neas.
+E, para Bruno Bettelheim, as pequenas velas e l�mpadas que a iluminam seriam vest�gios das antigas fogueiras que os pag�os do norte da Europa ateavam no alto das montanhas para antecipar a chegada do Sol e o fim do inverno.
+Mais 10%.
+Para 94, a ind�stria de m�quinas agr�colas prev� crescimento de 10%.
+Acordo na C�mara Setorial garante cr�dito de US$ 650 milh�es para o Finame Rural.
+Em contrapartida, a propor��o de empres�rios no partido, que era de 22%, hoje alcan�a a marca de 54%.
+� o caso, entre outros, do deputado S�rgio Machado (Villejack Jeans), do senador Teot�nio Vilela Filho (Mata Verde Agropecu�rio, Usina Seresta), do senador Albano Franco (presidente da Confedera��o Nacional da Ind�stria).
+Essa mudan�a no perfil social da bancada n�o ocorreu no maior partido conservador, o PFL, nem no PT, partido de esquerda.
+O produtor iniciou a colheita, em fevereiro passado, de 170 hectares de amendoim e 240 hectares de soja.
+Em 93, ele plantou 290 ha de amendoim e 97 ha de soja.
+Com o dinheiro do amendoim Guidi conseguiu comprar duas camionetes e um trator de 150 Hp, est� construindo uma casa em Pontal e ainda tem para receber US$ 470 mil referente � venda de 49,4 mil sacas.
+Parte desse dinheiro vai para a colheita deste ano.
+�Trabalho com amendoim h� 27 anos e sempre tive lucro.
+Nos primeiros dez anos, o lucro por safra alcan�ava 40%.
+Hoje, varia entre 12% e 15%�.
+O Banco apresentou em 1993 um n�vel de alavancagem alto, demonstrando maior agressividade nas suas opera��es.
+Os n�veis dos dep�sitos aumentaram 440%, em 1993, basicamente em fun��o de dep�sitos interfinanceiros e a prazo.
+M�RIO ALBERTO DIAS LOPES COELHO � consultor da Austin Asis.
+Existe ambiente pol�tico, social e econ�mico para se produzir autom�veis no Brasil?
+Sim, embora a abertura dos portos n�o esteja sendo feita impunemente.
+Em termos pol�ticos, o pa�s saiu das elei��es com um Estado mais vigoroso e democr�tico, em melhores condi��es de enfrentar a infla��o e o desemprego.
+Pode se reorientar para uma maior produ��o industrial e contribuir para a cria��o de mais emprego e riqueza, consequ�ncia natural do processo.
+Do ponto de vista l�gico, um plano de estabiliza��o equivale a uma promessa.
+O governo promete � sociedade que, de agora em diante, vai mudar de vida vai garantir o equil�brio das contas p�blicas e vai parar de abusar do seu monop�lio de cria��o de moeda.
+Se isso ser� cumprido na pr�tica, s� o tempo dir�.
+Mas, para que a promessa possa ser efetivamente cumprida, � fundamental que se acredite que o governo ir� de fato cumpri-la.
+�Al�m de brincar, � uma experi�ncia.
+Pela TV voc� v�.
+A diferen�a � que no videogame voc� sente.
+� como se estivesse l�, acrescenta Rodrigo.
+H� outros apelos da fita que seduzem os �gameman�acos�, especialmente os f�s de Ayrton Senna.
+No in�cio de cada corrida, por exemplo, as principais dicas do circuito s�o fornecidas pelo piloto.
+A imagem de Ayrton aparece em fotos digitalizadas.
+Um dos respons�veis pela sele��o das mo�as da Ford, que n�o quis se identificar, aponta o italiano Maldini (considerado pela imprensa europ�ia o �bumbum� mais bonito da Copa), que recebeu apenas um voto, como o mais bonito.
+�Este sim, seria modelo com certeza�, disse.
+Os reservas da sele��o brasileira Viola e Ronald�o e v�rios componentes da Nig�ria tamb�m foram apontados.
+�O Viola ou o Ronald�o seria um 'black man' perfeito� disse um dos modelos masculinos da Ford, que tamb�m n�o quis se identificar.
+Esteve assim com o astro da confer�ncia, lorde Keynes �um sujeito sutil, bom expositor, muito persuasivo�, conta Campos.
+A delega��o brit�nica era a mais forte em termos intelectuais, embora outros participantes tivessem dela uma opini�o algo debochada.
+Corria uma piada segundo a qual Keynes era inteligente demais para ser consistente; Dennis Robertson, outra estrela, era consistente demais para ser inteligente; e Lionel Robbins, o terceiro nome de prest�gio, n�o era nem inteligente nem consistente.
+Durante a negocia��o, ontem � tarde, Muller telefonou para o presidente do S�o Paulo, Fernando Casal de Rey, e se lamentou.
+�Ele reclamou de que n�o lhe pagariam o prometido, n�o lhe dariam um carro e que s� bancariam seis meses de aluguel de sua casa e n�o os quatro anos, como esperava�, disse o dirigente.
+Ela preferiu ser a primeira dama da ilha de Skorpios.
+Foi nesse papel que, encarnada por outra Jacqueline (Bisset), Jackie O. debutou na tela, 15 anos depois de ter-se transformado na mais pranteada e sedutora vi�va deste s�culo.
+Merecia coisa melhor.
+Al�m de muito ruim, o filme �O Magnata Grego� (The Greek Tycoon) dedicava mais aten��o ao seu segundo marido, interpretado por Anthony Quinn, que ali�s faria o papel de S�crates Onassis (sogro de Jackie) no telefilme �Onassis: The Richest Man in the World�, produzido em 1988.
+O presidente da Portuguesa, Manuel Pacheco, deve entrar ter�a-feira com uma a��o contra o Vasco da Gama.
+Ontem � tarde, Pacheco disse que esperar� at� ter�a-feira para que o clube carioca envie uma c�pia do seguro de vida previsto no contrato de empr�stimo do meia Dener, morto ter�a-feira.
+Mas, no Rio, o diretor de futebol do Vasco, Eurico Miranda, confirmou que o seguro n�o foi feito.
+Segundo Miranda, a legisla��o brasileira pro�be que um seguro desta natureza tenha como benefici�rio uma empresa ou associa��o.
+Resultado: press�o nos pre�os combinada com a falta de produtos.
+Se a queda da infla��o certamente produziria efeitos positivos na candidatura Fernando Henrique Cardoso, as prateleiras vazias municiam os advers�rios.
+� certo que, desta vez, o governo est� mais atento do que no Plano Cruzado e vai usar, segundo o ministro Rubens Ricupero, todos os artif�cios para evitar uma febre de consumo.
+Se vai conseguir, � outro problema.
+Mais uma vez, se v� que a deseduca��o � o problema que mais sai caro ao Brasil.
+PS Realmente � not�vel a evolu��o de Lula nos �ltimos 15 anos.
+Recebi ontem por fax entrevista que ele concedeu em 1979 para a revista �Especial�, onde exp�e sua vis�o sobre a mulher.
+�A gente n�o pode pensar em jogar a mulher no mercado de trabalho enquanto houver excesso de m�o-de-obra�.
+Ali�s, nessa entrevista ele, corretamente, defende a legaliza��o do aborto com argumentos razo�veis e, agora, por motivos eleitoreiros, cedeu � press�o da Igreja Cat�lica.
+A campanha eleitoral encerrou-se na noite de quinta-feira, com um pronunciamento de oito minutos de todos os candidatos.
+A propaganda eleitoral nas redes de TV custou ao uruguaio mais de US$ 14 milh�es (cerca de US$ 7,00 por cada eleitor).
+Pela lei, os partidos recebem recursos do governo de acordo com os resultados da �ltima elei��o.
+Acho importante notar que a Bienal mudou a maneira de conseguir patroc�nio cultural no pa�s, que antes era feito � base do �me d� um dinheiro a�.
+Vendendo cada sala especial para um patrocinador exclusivo, demos a ele um retorno muito mais palp�vel, com an�ncios em meios de comunica��o e o abatimento de impostos.
+Tratamos a Bienal como empresa.
+Folha -- O n�mero de visitantes pagantes da Bienal do dia da abertura (12 de outubro) at� o dia 7 deste m�s foi de 103 mil.
+Se a Bienal for at� o dia 11 de dezembro, dever� ter um p�blico total de no m�ximo 250 mil pagantes, al�m de quase 200 mil crian�as, seguindo a proje��o.
+O Sr. acha um bom resultado?
+Para o volante Dinho, um dos jogadores respons�veis pelo setor de marca��o do Santos, � imposs�vel segurar o ataque do S�o Paulo do t�cnico Tel� Santana.
+A sa�da, diz, �� partir para o ataque�.
+Sobre sua sa�da do S�o Paulo no in�cio do Campeonato Paulista o jogador diz que a quest�o � �assunto encerrado�.
+Depois da Portuguesa, esta tarde no Canind�, o Palmeiras far� tr�s jogos seguidos em seu est�dio: contra o Bragantino, ter�a-feira, a Ferrovi�ria, quinta-feira, e o Am�rica, domingo.
+�Se vencermos esses quatro jogos, chegaremos a 80% de aproveitamento dos pontos disputados.
+Acredito que, mantendo esse percentual, seremos campe�es�, disse Luxemburgo.
+Gianetti da Fonseca afirmou que a avalia��o tem que ser feita por �pares� (pesquisadores da mesma �rea), que n�o fa�am parte do grupo analisado.
+Freddy Tatoo vive atualmente em Bolonha e j� est� acostumado ao circuito internacional da tatuagem.
+Ele j� tatuou, pelo menos, mil pessoas.
+Pelo grupo brazuca, ainda estar�o presentes Tyes e Caio (Rio de Janeiro) e o especialista em �body piercing�, Andr� Meyer (S�o Paulo).
+Eles v�o ser indiciados por suposta corrup��o de menores, favorecimento � prostitui��o e c�rcere privado.
+O advogado Jos� Carlo Dri, que defende os respons�veis pela boate, se recusou a falar.
+WILSON BALDINI JR.
+O norte-americano Evander Holyfield coloca o t�tulo mundial dos pesos-pesados (acima de 86,183 quilos), vers�o Associa��o Mundial (AMB) e Federa��o Internacional (FIB), hoje, contra o seu compatriota Michael Moorer.
+Quatro vereadores na cidade e o vice-prefeito afirmaram que votar�o em Fernando Henrique Cardoso, embora seus partidos (PMDB e PPR) tenham candidatos pr�prios a presidente.
+�Voto no Fernando Henrique porque ele fez o Plano Real�, afirmou o vice-prefeito, Manoel Lopes Duarte, do PPR.
+A coreografia criada por Bil T. Jones para �An American Evening� chama-se �I Want to Cross Over�, com m�sica gospel cantada por Liz McComb.
+O cen�rio, de Donald Baechler, se comp�e de peda�os de um barco, uma pequena casa e um tubo de chamin�.
+Negro e portador do v�rus da Aids, Bill T. Jones � uma das express�es mais poderosas da dan�a contempor�nea.
+Agora, al�m de dirigir seu grupo sediado em Nova York, ele tamb�m vai atuar como core�grafo-residente do Ballet da �pera de Lyon, que acaba de nome�-lo para o cargo, antes ocupado por Maguy Marin.
+Ex-bailarino do grupo de Trisha Brown, Stephen Petronio � outra estrela do momento.
+Para �An American Evening� ele criou �Extra Veinous�, cujo t�tulo significa o contr�rio de �intravenoso�.
+A coreografia de Susan Marshall que o Ballet de Lyon dan�a hoje chama-se �Central Figure�, com m�sica de Philip Glass.
+�Inspira-se no mais antigo e melhor bailarino de minha companhia, que morreu no �ltimo ver�o�, ela diz.
+O tema de Julia Child era uma simples quiche, mas era pol�tico: mostrar aos paran�icos americanos que o ovo e a manteiga fazem pratos deliciosos e que a patrulha do colesterol deve dar espa�o ao prazer da comida.
+Marcella Hazan falou sobre alcachofras e catequizou o p�blico sobre a supremacia do �leo de oliva extra virgem para cozinhar, apesar da exorbit�ncia do pre�o.
+Patricia Wells ensinou o preparo de codornas marinadas e cuscuz marroquino e incentivava o p�blico a repartir �food experiences� (o nome moderno para receber amigos para comer) como ela mesma faz em sua casa na Provence (sul da Fran�a), regi�o em moda nos Estados Unidos.
+Os irm�os Gershwin eram t�o diferentes que se completavam �s maravilhas.
+George fazia a m�sica, Ira a letra, embora, como em toda parceria, cada qual desse palpites na especialidade do outro.
+Primeiro, George fazia a m�sica; Ira capturava o �esp�rito� da can��o e apunha-lhe t�tulo e letra.
+�s vezes sugeria a George uma mudan�a no andamento, para que este se adequasse melhor � letra.
+Um caso t�pico foi o de �Someone to Watch Over Me� (1926), que nasceu �brejeira� e Ira recomendou que ela se tornasse uma balada rom�ntica.
+George era um d�namo, bo�mio e namorador; Ira era mais recluso e solidamente casado.
+George era indiferente politicamente; Ira era �liberal�, com inclina��es socialistas.
+A versatilidade do George como compositor s� era igualada pela de Ira como letrista.
+Os dois eram bambas tanto nas can��es mais r�tmicas, alegres e humor�sticas, como nas mais �s�rias�, rom�nticas e profundas.
+Segundo a administra��o do aeroclube, a quadrilha conseguiu decolar �s 4h, sob forte nevoeiro e sem ilumina��o.
+O avi�o � do m�dico Domingues Braille, dono de uma cl�nica em S�o Jos� do Rio Preto.
+O jogador, sabendo que ganharia mais de US$ 250 mil, fez o que a voz ordenou.
+A roleta girou, e ele perdeu.
+�Droga�, disse a voz em sua cabe�a.
+Dividido em seis partes, o livro traz 32 cap�tulos que procuram abranger todos os recursos do programa.
+� bastante did�tico e utiliza bem desenhos para exemplificar efeitos especiais.
+O texto � curto e pr�tico e todos os cap�tulos s�o recheados de notas, dicas e alertas, ilustradas com as respectivas figuras �Note�, �Tip� e �Stop�.
+Com dire��o de Roberto de Oliveira, a Bandeirantes prepara um document�rio sobre a Campanha Contra a Fome e o trabalho do rapaz.
+O especial dever� ir ao ar no Brasil e em v�rios pa�ses poucos dias antes da elei��o para o Pr�mio Nobel da Paz.
+O mais grave, no entanto, s�o os problemas que os clientes poder�o enfrentar.
+A quem recorrer no caso de um descomprimento do que � prometido?
+Notamos ainda que, quando um passageiro brasileiro se inscreve num programa de milhagem de uma companhia a�rea internacional que serve o Brasil, passados 30 dias, recebe a proposta da Iapa.
+Ou seja, se tal procedimento n�o for ocasional e tiver realmente a coniv�ncia das transportadoras a�reas, estas est�o expressamente contribuindo para a concorr�ncia desleal e il�cita perante os agentes de viagens al�m de n�o preservar seus clientes, que n�o s�o consultados sobre se querem ou n�o receber tais propostas ou se autorizam a divulga��o de seus nomes e endere�os.
+Como partiu do PT o pedido de encontro com o presidente Nelson Mandela, o pr�prio partido assumiu os custos do trecho da viagem que inclui a �frica do Sul.
+Segundo a tesoureira da campanha de Lula, Tatau Godinho, os custos se resumem � passagem a�rea, j� que o candidato ficou hospedado na casa do embaixador do Brasil na �frica do Sul, Ant�nio Amaral de Sampaio.
+Ela n�o tem os valores exatos pagos pelo partido.
+MAIS RETIRADOS: O n�mero da esquerda representa a posi��o atual e o da direita, a posi��o na quinzena anterior.
+Consulta realizada nos dias 28 e 29/03/94 junto �s locadoras Cine Arte V�deo, Gentile V�deo Hobby V�deo, Over V�deo, Real V�deo, V�deo Clube do Brasil, V�deo Factory e Wolf V�deo.
+RECOMENDADOS: avalia��o feita a partir de fitas emprestadas por distribuidoras de todo o pa�s.
+Derrida -- A palavra engajado tem uma hist�ria.
+Quando a gente se diz engajado, corre o risco de evocar modelos anteriores e o engajamento hoje deve encontrar formas novas.
+Mas o trabalho no Parlamento � uma forma de engajamento, claro.
+Folha -- Obrigada pela entrevista.
+A not�cia veiculada por esta Folha (em 10/10/1994) de que o presidente eleito Fernando Henrique Cardoso pretende propor o fim da unicidade sindical (sindicato �nico representando a mesma categoria e na mesma base territorial, conforme est� previsto no artigo 8�, II da Constitui��o Federal de 1988) juntamente com o contrato coletivo de trabalho, demonstra sua coer�ncia com a moderniza��o das rela��es sociais.
+Com efeito, os estudiosos da mat�ria afirmam que a negocia��o coletiva de trabalho est� diretamente relacionada com o modelo de organiza��o sindical (cf. Amauri Mascaro Nascimento, em �Direito Sindical�, editora Saraiva, 1989, p�g. 313).
+N�o houve mortos.
+Cerca de 30 feridos leves foram atendidos na Santa Casa da cidade.
+O prefeito Mercedes Ribeiro de Miranda (PMDB) decretou estado de calamidade p�blica na cidade, que ficou sem �gua e luz durante 22 horas.
+Todos os anunciantes que aderiram � id�ia dos clubes infantis est�o atr�s de um mercado de dimens�es gigantescas: os miniconsumidores americanos na faixa de idade entre 4 e 12 anos gastaram no ano passado US$ 7,3 bilh�es das suas pr�prias mesadas e influenciaram suas fam�lias a comprar mais US$ 130 bilh�es.
+Tanto dinheiro parece tornar desprez�vel a �tica do marketing: a Delta recebe centenas de cartas de crian�as denunciando pais ou parentes que preferiram viajar por outras companhias a�reas, configurando um claro est�mulo ao �dedodurismo� precoce.
+Helcio Emerich � jornalista, publicit�rio e vice-presidente da ag�ncia Almap/BBDO.
+Fabiane recebeu CR$ 35 mil para usar camiseta da candidata e gritar o nome de Qu�rcia.
+�Deus me livre.
+N�o voto em Qu�rcia de jeito nenhum.
+Estou aqui a trabalho�, disse.
+Ela pert�ncia a um grupo de cem garotas de Bras�lia contratado para fazer propaganda de Ana Paula Junqueira.
+�Acho absurdo um pol�tico pagar algu�m para gritar seu nome�, reclama.
+O t�cnico irritou-se com uma pergunta e n�o a respondeu.
+O Brasil �nunca esteve em apuros�, segundo o t�cnico.
+A frente do glaciar, que fica na beira do rio, desprende blocos de gelo do tamanho de um pr�dio de 30 andares.
+D� para sentir a terra estremecer e o barulho parece o de um trov�o.
+Fomos convidados a fazer um minitreking, ou seja, caminhar cla�ando �crampones� (sapatos com pregos de ferro) sobre o glaciar.
+Durante o passeio, escut�vamos os ru�dos da movimenta��o lenta deste enorme rio de gelo e contempl�vamos os contrastes crom�ticos da superf�cie congelada refletindo o c�u azul.
+�� preciso ter em conta que a eleva��o das importa��es com a redu��o das al�quotas � estimada em 1% da pauta das importa��es brasileira.
+Esse percentual � razoavelmente pequeno porque significa US$ 250 milh�es�, disse.
+Amaral disse ainda que esse aumento de importa��es decorre do aumento da demanda que se exerce sobre toda a oferta de produtos.
+ENFERMAGEM -- Acontecer� no pr�ximo dia 6, no Ces-Senac (av. Tiradentes, 822), semin�rio sobre a caracteriza��o do pessoal de enfermagem no Estado de S�o Paulo.
+A presen�a deve ser confirmada at� o pr�ximo dia 5, pelo tel. 221-2155.
+OFICINA DE LIVROS -- A Oficina de Artes do Livro oferece 8 vagas para o curso �Papel Artesanal: Processamento de Fibras Vegetais�, que vai de 5 a 14 de abril.
+Informa��es pelo tel. 212-2051.
+O documento tamb�m sugere a cria��o de uma sistem�tica mensal de acompanhamento.
+�Elvis morreu quando entrou para o ex�rcito�.
+A caixa com cinco �CDs Elvis: From Nashville To Memphis� enterra em parte o velho mito explicitado pelo beatle.
+O pacote traz as grava��es essenciais do cantor na d�cada de 60, exatamente depois de sua baixa no ex�rcito norte-americano, em 6 de mar�o de 1960.
+Esses s�o os anos em que Elvis deixou de ser o rei do rock'n'roll para tornar-se o s�mbolo pr�-fabricado de uma gera��o que sonhava com Honolulu e pracinhas.
+Nessa �poca, o cantor forjou alguns de seus maiores sucessos como, para ficar num s� exemplo, �It's Now or Never�.
+Tamb�m nesse per�odo Elvis Aaron Presley sucumbiu ao monstro chamado Elvis Presley.
+O cantor n�o esteve livre do ass�dio e do culto de seus f�s nem mesmo durante os dois anos em que esteve ligado ao ex�rcito.
+E � justamente por isso que n�o se deve parar, preparando terreno ao futuro presidente.
+Al�m de aumentar o n�mero de cassa��es, deve-se estimular as demais CPIs sobre Empreiteiras, financiamentos eleitorais e CUT a CPI da CUT ter� o dom de trazer � tona poss�veis desvios da burocracia e da praga do corporativismo, o que, acreditem, vai acabar ajudando n�o apenas o PT mas a democracia.
+Mas a roubalheira � apenas a ponta do �lama�al�: o problema essencial, a grande delinqu�ncia, � a baixa taxa de seriedade e compromisso dos homens p�blicos, respons�vel por estarmos atolados em n�veis jamais vistos de mis�ria e viol�ncia e, como diz o salmo, � a� que �n�o se pode estar em p�.
+PS Milagre mesmo � gente como Jo�o Alves trocar a leitura de volantes de loteria pela B�blia quem sabe o poder p�blico n�o o ajuda, dando-lhe uma cela especial para aprofundar tais leituras.
+�Voc�s s�o pessoas de um Brasil que tem sinais da renova��o.
+Se Roseana, os senadores, deputados e prefeitos est�o me apoiando n�o � porque tenhamos firmado qualquer compromisso pessoal.
+Jamais ningu�m me pediu nada.
+Mas, nada mesmo�, disse FHC.
+�s 16h, antes de embarcar para a cidade de Cod� (MA), onde seria realizado um com�cio �s 18h de ontem, Fernando Henrique afirmou que vai aguardar o posicionamento do senador Jos� Sarney (PMDB-AP) em rela��o �s elei��es presidenciais.
+Folha -- Como voc� recebeu a not�cia de que seria substitu�do?
+Giovane -- Normalmente.
+Sele��o � isso, joga quem est� melhor, n�o tem esse neg�cio de nome.
+A melhor equipe � que deve jogar.
+... casos positivos de dengue em Monte Apraz�vel (38 km de Rio Preto-SP) foram divulgados ontem pelo Ersa (Escrit�rio Regional de Sa�de) de Rio Preto.
+A regi�o vive uma epidemia da doen�a.
+A cidade de Monte Apraz�vel tem 68 casos confirmados.
+Rio Preto tem tr�s casos positivos.
+Os policiais federais de Mato Grosso do Sul entraram em greve ontem, em ades�o ao movimento iniciado no Distrito Federal.
+A reivindica��o � de equipara��o salarial com a Pol�cia Civil do DF, o que representaria um reajuste de 300%.
+Em Mato Grosso, a paralisa��o est� prevista para come�ar hoje.
+Todas receberam beijos de Itamar.
+Fu foi a mais calorosa.
+Beijou, abra�ou duas vezes e conversou por alguns minutos com o conterr�neo presidente.
+�Eu torci muito por voc�, viu?�, disse Itamar a Fu.
+Cada usu�rio pode importar programas at� o valor de US$ 200 mil.
+Os pedidos at� US$ 20 mil dispensam guia de importa��o.
+Se o valor da compra for baixo, � melhor pedir encomenda por correio normal.
+Pelo correio expresso o frete custa pr�ximo de US$ 50.
+Se n�o quiser se aventurar pela importa��o direta, existem empresas, como a Brasoftware (tel. 011/253-1588) que importam programas sob encomenda.
+Segundo Ricardo Jord�o, gerente de marketing da Brasoftware, o produto -- entregue no m�ximo em 15 dias -- sai por um pre�o similar � importa��o feita pelo usu�rio.
+A taxa de servi�o da Brasoftware � coberta pelo desconto que obt�m junto ao fornecedor.
+ARRIET CHANIN -- A artista mostra 15 monotipias trabalhadas em papel artesanal de folha de bananeira e 12 gravuras em metal, que s�o instrumentos musicais e conchas.
+De seg a sex das 11h �s 19h e s�b das 10h �s 13h.
+Pre�os das obras: de R$ 50 a R$ 500.
+At� 30 de setembro.
+As novas impressoras a laser da HP v�m com um novo padr�o de velocidade 12 p�ginas por minuto (ppm) e s�o de 30% a 40% mais r�pidas que as da gera��o anterior.
+As LaserJet 4 Plus e 4M Plus substituem os modelos 4 e 4M.
+T�m resolu��o de 600 x 600 pontos por polegada (dpi), o que permite obter imagens com mais defini��o.
+O empres�rio e corretor de im�veis, Roberto Capuano, foi reeleito pela terceira vez presidente do Conselho Regional de Corretores de Im�veis do Estado de S�o Paulo (Creci).
+Ser� a sua quarta gest�o � frente do �rg�o.
+Enquanto Gloria Pires ficou nervosa em sua estr�ia como modette, o marido Orlando Moraes amou e pediu bis.
+De acordo com a Enciclop�dia Brit�nica, o linchamento � uma pr�tica que se verifica em momentos de instabilidade ou de amea�a de anarquia.
+A sensa��o de inseguran�a e a debilidade do poder p�blico eram sem d�vida caracter�sticas dos Estados Unidos do final do s�culo 18, quando (segundo a vers�o mais aceita) a palavra teria sido inventada a partir da pr�tica adotada por um juiz Lynch, de condenar e executar seus advers�rios sem o devido processo legal.
+Lamentavelmente s�o tamb�m caracter�sticas de diversas regi�es do Brasil de hoje.
+O b�rbaro linchamento de tr�s pessoas ocorrido esta semana no Paran� pode ter sido particularmente chocante e sem d�vida o foi, filmado e transmitido pela televis�o em toda a sua brutalidade, mas constitui apenas mais um na lista dos crimes desse tipo registrados no pa�s.
+Apenas na Bahia, por exemplo, o comando da Pol�cia Militar estadual informa que foram 350 casos nos �ltimos quatro anos.
+Formada por t�cnicos da Fazenda e das Minas e Energia, a comiss�o tem prazo de 60 dias para conluir o trabalho.
+O objetivo �e melhorar a rentabilidade do setor.
+Segundo o ministro das Minas e Energia, Alexis Stepanenko, essa melhoria dever� ser buscada atrav�s da redu��o de custos e aumento de efici�ncia.
+No dia sete de dezembro de 1941, avi�es japoneses praticamente destru�ram a frota norte-americana no Pac�fico, ancorada em Pearl Harbour, no Hava�.
+Os EUA, depois de um hist�rico discurso de seu presidente, Franklin Delano Roosevelt, entraram em guerra contra o Jap�o e seus aliados no Eixo, Alemanha e It�lia.
+O chamado �mundo livre� (que ent�o inclu�a tamb�m a URSS) unia-se contra o nazi-fascismo.
+E o Brasil?
+No Departamento de Estado, em Washington, desconfiava-se de Get�lio, ditador com uma pol�tica em certos momentos pr�xima ao fascismo mussoliniano.
+Vargas, na verdade, preferia ficar de fora, ou aliar-se com os vencedores.
+A Argentina, por exemplo, era bem mais pr�-nazi do que o Brasil.
+Welles foi mandado como uma esp�cie de embaixador cultural.
+N�o foi o �nico.
+E ele gostava de Roosevelt e detestava os nazistas.
+J� estou trabalhando com as principais figuras da vida pol�tica do pa�s com senadores e secret�rios de governo.
+Agora estou trabalhando no desenvolvimento de um programa para aumentar a contribui��o das pol�ticas p�blicas para o capital social.
+Mas s�o necess�rias estrat�gias da base da sociedade.
+A subst�ncia tamb�m � usada em v�rios medicamentos vendidos no Brasil, que dizem ser capazes de aliviar dores causadas por varizes e eliminar c�imbras noturnas em pessoas que sofrem problemas de circula��o nas pernas.
+Levantamento realizado entre pacientes que ingeriram c�psulas com a subst�ncia entre 1969 e 1992 indicou que 16 pessoas podem ter morrido em consequ�ncia de contra-indica��es causadas pelo sulfato de quinino.
+Conforme acordo autorizado pelo BC, o saldo da d�vida do Estado com a �Nossa Caixa� de US$ 1 bilh�o ter� que ser pago este ano, em doze parcelas mensais, atualizadas e com encargos.
+Corresponde a duas vezes e meia o patrimonio liquido da institui��o.
+O Banespa carrega US$ 8,1 bilh�es de cr�ditos do setor p�blico.
+Segundo o detetive Paulo Ara�jo, a delegacia recebeu um telefonema de homens que se identificaram como traficantes do morro Azul (no Flamengo).
+Segundo o detetive, eles amea�aram invadir a delegacia, no hor�rio de visita, para resgatar dez companheiros.
+Os fundos de commodities projetam para este m�s, na m�dia, rentabilidade bruta de 3,72%, segundo a Anbid.
+A rentabilidade l�quida, para saque em 1� de setembro, vai depender da varia��o da Ufir.
+Por enquanto, este indexador projeta varia��o de 1%, com o que os fundos de commodities renderiam 3,03%.
+Os norte-americanos que se reuniram ontem na C�mara de Com�rcio Brasil-Estados Unidos, em S�o Paulo, ficaram conformados com a derrota de sua sele��o.
+�J� estamos contentes por ter chegado �s oitavas-de-final�, disse Phillip Trent, 24, que assistiu o jogo ao lado do amigo Brian Fagerburg, 24.
+Ambos seguraram uma bandeira dos Estados Unidos durante a partida.
+O desembargador Doreste Batista, convidado pelo governador do Rio, Nilo Batista, para ser o supersecret�rio de Seguran�a do Estado, disse ontem que, se confirmado, vai pedir apoio do Ex�rcito para subir morros e colocar policiamento ostensivo nas ruas.
+At� o meio-dia de ontem, o desembargador estava tentando um contato com o governador para definir se aceitava ou n�o o cargo.
+Ao lado do ala Charles Barkley, �Shaq� � hoje um dos maiores astros do basquete mundial.
+� piv� do Orlando Magic, equipe da NBA, liga profissional de basquete dos EUA.
+Conquistou o t�tulo mundial deste ano com a sele��o de seu pa�s, o �Dream Team� (Time dos Sonhos) 2.
+Enfermeiro � uma designa��o v�lida apenas para quem concluiu o curso superior de enfermagem.
+Os outros profissionais da �rea s�o t�cnicos, auxiliares e atendentes.
+O t�cnico tem o segundo grau completo pode optar entre o curso t�cnico de enfermagem ou curso de forma��o de t�cnico.
+�Os limites da privatiza��o s�o as necessidades do governo.
+Nada est� exclu�do�, afirmou Montoro Filho.
+Para ele, a privatiza��o entende desde a venda de ativos p�blicos at� a��es de estatais.
+Disse que o governo vai vender �desde autom�vel at� participa��o acion�ria em empresas de energia el�trica�.
+O dinheiro das vendas pode ser utilizado tamb�m para o pagamento da d�vida do Banespa.
+E tudo, segundo a Jovem Pan, para fazer �proselitismo�, para �as promessas de sempre�.
+Fernando Henrique apareceu no r�dio e na televis�o dizendo que n�o existe nada contra o seu vice.
+E que, portanto, nem pensa numa troca de Guilherme Palmeira.
+Uma defesa bem parecida com aquela de Lula, no come�o das den�ncias contra Jos� Paulo Bisol.
+�N�o t�nhamos uma seguran�a exemplar, mas ela era suficiente para permitir o alvar� de licen�a dos bombeiros�, afirmou.
+Segundo ele, o alvar� foi queimado, com tudo o que se perdeu.
+Taranto n�o soube dizer se os parentes das v�timas poder�o pedir alguma indeniza��o.
+A MTV n�o exibiu ontem o desenho animado �Beavis e Butt-Head� por determina��o de liminar expedida pela Vara Central da Inf�ncia e Juventude, em S�o Paulo.
+Entre 20h e 20h30, hor�rio em que o �cartoon� costuma ir ao ar, a emissora transmitiu uma sequ�ncia de videoclipes.
+Antes, informou sobre a proibi��o da Justi�a num comunicado de 15 segundos.
+Os nativos ainda se utilizam de lan�as primitivas para pescar nas rasas piscinas naturais repletas de polvos e peixes tropicais.
+Mas h� a pesca comercial de atum controlada por norte-americanos de origem portuguesa, chineses de Taiwan e sul-coreanos em embarca��es de 1,2 tonelada que valem at� US$ 13 milh�es.
+Patologistas de hospital de Riverside (Calif�rnia) tiveram que vestir roupas especiais para autopsiar corpo de uma mulher que teria qu�micos t�xicos no sangue, porque o cheiro os fez desmaiar.
+Cristou representa os clubes alem�es do Bayern Leverkusen e Shalk-04.
+O empres�rio disse que pretende ver tamb�m outros jogadores brasileiros, entre eles Cafu, do S�o Paulo, Mirandinha, do Paysandu, e Claudinho, da Ponte Preta.
+O caso nasceu de uma �vendetta�, obra e desgra�a de um certo Francesco Farina, dono do Modena, um time rebaixado � 3� divis�o.
+Farina tentou salvar o Modena na pol�tica e no tapet�o.
+Obviamente, n�o conseguiu.
+Por isso, metralhou Antonio Matarrese, presidente da federa��o.
+Farina acusou os clubes de sonegarem impostos com a cumplicidade da federa��o.
+Para multiplicar o porte das suas den�ncias, ele apresentou as suas perora��es em dois s�tios diferentes, �s procuradorias de Roma e de Mil�o.
+Muitas de elas s�o inconsequentes, improv�veis, sem documenta��o.
+De todo modo, na Bota, � Justi�a cabe a miss�o de investigar, antes de meramente julgar.
+Gloria Attanasio, magistrada da capital, meramente decidiu o que a sua fun��o lhe exigia, e enviou 250 detetives �s sedes dos tais 34 clubes para, l�, coletarem os livros cont�beis e os recibos correspondentes ao versamento dos impostos.
+Para a constru��o do palco foram utilizadas 170 toneladas de ferro e alum�nio.
+A quantidade de ferro daria para construir 180 carros iguais a um Santana.
+E o volume de alum�nio daria para fazer 275 mil latas de refrigerante.
+O palco foi criado por Mark Fischer, respons�vel pela montagem do palco da �ltima turn� dos Stones �Steel Wheels�, do novo show do Pink Floyd e do espet�culo ZooTV do grupo irland�s U2.
+Apesar da popularidade, Chico de Miguel perdeu as duas �ltimas elei��es para a Prefeitura de Itabaiana, que passou a ser controlada por pol�ticos locais ligados ao governador Jo�o Alves.
+Ele atribui as derrotas ao Judici�rio, acusado de auxiliar seus opositores.
+N�o explica como, mas garante que n�o vai perder a pr�xima elei��o em Itabaiana.
+O deputado afirma n�o ser um homem rico.
+�Hoje, tenho s� uma fazenda e pouco mais de 500 cabe�as de gado.
+Mas tudo que ganho reverto tamb�m em ajuda para o povo, que recebe rem�dios, feira e dinheiro quando est� precisando.
+Nunca fui ego�sta�, afirma.
+Outros institutos mostram n�meros diferentes, mas n�o s�o divulgados pela televis�o.
+A Liga de Assist�ncia e Recupera��o, �rg�o ligado � Prefeitura de Salvador, est� desenvolvendo um projeto para a confec��o de brinquedos a partir de sucatas.
+Crian�as de 11 a 14 anos v�o aprender a transformar sucatas em brinquedos educativos.
+A Secretaria de Sa�de da Para�ba informou que Jo�o Pessoa est� entre as cinco capitais brasileiras que atingiram 90% de cobertura vacinal na �ltima campanha de multivacina��o, realizada em 90.
+Neste ano, a secretaria espera vacinar cerca de 480 mil crian�as.
+A R�ssia vetou documento da Confer�ncia sobre Seguran�a e Coopera��o na Europa sobre a ex-rep�blica iugoslava da B�snia.
+O veto impediu a declara��o que pedia aos s�rvios da B�snia que parassem ataques no encrave mu�ulmano de Bihac.
+A reuni�o terminou ontem em Budapeste.
+�Se o pre�o do �nibus for convertido pela tabela da montadora, o impacto ser� de 10% a 12% na tarifa�, diz Dias.
+As tabelas das montadoras, segundo ele, est�o 40% acima do pre�o de mercado.
+O F�rum j� tem os pre�os m�ximos pagos no mercado.
+�nibus leve, US$ 55 mil, para o pesado, US$ 70 mil.
+Antes do Mundial, a delega��o brasileira participa dos Jogos da Amizade que t�m in�cio dia 23, na R�ssia.
+William Gibson � um escritor normalmente associado � fic��o cient�fica.
+De fato, a maioria de suas hist�rias acontece num tempo algo � frente do presente, mas Gibson rompe completamente com a id�ia fundadora da FC no cinema ou na literatura de �futuro� como evolu��o.
+A estrat�gia de Gibson consiste em projetar no futuro o que j� est� latente no mundo contempor�neo.
+Nos romances e contos do escritor, o cen�rio � muito pr�ximo do atual, s� que visto com uma certa distor��o alucin�gena, um certo exagero de imagina��o.
+Valores m�dios das loca��es comerciais aumentam 50,16%.
+Os valores m�dios das loca��es de im�veis n�o residenciais na cidade de S�o Paulo subiram em junho 50,16%, segundo a Hubert Im�veis e Administra��o Ltda.
+A evolu��o foi menos acelerada que a verificada em maio (55,16%).
+A maior desacelera��o ocorreu na regi�o da Avenida Faria Lima.
+A Pol�cia Federal prendeu ontem em Bras�lia Vicente Wilson Rivera Ramos, o �Vicentico�, filho de um dos chefes do Cartel de C�li (organiza��o colombiana que trafica drogas).
+�Vicentico� havia escapado do cerco da PF, no domingo, quando foram apreendidas 7,5 toneladas de coca�na em Guara� (TO).
+Filho de Vicente Rivera Gonzalez, ele j� foi condenado na Holanda a 14 anos de pris�o, segundo a PF.
+Os receios de Durante quanto � rea��o da plat�ia eram compartilhados pelo restante da comitiva.
+Documentos reservados da seguran�a da Presid�ncia da Rep�blica alertavam, na semana anterior, para a possibilidade do presidente ouvir vaias.
+Mais que isso: os pap�is da seguran�a afirmavam que, em raz�o da infla��o alta, n�o se deveria descartar sequer o risco do p�blico atirar objetos no camarote presidencial.
+Em telefonema a Maur�cio Corr�a, ministro da Justi�a, o governador do Rio, Leonel Brizola, desaconselhou a ida de Itamar ao Samb�dromo.
+�A rea��o do p�blico � imprevis�vel�, disse.
+Avisado do alerta do governador, Itamar desdenhou os riscos e manteve a viagem.
+Corr�a trocaria a preocupa��o pela descontra��o.
+Agarrado ao copo de u�sque, foi outro destaque da noite.
+A Kodak ainda passou a atender encomendas de pe�as pl�sticas para a Philips, Wapsa e NGK, entre 16 clientes.
+Uma nova divis�o de servi�os, batizada de Kis, opera na administra��o de copiadoras e outros equipamentos de imagem para grandes empresas.
+�� um nicho promissor�.
+Segundo Galan, os novos produtos e servi�os acrescentar�o 10% ao faturamento, que foi de US$ 360 milh�es em 93.
+�No pr�ximo ano ser�o mais 15%�, prev�.
+White -- Acho que sim.
+A pr�pria psican�lise est� baseada numa concep��o tropol�gica da consci�ncia humana.
+�A Interpreta��o dos Sonhos�, um dos textos fundadores da psican�lise, apresenta toda uma tropologia do ato de sonhar e d� a base para uma po�tica que combina a teoria da tropologia com a no��o da inven��o po�tica.
+A id�ia de tropos � provis�ria.
+Uma das coisas que os linguistas ainda precisam estudar � a teoria dos tropos.
+Roman Jakobson trabalhou nessa dire��o.
+Lacan desenvolveu os conceitos de met�fora e meton�mia para caracterizar certos modos de consci�ncia.
+N�o se pense que esta ruptura modernizante passa pelo PT.
+O partido � um aglomerado heterog�neo, que s� se mant�m unido pela perspectiva de vit�ria de Lula.
+Teses modernas e sociais como o controle civil sobre o sistema de sa�de esbarram em resist�ncias enormes dos setores hegem�nicos do PT, sob o argumento de que amarrariam a atua��o do governo, logo agora que o partido se prepara para assumir o poder.
+Tampouco passa por FHC, representante da contemporiza��o bem explicada, ou por Antonio Britto cujo voto, em favor da anistia total aos agricultores, semana passada, ressuscitou os piores progn�sticos sobre seu estilo.
+O motorista teria perdido o controle do carro, batido em um Monza e em uma �rvore e capotado na rua En�as Luiz Carlos Barbante.
+Maria Elisa Flora Demarchi, 15, est� internada em estado grave no Hospital do Mandaqui.
+�O boneco servir� para criar o que chamamos de 'efeito aur�ola' no motorista.
+Pesquisas comprovam que quando o motorista v� a fiscaliza��o ele reduz a velocidade por pelo menos tr�s quil�metros�, diz Lehfeld.
+O boneco vai ser colocado em locais sem congestionamento.
+Vai �trabalhar� das 7h �s 17h30, inclusive nos finais de semana.
+A perman�ncia em cada cabine ainda n�o foi definida.
+Perry descreveu tr�s fases para a opera��o: a entrada no Haiti e a conquista de todas as posi��es estrat�gicas (com dura��o de seis a horas a dois dias), a cria��o de �um ambiente seguro� (de tr�s a seis meses) e a transfer�ncia do controle da situa��o para a ONU.
+O presidente Bill Clinton passou a maior parte do dia de ontem em reuni�es com seus assessores militares e de seguran�a nacional na Casa Branca.
+Vila Madalena e Pinheiros ganharam novas op��es noturnas nesta semana.
+O bar e restaurante Lanterna inaugurou na ter�a-feira um espa�o para dan�ar.
+Agora o jardim entre a galeria e o sal�o do restaurante propriamente dito tem um bar com som dos DJs Bidi, Bartolo e Hori.
+O tr�nsito nas avenidas Paulista e Brigadeiro Lu�s Ant�nio (zona central de S�o Paulo) parou no dia 9 de setembro de 86.
+O aposentado Eliseu Francisco de Lyra (na �poca com 44 anos), depois de discutir com o irm�o, saiu do carro, subiu no cap�, tirou as roupas e sapateou.
+A AI, organismo internacional com mais de 1,1 milh�o de membros, divulgou o documento ontem em S�o Paulo.
+O relat�rio, referente ao ano de 1993, tamb�m destaca a impunidade dos acusados pelos massacres do Carandiru (outubro de 92), da Candel�ria (julho de 93) e de Vig�rio Geral (agosto de 93).
+O apelo pop do evento j� est� causando rea��es.
+Ohtake diz que at� o compositor e cantor baiano Caetano Veloso manifestou desejo de assistir ao show de Nusrat, nesta segunda-feira, em S�o Paulo.
+�Um monte de artistas vai querer ver os concertos�, diz Ohtake.
+O secret�rio espera a resposta do p�blico.
+�Toda plat�ia de m�sica erudita � tradicionalista�, explica.
+�Em Campos do Jord�o tamb�m.
+O p�blico de l� � extremamente treinado em m�sica do s�culo 19.
+Para mim � muito estimulante oferecer uma id�ia espacial para um trabalho que come�ou a partir de uma id�ia musical.
+Basicamente, � o que tenho feito com os m�sicos com os quais trabalho.
+O que mudou talvez seja meu vocabul�rio, que est� menos estruturado.
+A din�mica, a maneira como os movimentos s�o constru�dos, hoje � diferente.
+1. Curr�culo com mais de tr�s p�ginas.
+2. Que o selecionador tenha que ler todo o curr�culo para descobrir o perfil do profissional.
+O cr�tico (�/deve ser), um insolente da raz�o?
+Um (mecenas) nobre que deifica o saber antes e mistifica o dar depois ou ele deve ser um iconoclasta, espalhar o que sabe enquanto o tempo faz das novidades fatos descart�veis.
+Ser� que j� n�o nos basta a humilde compuls�o passiva dos leitores que aceitam nos ler para serem informados?
+Mas o leitor tamb�m sabe se vingar com o desd�m e a indiferen�a com que � capaz de folhear as p�ginas da sua busca.
+Um cr�tico de jornal cotidiano deve se conformar com o papel de narrador e seu destaque secund�rio ao simples gesto de uma p�gina ser virada e ficar para tr�s.
+A not�cia � um fato, n�o um exerc�cio acad�mico.
+A dimens�o do fato est� na relatividade da sua import�ncia para cada leitor e n�o no espa�o narc�sico que ocupa sob uma assinatura.
+Em Medicina, ganharam Richard Dart e Richard Gustafson, da Universidade de Arizona por fracassarem ao tratar um paciente mordido por uma cascavel com choques el�tricos aplicados no l�bio durante cinco minutos.
+Em F�sica, ganhou a Ag�ncia Meteorol�gica Japonesa por um estudo de sete anos sobre a hip�tese de terremotos serem causados por peixes rebolando suas caudas.
+Na madrugada de ontem, os pr�dios foram vigiados a dist�ncia por homens da Rota, que estavam sem farda.
+Segundo o coronel, eles viram o armamento da quadrilha ser retirado de um dos pr�dios em um Gol.
+Esse carro foi achado no in�cio da tarde em Engenheiro Marsilac (zona sul).
+Com mandados de busca e apreens�o, os homens da Rota invadiram os tr�s apartamentos.
+N�o houve tiroteio.
+De acordo com o coronel, a pol�cia descobriu a quadrilha atrav�s de uma den�ncia an�nima.
+Eduardo Louren�o -- N�o se deve esperar uma interven��o que tenha efeitos imediatos como a dos pol�ticos.
+Os escritores aqui reunidos pretendem alertar a comunidade internacional sobre os ataques sofridos pela liberdade de pensar e de escrever em v�rios pa�ses do mundo.
+Os exemplos mais c�lebres e tr�gicos s�o os de Rushdie e Nasreen.
+O nosso protesto � de ordem moral, temos a obriga��o de defender uma das grandes tradi��es da nossa civiliza��o, que � a da liberdade de express�o.
+Jos� Saramago -- A contribui��o vai depender do eco que o Parlamento possa ter na opini�o p�blica.
+Podemos dizer coisas importantes, tomar grandes decis�es, mas se n�o tiver repercuss�o...
+Tudo depende da capacidade que o Parlamento tiver de transmitir as suas id�ias � imprensa, ao r�dio e � televis�o.
+N�o sei se os jornalistas est�o conscientes da grande responsabilidade que t�m.
+Os seus 100 quil�metros j� levaram este ano 200 mil toneladas de soja.
+Custo: US$ 8 por tonelada.
+No Sul, US$ 24!
+Por que n�o publicar que o PT fez um semin�rio no Nordeste sobre ela e concluiu pela necessidade de sua constru��o?
+Esta � uma obra do Brasil, transformadora da estrutura do pa�s.
+N�o comporta qualquer reserva, e bendito o pa�s em que os homens p�blicos exigem dos candidatos a solu��o de problemas nacionais, minorar a fome, dando leite �s crian�as que se alimentam de lixo, e estradas para desenvolver o pa�s.
+O ex-jogador assumiu o cargo ontem em substitui��o a Carlinhos, demitido ap�s a derrota para o Bahia (0 a 1), anteontem.
+�As torcidas partem para cima da PM a fim de que nos concentremos num s� local.
+Entre os denunciados por corrup��o passiva est�o o procurador de Justi�a aposentado Aldegy do Nascimento, 26 delegados, inspetores e peritos da Pol�cia Civil e Ary Chagas de Aguiar, assessor da promotora L�cia Atalla.
+Os 38 nomes fazem parte de um aditamento � primeira den�ncia feita pelo Minist�rio P�blico no esc�ndalo do bicho, em junho.
+O pai de Rom�rio foi sequestrado na segunda-feira � noite, no Rio.
+Os sequestradores estariam exigindo US$ 7 milh�es (cerca de CR$ 9,6 bilh�es) para libert�-lo.
+Abalado, o jogador amea�ou n�o disputar a Copa do Mundo, em junho e julho, nos EUA, caso o pai n�o seja libertado.
+O s�timo filme de Spike Lee acerta nos figurinos e trejeitos da �poca e resgata preciosidades do soul negro dos anos 70.
+As m�sicas incluem The Jackson 5, Stevie Wonder Jimi Hendrix, James Brown e outros 18 bons nomes.
+O problema � a hist�ria.
+Quase nada acontece.
+H� algumas brigas engra�adas entre irm�os, cenas de separa��o conjugal e uma morte.
+Em algumas passagens, h� di�logos intermin�veis e vazios.
+Durante o almo�o familiar, por exemplo, fala-se muito sobre quase nada e ningu�m mastiga.
+Soa for�ado demais.
+N�o se sabe ainda como Oosterbroek morreu.
+Seu corpo n�o tinha marcas de tiros.
+Aparentemente ele quebrou o pesco�o quando tentava fugir do fogo cruzado.
+A pol�cia anunciou a liberta��o de seis homens que estavam trancados no por�o de um pr�dio do CNA em Johannesburgo.
+Segundo o porta-voz da pol�cia, Dave Bruce, os seis foram torturados.
+O deputado Adroaldo Streck (PSDB-RS) ficou encarregado de confirmar com Britto o encontro.
+Britto j� informou a Streck que � favor�vel, mas ficou de discutir a quest�o com o PMDB ga�cho.
+Camila viajava no colo de Dalva, na frente do ve�culo.
+O corpo de Camila foi lan�ado para fora do carro.
+A menina morreu na hora.
+Dalva seria submetida ontem a cirurgia no hospital de S�o Paulo.
+A fam�la voltava para casa, em American�polis, depois de participar de casamento na Barra Funda e deu carona a amigos.
+Outras cinco pessoas ficaram feridas.
+Niemeyer -- �, foi em 1936.
+Folha -- o que o sr. pode dizer dele em termos pessoais?
+Jazz rap?
+Esque�a.
+Ou melhor, n�o esque�a porque ainda se faz muita coisa boa, mas deixe de querer ser bacana citando-o como a �ltima tend�ncia.
+Para aquele verniz atualizador, passe desde j� a usar a express�o �blues rap�.
+E, para um efeito extra, cite G. Love and Special Sauce.
+Seu �lbum de estr�ia, lan�ado no �ltimo ver�o americano, causou pouco ou nenhum impacto na parada.
+Mas uma boa escava��o nas se��es de cr�ticas de revistas de m�sica importadas revela a verdadeira adora��o que G. Love estimulou entre os jornalistas especializados.
+O secret�rio-geral da Presid�ncia, Mauro Durante, tamb�m tem emprego garantido.
+Itamar quis encaix�-lo no Tribunal Superior do Trabalho, mas a OAB (Ordem dos Advogados do Brasil) n�o deixou.
+O presidente articulou, ent�o, lobby certeiro em dire��o ao Sebrae (Servi�o de Apoio � Pequena e M�dia Empresa) e ainda pediu a FHC que o ajudasse a p�r Durante na presid�ncia da entidade.
+Esses dados fazem parte do livro �Conflitos no Campo Brasil 1993�, lan�ado na ABI (Associa��o Brasileira de Imprensa), no Rio de Janeiro, pela CPT (Comiss�o Pastoral da Terra), �rg�o da Igreja.
+Constam do livro 15 quadros estat�sticos relacionando os nomes das v�timas e os respons�veis por crimes ligados a conflitos de terra.
+As causas da viol�ncia no campo tamb�m s�o discutidas.
+O diretor executivo do FMI tamb�m enfatizou a necessidade da queda dos juros no Brasil para incentivar a volta de investimentos maci�os nos setor produtivo.
+A uma pergunta sobre se a volta dos investimentos externos n�o dependeria do selo de aprova��o do Plano Real atrav�s de um acordo formal entre o FMI e o Brasil, Camdessus disse que o que vai fazer a taxa de juros cair n�o � um acordo com o FMI, mas a credibilidade interna e externa do Plano Real.
+Bancos e financeiras receberam bem a resolu��o 2.071 do Conselho Monet�rio Nacional.
+Baixada sexta-feira, ela cria a taxa de juro flutuante.
+Abre a perspectiva de aplica��es por prazo mais longos.
+O juro flutante permite reavaliar periodicamente as taxas de juros das aplica��es e empr�stimos.
+A interven��o diplom�tica russa foi decisiva para evitar o envolvimento ocidental na guerra civil na B�snia.
+Ao se oferecer para intermediar o conflito, a R�ssia deu aos seus aliados s�rvios a possibilidade de uma retirada honrosa.
+Isso praticamente eliminou a amea�a de ataques a�reos da Otan.
+Essa � a avalia��o do Instituto Internacional de Estudos Estrat�gicos de Londres (IIEE), um dos mais importantes centros de pesquisa militar do Ocidente.
+�Um ataque agora n�o faz mais sentido, desde que os s�rvios continuem a recuar�, disse � Folha o coronel Andrew Duncan, do IIEE.
+O espantoso � que fora da pol�cia tamb�m se viu o �mal� no incidente de Cavalera.
+O jornal carioca �O Globo� recolheu depoimentos sobre a �pisada� n�o confirmada nem pela pol�cia e explicada por Cavalera como um simples trope�o, normal na movimenta��o de palco, em seu depoimento na delegacia.
+Todos, contra ou a favor, partem do princ�pio de que aquele foi um gesto de �protesto�.
+�Foi um protesto moleque.
+S� poderia ter partido de uma cabe�a de merda�.
+Um psicanalista (sic), Jos� Nazar, sugere, acreditem se quiser, nada mais nada menos do que linchamento como puni��o.
+O jurista Miguel Reale Jr. prefere um �pessedebismo� pedag�gico, ao aprovar a atitude da pol�cia em det�-lo como exemplo para a juventude.
+A tumorectomia �garante sobrevida compar�vel e melhor qualidade de vida do que a mastectomia total�, escreveu a chefe do novo estudo, Anna Lee-Feldstein.
+Cientistas da Universidade da Calif�rnia em Irvine (EUA) publicaram a pesquisa na �ltima edi��o da revista da Associa��o M�dica Norte-americana, (�Jama�).
+Com isso, o Cruzeiro deve come�ar a partida com Michelli, Magno, Derlan, Marcus Vin�cius e Anderson; Emiliano, Juliano e Anderson Le�o; Ricardinho, N�lson e Herbert.
+A equipe vem treinando junta h� pouco tempo.
+O atacante N�lson e meia Anderson Le�o chegaram agora ao time e o ponta-direita Ricardinho, o meia Juliano e o zagueiro Derlan foram promovidos no final de 93.
+Rec�m-promovido, Derlan, 17, j� � um dos destaques do Cruzeiro por sua habilidade nas sa�das da defesa para o ataque.
+O jogador chegou no clube h� dez meses e, em novembro, foi para a equipe de juniores.
+�Estamos muito motivados.
+A casa vai estar cheia e n�s vamos mostrar porque viemos disputar a Copa�, afirma.
+Como sua musa Iris gosta do tema, Silvio Santos vai agora todos os weekends ao teatro.
+Depois da pe�a de Leilah Assump��o, os dois foram juntinhos assitir a �Aluga-se Um Namorado�.
+Amaram.
+Par�grafo 3�. -- Nos contratos celebrados ou convertidos em URV, em que haja cl�usula de reajuste de valor por �ndice de pre�os ou por �ndice que reflita a varia��o ponderada dos custos dos insumos utilizados, o c�lculo desses �ndices, para efeitos de reajuste, dever� ser nesta moeda at� a emiss�o do real e, da� em diante, em real, observado o art. 38 da lei n�. 8.880 de 27.05.94.
+Par�grafo 4�. -- A Taxa Referencial -- TR -- somente poder� ser utilizada nas opera��es realizadas nos mercados financeiro, de valores mobili�rios, de seguros, de previd�ncia privada e de futuros.
+A sensa��o que se tem quando se est� andando num tren� puxado por c�es � semelhante � do esqui aqu�tico.
+Na minha primeira tentativa, fiz exatamente a mesma coisa que na primeira vez que pratiquei esqui aqu�tico: ca� e me soltei.
+Precisei cair mais algumas vezes at� aprender a usar o breque.
+Al�m de dominar esta habilidade, as duas �nicas coisas realmente necess�rias para se viajar de tren� pelo Alasca eram estar em razo�vel boa forma f�sica e ter mais de 13 anos de idade.
+Eu e meu marido, advogados de meia idade, apenas um pouco fora de forma, satisfaz�amos essas condi��es.
+Todos os pl�nios costumam almo�ar no Esplanada Grill, refei��es de US$ 40, em m�dia.
+Os motivos v�o do �bom atendimento� � certeza de achar ali �pessoas como n�s�.
+E se encontram � noite no bar Cabral, embora n�o se conhe�am.
+N�o � necess�rio.
+Os pl�nios t�m no Cabral, aberto no final de 1992 numa travessa escondida da avenida Cidade Jardim, sua meca.
+Luciano Huck, 22 um dos propriet�rios do bar, � o profeta da turma.
+L�, os pl�nios fazem amigos, influenciam pessoas, rev�em colegas, contam as novidades -- as �ltimas viagens para os Estados Unidos, quem comprou carro novo, quem �vai ter� que come�ar a trabalhar com o pai, os que ainda n�o conseguiram telefone celular.
+Pedem cerveja, tequila e batidas, num gasto m�dio de US$ 30 por noite, e falam mal dos mauricinhos que, como diz Rubinho Gimenes, s�o �cheios de querer ser�.
+Eles n�o.
+Eles s�o.
+O jovem corretor de seguros Kiko Villela, 22, �.
+Cursando economia na Faap, Kiko espera ansioso o seu telefone celular, que �est� para sair�.
+�� imprescind�vel hoje em dia�, regulamenta.
+O brinquedinho eletr�nico tem uma fun��o interessante no mundo dos pl�nios: substitui o antigo �torpedo�, bilhete desferido entre jovens em bares.
+Em vez de mandar um papel pelo gar�om com cantadas escritas, o pl�nio liga para a pl�nia de sua predile��o e pronto -- est� feito o contato imediato.
+Entretanto, h� possibilidade de queda nas cota��es externas nos pr�ximos meses, em especial no segundo semestre, quando se verifica a colheita de arroz no hemisf�rio Norte.
+A queda das cota��es internacionais viabilizaria o ingresso do produto a pre�os mais baixos, fato que impediria maiores avan�os das cota��es internas.
+Segundo a previs�o da Conab, a produ��o nacional de arroz deve chegar a 10,6 milh�es de toneladas de gr�os.
+No ano passado, o pa�s colheu 9,5 milh�es de toneladas.
+O diretor-presidente da Yashica do Brasil, Kazuo Tamura, 56, morreu em um acidente de carro ontem de manh� em S�o Paulo.
+Ele dirigia sozinho um Santana 90 em alta velocidade na avenida do Estado, no Cambuci (regi�o central), quando, provavelmente, perdeu o controle do carro e bateu num poste.
+Isoladamente, os EUA e o Jap�o, por exemplo, praticaram al�quotas ainda menores 5,46% e 4,79%, respectivamente.
+O levantamento mostra que a Constitui��o de 1988 elevou de 15,27% para 20,53% a al�quota m�dia sobre o consumo no Brasil em 1989.
+O ministro da Aeron�utica, L�lio Viana L�bo, aproveitou a comemora��o do Dia do Soldado ontem para queixar-se das dificuldades salariais dos militares.
+�Vivemos ainda dias dif�ceis, num contexto econ�mico, social e pol�tico em que as necessidades b�sicas de nossas for�as ( ...) t�m sofrido severamente as agruras que toda na��o atravessa�, disse, em sauda��o enviada ao Ex�rcito.
+Os f�s de Frank Zappa v�o ter uma boa surpresa em janeiro.
+O lan�amento de um CD com uma colet�nea de m�sicas de Zappa interpretadas por bandas cover dos Estados Unidos, Alemanha, B�lgica, Inglaterra, Holanda, It�lia, Su�cia e Brasil (com a banda The Central Scrutinizer).
+A banda brasileira nasceu em 1990 para fazer uma homenagem a Zappa.
+�Era um cara que sempre apostou no que acreditava.
+Lutava contra a bestialidade das pessoas e sempre mostrou isso de uma forma bizarra�, conta Mano Bap, 29, vocalista da The Central Scrutinizer.
+Para o diretor de cr�dito da Febraban, Christoph Heinrich Von Beackedorff, �n�o h� novos tomadores, s� est�o sendo rolados os empr�stimos que est�o vencendo�.
+Ambos apontam o baixo volume de capta��o de recursos com a venda de CDBs aos investidores como o principal fator de inseguran�a dos bancos na defini��o dos custos dos empr�stimos.
+Desse modo, apenas parte do fundo de viagem seria empregada na compra de �traveler cheques� ou c�dulas.
+O restante ficaria em uma aplica��o com anivers�rio coincidindo com o vencimento da fatura mensal do cart�o.
+O caso passou a ser investigado porque pessoas que foram ao vel�rio de Pereira teriam dito que o cad�ver suava e teve sua pele arrepiada dentro do caix�o.
+A fam�lia foi alertada mas n�o chamou nenhum m�dico e fez o enterro depois de dez horas de vel�rio.
+No cinema, sorte do espectador que ter� n�o uma, mas duas mostras internacionais: a Internacional de Cinema, que agora vai ser competitiva e tem Pedro Almod�var e Quentin Tarantino entre os convidados, e a Banco Nacional de Cinema, que ser� mais que uma amostra do evento com sede no Rio, com um ciclo em homenagem ao rei do filme �B�, Roger Corman, tamb�m presente.
+Shows h� para todos os ouvidos.
+O 9� Free Jazz Festival ampliou seu leque e programou uma noite trepidante sob o comando do �godfather of soul� James Brown e cedeu espa�o para a modernidade de Guru, US3 e dos Digable Planets.
+Fora do festival, o De La Soul � a atra��o mais promissora.
+�Como eles aumentaram 6,28% nas duas semanas, acertamos que eles iriam parar os aumentos.
+Dallari recebeu tamb�m a Associa��o Brasileira da Ind�stria de M�quinas.
+A entidade tamb�m se queixou de aumentos que variam de 5,5% a 11,5% no custo de alguns fundidos, forjados, a�o plano e rolamentos.
+Foi ao final de um ano e tr�s meses que o prefeito teve sua melhor avalia��o.
+Recebeu 27% de �timo e bom, 30% de ruim e p�ssimo e 42% de avalia��o regular.
+A boa performance de Maluf em mar�o �ltimo coincidiu com a fase em que ele anunciava sua candidatura � presid�ncia e investia em inaugura��es de grandes obras, como o t�nel do rio Pinheiros.
+A Gelaguela maior fabricante nacional de sobremesas individuais geladas est� investindo US$ 200 mil no sistema de franquias.
+A empresa, que faturou US$ 1 milh�o em 93, vai inaugurar at� setembro mais quatro unidades da Gelaguela Sobremesas.
+O objetivo � consolidar a marca.
+Esta taxa, no entanto, n�o reflete o que est� acontecendo de fato com os pre�os nas �ltimas semanas.
+Isto porque os �ndices de infla��o s� captam totalmente as mudan�as de pre�os 30 dias ap�s o aumento.
+� sempre considerada a m�dia de quatro semanas em rela��o �s quatro anteriores.
+A partir de agora, a taxa deve sempre recuar.
+Os pre�os estiveram praticamente est�veis nas �ltimas semanas.
+Juarez Rizzieri, coordenador do �ndice de Pre�os ao Consumidor da Fipe, prev� para agosto uma taxa entre 1% e 2%.
+A inten��o � votar e aprovar a MP na pr�xima semana, entre os dias 26 e 27.
+O pr�prio presidente Itamar Franco pedir� empenho de todos os ministros para mobilizar os seus partidos para a vota��o.
+O ministro Ricupero ficou satisfeito com aprova��o da MP da URV na comiss�o mista do Congresso que examinou a medida.
+Para o ministro, disse Simon, as mudan�as feitas s�o assimil�veis pelo plano.
+Os novos propriet�rios ser�o integrados � rede de franquia da Pakalolo, que conta atualmente com 76 pontos de venda.
+Tamb�m ser�o submetidas ao mesmo processo as 28 lojas da Body for Sure, a grife esportiva do grupo.
+�Eu acredito no efeito barriga no balc�o�, afirma o empres�rio Humberto Nastari, 37.
+Na condi��o de senhor de meia-idade com id�ias adolescentes, tio Dave vem observando com bastante inquieta��o que seu sentido do paladar parece estar se esvaindo pelo ralo.
+Peixe passou a ter gosto de carne para mim e uma noite destas, no Rodeio, eu confundi salsicha caseira com frango kebab.
+Chocante.
+Constrangedor.
+Ent�o me deparei com um estudo que afirma que algumas pessoas realmente t�m um paladar mais agu�ado do que outras.
+E, horror dos horrores, s�o as mulheres que t�m o paladar mais apurado do que os homens (pare de dar essas risadinhas, voc� a� no fundo da classe).
+Parece que a Universidade Yale, aquela que nos deu o inesquec�vel George Bush, comprovou a exist�ncia de �superdegustadores�, ou seja pessoas que possuem um n�mero de papilas gustativas muito al�m do normal.
+�Adventure in Castle� � um programa para aprender matem�tica.
+Em ingl�s, traz no��es sobre c�lculos de soma, subtra��o, multiplica��o e divis�o.
+Custa US$49, na Brasoftware.
+Tel. (011) 253-1588.
+�Journey in the Universe� � um programa em ingl�s, voltado para crian�as que querem aprender no��es de astronomia.
+Est� sendo vendido na Brasoftware em promo��o.
+Sai de US$ 79 por US$ 49.
+�MacMillan Dictionary for Children� � um dicion�rio de ingl�s em CD-ROM.
+Conta com recursos de figuras e sons, que mostram o significado das palavras.
+Custa US$45 na Multim�dia Center (tel 011 959-2650).
+Ele disse, no entanto, que a lei antitruste aprovada pelo Congresso reflete o clima que o pa�s est� vivendo nessa fase que antecede o lan�amento do real.
+Mas afirmou que a lei antitruste aprovada pelo Congresso tem as deforma��es t�picas da situa��o conjuntural.
+Apesar da declara��o dos separatistas, o governo brit�nico continua disposto a iniciar as negocia��es com o Sinn Fein, bra�o pol�tico do IRA, antes do Natal.
+O ministro brit�nico para a Irlanda do Norte, Patrick Mayhew, afirmou ontem em Belfast que o cronograma do processo de paz ser� mantido.
+Mayhew disse que o IRA poderia demonstrar que realmente n�o autorizou a opera��o de Newry devolvendo os US$ 210 mil levados durante o assalto.
+FHC -- Esse comit� faz tudo errado.
+Ruth -- Por isso perdemos as elei��es.
+(FHC come�a a rir).
+Sabe por que eu amo tanto voc�, querido?
+Porque est� sempre rindo.
+(Come�am ambos a chorar.
+Um estudo do Corecon (Conselho Regional de Economia), do Rio de Janeiro, divulgado ontem, est� projetando, na hip�tese de ado��o do real em abril, uma infla��o de 2,3$%.
+Com base nesta proje��o, o estudo prev� uma infla��o com a nova moeda de 30% em 94.
+O SPQ (Sistema de Proje��es Qualificadas) resulta da m�dia de opini�es de 16 economistas que fazem an�lise de conjuntura.
+Segundo o presidente do Corecon, H�lio Portocarrero, a infla��o do real n�o ser� puxada pelos oligop�lios, apontados pela equipe econ�mica do governo como os principais vil�es na subida de pre�os ap�s a implanta��o da URV.
+Para ele, a alta de pre�os na fase da nova moeda dever� ser impulsionada pelos setores competitivos como o agr�cola e o de servi�os.
+O projeto ainda n�o � oficial.
+Segundo o professor Vicente Amato Neto, 66, �h� informa��es que todos ouvem, no sentido de fazer o estacionamento�.
+Ontem, a diretoria da atl�tica e Amato Neto que iniciou a organiza��o dos alunos para a preserva��o do clube se reuniram para discutir alternativas para a �rea.
+Cedras n�o tem o controle de sua pol�cia.
+Para evitar dist�rbios, vamos distribuir �s TVs uma fita para mostrar � popula��o o que viemos fazer aqui.
+Esperamos uma mudan�a de atitude da pol�cia.
+Est�o ocorrendo viola��es dos direitos humanos.
+Se o sr. Cedras n�o tomar os passos apropriados, n�s vamos dizer num clima de respeito que passos ele deve dar.
+Tamb�m vamos distribuir 1 milh�o de cestas b�sicas, para matar a fome da popula��o at� a transi��o para a democracia.
+A Roma venceu ontem a Lazio, tamb�m na capital italiana, por 3 a 0, na partida mais importante e empolgante da d�cima primeira rodada do Campeonato Italiano de 1994/95.
+Animado com um teste de vesti�rio, o tcheco Zdenek Zeman, t�cnico da Lazio, colocou em campo o atacante croata Alan Boksic, cuja volta ao time estava prevista para a pr�xima semana.
+No mercado futuro do �ndice Bovespa, a cota��o para maio ficou em 20.000 pontos, projetando rentabilidade de 66,78% ao m�s.
+No mercado futuro de d�lar, a expectativa de desvaloriza��o cambial para abril ficou em 42,40%, contra 42,53% no dia anterior.
+O comportamento das pessoas no trabalho est� passando por uma nova e importante altera��o.
+Gest�o participativa, times de qualidade e outros conceitos foram assimilados pelas empresas ao mesmo tempo em que elas passaram a exigir funcion�rios mais abertos ao di�logo e cientes da import�ncia do cumprimento das regras de boas maneiras dentro do ambiente de trabalho.
+�As empresas notaram que sua imagem � a maior prejudicada quando um funcion�rio comete uma indelicadeza em p�blico e diante de um cliente ou subordinado�, afirma o consultor de marketing pessoal Otto Reiter, 69.
+�O refinamento do executivo � mais um diferencial na hora da sua avalia��o�, completa.
+Nos Estados Unidos, universidades e algumas empresas chegam a oferecer cursos de boas maneiras nos neg�cios.
+No Brasil, essa atitude ainda depende da iniciativa de cada um.
+A quest�o que enfrentaram era retomar a arte intensamente dram�tica de sua tradi��o sem se escravizar a ela.
+A solu��o que deram, obter equil�brio entre figura��o e abstra��o em que a profundidade n�o � nem cl�ssica (pers -- pectiva) nem modernista (planificada), mas sugerida por jogos crom�ticos semilivres �s vezes livres demais, como voc� vai ver.
+Cada um o fez a seu modo.
+Dos cinco, Baselitz, Immendorff e Kirkeby fizeram melhor.
+Kirkeby, que vem ao Brasil para a abertura, tamb�m ter� sala especial na 22� Bienal, a partir do dia 12.
+Tanta coincid�ncia s� pode ser instrutiva; pode mostrar mais a Alemanha que qualquer contato pol�tico afinal, como dizem eles, o ser reside na linguagem.
+Habite-se.
+Al�m da proposta de 18 cassa��es, o relat�rio final da CPI sugere que oito deles sejam objeto de processo criminal.
+S�o os deputados que, segundo o texto, devem ter os �elementos comprobat�rios encaminhados ao Minist�rio P�blico�.
+Est�o nesse grupo Ricardo Fiuza (PFL-PE), Jo�o Alves (sem partido-BA), Cid Carvalho (PMDB-MA), Jos� Geraldo (PMDB-MG), Manoel Moreira (PMDB-SP), F�bio Raunheitti (PTB-RJ), �zio Ferreira (PFL-AM) e Paulo Portugal (PP-RJ).
+Al�m disso, avaliou como fraca a organiza��o existente at� ontem na campanha.
+A partir da pr�xima semana, v�o viajar para os Estados os membros da coordena��o que n�o pertenceram ao n�cleo que passa a deter as decis�es.
+Cerca de 300 pessoas foram convidadas para a cerim�nia.
+De esse total, 40 s�o parentes e amigos do ex-jogador.
+Entre os convidados est�o o �rei da soja�, Olacyr de Moraes, o empres�rio Alfredo Saad e o comandante Rolim Amaro, dono da empresa a�rea TAM.
+Os funcion�rios da prefeitura reclamam que L�dice est� adiando desde maio o pagamento das perdas salariais da categoria.
+Eles tentaram agendar um encontro com Lula para pedir que ele ajudasse na negocia��o.
+Uma reprodu��o gigantesca da �Maja Desnuda�, de Goya, ocupa uma das paredes da casa.
+No banheiro, um funcion�rio vende balas, cigarros, chocolates, �gua mineral, rem�dios (Engov) e, claro, camisinhas.
+4 de novembro -- morto em emboscada o l�der da ALN e ex-deputado Carlos Marighella.
+19 de junho -- M�dici anuncia a constru��o da rodovia Transamaz�nica.
+As quest�es colocadas pela �body art� nos anos 60 foram retomadas agora como reflex�o sobre uma realidade em que os corpos est�o submetidos a um trabalho de transforma��o que real�a o sentido tr�gico.
+A Aids foi apenas um catalisador para a volta do pr�prio corpo como objeto est�tico em diversos trabalhos.
+�O v�deo � uma experi�ncia f�sica, ( ...) tem um efeito direto nos corpos das pessoas.
+O v�deo pode ser um instrumento poderoso para tocar as pessoas diretamente, na percep��o, em �reas que a cultura ocidental n�o leva em conta como um caminho para o conhecimento.
+Desde a Idade M�dia, esse caminho � feito atrav�s do intelecto e n�o do corpo.
+O corpo foi negligenciado�, disse Viola � Folha em 92.
+A a��o alega que a Petroplastic n�o considera a Dow acionista da Triunfo.
+E comprova que a Petroplastic recebeu US$ 5 milh�es para concordar com a transfer�ncia, para a Dow, do controle da Atochem (fundadora da Triunfo).
+Boris Gorentzvaig, da Petroplastic, diz que os US$ 5 milh�es eram adiantamento em �acordo de gaveta� com a Dow.
+�Mas a Dow dos EUA n�o quis cumprir o contrato e transferir tecnologia a uma concorrente�, diz.
+O �rg�o respons�vel pelo mercado de telebingos � a Susep (Superintend�ncia de Seguros Privados), uma autarquia federal subordinanda ao Minist�rio da Fazenda.
+�Compramos o t�tulo para verificar, levei para casa e n�o enxerguei nada�, disse ontem Vera Melo Ara�jo, chefe do Departamento de Fiscaliza��o da Susep.
+F�bio Igel, Jo�o Herreras e Marcelo Loureiro pilotam festa no Cabral.
+� em prol do Fundo Social de Solidariedade a apresenta��o de hoje de �Mulher, a Melhor Op��o de Investimento� no Teatro It�lia.
+Divulgado na semana passada, o fraco resultado em novembro da produ��o industrial e das novas encomendas �s ind�strias acabou com as esperan�as de que a Alemanha tivesse sa�do da pior recess�o do p�s-guerra no �ltimo trimestre do ano.
+Os dados detalhados sobre o desempenho do PIB entre outubro e dezembro n�o ser�o revelados at� mar�o.
+Os economistas afirmam que a recupera��o puxada pelas exporta��es que o governo previa para o segundo semestre de 1993 n�o aconteceu.
+A federa��o das pequenas e m�dias empresas revelou numa pesquisa que seus filiados est�o pr�ximos do p�nico em rela��o � situa��o econ�mica.
+Suas expectativas para o futuro pr�ximo da economia s�o as piores em dez anos.
+N�o preciso, governador, porque, sabendo da hospitalidade paranaense, eu nem trouxe carteira.
+Os anos 80 foram um divisor de �guas na industrializa��o brasileira.
+Vera L�cia Barbosa, 40, morreu na �ltima quarta presa pelo cinto de seguran�a ao carro em que estava, que caiu no rio Tiet�.
+O acidente foi �s 21h50, perto da Ponte Cruzeiro do Sul (zona norte).
+As outras tr�s pessoas que estavam no carro conseguiram se salvar.
+O tombamento de um caminh�o no km 226 da pista Rio-S�o Paulo da via Dutra �s 5h de ontem provocou engarrafamento na rodovia.
+O acidente causou a morte do menino Ren� Scorsa, que estava no Logus dirigido por S�rgio Duran, 33, que bateu no caminh�o.
+�Sobral, embora seja uma das principais cidades do Cear�, ainda n�o disp�e de um sistema adequado de esgotos.
+O projeto de canaliza��o de um c�rrego no munic�pio de Contagem ( ...) � indispens�vel para os moradores porque, durante o per�odo de chuvas, o rio transborda e alaga alguns bairros de Contagem�.
+Nenhum munic�pio brasileiro, nem mesmo S�o Paulo, �disp�e de um sistema adequado de esgotos�.
+Enchentes d�o-se pelo pa�s inteiro.
+A escolha t�o particular de Sobral, que por ser �uma das principais cidades do Cear� � tamb�m uma grande concentra��o eleitoral do Estado, coincide com sua escolha para a a��o de campanha que l� fez o candidato Fernando Henrique no fim-de-semana.
+A mineira Contagem � outra grande concentra��o eleitoral, com prefeito do PSDB e onde o partido, j� como propaganda, fez a conven��o de escolha de Fernando Henrique.
+Covas -- Quando se falando em emprego, n�o se tratando s� de uma quest�o administrativa.
+Quando se fala do Real, fala-se de posi��es que traduzem aquilo que deve ser o papel de S�o Paulo.
+Folha -- Pa�ses como a Argentina, que passaram por ajustes parecidos com o do Brasil, t�m hoje um desemprego recorde ...
+Qual a mat�ria odiada na escola?
+Lugar de que mais gosta?
+Os dados s�o do SCI (Servi�o de Seguran�a ao Cr�dito e Informa��es) que armazena informa��es comerciais sobre 1,6 milh�o de empresas no Brasil.
+Do volume de protestos, a regi�o Sudeste respondeu por 39,1% do total e a Sul, 23,5%.
+O Estado de S�o Paulo representou 22,8% dos protestos.
+O conselho tamb�m manteve multas aplicadas � Ind�strias J.B. Duarte e a seus dirigentes pela pr�tica de atividades restritas a institui��es financeiras.
+A corretora Antonio Delapieve e seus dirigentes foram multados por negociar t�tulos de renda fixa a pre�os superavaliados.
+Hoje, os mediadores internacionais lorde Carrington e Henry Kissinger (ex-ministros do exterior do Reino Unido e EUA, respectivamente) se encontram com os l�deres rivais negros Nelson Mandela (CNA) e Mangosuthu Buthelezi (zulu).
+Pelo menos dez pessoas morreram e 80 se feriram no terceiro dia consecutivo de luta entre fac��es rivais em Kabul, Afeganist�o.
+Tropas do presidente Burhanuddin Rabbani enfrentam soldados do primeiro-ministro Gulbuddin Hekmatyar.
+Biotecnologia -- busca o aprimoramento e cria��o de novos produtos a partir da manipula��o de animais e plantas destinados � sa�de e est�tica humana.
+Apesar de ter conseguido a guarda de Natasha, Concei��o est� aflita.
+�Tenho os pap�is, mas n�o tenho a minha filha.
+Agora s� deixo a Fran�a com ela.
+Esse � o meu grito�, se inflama.
+Segundo Concei��o, tem existido certa indiferen�a da pol�cia local em rela��o ao caso.
+A estrat�gia montada por Qu�rcia tem o objetivo de ganhar a pr�via do PMDB que vai definir o candidato do partido � Presid�ncia no pr�ximo dia 15, bem como manter o PMDB unido durante a campanha.
+Os ataques a Sarney s� ser�o feitos se a contabilidade quercista constatar que a vit�ria na pr�via est� amea�ada.
+Esse � o tema mais discutido na imprensa do pa�s; anteontem, ele foi objeto de debate especial do gabinete de governo japon�s.
+No caso de ontem, um garoto de 14 anos foi encontrado enforcado numa viga em sua escola em Yorii, 50 km a sudoeste de T�quio.
+A RBS tem ainda tr�s concess�es de TV em Santa Catarina e 30% da TV Cachoeiro, de Cachoeiro do Itapemirim, no Esp�rito Santo.
+O �nico caso que se tem registro de aplica��o rigorosa da lei ocorreu no final da d�cada de 60.
+Destaca que o Partido Liberal Democr�tico (PLD), derrubado do poder no ano passado, dever� apresentar seu presidente, Yohei Kono, como candidato a primeiro-ministro do Jap�o.
+Publica documentos comprados de James McDougal, ex-s�cio de Bill e Hillary Clinton na empresa Whitewater.
+Os documentos indicam que a primeira-dama dos EUA estava mais envolvida no dia-a-dia da administra��o da firma do que fora admitido at� agora.
+Depois de uma disputa acirrada entre as ind�strias para conquistar os melhores mercados, o ranking das gigantes multinacionais que dividem o bolo mudou.
+A japonesa NEC continua na lideran�a.
+O segundo lugar foi conquistado pela Ericsson, que em agosto de 93 ocupava a quarta posi��o em contratos assinados para fornecimento de equipamentos.
+A NEC tem mais de 55% (US$ 455 milh�es) dos contratos assinados pelas concession�rias da Telebr�s, incluindo a� a segunda fase da implanta��o do sistema m�vel em S�o Paulo e a terceira no Rio, previstas para 94.
+O engenheiro Bruno Maranh�o, l�der do Movimento por Terra, Trabalho e Liberdade, pertence a uma das fam�lias que dominam os latif�ndios no Nordeste.
+Uma invas�o do grupo, feita h� dois anos, em Pernambuco, na usina Massauassu, prejudicou justamente a fam�lia Maranh�o.
+Depois de conquistar seu pr�mio de estr�ia no concurso Smirnoff em S�o Paulo, o jovem estilista S�rgio Machado emplacou mais uma.
+� dele o segundo lugar do Smirnoff International Fashion Awards, disputado anteontem em Dublin.
+�Alphaville� � uma mistura de fic��o cient�fica, filme noir, e a irrever�ncia das hist�rias em quadrinhos.
+A hist�ria narra a ida de um agente secreto � desumana cidade futurista de Alphaville, controlada por um c�rebro eletr�nico que baniu totalmente os conceitos de amor e solidariedade.
+Para construir o ambiente futurista do filme, Godard recorreu quase que totalmente � fotografia.
+A c�mera de Raoul Coutard conseguiu transformar Paris numa cidade g�lida, sem contar a ousadia de montar sequ�ncias inteiras em negativo.
+O experimentalismo do filme n�o esconde sua proposi��o simb�lica, a aliena��o na sociedade tecnol�gica.
+Uma audi�ncia hoje na 7� Junta da Justi�a do Trabalho em Bel�m reabre a discuss�o sobre a obrigatoriedade do diploma espec�fico para o exerc�cio do jornalismo.
+O Sindicato dos Jornalistas do Par� exige a demiss�o dos cerca de 150 jornalistas irregulares (que n�o t�m curso universit�rio de Jornalismo) das empresas de Bel�m.
+Como o Estado n�o tem recursos, nem vai ter para todos, o atendimento acaba deixando de fora justamente os mais pobres, que n�o conseguem acesso aos servi�os p�blicos.
+O problema � pol�tico porque envolve, por exemplo, a gratuidade da educa��o, da sa�de, da previd�ncia m�nima.
+Excetuada a educa��o b�sica de primeiro grau, que deve ser gratuita e obrigat�ria, para todo o resto � preciso discriminar quem pode pagar.
+Dados do IBGE (Instituto Brasileiro de Geografia e Estat�stica), de 1989, mostram que 30,6 milh�es de brasileiros.
+Dos fumantes, 18,1 milh�es s�o homens e 12,5 milh�es, mulheres.
+O homem fuma entre 11 e 20 cigarros por dia e a mulher, entre 5 e 10.
+A PF (Pol�cia Federal) indiciou ontem duas pessoas por vender carro com �gio.
+Foram indiciados o dono da revendedora 3.000 Autom�veis, Leonardo Romanioli Filho, e o vendedor Lu�s Silveira, da concession�ria Frame.
+Os dois foram presos quando o taxista Jos� Fioravanti, 61, foi entregar US$ 1.000 a Romanioli como �gio da compra de um Santana Quantum.
+Ele estava acompanhados de tr�s agentes da PF.
+Um deles era seu filho, S�rgio Fioravanti.
+A partir desta edi��o, a tabela com o levantamento dos indicadores Folha-Sebrae passa a incluir a compara��o da capacidade ocupada no m�s pesquisado com o mesmo m�s no ano anterior.
+O uso da compara��o anualizada permite verificar a varia��o da atividade industrial sem a influ�ncia de fatores sazonais.
+Com a inclus�o de mais esse dado comparativo, a tabela fica mais completa.
+Ela tem tamb�m textos com explica��es sobre cada item pesquisado.
+A vers�o mais corrente sobre o in�cio do enriquecimento de Qu�rcia � de que ele comprou terrenos em �reas de periferia que seriam beneficiadas pelos projetos de melhoria urbana da Prefeitura.
+Na sua administra��o, executou um plano vi�rio com largas avenidas interligando os bairros, que provocaram um boom imobili�rio.
+Nicolau diz que hoje o produtor de leite C recebe US$ 0,22, enquanto o custo est� em US$ 0,24.
+Dallari orientou os produtores a negociarem com a ind�stria.
+Esses aspectos podem ser observados nos relatos das pacientes: �quando tenho que arrumar a casa e n�o estou com vontade, escuto vozes que me incentivam a come�ar a tarefa.
+Depois de um tempo, as vozes aumentam e acabo ficando atordoada e n�o consigo terminar o que comecei�.
+Estes problemas n�o s�o frequentes nos homens, pois al�m de terem uma possibilidade menor de casarem e terem filhos, esses n�o s�o os tipos de pap�is socialmente esperados para o sexo masculino.
+O restaurante, especializado em carnes e saladas, oferecer� uma garrafa de champanhe espanhol para os casais na noite do dia 12.
+Quem n�o quiser beber durante o jantar, poder� levar o champanhe para casa.
+�As pessoas est�o cansadas do hype que se cria em torno das roupas, dos estilistas e do luxo abusivo.
+Conforto � fundamental.
+Eu n�o me associo com moda.
+Para mim roupa e moda s�o coisas diferentes�.
+Folha -- o que a mulher precisa hoje, em termos de roupas?
+Desorganiza��o e viol�ncia marcam torneio que consolida o 'efeito Copa'.
+Os clubes de futebol do Brasil est�o valorizando cada vez mais a posse de bola e a utiliza��o de lan�amentos como armas para se chegar ao ataque.
+� o que confirmam as estat�sticas do Campeonato Brasileiro, encerrado ontem com o novo t�tulo palmeirense.
+Emily Lloyd vai ser a desbocada punk �Tank Girl�, dos gibis ingleses.
+Ainda na Inglaterra, Spielberg negocia os direitos da s�rie de �TV Doctor Who�.
+Chris Columbus est� ligado � adapta��o dos super-her�is �Quarteto Fant�stico�, que inauguraram a era Marvel nos gibis h� 30 anos.
+Mas o or�amento de US$ 5 milh�es insinua que a informa��o n�o � s�ria.
+S�o comuns casos como a filmagem do gibi �Sargento Rock�, que teve os direitos comprados por Joel Silver h� cinco anos.
+Arnold Schwarzenegger deu entrevistas como ator principal, foi substitu�do por Bruce Willis h� dois anos, mas o projeto nunca saiu do papel.
+Na campanha eleitoral, Alencar e Nilo estiveram em lados opostos quanto � a��o das For�as Armadas no combate ao crime.
+Alencar defendeu a ado��o do estado de defesa, que prev� a suspens�o de garantias individuais.
+Trata-se, segundo ele, de um �instrumento de defesa da democracia�.
+Ao contr�rio, nesta situa��o econ�mica de extrema gravidade, todos gostariam de ajudar o pa�s.
+Para isso, por�m, � necess�rio que o nosso Banco Central busque n�o a �independ�ncia� do Tesouro, que � imposs�vel, mas que deixe de �quebrar-lo� com a sua pol�tica agressiva de juros.
+O shopping West Plaza j� come�ou sua promo��o de Dia das M�es.
+Diariamente, est� promovendo desfiles de moda para seus consumidores.
+Os desfiles, com roupas a venda no pr�prio shopping, visam orientar o consumidor a respeito da moda.
+Estilistas estar�o � disposi��o dos clientes.
+Segundo ele, isso aconteceu ap�s a pris�o de soldados venezuelanos em territ�rio brasileiro, em fevereiro de 1993.
+�Depois desse incidente, as For�as Armadas venezuelanas tiveram de esfriar suas a��es na fronteira com o Brasil.
+A Guarda Nacional da Venezuela passou a conversar com os �ndios e a convenc�-los de fazer o papel dela, ou seja, reprimir os garimpeiros�, diz Altino.
+A maioria dos bons jogadores brasileiros atuava em clubes profissionais, filiados � FBF.
+N�o houve acordo para uma tr�gua durante a Copa.
+A solu��o foi negociar diretamente com os jogadores.
+A CBD aliciou Le�nidas (Vasco), Luizinho e Waldemar de Brito (S�o Paulo), entre outros.
+Era uma �poca em que os cariocas imperavam em campo e fora dele.
+Por raz�es que a pr�pria raz�o desconhece, no sorteio do local do jogo decisivo, a bolinha quase sempre apontava o assustador est�dio de S�o Janu�rio.
+O torcedor paulista sonhava com uma era futura, em que S�o Paulo chegaria � supremacia, se poss�vel de forma arrasadora.
+Passaram-se uns 50 anos e o sonho se realizou, trazendo alegria e tamb�m, inesperadamente, frustra��o.
+Ficamos todos , isto �, os torcedores paulistas que vem dos idos de 1940, como o americano m�dio, no fim da Guerra Fria.
+J� n�o h� o imp�rio do mal para combater.
+Vasco, Botafogo, Fluminense e at� o Flamengo s�o fantasmas do passado.
+O bombeiro suspeita que o golfinho tenha morrido afogado.
+Ele teria ficado preso em uma rede de pesca sem condi��es de subir � tona para respirar.
+Os bombeiros pediram ajuda ao Cebimar (Centro de Biologia Marinha) da USP para apurar as causas da morte do golfinho.
+At� as 17h, a Prefeitura de Caraguatatuba n�o havia retirado o golfinho da praia.
+O governo vai usar a URV para corrigir impostos, garantiu FHC.
+�A URV � produzida por tr�s �ndices, e um dos �ndices � o mesmo da Ufir.
+Vai ter paridade, j� desde o come�o�, explicou.
+Quanto �s tarifas p�blicas, n�o ser�o fixadas em URV para evitar reajustes di�rios, mas ter�o corre��es peri�dicas pelo indexador.
+�Do ponto de vista de prote��o ao consumidor, vou reajustar o pre�o da tarifa p�blica no dia normal como se ela fosse URV.
+Ou seja, pela m�dia real dos �ltimos quatro meses�, disse.
+�Onze milh�es de aposentados ganham m�nimo.
+Esses v�o ter um aumento real, pelo que disse o ministro Cutolo (S�rgio, da Previd�ncia), na passagem para a URV, de 17%.
+E alguns v�o ter de 30%�.
+Ele atribuiu a discuss�o sobre o valor dos benef�cios a uma �ilus�o monet�ria�.
+�As pessoas pensam que recebem US$ 80, mas 30 dias depois esse valor � muito menor�.
+Encerrada a festa, Maluly viu o rapaz esperando.
+O deputado se aproximou.
+Eu precisava de uma ajuda do senhor, deputado ...
+O Pr�ncipe Charles, do Reino Unido, fez respira��o boca-a-boca num boneco (foto) na inaugura��o do novo pr�dio do Centro de Ambul�ncias de Wellington, capital da Nova Zel�ndia.
+O homem que atacou Charles anteontem n�o ter� direito a fian�a e ficar� preso pelo menos at� o fim da visita de cinco dias do pr�ncipe ao pa�s.
+Litu�nia e Ucr�nia elogiam a Otan Os presidentes da Litu�nia, Algirdas Zebrauskas, e da Ucr�nia, Leonid Kravtchuk, elogiaram ontem o programa de �Parceria pela Paz� da Otan (alian�a militar ocidental, liderada pelos EUA), que prev� opera��es conjuntas com pa�ses do Leste Europeu.
+Segundo eles, os la�os com a alian�a n�o v�o afetar as rela��es com a R�ssia.
+N�o � verdade, como diz Gusm�o, que a miss�o enviada a Israel tenha feito �confronto de pre�os�.
+O relat�rio sigiloso n�o traz qualquer lista de pre�os.
+As evid�ncias de superfaturamento foram levantadas pelos professores Armando Lagan� e M�rcio Rillo, da USP, a pedido da Folha.
+Sustentadas em depoimentos � PF, foram confirmadas em tr�s per�cias.
+A �ltima delas, com a participa��o de tr�s cientistas, comprovou superfaturamento de 343%.
+Stephen Freehill, 16 (foto), compareceu ontem a um tribunal de Cingapura para uma audi�ncia preliminar.
+Ele � acusado de vandalismo.
+Pode ser condenado ao a�oite, como seu compatriota Michael Fay, 18, preso na mesma ocasi�o.
+Fay foi punido com quatro vergastadas e agora cumpre pena de pris�o.
+For�as do norte do I�men anunciaram a tomada da base a�rea de Al Anad, a principal dos sul-iemenitas.
+A base fica a apenas 50 km de �den, a capital do sul.
+A guerra come�ou h� 13 dias por causa de um confronto entre o presidente Ali Abdulllah Saleh e seu vice.
+�N�o damos conta de atend�-los.
+A resposta ao pedido de ajuda foi gigantesca�, diz Virginia de la Guardia, porta-voz da M�dicos Sem Fronteiras.
+A maioria dos volunt�rios s�o jovens.
+Mas, comenta Enrique Albizu, presidente da Medicus Mundi, �� precisamente nas situa��es de emerg�ncia que necessitamos de gente mais especializada�.
+O Santos volta a viver um clima tenso para o pr�ximo jogo, domingo contra o Santo Andr�, em busca da primeira vit�ria no Campeonato Paulista.
+O t�cnico Pepe, irritado com a atua��o do time no empate em 1 a 1 com Ituano anteontem, disse �que est� faltando tranquilidade e talento aos jogadores�.
+Pepe ainda tem esperan�a que o centroavante Guga renove o seu contrato, vencido em 31 de dezembro.
+�Se isso acontecer, tem que ser r�pido, do contr�rio n�o adianta mais�, afirmou.
+Nascido no dia 23 de maio de 1972, em S�o Paulo, Rubens Gon�alves Barrichello come�ou sua carreira automobil�stica no kart, em 81, quando foi vice-campe�o j�nior da capital paulista.
+Em 1989, ap�s ganhar diversos t�tulos paulistas e brasileiros no kart, Barrichello disputou o Campeonato Brasileiro de F�rmula Ford, terminando em quarto lugar.
+A Sunab (Superintend�ncia Nacional de Abastecimento) autuou s�bado duas lojas do ParK Shopping, em Bras�lia.
+Ambas n�o apresentavam pre�os de produtos expostos nas vitrines.
+A multa para lojistas que descumprirem a determina��o pode chegar a R$ 128 mil.
+Segundo Eduardo Lago, superintendente do org�o, j� foram aplicadas 400 multas em todo o pa�s desde o in�cio do m�s.
+A Mercedes-Benz deve instalar a linha de montagem do Swatch, carro compacto com design do fabricante dos rel�gios da marca, em Sarreguemines, cidade do norte da Fran�a.
+A decis�o foi tomada pelo conselho da montadora alem�, na �ltima sexta-feira.
+O an�ncio oficial ser� feito ap�s o dia 20.
+Beckenbauer aconselha o lateral a conversar com o treinador, procurando reverter a situa��o.
+O caso Effenberg, jogador cortado por ter feito gesto obsceno � torcida, tamb�m foi criticado.
+�Foi um ato muito rigoroso.
+�Beckenbauer tem o direito de dizer o que quiser�.
+Rodado no in�cio da d�cada de 50, �poca �urea do macarthismo persegui��o a artistas e intelectuais comunistas ou acusados de simpatizar com essa ideologia liderada pelo senador Joseph McCarthy nos EUA, o filme ganha vers�o para CD-ROM, com direito a um extenso material de refer�ncia.
+Pre�o: US$ 74,71.
+O filme narra uma greve de mineiros em uma cidade do Novo M�xico, nos EUA.
+H� uma trama paralela, com Ramon e Esperanza, casal que ap�s a greve se separa.
+Caso isso ocorresse, eles seriam colocados em liberdade t�o logo chegassem no Canad�.
+Documento do Minist�rio das Rela��es Exteriores do Canad�, obtido pela Folha, d� apoio ao lobby (grupo de press�o) feito pela Embaixada do Canad� junto a senadores e deputados brasileiros para que os canadenses sejam expulsos.
+Segundo o c�nsul, era crescente a press�o do Congresso norte-americano sobre o Departamento de Estado para que se adotasse uma pol�tica id�ntica � do Brasil.
+�Queremos dar um visto por dez anos.
+� mais eficiente do que por apenas quatro anos.
+Estamos sendo for�ados, ao inv�s, a reduzir o prazo para tr�s meses.
+� uma situa��o absurda�, disse Taylor, 49.
+Um jogador que estava nos planos de Pepe era o meia Carlos Alberto Dias, que acabou indo para o Flamengo, j� que a diretoria do Santos vetou o seu nome.
+No momento, Pel�, o maior jogador da hist�ria da equipe, � tamb�m a maior esperan�a de um time forte.
+Durante esta semana, o agora dirigente esteve no Jap�o, em busca de uma empresa que, no futuro, invista no Santos.
+Recentemente, o time firmou um contrato de US$ 50 mil mensais com a Lousano, que vale at� o fim do Campeonato Paulista.
+O aumento das despesas acontece at� mesmo nos minist�rios da Fazenda e do Planejamento, respons�veis pela montagem do Or�amento federal.
+No Minist�rio da Fazenda, as despesas crescer�o de US$ 1,13 bilh�es em 93 para US$ 1,36 bilh�es em 94 (ou mais 20,07%).
+No Planejamento, o crescimento � de 33,19%, elevando as despesas de US$ 87,6 milh�es em 93 para US$ 116,69 milh�es em 94.
+O deputado Aloizio Mercadante (PT/SP) divulgou ontem n�meros do Tesouro sobre a execu��o or�ament�ria de janeiro a novembro de 1993, demonstrando que o governo conseguiu um super�vit operacional de US$ 9,2 bilh�es neste per�odo.
+Embora esteja a quase 40 km do est�dio, o hotel foi escolhido porque est� pr�ximo da California State University, onde o Brasil treinar� durante esta semana.
+A chegada ao hotel foi �s 12h10 (16h10 em Bras�lia).
+Toda a delega��o subiu diretamente para os quartos.
+Os jogadores e Parreira n�o falaram com a imprensa.
+Para estimular a participa��o, a cada dois dias de trabalho as meninas ganham uma boneca.
+A Rep�blica Movimento de Ema�s trabalha h� 20 anos com adolescentes carentes.
+Cheia de energia, a interpreta��o da Orquestra Estadual da Hungria vem amortecida um pouco pela artificialidade da grava��o, de quase 15 anos atr�s, mas compensa a sonoridade de est�dio com o vigor.
+A Sinfonia toda � uma esp�cie de demoniza��o da Nona de Beethoven, que � muito mais seu assunto do que o Fausto de Goethe.
+Nisto tamb�m na resist�ncia e repeti��o de Beethoven Liszt � um foco importante, com reflexos na obra de Tchaikovsky, Scriabin e Strauss.
+N�o poderia haver realiza��o maior para um compositor: substituir Beethoven, ou ter pelo menos a ilus�o de ser um novo ponto de partida.
+Para Simon Franco, 55, presidente da Simon Franco Recursos Humanos, �n�o se pode jogar fora 15 anos de experi�ncia�.
+Segundo ele, Carvalho n�o est� avaliando que pode ampliar seu espectro de op��es sem a necessidade de uma mudan�a radical.
+A reprodu��o assistida est� regulamentada no Brasil desde novembro de 1992, embora sua pr�tica tenha sido iniciada oito anos antes.
+Determinadas pelo Conselho Federal de Medicina (CFM) as normas �ticas foram inspiradas nas de outros pa�ses Estados Unidos, Fran�a e It�lia, por exemplo.
+Mas, como nesses pa�ses, os avan�os nas pesquisas m�dicas tornaram os c�digos de �tica ultrapassados.
+A possibilidade da gravidez p�s-menopausa e o uso de �vulos de fetos abortados (veja texto � p�g. 6) s�o exemplos mais gritantes.
+�Ficaram de fora de nossa regulamenta��o�, disse � Folha Antonio Henrique Pedrosa Neto, diretor do CFM.
+Ele afirmou, no entanto, que pelo menos a �gravidez p�s-menopausa ser� objeto de nova resolu��o�.
+Pol�cia Civil encontrou dois homens amarrados, amorda�ados e de olhos vendados, em um barraco da favela Nova Bras�lia (zona norte).
+Mauro Pereira e Gilson Paulino dos Santos disseram ter sido sequestrados por traficantes, que os teriam confundido com policiais.
+Fiscais do Tribunal Regional Eleitoral e PMs do Batalh�o de Choque foram recebidos a tiros ontem � tarde na Vila do Jo�o, Manguinhos (zona norte do Rio).
+Os fiscais retirar propaganda irregular.
+No tiroteio, morreu Adriano Herculano da Silva, 18.
+O casamento ser� realizado pelo reverendo Onaldo Pereira, 38, ordenado h� oito anos nos Estados Unidos.
+O pastor � respons�vel no Brasil pela Comunidade Pacifista Crist�, fundada na Alemanha em 1708.
+A sele��o brasileira feminina de basquete � a campe� do Torneio Internacional de Basquete, um quadrangular amistoso disputado em Recife (PE) e que contou com a participa��o das sele��es de Cuba, Argentina e Eslov�nia.
+O time comandado pelo t�cnico Miguel �ngelo da Luz ficou com o t�tulo ao derrotar anteontem � noite na final a forte sele��o de Cuba por 115 a 93 (54 a 44 no primeiro tempo).
+A cestinha do jogo foi Hort�ncia, que marcou 38 pontos.
+Al�m da fazenda do Sabi�, participaram do leil�o a Terra Boa e a Mata Velha, tradicionais criadoras de nelore.
+O principal destaque na pista foi a vaca Santya, selecionada para ser doadora em transfer�ncia de embri�es, que foi vendida por 40,2 mil URVs.
+Filha do touro Chummak, ela est� prenha e foi comercializada com uma bezerra ao p�.
+�A Times Square � tamb�m um ponto de encontro e uma alternativa teen�, afirma o diretor da Multiplan.
+Para Spinelli, o espa�o representa uma nova tend�ncia para as �reas de conveni�ncia dos shoppings.
+Paula Toller Isso � uma coisa que a gravadora vinha sugerindo para a gente h� muito tempo.
+Gostamos da id�ia, resolvemos pegar velhos sucessos e fazer neste formato.
+Grava��es ac�sticas se encaixam com o nosso tipo de som.
+Sempre tivemos can��es mais lentas.
+Bruno Fortunato A gente sempre teve m�sicas que d�o para tocar no viol�o.
+A inten��o � louv�vel e esta Folha tem denunciado sempre a ciranda financeira como obst�culo a uma aut�ntica estabiliza��o.
+O momento e a forma escolhidos para mudar o sistema financeiro, entretanto, dependem fortemente de vari�veis pol�ticas.
+Est� em estudo no Minist�rio da Previd�ncia um conjunto de medidas para desestimular o aumento de aposentadorias no pr�ximo ano.
+Elas ser�o oferecidas a Itamar, como alternativa � propalada quebra do caixa previdenci�rio.
+Causou m� impress�o, na quarta, a participa��o do ministro da Justi�a, Alexandre Dupeyrat, na reuni�o que tratou do projeto sobre abuso do poder econ�mico.
+Chegou atrasado, foi duro com seus cr�ticos e saiu antes do fim.
+Em �Olhos de Serpente� (Snake Eyes) de Abel Ferrara, Madonna faz o papel dela mesma: uma falsa atriz, falsa loira, que paga para estar num filme.
+� de longe sua melhor atua��o no cinema.
+Ferrara p�e o excelente Harvey Keitel para interpretar o seu papel: o do diretor que faz da falsa loira uma verdadeira atriz.
+�Olhos de Serpente� � o filme dentro do filme dentro do filme.
+Eddie (Harvey Keitel) dirige a hist�ria de um casal de ricos drogados em crise (Madonna e James Russo).
+Eddie tamb�m est� em crise com sua mulher (interpretada pela mulher de Abel Ferrara) e n�o resiste � tenta��o da atriz, que paga para ele filmar (Madonna � produtora de �Olhos de Serpente�).
+Caso o tetracampe�o seja o Brasil (seja feita a vossa vontade assim na Terra como nos gramados), confirma-se que os �ltimos ser�o os primeiros.
+Vi, num jogo quase inacredit�vel da Copa, o Eire jogar futebol muito melhor do que a It�lia.
+Venceu por 1 a 0.
+J� bastava, para orgulho da Irlanda, que a Inglaterra sequer tivesse chegado � Copa.
+Mas � na Republica Dominicana que o cruzeiro atinge seu ponto alto.
+O navio ancora numa praia, Serena Cay, que se torna exclusiva dos 1.600 passageiros.
+Na areia h� apenas espregui�adeiras, coqueiros e barraquinhas de artesanato.
+O mar � cristalino.
+Quem quiser, vai em excurs�o at� o resort local, Casa de Campo.
+Sem dizer que o seu clube dispensou, humilhantemente, o seu arqueiro Zubizarreta -- depois da derrota para o Milan, 0 a 4, na decis�o da Copa da Europa.
+�Chegamos at� aqui, claro, tamb�m gra�as a Cruyff�.
+S� Salinas discordou dessa posi��o.
+Dos dez cestinhas da atual temporada, apenas um joga fora do garraf�o.
+Dos cinco primeiros artilheiros, quatro s�o superpiv�s.
+Todos eles estar�o esta noite em Minneapolis.
+Chance imperd�vel para aqueles que apreciam enterradas, tocos e a mais primitiva troca de porradas.
+A NBA nem existia e o profissionalismo no esporte apenas engatinhava quando apareceu o primeiro gigante do basquete norte-americano.
+Seu nome: George Mikan.
+Sua virtude: 2,08 m.
+Em torno do gabinete presidencial no Planalto ele quer os dois maiores colaboradores na campanha eleitoral: o amigo, empres�rio e secret�rio-geral do PSDB S�rgio Motta e o ex-reitor da Unicamp e coordenador do programa de governo Paulo Renato de Souza.
+Cardoso diz que a frente para sua elei��o foi formada exclusivamente em torno de um programa.
+�O n�mero � mais alarmante se pensarmos que daria para abastecer toda a popula��o brasileira por oito meses�, diz Dalmo Rosalen, da Secretaria do Meio Ambiente.
+H� cidades que enfrentam programas de racionamento de �gua.
+Um exemplo � a bacia do Vale do Piracicaba (a 170 quil�metros a noroeste de S�o Paulo).
+Flach Hoje est�o estabelecidas penas de pris�o, mesmo que eventualmente n�o sejam cumpridas ou sejam convertidas em presta��o de servi�os � comunidade.
+Mas eu imagino que deveria se dotar, pelo menos o Judici�rio, de medidas que possibilitassem a concess�o do perd�o judicial.
+Folha -- O senhor acha que o usu�rio � quest�o de sa�de, de educa��o ou de pol�cia?
+Flach No terreno das id�ias, eu priorizo como uma quest�o de educa��o e de sa�de.
+Eu estou muito preocupado em estimular as atividades preventivas.
+Nossa preocupa��o � um projeto que estimule no jovem uma vida sem drogas, promovendo-se um comportamento construtivo.
+A espanhola Arantxa Sanchez, tenista com mlehor desempenho nesta temporada, foi eliminada na primeira rodad do Virginia Slims Masters em Nova York, que re�ne as 16 melhores jogadoras do ano.
+Sanchez, n� 2 do ranking mundial, perdeu num jogo dram�tico para a francesa Julie Halard por 6/2, 1/6 e 7/6, com 7/2 no tie-break decisivo.
+Na 22� posi��o do ranking, Halard foi empurrada pelo p�blico no Madison Square Garden.
+�Se for uma greve s� por grevismo, da� prejudica a quem � favor�vel a isso.
+A�, poderia afetar o Lula, n�o a mim�, disse FHC.
+O candidato tucano a presidente deu essa declara��o ontem � tarde, depois de discursar para cerca de 550 empres�rios na hora do almo�o (leia texto nesta p�gina).
+A PMD (psicose man�aco-depressiva) � uma doen�a psiqui�trica que leva a uma altera��o abrupta do comportamento.
+As pessoas com PMD oscilam entre dois p�los.
+T�m fases de depress�o e de euforia (mania).
+Nem todos passam pelas duas fases.
+Os irm�os Osny Silveira Neto e Guilherme Silveira abrem hoje o bar Hor�cio no Itaim.
+Jos� Arthur Giannotti, Marilena Chaui, Gloria Kalil e Jorge da Cunha Lima faziam parte de plat�ia-cabe�a que acompanhou anteontem no Masp a palestra de Claude Lefort apresentado por S�rgio Cardoso.
+Essa �poca marca a forma��o da base da moral samurai, resultado da disciplina f�sica e mental do zen-budismo, dos ditames do confuncionismo e do esp�rito militarista reinante.
+Os EUA entram na vida japonesa em meados do s�culo 19, quando obrigam o �shogun� a assinar um tratado de com�rcio.
+Esse sinal de fraqueza associado a crises internas levam � queda do sistema em 1867.
+Um ano depois, a capital passa a ser T�quio.
+A hidrogin�stica � uma alternativa para quem n�o tem acompanhante.
+A professora Cl�udia Morgado, 25, da academia Competition, recomenda exerc�cios leves, como a corrida.
+�O importante � levantar bem o joelho, manter o ritmo e encostar o calcanhar no ch�o a cada movimento�, diz.
+�Primeiro foi uma oportunidade de esclarecer muitos pontos que eu n�o conhecia antes.
+Mesmo entendendo suas explica��es sobre o programa de governo, entendo que ele tinha que ter um programa de governo, que vai dar um meio de cobrar depois.
+Um programa � um planejamento para se atingir certas metas.
+A falta de um programa poderia dar a impress�o de n�o ter metas.
+�Gostei da sua posi��o sobre a religi�o.
+Mesmo criticado, ele defende sua posi��o claramente em favor dos evang�licos e de seu ponto de vista religioso.
+Em muitos momentos ele simplesmente deixou de se posicionar.
+Talvez ele n�o tenha id�ias elaboradas sobre os problemas do Estado.
+�Ele foi mais direto, mais atacante, deu respostas mais profundas e melhores.
+Minha expectativa n�o foi atendida em v�rios pontos.
+Redes, multim�dia, sistemas operacionais ou inform�tica na educa��o.
+Esses s�o alguns dos temas que estar�o sendo discutidos no Congresso da Fenasoft, promovido paralelamente � feira.
+O congresso � dividido em tr�s temas, com o objetivo de atender de profissionais de inform�tica a executivos e empres�rios interessados em novas tecnologias, pasando pelo usu�rio final.
+O t�cnico da Holanda, Dick Advocaat, disse que sua preocupa��o � anular o esquema defensivo da Irlanda.
+Ele afirmou que pretende refor�ar seu meio-campo, mas n�o adiantou a escala��o da equipe.
+Folha -- O senhor foi indiciado em inqu�rito da Pol�cia Federal por manipula��o de arbitragem, estelionato e forma��o de quadrilha.
+O senhor teme ser condenado?
+Miranda -- Em rela��o a mim isso n�o vai dar absolutamente em nada.
+N�o sei em rela��o a os outros indiciados.
+Me indiciaram precipitadamente, a�odadamente.
+� coisa orquestrada, dirigida.
+Com prega��es que duram at� dois minutos.
+Os candidatos da Bahia est�o encontrando certa dificuldade em contratar cantores para animar suas campanhas.
+O maior aumento aconteceu em direito (mais 366 candidatos) e a maior queda, em engenharia (menos 980).
+O curso que mais cresceu foi odontologia na USP em Bauru 83,5%.
+O n�mero total de candidatos no vestibular caiu em rela��o ao ano passado.
+De 140.518 inscritos, a Fuvest registrou 139.369.
+A inten��o de Fleury � vender as a��es para equilibrar as finan�as e come�ar a pagar a d�vida do Estado, estimada em R$ 31 bilh�es.
+Segundo a assessoria de imprensa do governador, Fleury j� pediu um levantamento de pre�o das a��es, junto a consultores especializados, para estabelecer o valor m�nimo das a��es, que v�o ser levadas a leil�o.
+O ministro da Fazenda, Rubens Ricupero, disse que o resultado do IPC-r �n�o � uma surpresa�.
+Ele explicou que o �ndice foi calculado do dia 16 de junho (ainda sobre cruzeiros reais) ao dia 14 deste m�s.
+O IPC-r de julho, segundo o ministro, �tem muito pouco da nova moeda�.
+�Normalmente n�s utilizamos dados hist�ricos sobre a produtividade em cada regi�o, al�m de informa��es de agricultores�, diz.
+Segundo Formaggio, est�o em estudo outras formas de obter a produtividade das culturas.
+Ele estima em dez horas o tempo necess�rio para classificar as �reas e culturas plantadas em uma extens�o de 180 quil�metros.
+O prefeito de Juazeiro, Manoel Salviano (PSDB), disse que qualquer funcion�rio que tenha participado da recep��o �foi espontaneamente, sem nenhuma press�o da prefeitura�.
+Acompanhada de Renata Queir�z, mulher do candidato do PSDB ao governo do Cear�, Tasso Jereissati, e de Patr�cia Gomes, mulher do ministro da Fazenda, Ciro Gomes, Ruth Cardoso visitou tr�s cidades cearenses antes de Juazeiro.
+Entre as mais de 140 participantes da feira, estavam presentes companhias como a Xerox, Kodak, QMS, Digital e Pennant, empresa da IBM para a �rea de impress�o.
+A Kodak, por exemplo, apresentou as impressoras/copiadoras ColorEdge 1.560 e 1.565, capazes de copiar em cores frente e verso de documentos de v�rias p�ginas.
+Podem produzir pequenas brochuras, mala direta, folhetos, realizando at� sete impress�es por minuto.
+Kim E sobre como eu fiquei totalmente nua o tempo todo.
+Assista ao filme de novo: voc� n�o vai me ver completamente nua a n�o ser numa cena, no final do strip tease.
+E por tr�s ...
+Est� tudo na sua cabe�a.
+Como quando eu conheci Alec.
+Jurava que ele usava um chap�u de cowboy.
+Alec Nunca usei um chap�u daqueles na vida!
+Nasci em Lono Island.
+O que um chap�u de cowboy estaria fazendo ali?
+Apesar de haver anunciado que o treino da Bulg�ria seria aberto ao p�blico, os dirigentes mudaram de id�ia pouco antes do in�cio do treinamento e proibiram a presen�a de torcedores no campo da Southern Methodist University, em Dallas.
+Os dirigentes da sele��o pediram aos seguran�as da universidade que afastassem o p�blico, composto em sua maior parte por jovens, que j� se preparava para acompanhar os treinos.
+� noite, o projeto prev� aulas de alfabetiza��o para jovens de 14 a 22 anos nos Cieps.
+Na elei��o de 89, Leonel Brizola prometia a constru��o de 10 mil Cieps.
+No projeto n�o h� defini��o de quantos podem ser constru�dos.
+O custo m�dio de cada um dos Cieps no Rio de Janeiro foi de cerca de US$ 1 milh�o.
+�Nossa preocupa��o � promover o desenvolvimento.
+E como n�o h� crescimento sem educa��o, a causa das causas para n�s � a eleva��o do n�vel educacional do povo brasileiro.
+Claro sem descuidar de tudo mais, mas vendo com clareza a quest�o das prioridades�, declara Brizola.
+Em Manaus, o representante da fam�lia Gra�a � Manuel Tavares da Gra�a, candidato derrotado a deputado estadual pelo PP.
+Paix�o � irm�o de Jos� Tavares da Gra�a, preso em agosto de 93 em Bel�m (PA) com 435 quilos de coca�na.
+� tamb�m primo de Curica e de Floriano Gra�a.
+Fui procurado pelos ladr�es para comprar a carga.
+Fomos eu, o Violim, e os investigadores Gilberto Brito, Euripedes Tozzo, Mauro, Marc�o e o Adner.
+Prenderam todos e apreenderam a mercadoria.
+Eles desviaram quatro toneladas da carga.
+Quatro Kombis foram para o Deic.
+S� isso foi devolvido para a Riachuelo.
+No realismo moderno de John Cassavettes (1929-89), por exemplo, o ator constru�a o personagem em tempo real diante da c�mera, baseado no improviso.
+Cada hesita��o, cada sil�ncio, cada desvio do olhar era um ganho na contabilidade do sublime que o diretor-autor planejava.
+Na semana passada, o jogador Buchwald criticou a escala��o de Matthaeus como l�bero, pedindo que ele jogasse no meio-campo.
+O t�cnico da Cor�ia, Kim Ho, afirmou que, no primeiro tempo, a equipe estava �intimidada pela Alemanha, o que � natural, pois eles s�o campe�es do mundo�.
+O secret�rio-geral do Conselho de Seguran�a da ONU, Boutros Boutros-Ghali, pediu que as pot�ncias que negociam um tratado de paz com as fac��es em luta na B�snia formem uma for�a-tarefa para agir na regi�o.
+Boutros-Ghali teme que as pot�ncias se retirem e suspendam san��es contra a S�rvia pondo em risco outras partes da ex-Iugosl�via se o plano for aceito.
+Folha -- Qual o segredo da vit�ria?
+Gilbert -- Acho que Andre usou essa mesma press�o a seu favor.
+Ele funciona desse jeito.
+Se voc� bate nele, o revide vem forte, pode apostar.
+Alguns tenistas s�o assim, usam sua carga emocional como combust�vel.
+Haver� mais e melhores escolas, laborat�rios, universidades e bolsas de estudo.
+Pequenas e m�dias empresas, For�as Armadas, deficientes f�sicos e artistas ter�o apoio e recursos.
+Segundo FHC, o Brasil possui esses recursos, muitos de eles naturais, e vantagens comparativas para realizar essas metas.
+Segundo ele, o tubo era independente do sistema de propuls�o do submarino e o vapor que escapou n�o � radiativo.
+O Emeraude n�o carrega armamento nuclear.
+A energia nuclear � usada s� para movimentar a embarca��o.
+Entre cada parte do submarino h� isolamento, evitando que acidentes numa parte atinjam as outras.
+O rela��es p�blicas da corpora��o, tenente-coronel Fernando Belo, disse que o encontro foi uma oportunidade para agradecer e elogiar a colabora��o da PM.
+Belo negou que na reuni�o tenha sido discutida a incorpora��o aos quadros da PM de cerca de 2.000 soldados do Ex�rcito.
+A possibilidade foi levantada em encontro do governador eleito, Marcello Alencar (PSDB), com oficiais do CML.
+As emissoras se negaram a se retratar e o juiz amea�ou �terminar com a cobertura do julgamento�, sem especificar como.
+Ito pode anunciar hoje que a transmiss�o est� proibida.
+A lei do Estado da Calif�rnia d� ao juiz de cada caso autoridade para impedir a presen�a de c�maras no tribunal se achar que ela prejudica a Justi�a.
+ANGOLA, UMA TRAG�DIA ESQUECIDA -- O fot�grafo Andr� Penner exp�e 29 fotos coloridas que retratam o cotidiano da guerra civil em Angola.
+Abertura hoje �s 19h30.
+Seg a sex das 9h30 �s 18h30.
+Pre�o das fotos: US$ 300.
+At� 17 de junho.
+BASTIDORES DOS MUSEUS -- O trabalho feito pelos fot�grafos Eduardo Castanho, Saul Queiroz, Rog�rio Voltan, Ricardo Hantzchel, Fausto Chermont e Eliana Lopes retrata o que as pessoas n�o v�em ao visitar um museu.
+Na Anhanguera, que une S�o Paulo a Ribeir�o Preto, costuma haver neblina pesada entre o km 21 e o km 43, tamb�m perto de Perus e Cajamar.
+Isto ocorre principalmente � noite e a visibilidade pode chegar a zero.
+A Rodovia Castelo Branco, que segue para o oeste do Estado, apresenta v�rios pontos com neblina.
+O mais perigoso � na regi�o da Serra das Conchas, entre o km 129 e o km 162, logo ap�s Tatu�, para quem vai para o interior.
+O paulistano n�o acredita em queda de infla��o nos pr�ximos meses.
+O �ndice deve aumentar, segundo 45% dos entrevistados pelo DataFolha.
+A impress�o de que o �ndice de alta dos pre�os cai se restringe a 27% das respostas.
+Tudo fica como est� na opini�o de 23%, enquanto 5% n�o sabem responder.
+A derrota (1 a 0) para a fraca sele��o dos Estados Unidos conturbou o ambiente da sele��o mexicana.
+Um grupo de torcedores arremessou ovos podres e insultos contra o time, na volta para a Cidade do M�xico na noite de domingo passado.
+�Nascidos para perder�.
+Segundo a ag�ncia de noticias Notimex, a rea��o pode ser uma amostra de que como vai reagir a torcida se o M�xico n�o se classificar para a segunda fase da Copa.
+�Adoraria ser candidata.
+Tenho certeza de que se pudesse concorrer a algum cargo, me elegeria.
+Eu adoro pol�tica�, confessa Mercedes Rossi de Almeida, 73, a m�e do candidato ao governo de S�o Paulo Francisco Rossi (PDT).
+O painel vai funcionar das 8h �s 22h.
+O p�blico alvo s�o os tr�s milh�es de pedestres que circulam diariamente na regi�o.
+Empresa funciona das 9h �s 19h, diariamente.
+Na Samp, dois titulares est�o fora de combate: o zagueiro Marco Rossi e o armador s�rvio Jugovic.
+Pior, o mister sueco Sven-Goran Eriksson tamb�m n�o poder� contar com o garoto Bertarelli, contratura na coxa direita.
+Retorna ao banco o bom volante eslov�nio Katanec.
+E, depois de uma semana com gripe, recupera a sua forma o holand�s Ruud Gullit, artilheiro do time com 14 tentos.
+Na terceira coloca��o, 34 pontos, a Juventus de Turim visita o Genoa, 22, no limiar do rebaixamento.
+A Juve vive a transi��o para uma nova temporada.
+Acabou-se a gest�o do ex-craque Giampiero Boniperti na administra��o do clube.
+Em seu lugar, vai assumindo um outro ex-atleta, Roberto Bettega.
+Com Boniperti se despede o t�cnico Giovanni Trapattoni, que ceder� o posto a Marcello Lippi, hoje no Napoli.
+A torcida da �Senhora� se conforta com a melhora de Roberto Baggio, que n�o ter� de operar o joelho.
+O projeto vem sendo implementado, em car�ter experimental, em tr�s escolas: Morumbi, Visconde de Porto Seguro (particulares) e Godofredo Furtado (estadual).
+�O programa ter� a��o integrada com pais e alunos atrav�s da forma��o de grupos de informa��o�, diz Antonio Carlos Gomes da Silva, superintendente do HC.
+4) A realiza��o no Cairo (Egito) da Confer�ncia Internacional sobre Popula��o e Desenvolvimento da ONU tem como maior objetivo a divulga��o mundial de pol�ticas para o controle de natalidade como forma de redu��o da pobreza.
+A popula��o mundial est� em torno de 5,66 bilh�es.
+VERA L�CIA DA COSTA ANTUNES � coordenadora de geografia do curso e col�gio Objetivo.
+Com a determina��o da Justi�a, os empregados ficam impedidos de fazer o pagamento das a��es.
+O chefe da ger�ncia jur�dica do BNDES, em S�o Paulo, Arnaldo Montenegro, 43 disse que ser� impetrada uma a��o de agravo de instrumento com pedido de reconsidera��o ao juiz Andrade Martins.
+O banco n�o informou porque o sigilo n�o foi mantido tamb�m para os grupos estrangeiros.
+A rela��o entre os dois piorou quando Fishel arranjou uma amante, no final dos anos 80.
+Ele teria deixado a mulher e trazido a amante para Roma.
+�V�rias vezes os dois foram pegos fazendo amor na minha cl�nica.
+Eu tolerei pois n�o quis me intrometer na vida privada dos meus funcion�rios�, disse Antinori, para quem Fishel era um excelente t�cnico.
+�Era um miser�vel quando chegou.
+Em poucos anos ganhou mais de US$ 1 milh�o�.
+Ap�s v�rias diverg�ncias com Fishel, Antinori o demitiu em 1990.
+Nessa �poca, Fishel teria conhecido o ginecologista brit�nico Robert Winston, a quem convidou para um encontro cient�fico promovido por Antinori.
+O cineasta italiano Franco Zefirelli revelou que, para iludir o servi�o militar durante o per�odo fascista, se tornou um partisan e acabou matando um soldado alem�o.
+As sinaliza��es que tem dado o presidente eleito, Fernando Henrique Cardoso, sobre a reforma patrimonial do Estado s�o auspiciosas.
+Mas � tamb�m preciso a sociedade mobilizar-se para sensibilizar o futuro Congresso Nacional na promo��o de reformas constitucionais indispens�veis ao �xito das mudan�as pretendidas.
+O patrim�nio p�blico n�o � fetiche a ser adorado e sim bem a ser utilizado principalmente em favor de quem n�o tem acesso a escolas, nem � assist�ncia m�dica, nem a empregos.
+Abram Szajman, 58, � presidente da Federa��o do Com�rcio do Estado de S�o Paulo e do Conselho Deliberatico do Sebrae-SP (Servi�o Brasileiro de Apoio �s Micro e Pequenas Empresas).
+Al�m de Mauro Salles que surpreendeu a galera ao revelar seu c�t� de fot�grafo profissional no �nico da carreira, o foco principal do encontro foi a mulher de J.R.Duran, Alexandra Brochen.
diff --git a/nltk/test/framenet.doctest b/nltk/test/framenet.doctest
new file mode 100644
index 0000000..b68d982
--- /dev/null
+++ b/nltk/test/framenet.doctest
@@ -0,0 +1,239 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+========
+FrameNet
+========
+
+The FrameNet corpus is a lexical database of English that is both human-
+and machine-readable, based on annotating examples of how words are used
+in actual texts. FrameNet is based on a theory of meaning called Frame
+Semantics, deriving from the work of Charles J. Fillmore and colleagues.
+The basic idea is straightforward: that the meanings of most words can
+best be understood on the basis of a semantic frame: a description of a
+type of event, relation, or entity and the participants in it. For
+example, the concept of cooking typically involves a person doing the
+cooking (Cook), the food that is to be cooked (Food), something to hold
+the food while cooking (Container) and a source of heat
+(Heating_instrument). In the FrameNet project, this is represented as a
+frame called Apply_heat, and the Cook, Food, Heating_instrument and
+Container are called frame elements (FEs). Words that evoke this frame,
+such as fry, bake, boil, and broil, are called lexical units (LUs) of
+the Apply_heat frame. The job of FrameNet is to define the frames
+and to annotate sentences to show how the FEs fit syntactically around
+the word that evokes the frame.
+
+------
+Frames
+------
+
+A Frame is a script-like conceptual structure that describes a
+particular type of situation, object, or event along with the
+participants and props that are needed for that Frame. For
+example, the "Apply_heat" frame describes a common situation
+involving a Cook, some Food, and a Heating_Instrument, and is
+evoked by words such as bake, blanch, boil, broil, brown,
+simmer, steam, etc.
+
+We call the roles of a Frame "frame elements" (FEs) and the
+frame-evoking words are called "lexical units" (LUs).
+
+FrameNet includes relations between Frames. Several types of
+relations are defined, of which the most important are:
+
+- Inheritance: An IS-A relation. The child frame is a subtype
+ of the parent frame, and each FE in the parent is bound to
+ a corresponding FE in the child. An example is the
+ "Revenge" frame which inherits from the
+ "Rewards_and_punishments" frame.
+
+- Using: The child frame presupposes the parent frame as
+ background, e.g the "Speed" frame "uses" (or presupposes)
+ the "Motion" frame; however, not all parent FEs need to be
+ bound to child FEs.
+
+- Subframe: The child frame is a subevent of a complex event
+ represented by the parent, e.g. the "Criminal_process" frame
+ has subframes of "Arrest", "Arraignment", "Trial", and
+ "Sentencing".
+
+- Perspective_on: The child frame provides a particular
+ perspective on an un-perspectivized parent frame. A pair of
+ examples consists of the "Hiring" and "Get_a_job" frames,
+ which perspectivize the "Employment_start" frame from the
+ Employer's and the Employee's point of view, respectively.
+
+To get a list of all of the Frames in FrameNet, you can use the
+`frames()` function. If you supply a regular expression pattern to the
+`frames()` function, you will get a list of all Frames whose names match
+that pattern:
+
+ >>> from pprint import pprint
+ >>> from nltk.corpus import framenet as fn
+ >>> len(fn.frames())
+ 1019
+ >>> pprint(fn.frames(r'(?i)medical'))
+ [<frame ID=256 name=Medical_specialties>, <frame ID=257 name=Medical_instruments>, ...]
+
+To get the details of a particular Frame, you can use the `frame()`
+function passing in the frame number:
+
+ >>> from pprint import pprint
+ >>> from nltk.corpus import framenet as fn
+ >>> f = fn.frame(256)
+ >>> f.ID
+ 256
+ >>> f.name
+ 'Medical_specialties'
+ >>> f.definition # doctest: +ELLIPSIS
+ "This frame includes words that name ..."
+ >>> len(f.lexUnit)
+ 29
+ >>> pprint(sorted([x for x in f.FE]))
+ ['Affliction', 'Body_system', 'Specialty', 'Type']
+ >>> pprint(f.frameRelations)
+ [<Parent=Cure -- Using -> Child=Medical_specialties>]
+
+The `frame()` function shown above returns a dict object containing
+detailed information about the Frame. See the documentation on the
+`frame()` function for the specifics.
+
+You can also search for Frames by their Lexical Units (LUs). The
+`frames_by_lemma()` function returns a list of all frames that contain
+LUs in which the 'name' attribute of the LU matchs the given regular
+expression. Note that LU names are composed of "lemma.POS", where the
+"lemma" part can be made up of either a single lexeme (e.g. 'run') or
+multiple lexemes (e.g. 'a little') (see below).
+
+ >>> from nltk.corpus import framenet as fn
+ >>> fn.frames_by_lemma(r'(?i)a little')
+ [<frame ID=189 name=Quantity>, <frame ID=2001 name=Degree>]
+
+-------------
+Lexical Units
+-------------
+
+A lexical unit (LU) is a pairing of a word with a meaning. For
+example, the "Apply_heat" Frame describes a common situation
+involving a Cook, some Food, and a Heating Instrument, and is
+_evoked_ by words such as bake, blanch, boil, broil, brown,
+simmer, steam, etc. These frame-evoking words are the LUs in the
+Apply_heat frame. Each sense of a polysemous word is a different
+LU.
+
+We have used the word "word" in talking about LUs. The reality
+is actually rather complex. When we say that the word "bake" is
+polysemous, we mean that the lemma "bake.v" (which has the
+word-forms "bake", "bakes", "baked", and "baking") is linked to
+three different frames:
+
+- Apply_heat: "Michelle baked the potatoes for 45 minutes."
+
+- Cooking_creation: "Michelle baked her mother a cake for her birthday."
+
+- Absorb_heat: "The potatoes have to bake for more than 30 minutes."
+
+These constitute three different LUs, with different
+definitions.
+
+Multiword expressions such as "given name" and hyphenated words
+like "shut-eye" can also be LUs. Idiomatic phrases such as
+"middle of nowhere" and "give the slip (to)" are also defined as
+LUs in the appropriate frames ("Isolated_places" and "Evading",
+respectively), and their internal structure is not analyzed.
+
+Framenet provides multiple annotated examples of each sense of a
+word (i.e. each LU). Moreover, the set of examples
+(approximately 20 per LU) illustrates all of the combinatorial
+possibilities of the lexical unit.
+
+Each LU is linked to a Frame, and hence to the other words which
+evoke that Frame. This makes the FrameNet database similar to a
+thesaurus, grouping together semantically similar words.
+
+In the simplest case, frame-evoking words are verbs such as
+"fried" in:
+
+ "Matilde fried the catfish in a heavy iron skillet."
+
+Sometimes event nouns may evoke a Frame. For example,
+"reduction" evokes "Cause_change_of_scalar_position" in:
+
+ "...the reduction of debt levels to $665 million from $2.6 billion."
+
+Adjectives may also evoke a Frame. For example, "asleep" may
+evoke the "Sleep" frame as in:
+
+ "They were asleep for hours."
+
+Many common nouns, such as artifacts like "hat" or "tower",
+typically serve as dependents rather than clearly evoking their
+own frames.
+
+Details for a specific lexical unit can be obtained using this class's
+`lus()` function, which takes an optional regular expression
+pattern that will be matched against the name of the lexical unit:
+
+ >>> from pprint import pprint
+ >>> from nltk.corpus import framenet as fn
+ >>> len(fn.lus())
+ 11829
+ >>> pprint(fn.lus(r'(?i)a little'))
+ [<lu ID=14744 name=a little bit.adv>, <lu ID=14733 name=a little.n>, ...]
+
+You can obtain detailed information on a particular LU by calling the
+`lu()` function and passing in an LU's 'ID' number:
+
+ >>> from pprint import pprint
+ >>> from nltk.corpus import framenet as fn
+ >>> fn.lu(256).name
+ 'foresee.v'
+ >>> fn.lu(256).definition
+ 'COD: be aware of beforehand; predict.'
+ >>> fn.lu(256).frame.name
+ 'Expectation'
+ >>> fn.lu(256).lexemes[0].name
+ 'foresee'
+
+Note that LU names take the form of a dotted string (e.g. "run.v" or "a
+little.adv") in which a lemma preceeds the "." and a part of speech
+(POS) follows the dot. The lemma may be composed of a single lexeme
+(e.g. "run") or of multiple lexemes (e.g. "a little"). The list of
+POSs used in the LUs is:
+
+v - verb
+n - noun
+a - adjective
+adv - adverb
+prep - preposition
+num - numbers
+intj - interjection
+art - article
+c - conjunction
+scon - subordinating conjunction
+
+For more detailed information about the info that is contained in the
+dict that is returned by the `lu()` function, see the documentation on
+the `lu()` function.
+
+-------------------
+Annotated Documents
+-------------------
+
+The FrameNet corpus contains a small set of annotated documents. A list
+of these documents can be obtained by calling the `documents()` function:
+
+ >>> from pprint import pprint
+ >>> from nltk.corpus import framenet as fn
+ >>> docs = fn.documents()
+ >>> len(docs)
+ 78
+ >>> pprint(sorted(docs[0].keys()))
+ ['ID', 'corpid', 'corpname', 'description', 'filename']
+
+Detailed information about each sentence contained in each document can
+be obtained by calling the `annotated_document()` function and supplying
+the 'ID' number of the document. For detailed information about the info
+that is for each document, see the documentation on the
+`annotated_document()` function.
+
diff --git a/nltk/test/generate.doctest b/nltk/test/generate.doctest
new file mode 100644
index 0000000..4840599
--- /dev/null
+++ b/nltk/test/generate.doctest
@@ -0,0 +1,67 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+===============================================
+Generating sentences from context-free grammars
+===============================================
+
+An example grammar:
+
+ >>> from nltk.parse.generate import generate, demo_grammar
+ >>> from nltk import CFG
+ >>> grammar = CFG.fromstring(demo_grammar)
+ >>> print(grammar)
+ Grammar with 13 productions (start state = S)
+ S -> NP VP
+ NP -> Det N
+ PP -> P NP
+ VP -> 'slept'
+ VP -> 'saw' NP
+ VP -> 'walked' PP
+ Det -> 'the'
+ Det -> 'a'
+ N -> 'man'
+ N -> 'park'
+ N -> 'dog'
+ P -> 'in'
+ P -> 'with'
+
+The first 10 generated sentences:
+
+ >>> for sentence in generate(grammar, n=10):
+ ... print(' '.join(sentence))
+ the man slept
+ the man saw the man
+ the man saw the park
+ the man saw the dog
+ the man saw a man
+ the man saw a park
+ the man saw a dog
+ the man walked in the man
+ the man walked in the park
+ the man walked in the dog
+
+All sentences of max depth 4:
+
+ >>> for sentence in generate(grammar, depth=4):
+ ... print(' '.join(sentence))
+ the man slept
+ the park slept
+ the dog slept
+ a man slept
+ a park slept
+ a dog slept
+
+The number of sentences of different max depths:
+
+ >>> len(list(generate(grammar, depth=3)))
+ 0
+ >>> len(list(generate(grammar, depth=4)))
+ 6
+ >>> len(list(generate(grammar, depth=5)))
+ 42
+ >>> len(list(generate(grammar, depth=6)))
+ 114
+ >>> len(list(generate(grammar)))
+ 114
+
diff --git a/nltk/test/gluesemantics.doctest b/nltk/test/gluesemantics.doctest
new file mode 100644
index 0000000..1aa111e
--- /dev/null
+++ b/nltk/test/gluesemantics.doctest
@@ -0,0 +1,381 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==============================================================================
+ Glue Semantics
+==============================================================================
+
+.. include:: ../../doc/definitions.rst
+
+
+======================
+Linear logic
+======================
+
+ >>> from nltk.sem import logic
+ >>> from nltk.sem.glue import *
+ >>> from nltk.sem.linearlogic import *
+
+ >>> from nltk.sem.linearlogic import Expression
+ >>> llexpr = Expression.fromstring
+
+Parser
+
+ >>> print(llexpr(r'f'))
+ f
+ >>> print(llexpr(r'(g -o f)'))
+ (g -o f)
+ >>> print(llexpr(r'(g -o (h -o f))'))
+ (g -o (h -o f))
+ >>> print(llexpr(r'((g -o G) -o G)'))
+ ((g -o G) -o G)
+ >>> print(llexpr(r'(g -o f)(g)'))
+ (g -o f)(g)
+ >>> print(llexpr(r'((g -o G) -o G)((g -o f))'))
+ ((g -o G) -o G)((g -o f))
+
+Simplify
+
+ >>> print(llexpr(r'f').simplify())
+ f
+ >>> print(llexpr(r'(g -o f)').simplify())
+ (g -o f)
+ >>> print(llexpr(r'((g -o G) -o G)').simplify())
+ ((g -o G) -o G)
+ >>> print(llexpr(r'(g -o f)(g)').simplify())
+ f
+ >>> try: llexpr(r'(g -o f)(f)').simplify()
+ ... except LinearLogicApplicationException as e: print(e)
+ ...
+ Cannot apply (g -o f) to f. Cannot unify g with f given {}
+ >>> print(llexpr(r'(G -o f)(g)').simplify())
+ f
+ >>> print(llexpr(r'((g -o G) -o G)((g -o f))').simplify())
+ f
+
+Test BindingDict
+
+ >>> h = ConstantExpression('h')
+ >>> g = ConstantExpression('g')
+ >>> f = ConstantExpression('f')
+
+ >>> H = VariableExpression('H')
+ >>> G = VariableExpression('G')
+ >>> F = VariableExpression('F')
+
+ >>> d1 = BindingDict({H: h})
+ >>> d2 = BindingDict({F: f, G: F})
+ >>> d12 = d1 + d2
+ >>> all12 = ['%s: %s' % (v, d12[v]) for v in d12.d]
+ >>> all12.sort()
+ >>> print(all12)
+ ['F: f', 'G: f', 'H: h']
+
+ >>> BindingDict([(F,f),(G,g),(H,h)]) == BindingDict({F:f, G:g, H:h})
+ True
+
+ >>> d4 = BindingDict({F: f})
+ >>> try: d4[F] = g
+ ... except VariableBindingException as e: print(e)
+ Variable F already bound to another value
+
+Test Unify
+
+ >>> try: f.unify(g, BindingDict())
+ ... except UnificationException as e: print(e)
+ ...
+ Cannot unify f with g given {}
+
+ >>> f.unify(G, BindingDict()) == BindingDict({G: f})
+ True
+ >>> try: f.unify(G, BindingDict({G: h}))
+ ... except UnificationException as e: print(e)
+ ...
+ Cannot unify f with G given {G: h}
+ >>> f.unify(G, BindingDict({G: f})) == BindingDict({G: f})
+ True
+ >>> f.unify(G, BindingDict({H: f})) == BindingDict({G: f, H: f})
+ True
+
+ >>> G.unify(f, BindingDict()) == BindingDict({G: f})
+ True
+ >>> try: G.unify(f, BindingDict({G: h}))
+ ... except UnificationException as e: print(e)
+ ...
+ Cannot unify G with f given {G: h}
+ >>> G.unify(f, BindingDict({G: f})) == BindingDict({G: f})
+ True
+ >>> G.unify(f, BindingDict({H: f})) == BindingDict({G: f, H: f})
+ True
+
+ >>> G.unify(F, BindingDict()) == BindingDict({G: F})
+ True
+ >>> try: G.unify(F, BindingDict({G: H}))
+ ... except UnificationException as e: print(e)
+ ...
+ Cannot unify G with F given {G: H}
+ >>> G.unify(F, BindingDict({G: F})) == BindingDict({G: F})
+ True
+ >>> G.unify(F, BindingDict({H: F})) == BindingDict({G: F, H: F})
+ True
+
+Test Compile
+
+ >>> print(llexpr('g').compile_pos(Counter(), GlueFormula))
+ (<ConstantExpression g>, [])
+ >>> print(llexpr('(g -o f)').compile_pos(Counter(), GlueFormula))
+ (<ImpExpression (g -o f)>, [])
+ >>> print(llexpr('(g -o (h -o f))').compile_pos(Counter(), GlueFormula))
+ (<ImpExpression (g -o (h -o f))>, [])
+
+
+======================
+Glue
+======================
+
+Demo of "John walks"
+--------------------
+
+ >>> john = GlueFormula("John", "g")
+ >>> print(john)
+ John : g
+ >>> walks = GlueFormula(r"\x.walks(x)", "(g -o f)")
+ >>> print(walks)
+ \x.walks(x) : (g -o f)
+ >>> print(walks.applyto(john))
+ \x.walks(x)(John) : (g -o f)(g)
+ >>> print(walks.applyto(john).simplify())
+ walks(John) : f
+
+
+Demo of "A dog walks"
+---------------------
+
+ >>> a = GlueFormula("\P Q.some x.(P(x) and Q(x))", "((gv -o gr) -o ((g -o G) -o G))")
+ >>> print(a)
+ \P Q.exists x.(P(x) & Q(x)) : ((gv -o gr) -o ((g -o G) -o G))
+ >>> man = GlueFormula(r"\x.man(x)", "(gv -o gr)")
+ >>> print(man)
+ \x.man(x) : (gv -o gr)
+ >>> walks = GlueFormula(r"\x.walks(x)", "(g -o f)")
+ >>> print(walks)
+ \x.walks(x) : (g -o f)
+ >>> a_man = a.applyto(man)
+ >>> print(a_man.simplify())
+ \Q.exists x.(man(x) & Q(x)) : ((g -o G) -o G)
+ >>> a_man_walks = a_man.applyto(walks)
+ >>> print(a_man_walks.simplify())
+ exists x.(man(x) & walks(x)) : f
+
+
+Demo of 'every girl chases a dog'
+---------------------------------
+
+Individual words:
+
+ >>> every = GlueFormula("\P Q.all x.(P(x) -> Q(x))", "((gv -o gr) -o ((g -o G) -o G))")
+ >>> print(every)
+ \P Q.all x.(P(x) -> Q(x)) : ((gv -o gr) -o ((g -o G) -o G))
+ >>> girl = GlueFormula(r"\x.girl(x)", "(gv -o gr)")
+ >>> print(girl)
+ \x.girl(x) : (gv -o gr)
+ >>> chases = GlueFormula(r"\x y.chases(x,y)", "(g -o (h -o f))")
+ >>> print(chases)
+ \x y.chases(x,y) : (g -o (h -o f))
+ >>> a = GlueFormula("\P Q.some x.(P(x) and Q(x))", "((hv -o hr) -o ((h -o H) -o H))")
+ >>> print(a)
+ \P Q.exists x.(P(x) & Q(x)) : ((hv -o hr) -o ((h -o H) -o H))
+ >>> dog = GlueFormula(r"\x.dog(x)", "(hv -o hr)")
+ >>> print(dog)
+ \x.dog(x) : (hv -o hr)
+
+Noun Quantification can only be done one way:
+
+ >>> every_girl = every.applyto(girl)
+ >>> print(every_girl.simplify())
+ \Q.all x.(girl(x) -> Q(x)) : ((g -o G) -o G)
+ >>> a_dog = a.applyto(dog)
+ >>> print(a_dog.simplify())
+ \Q.exists x.(dog(x) & Q(x)) : ((h -o H) -o H)
+
+The first reading is achieved by combining 'chases' with 'a dog' first.
+Since 'a girl' requires something of the form '(h -o H)' we must
+get rid of the 'g' in the glue of 'see'. We will do this with
+the '-o elimination' rule. So, x1 will be our subject placeholder.
+
+ >>> xPrime = GlueFormula("x1", "g")
+ >>> print(xPrime)
+ x1 : g
+ >>> xPrime_chases = chases.applyto(xPrime)
+ >>> print(xPrime_chases.simplify())
+ \y.chases(x1,y) : (h -o f)
+ >>> xPrime_chases_a_dog = a_dog.applyto(xPrime_chases)
+ >>> print(xPrime_chases_a_dog.simplify())
+ exists x.(dog(x) & chases(x1,x)) : f
+
+Now we can retract our subject placeholder using lambda-abstraction and
+combine with the true subject.
+
+ >>> chases_a_dog = xPrime_chases_a_dog.lambda_abstract(xPrime)
+ >>> print(chases_a_dog.simplify())
+ \x1.exists x.(dog(x) & chases(x1,x)) : (g -o f)
+ >>> every_girl_chases_a_dog = every_girl.applyto(chases_a_dog)
+ >>> r1 = every_girl_chases_a_dog.simplify()
+ >>> r2 = GlueFormula(r'all x.(girl(x) -> exists z1.(dog(z1) & chases(x,z1)))', 'f')
+ >>> r1 == r2
+ True
+
+The second reading is achieved by combining 'every girl' with 'chases' first.
+
+ >>> xPrime = GlueFormula("x1", "g")
+ >>> print(xPrime)
+ x1 : g
+ >>> xPrime_chases = chases.applyto(xPrime)
+ >>> print(xPrime_chases.simplify())
+ \y.chases(x1,y) : (h -o f)
+ >>> yPrime = GlueFormula("x2", "h")
+ >>> print(yPrime)
+ x2 : h
+ >>> xPrime_chases_yPrime = xPrime_chases.applyto(yPrime)
+ >>> print(xPrime_chases_yPrime.simplify())
+ chases(x1,x2) : f
+ >>> chases_yPrime = xPrime_chases_yPrime.lambda_abstract(xPrime)
+ >>> print(chases_yPrime.simplify())
+ \x1.chases(x1,x2) : (g -o f)
+ >>> every_girl_chases_yPrime = every_girl.applyto(chases_yPrime)
+ >>> print(every_girl_chases_yPrime.simplify())
+ all x.(girl(x) -> chases(x,x2)) : f
+ >>> every_girl_chases = every_girl_chases_yPrime.lambda_abstract(yPrime)
+ >>> print(every_girl_chases.simplify())
+ \x2.all x.(girl(x) -> chases(x,x2)) : (h -o f)
+ >>> every_girl_chases_a_dog = a_dog.applyto(every_girl_chases)
+ >>> r1 = every_girl_chases_a_dog.simplify()
+ >>> r2 = GlueFormula(r'exists x.(dog(x) & all z2.(girl(z2) -> chases(z2,x)))', 'f')
+ >>> r1 == r2
+ True
+
+
+Compilation
+-----------
+
+ >>> for cp in GlueFormula('m', '(b -o a)').compile(Counter()): print(cp)
+ m : (b -o a) : {1}
+ >>> for cp in GlueFormula('m', '((c -o b) -o a)').compile(Counter()): print(cp)
+ v1 : c : {1}
+ m : (b[1] -o a) : {2}
+ >>> for cp in GlueFormula('m', '((d -o (c -o b)) -o a)').compile(Counter()): print(cp)
+ v1 : c : {1}
+ v2 : d : {2}
+ m : (b[1, 2] -o a) : {3}
+ >>> for cp in GlueFormula('m', '((d -o e) -o ((c -o b) -o a))').compile(Counter()): print(cp)
+ v1 : d : {1}
+ v2 : c : {2}
+ m : (e[1] -o (b[2] -o a)) : {3}
+ >>> for cp in GlueFormula('m', '(((d -o c) -o b) -o a)').compile(Counter()): print(cp)
+ v1 : (d -o c) : {1}
+ m : (b[1] -o a) : {2}
+ >>> for cp in GlueFormula('m', '((((e -o d) -o c) -o b) -o a)').compile(Counter()): print(cp)
+ v1 : e : {1}
+ v2 : (d[1] -o c) : {2}
+ m : (b[2] -o a) : {3}
+
+
+Demo of 'a man walks' using Compilation
+---------------------------------------
+
+Premises
+
+ >>> a = GlueFormula('\\P Q.some x.(P(x) and Q(x))', '((gv -o gr) -o ((g -o G) -o G))')
+ >>> print(a)
+ \P Q.exists x.(P(x) & Q(x)) : ((gv -o gr) -o ((g -o G) -o G))
+
+ >>> man = GlueFormula('\\x.man(x)', '(gv -o gr)')
+ >>> print(man)
+ \x.man(x) : (gv -o gr)
+
+ >>> walks = GlueFormula('\\x.walks(x)', '(g -o f)')
+ >>> print(walks)
+ \x.walks(x) : (g -o f)
+
+Compiled Premises:
+
+ >>> counter = Counter()
+ >>> ahc = a.compile(counter)
+ >>> g1 = ahc[0]
+ >>> print(g1)
+ v1 : gv : {1}
+ >>> g2 = ahc[1]
+ >>> print(g2)
+ v2 : g : {2}
+ >>> g3 = ahc[2]
+ >>> print(g3)
+ \P Q.exists x.(P(x) & Q(x)) : (gr[1] -o (G[2] -o G)) : {3}
+ >>> g4 = man.compile(counter)[0]
+ >>> print(g4)
+ \x.man(x) : (gv -o gr) : {4}
+ >>> g5 = walks.compile(counter)[0]
+ >>> print(g5)
+ \x.walks(x) : (g -o f) : {5}
+
+Derivation:
+
+ >>> g14 = g4.applyto(g1)
+ >>> print(g14.simplify())
+ man(v1) : gr : {1, 4}
+ >>> g134 = g3.applyto(g14)
+ >>> print(g134.simplify())
+ \Q.exists x.(man(x) & Q(x)) : (G[2] -o G) : {1, 3, 4}
+ >>> g25 = g5.applyto(g2)
+ >>> print(g25.simplify())
+ walks(v2) : f : {2, 5}
+ >>> g12345 = g134.applyto(g25)
+ >>> print(g12345.simplify())
+ exists x.(man(x) & walks(x)) : f : {1, 2, 3, 4, 5}
+
+---------------------------------
+Dependency Graph to Glue Formulas
+---------------------------------
+ >>> from nltk.corpus.reader.dependency import DependencyGraph
+
+ >>> depgraph = DependencyGraph("""1 John _ NNP NNP _ 2 SUBJ _ _
+ ... 2 sees _ VB VB _ 0 ROOT _ _
+ ... 3 a _ ex_quant ex_quant _ 4 SPEC _ _
+ ... 4 dog _ NN NN _ 2 OBJ _ _
+ ... """)
+ >>> gfl = GlueDict('nltk:grammars/sample_grammars/glue.semtype').to_glueformula_list(depgraph)
+ >>> for gf in gfl:
+ ... print(gf)
+ \x y.sees(x,y) : (f -o (i -o g))
+ \P Q.exists x.(P(x) & Q(x)) : ((fv -o fr) -o ((f -o F2) -o F2))
+ \x.John(x) : (fv -o fr)
+ \x.dog(x) : (iv -o ir)
+ \P Q.exists x.(P(x) & Q(x)) : ((iv -o ir) -o ((i -o I5) -o I5))
+ >>> glue = Glue()
+ >>> for r in sorted([r.simplify().normalize() for r in glue.get_readings(glue.gfl_to_compiled(gfl))], key=str):
+ ... print(r)
+ exists z1.(John(z1) & exists z2.(dog(z2) & sees(z1,z2)))
+ exists z1.(dog(z1) & exists z2.(John(z2) & sees(z2,z1)))
+
+-----------------------------------
+Dependency Graph to LFG f-structure
+-----------------------------------
+ >>> from nltk.sem.lfg import FStructure
+ >>> fstruct = FStructure.read_depgraph(depgraph)
+ >>> print(fstruct)
+ f:[pred 'sees'
+ obj h:[pred 'dog'
+ spec 'a']
+ subj g:[pred 'John']]
+
+---------------------------------
+LFG f-structure to Glue
+---------------------------------
+ >>> for gf in fstruct.to_glueformula_list(GlueDict('nltk:grammars/sample_grammars/glue.semtype')): # doctest: +SKIP
+ ... print(gf)
+ \x y.sees(x,y) : (i -o (g -o f))
+ \x.dog(x) : (gv -o gr)
+ \P Q.exists x.(P(x) & Q(x)) : ((gv -o gr) -o ((g -o G3) -o G3))
+ \P Q.exists x.(P(x) & Q(x)) : ((iv -o ir) -o ((i -o I4) -o I4))
+ \x.John(x) : (iv -o ir)
+
+.. see gluesemantics_malt.doctest for more
diff --git a/nltk/test/gluesemantics_malt.doctest b/nltk/test/gluesemantics_malt.doctest
new file mode 100644
index 0000000..87b2b17
--- /dev/null
+++ b/nltk/test/gluesemantics_malt.doctest
@@ -0,0 +1,68 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+.. see also: gluesemantics.doctest
+
+==============================================================================
+ Glue Semantics
+==============================================================================
+
+ >>> from nltk.sem.glue import *
+ >>> nltk.sem.logic._counter._value = 0
+
+--------------------------------
+Initialize the Dependency Parser
+--------------------------------
+ >>> from nltk.parse.malt import MaltParser
+
+ >>> tagger = RegexpTagger(
+ ... [('^(John|Mary)$', 'NNP'),
+ ... ('^(sees|chases)$', 'VB'),
+ ... ('^(a)$', 'ex_quant'),
+ ... ('^(every)$', 'univ_quant'),
+ ... ('^(girl|dog)$', 'NN')
+ ... ])
+ >>> depparser = MaltParser(tagger=tagger)
+
+--------------------
+Automated Derivation
+--------------------
+ >>> glue = Glue(depparser=depparser)
+ >>> readings = glue.parse_to_meaning('every girl chases a dog'.split())
+ >>> for reading in sorted([r.simplify().normalize() for r in readings], key=str):
+ ... print(reading.normalize())
+ all z1.(girl(z1) -> exists z2.(dog(z2) & chases(z1,z2)))
+ exists z1.(dog(z1) & all z2.(girl(z2) -> chases(z2,z1)))
+
+ >>> drtglue = DrtGlue(depparser=depparser)
+ >>> readings = drtglue.parse_to_meaning('every girl chases a dog'.split())
+ >>> for reading in sorted([r.simplify().normalize() for r in readings], key=str):
+ ... print(reading)
+ ([],[(([z1],[girl(z1)]) -> ([z2],[dog(z2), chases(z1,z2)]))])
+ ([z1],[dog(z1), (([z2],[girl(z2)]) -> ([],[chases(z2,z1)]))])
+
+--------------
+With inference
+--------------
+
+Checking for equality of two DRSs is very useful when generating readings of a sentence.
+For example, the ``glue`` module generates two readings for the sentence
+*John sees Mary*:
+
+ >>> from nltk.sem.glue import DrtGlue
+ >>> readings = drtglue.parse_to_meaning('John sees Mary'.split())
+ >>> for drs in sorted([r.simplify().normalize() for r in readings], key=str):
+ ... print(drs)
+ ([z1,z2],[John(z1), Mary(z2), sees(z1,z2)])
+ ([z1,z2],[Mary(z1), John(z2), sees(z2,z1)])
+
+However, it is easy to tell that these two readings are logically the
+same, and therefore one of them is superfluous. We can use the theorem prover
+to determine this equivalence, and then delete one of them. A particular
+theorem prover may be specified, or the argument may be left off to use the
+default.
+
+ >>> readings[0].equiv(readings[1])
+ True
+
+
diff --git a/nltk/test/gluesemantics_malt_fixt.py b/nltk/test/gluesemantics_malt_fixt.py
new file mode 100644
index 0000000..8e28d5b
--- /dev/null
+++ b/nltk/test/gluesemantics_malt_fixt.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+def setup_module(module):
+ from nose import SkipTest
+ from nltk.parse.malt import MaltParser
+
+ try:
+ depparser = MaltParser()
+ except LookupError:
+ raise SkipTest("MaltParser is not available")
diff --git a/nltk/test/grammar.doctest b/nltk/test/grammar.doctest
new file mode 100644
index 0000000..0a9f394
--- /dev/null
+++ b/nltk/test/grammar.doctest
@@ -0,0 +1,48 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+===============
+Grammar Parsing
+===============
+
+Grammars can be parsed from strings:
+
+ >>> from nltk import CFG
+ >>> grammar = CFG.fromstring("""
+ ... S -> NP VP
+ ... PP -> P NP
+ ... NP -> Det N | NP PP
+ ... VP -> V NP | VP PP
+ ... Det -> 'a' | 'the'
+ ... N -> 'dog' | 'cat'
+ ... V -> 'chased' | 'sat'
+ ... P -> 'on' | 'in'
+ ... """)
+ >>> grammar
+ <Grammar with 14 productions>
+ >>> grammar.start()
+ S
+ >>> grammar.productions() # doctest: +NORMALIZE_WHITESPACE
+ [S -> NP VP, PP -> P NP, NP -> Det N, NP -> NP PP, VP -> V NP, VP -> VP PP,
+ Det -> 'a', Det -> 'the', N -> 'dog', N -> 'cat', V -> 'chased', V -> 'sat',
+ P -> 'on', P -> 'in']
+
+Probabilistic CFGs:
+
+ >>> from nltk import PCFG
+ >>> toy_pcfg1 = PCFG.fromstring("""
+ ... S -> NP VP [1.0]
+ ... NP -> Det N [0.5] | NP PP [0.25] | 'John' [0.1] | 'I' [0.15]
+ ... Det -> 'the' [0.8] | 'my' [0.2]
+ ... N -> 'man' [0.5] | 'telescope' [0.5]
+ ... VP -> VP PP [0.1] | V NP [0.7] | V [0.2]
+ ... V -> 'ate' [0.35] | 'saw' [0.65]
+ ... PP -> P NP [1.0]
+ ... P -> 'with' [0.61] | 'under' [0.39]
+ ... """)
+
+Chomsky Normal Form grammar (Test for bug 474)
+
+ >>> g = CFG.fromstring("VP^<TOP> -> VBP NP^<VP-TOP>")
+ >>> g.productions()[0].lhs()
+ VP^<TOP>
diff --git a/nltk/test/grammartestsuites.doctest b/nltk/test/grammartestsuites.doctest
new file mode 100644
index 0000000..3c18e46
--- /dev/null
+++ b/nltk/test/grammartestsuites.doctest
@@ -0,0 +1,109 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==========================
+ Test Suites for Grammars
+==========================
+
+Sentences in the test suite are divided into two classes:
+
+- grammatical (*accept*) and
+- ungrammatical (*reject*).
+
+If a sentence should parse accordng to the grammar, the value of
+``trees`` will be a non-empty list. If a sentence should be rejected
+according to the grammar, then the value of ``trees`` will be ``None``.
+
+ >>> from nltk.parse import TestGrammar
+ >>> germantest1 = {}
+ >>> germantest1['doc'] = "Tests for person agreement"
+ >>> germantest1['accept'] = [
+ ... 'ich komme',
+ ... 'ich sehe mich',
+ ... 'du kommst',
+ ... 'du siehst mich',
+ ... 'sie kommt',
+ ... 'sie sieht mich',
+ ... 'ihr kommt',
+ ... 'wir kommen',
+ ... 'sie kommen',
+ ... 'du magst mich',
+ ... 'er mag mich',
+ ... 'du folgst mir',
+ ... 'sie hilft mir',
+ ... ]
+ >>> germantest1['reject'] = [
+ ... 'ich kommt',
+ ... 'ich kommst',
+ ... 'ich siehst mich',
+ ... 'du komme',
+ ... 'du sehe mich',
+ ... 'du kommt',
+ ... 'er komme',
+ ... 'er siehst mich',
+ ... 'wir komme',
+ ... 'wir kommst',
+ ... 'die Katzen kommst',
+ ... 'sie komme',
+ ... 'sie kommst',
+ ... 'du mag mich',
+ ... 'er magst mich',
+ ... 'du folgt mir',
+ ... 'sie hilfst mir',
+ ... ]
+ >>> germantest2 = {}
+ >>> germantest2['doc'] = "Tests for number agreement"
+ >>> germantest2['accept'] = [
+ ... 'der Hund kommt',
+ ... 'die Hunde kommen',
+ ... 'ich komme',
+ ... 'wir kommen',
+ ... 'ich sehe die Katzen',
+ ... 'ich folge den Katzen',
+ ... 'ich sehe die Katzen',
+ ... 'ich folge den Katzen',
+ ... 'wir sehen die Katzen',
+ ... 'wir folgen den Katzen'
+ ... ]
+ >>> germantest2['reject'] = [
+ ... 'ich kommen',
+ ... 'wir komme',
+ ... 'der Hunde kommt',
+ ... 'der Hunde kommen',
+ ... 'die Katzen kommt',
+ ... 'ich sehe der Hunde',
+ ... 'ich folge den Hund',
+ ... 'ich sehen der Hunde',
+ ... 'ich folgen den Hund',
+ ... 'wir sehe die Katzen',
+ ... 'wir folge den Katzen'
+ ... ]
+ >>> germantest3 = {}
+ >>> germantest3['doc'] = "Tests for case government and subcategorization"
+ >>> germantest3['accept'] = [
+ ... 'der Hund sieht mich',
+ ... 'der Hund kommt',
+ ... 'ich sehe den Hund',
+ ... 'ich helfe dem Hund',
+ ... ]
+ >>> germantest3['reject'] = [
+ ... 'ich sehe',
+ ... 'ich helfe',
+ ... 'ich komme den Hund',
+ ... 'ich sehe den Hund die Katzen',
+ ... 'du hilfst mich',
+ ... 'du siehst mir',
+ ... 'du siehst ich',
+ ... 'der Hunde kommt mich',
+ ... 'die Hunde sehe die Hunde',
+ ... 'der Hund sehe die Hunde',
+ ... 'ich hilft den Hund',
+ ... 'ich hilft der Hund',
+ ... 'ich sehe dem Hund',
+ ... ]
+ >>> germantestsuites = [germantest1, germantest2, germantest3]
+ >>> tester = TestGrammar('grammars/book_grammars/german.fcfg', germantestsuites)
+ >>> tester.run()
+ Tests for person agreement: All tests passed!
+ Tests for number agreement: All tests passed!
+ Tests for case government and subcategorization: All tests passed!
diff --git a/nltk/test/index.doctest b/nltk/test/index.doctest
new file mode 100644
index 0000000..cdfd104
--- /dev/null
+++ b/nltk/test/index.doctest
@@ -0,0 +1,100 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+.. _align howto: align.html
+.. _ccg howto: ccg.html
+.. _chat80 howto: chat80.html
+.. _childes howto: childes.html
+.. _chunk howto: chunk.html
+.. _classify howto: classify.html
+.. _collocations howto: collocations.html
+.. _compat howto: compat.html
+.. _corpus howto: corpus.html
+.. _data howto: data.html
+.. _dependency howto: dependency.html
+.. _discourse howto: discourse.html
+.. _drt howto: drt.html
+.. _featgram howto: featgram.html
+.. _featstruct howto: featstruct.html
+.. _framenet howto: framenet.html
+.. _generate howto: generate.html
+.. _gluesemantics howto: gluesemantics.html
+.. _gluesemantics_malt howto: gluesemantics_malt.html
+.. _grammar howto: grammar.html
+.. _grammartestsuites howto: grammartestsuites.html
+.. _index howto: index.html
+.. _inference howto: inference.html
+.. _internals howto: internals.html
+.. _japanese howto: japanese.html
+.. _logic howto: logic.html
+.. _metrics howto: metrics.html
+.. _misc howto: misc.html
+.. _nonmonotonic howto: nonmonotonic.html
+.. _parse howto: parse.html
+.. _portuguese_en howto: portuguese_en.html
+.. _probability howto: probability.html
+.. _propbank howto: propbank.html
+.. _relextract howto: relextract.html
+.. _resolution howto: resolution.html
+.. _semantics howto: semantics.html
+.. _simple howto: simple.html
+.. _stem howto: stem.html
+.. _tag howto: tag.html
+.. _tokenize howto: tokenize.html
+.. _toolbox howto: toolbox.html
+.. _tree howto: tree.html
+.. _treetransforms howto: treetransforms.html
+.. _util howto: util.html
+.. _wordnet howto: wordnet.html
+.. _wordnet_lch howto: wordnet_lch.html
+
+===========
+NLTK HOWTOs
+===========
+
+* `align HOWTO`_
+* `ccg HOWTO`_
+* `chat80 HOWTO`_
+* `childes HOWTO`_
+* `chunk HOWTO`_
+* `classify HOWTO`_
+* `collocations HOWTO`_
+* `compat HOWTO`_
+* `corpus HOWTO`_
+* `data HOWTO`_
+* `dependency HOWTO`_
+* `discourse HOWTO`_
+* `drt HOWTO`_
+* `featgram HOWTO`_
+* `featstruct HOWTO`_
+* `framenet HOWTO`_
+* `generate HOWTO`_
+* `gluesemantics HOWTO`_
+* `gluesemantics_malt HOWTO`_
+* `grammar HOWTO`_
+* `grammartestsuites HOWTO`_
+* `index HOWTO`_
+* `inference HOWTO`_
+* `internals HOWTO`_
+* `japanese HOWTO`_
+* `logic HOWTO`_
+* `metrics HOWTO`_
+* `misc HOWTO`_
+* `nonmonotonic HOWTO`_
+* `parse HOWTO`_
+* `portuguese_en HOWTO`_
+* `probability HOWTO`_
+* `propbank HOWTO`_
+* `relextract HOWTO`_
+* `resolution HOWTO`_
+* `semantics HOWTO`_
+* `simple HOWTO`_
+* `stem HOWTO`_
+* `tag HOWTO`_
+* `tokenize HOWTO`_
+* `toolbox HOWTO`_
+* `tree HOWTO`_
+* `treetransforms HOWTO`_
+* `util HOWTO`_
+* `wordnet HOWTO`_
+* `wordnet_lch HOWTO`_
diff --git a/nltk/test/inference.doctest b/nltk/test/inference.doctest
new file mode 100644
index 0000000..08d5df2
--- /dev/null
+++ b/nltk/test/inference.doctest
@@ -0,0 +1,533 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+====================================
+Logical Inference and Model Building
+====================================
+
+ >>> from nltk import *
+ >>> from nltk.sem.drt import DrtParser
+ >>> from nltk.sem import logic
+ >>> logic._counter._value = 0
+
+------------
+Introduction
+------------
+
+Within the area of automated reasoning, first order theorem proving
+and model building (or model generation) have both received much
+attention, and have given rise to highly sophisticated techniques. We
+focus therefore on providing an NLTK interface to third party tools
+for these tasks. In particular, the module ``nltk.inference`` can be
+used to access both theorem provers and model builders.
+
+---------------------------------
+NLTK Interface to Theorem Provers
+---------------------------------
+
+The main class used to interface with a theorem prover is the ``Prover``
+class, found in ``nltk.api``. The ``prove()`` method takes three optional
+arguments: a goal, a list of assumptions, and a ``verbose`` boolean to
+indicate whether the proof should be printed to the console. The proof goal
+and any assumptions need to be instances of the ``Expression`` class
+specified by ``nltk.sem.logic``. There are currently three theorem provers
+included with NLTK: ``Prover9``, ``TableauProver``, and
+``ResolutionProver``. The first is an off-the-shelf prover, while the other
+two are written in Python and included in the ``nltk.inference`` package.
+
+ >>> lp = LogicParser()
+ >>> p1 = lp.parse('man(socrates)')
+ >>> p2 = lp.parse('all x.(man(x) -> mortal(x))')
+ >>> c = lp.parse('mortal(socrates)')
+ >>> Prover9().prove(c, [p1,p2])
+ True
+ >>> TableauProver().prove(c, [p1,p2])
+ True
+ >>> ResolutionProver().prove(c, [p1,p2], verbose=True)
+ [1] {-mortal(socrates)} A
+ [2] {man(socrates)} A
+ [3] {-man(z2), mortal(z2)} A
+ [4] {-man(socrates)} (1, 3)
+ [5] {mortal(socrates)} (2, 3)
+ [6] {} (1, 5)
+ <BLANKLINE>
+ True
+
+---------------------
+The ``ProverCommand``
+---------------------
+
+A ``ProverCommand`` is a stateful holder for a theorem
+prover. The command stores a theorem prover instance (of type ``Prover``),
+a goal, a list of assumptions, the result of the proof, and a string version
+of the entire proof. Corresponding to the three included ``Prover``
+implementations, there are three ``ProverCommand`` implementations:
+``Prover9Command``, ``TableauProverCommand``, and
+``ResolutionProverCommand``.
+
+The ``ProverCommand``'s constructor takes its goal and assumptions. The
+``prove()`` command executes the ``Prover`` and ``proof()``
+returns a String form of the proof
+If the ``prove()`` method has not been called,
+then the prover command will be unable to display a proof.
+
+ >>> prover = ResolutionProverCommand(c, [p1,p2])
+ >>> print(prover.proof()) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ File "...", line 1212, in __run
+ compileflags, 1) in test.globs
+ File "<doctest nltk/test/inference.doctest[10]>", line 1, in <module>
+ File "...", line ..., in proof
+ raise LookupError("You have to call prove() first to get a proof!")
+ LookupError: You have to call prove() first to get a proof!
+ >>> prover.prove()
+ True
+ >>> print(prover.proof())
+ [1] {-mortal(socrates)} A
+ [2] {man(socrates)} A
+ [3] {-man(z4), mortal(z4)} A
+ [4] {-man(socrates)} (1, 3)
+ [5] {mortal(socrates)} (2, 3)
+ [6] {} (1, 5)
+ <BLANKLINE>
+
+The prover command stores the result of proving so that if ``prove()`` is
+called again, then the command can return the result without executing the
+prover again. This allows the user to access the result of the proof without
+wasting time re-computing what it already knows.
+
+ >>> prover.prove()
+ True
+ >>> prover.prove()
+ True
+
+The assumptions and goal may be accessed using the ``assumptions()`` and
+``goal()`` methods, respectively.
+
+ >>> prover.assumptions()
+ [<ApplicationExpression man(socrates)>, <AllExpression all x.(man(x) -> mortal(x))>]
+ >>> prover.goal()
+ <ApplicationExpression mortal(socrates)>
+
+The assumptions list may be modified using the ``add_assumptions()`` and
+``retract_assumptions()`` methods. Both methods take a list of ``Expression``
+objects. Since adding or removing assumptions may change the result of the
+proof, the stored result is cleared when either of these methods are called.
+That means that ``proof()`` will be unavailable until ``prove()`` is called and
+a call to ``prove()`` will execute the theorem prover.
+
+ >>> prover.retract_assumptions([lp.parse('man(socrates)')])
+ >>> print(prover.proof()) # doctest: +ELLIPSIS
+ Traceback (most recent call last):
+ File "...", line 1212, in __run
+ compileflags, 1) in test.globs
+ File "<doctest nltk/test/inference.doctest[10]>", line 1, in <module>
+ File "...", line ..., in proof
+ raise LookupError("You have to call prove() first to get a proof!")
+ LookupError: You have to call prove() first to get a proof!
+ >>> prover.prove()
+ False
+ >>> print(prover.proof())
+ [1] {-mortal(socrates)} A
+ [2] {-man(z6), mortal(z6)} A
+ [3] {-man(socrates)} (1, 2)
+ <BLANKLINE>
+ >>> prover.add_assumptions([lp.parse('man(socrates)')])
+ >>> prover.prove()
+ True
+
+-------
+Prover9
+-------
+
+Prover9 Installation
+~~~~~~~~~~~~~~~~~~~~
+
+You can download Prover9 from http://www.cs.unm.edu/~mccune/prover9/.
+
+Extract the source code into a suitable directory and follow the
+instructions in the Prover9 ``README.make`` file to compile the executables.
+Install these into an appropriate location; the
+``prover9_search`` variable is currently configured to look in the
+following locations:
+
+ >>> p = Prover9()
+ >>> p.binary_locations() # doctest: +NORMALIZE_WHITESPACE
+ ['/usr/local/bin/prover9',
+ '/usr/local/bin/prover9/bin',
+ '/usr/local/bin',
+ '/usr/bin',
+ '/usr/local/prover9',
+ '/usr/local/share/prover9']
+
+Alternatively, the environment variable ``PROVER9HOME`` may be configured with
+the binary's location.
+
+The path to the correct directory can be set manually in the following
+manner:
+
+ >>> config_prover9(path='/usr/local/bin') # doctest: +SKIP
+ [Found prover9: /usr/local/bin/prover9]
+
+If the executables cannot be found, ``Prover9`` will issue a warning message:
+
+ >>> p.prove() # doctest: +SKIP
+ Traceback (most recent call last):
+ ...
+ LookupError:
+ ===========================================================================
+ NLTK was unable to find the prover9 executable! Use config_prover9() or
+ set the PROVER9HOME environment variable.
+ <BLANKLINE>
+ >> config_prover9('/path/to/prover9')
+ <BLANKLINE>
+ For more information, on prover9, see:
+ <http://www.cs.unm.edu/~mccune/prover9/>
+ ===========================================================================
+
+
+Using Prover9
+~~~~~~~~~~~~~
+
+The general case in theorem proving is to determine whether ``S |- g``
+holds, where ``S`` is a possibly empty set of assumptions, and ``g``
+is a proof goal.
+
+As mentioned earlier, NLTK input to ``Prover9`` must be
+``Expression``\ s of ``nltk.sem.logic``. A ``Prover9`` instance is
+initialized with a proof goal and, possibly, some assumptions. The
+``prove()`` method attempts to find a proof of the goal, given the
+list of assumptions (in this case, none).
+
+ >>> goal = lp.parse('(man(x) <-> --man(x))')
+ >>> prover = Prover9Command(goal)
+ >>> prover.prove()
+ True
+
+Given a ``ProverCommand`` instance ``prover``, the method
+``prover.proof()`` will return a String of the extensive proof information
+provided by Prover9, shown in abbreviated form here::
+
+ ============================== Prover9 ===============================
+ Prover9 (32) version ...
+ Process ... was started by ... on ...
+ ...
+ The command was ".../prover9 -f ...".
+ ============================== end of head ===========================
+
+ ============================== INPUT =================================
+
+ % Reading from file /var/...
+
+
+ formulas(goals).
+ (all x (man(x) -> man(x))).
+ end_of_list.
+
+ ...
+ ============================== end of search =========================
+
+ THEOREM PROVED
+
+ Exiting with 1 proof.
+
+ Process 6317 exit (max_proofs) Mon Jan 21 15:23:28 2008
+
+
+As mentioned earlier, we may want to list some assumptions for
+the proof, as shown here.
+
+ >>> g = lp.parse('mortal(socrates)')
+ >>> a1 = lp.parse('all x.(man(x) -> mortal(x))')
+ >>> prover = Prover9Command(g, assumptions=[a1])
+ >>> prover.print_assumptions()
+ all x.(man(x) -> mortal(x))
+
+However, the assumptions are not sufficient to derive the goal:
+
+ >>> print(prover.prove())
+ False
+
+So let's add another assumption:
+
+ >>> a2 = lp.parse('man(socrates)')
+ >>> prover.add_assumptions([a2])
+ >>> prover.print_assumptions()
+ all x.(man(x) -> mortal(x))
+ man(socrates)
+ >>> print(prover.prove())
+ True
+
+We can also show the assumptions in ``Prover9`` format.
+
+ >>> prover.print_assumptions(output_format='Prover9')
+ all x (man(x) -> mortal(x))
+ man(socrates)
+
+ >>> prover.print_assumptions(output_format='Spass')
+ Traceback (most recent call last):
+ . . .
+ NameError: Unrecognized value for 'output_format': Spass
+
+Assumptions can be retracted from the list of assumptions.
+
+ >>> prover.retract_assumptions([a1])
+ >>> prover.print_assumptions()
+ man(socrates)
+ >>> prover.retract_assumptions([a1])
+
+Statements can be loaded from a file and parsed. We can then add these
+statements as new assumptions.
+
+ >>> g = lp.parse('all x.(boxer(x) -> -boxerdog(x))')
+ >>> prover = Prover9Command(g)
+ >>> prover.prove()
+ False
+ >>> import nltk.data
+ >>> new = nltk.data.load('grammars/sample_grammars/background0.fol')
+ >>> for a in new:
+ ... print(a)
+ all x.(boxerdog(x) -> dog(x))
+ all x.(boxer(x) -> person(x))
+ all x.-(dog(x) & person(x))
+ exists x.boxer(x)
+ exists x.boxerdog(x)
+ >>> prover.add_assumptions(new)
+ >>> print(prover.prove())
+ True
+ >>> print(prover.proof()) # doctest: +ELLIPSIS
+ ============================== prooftrans ============================
+ Prover9 (...) version ...
+ Process ... was started by ... on ...
+ ...
+ The command was ".../prover9".
+ ============================== end of head ===========================
+ <BLANKLINE>
+ ============================== end of input ==========================
+ <BLANKLINE>
+ ============================== PROOF =================================
+ <BLANKLINE>
+ % -------- Comments from original proof --------
+ % Proof 1 at ... seconds.
+ % Length of proof is 13.
+ % Level of proof is 4.
+ % Maximum clause weight is 0.000.
+ % Given clauses 0.
+ <BLANKLINE>
+ <BLANKLINE>
+ 1 (all x (boxerdog(x) -> dog(x))). [assumption].
+ 2 (all x (boxer(x) -> person(x))). [assumption].
+ 3 (all x -(dog(x) & person(x))). [assumption].
+ 6 (all x (boxer(x) -> -boxerdog(x))). [goal].
+ 8 -boxerdog(x) | dog(x). [clausify(1)].
+ 9 boxerdog(c3). [deny(6)].
+ 11 -boxer(x) | person(x). [clausify(2)].
+ 12 boxer(c3). [deny(6)].
+ 14 -dog(x) | -person(x). [clausify(3)].
+ 15 dog(c3). [resolve(9,a,8,a)].
+ 18 person(c3). [resolve(12,a,11,a)].
+ 19 -person(c3). [resolve(15,a,14,a)].
+ 20 $F. [resolve(19,a,18,a)].
+ <BLANKLINE>
+ ============================== end of proof ==========================
+
+----------------------
+The equiv() method
+----------------------
+
+One application of the theorem prover functionality is to check if
+two Expressions have the same meaning.
+The ``equiv()`` method calls a theorem prover to determine whether two
+Expressions are logically equivalent.
+
+ >>> a = lp.parse(r'exists x.(man(x) & walks(x))')
+ >>> b = lp.parse(r'exists x.(walks(x) & man(x))')
+ >>> print(a.equiv(b))
+ True
+
+The same method can be used on Discourse Representation Structures (DRSs).
+In this case, each DRS is converted to a first order logic form, and then
+passed to the theorem prover.
+
+ >>> dp = DrtParser()
+ >>> a = dp.parse(r'([x],[man(x), walks(x)])')
+ >>> b = dp.parse(r'([x],[walks(x), man(x)])')
+ >>> print(a.equiv(b))
+ True
+
+
+--------------------------------
+NLTK Interface to Model Builders
+--------------------------------
+
+The top-level to model builders is parallel to that for
+theorem-provers. The ``ModelBuilder`` interface is located
+in ``nltk.inference.api``. It is currently only implemented by
+``Mace``, which interfaces with the Mace4 model builder.
+
+Typically we use a model builder to show that some set of formulas has
+a model, and is therefore consistent. One way of doing this is by
+treating our candidate set of sentences as assumptions, and leaving
+the goal unspecified.
+Thus, the following interaction shows how both ``{a, c1}`` and ``{a, c2}``
+are consistent sets, since Mace succeeds in a building a
+model for each of them, while ``{c1, c2}`` is inconsistent.
+
+ >>> a3 = lp.parse('exists x.(man(x) and walks(x))')
+ >>> c1 = lp.parse('mortal(socrates)')
+ >>> c2 = lp.parse('-mortal(socrates)')
+ >>> mace = Mace()
+ >>> print(mace.build_model(None, [a3, c1]))
+ True
+ >>> print(mace.build_model(None, [a3, c2]))
+ True
+
+We can also use the model builder as an adjunct to theorem prover.
+Let's suppose we are trying to prove ``S |- g``, i.e. that ``g``
+is logically entailed by assumptions ``S = {s1, s2, ..., sn}``.
+We can this same input to Mace4, and the model builder will try to
+find a counterexample, that is, to show that ``g`` does *not* follow
+from ``S``. So, given this input, Mace4 will try to find a model for
+the set ``S' = {s1, s2, ..., sn, (not g)}``. If ``g`` fails to follow
+from ``S``, then Mace4 may well return with a counterexample faster
+than Prover9 concludes that it cannot find the required proof.
+Conversely, if ``g`` *is* provable from ``S``, Mace4 may take a long
+time unsuccessfully trying to find a counter model, and will eventually give up.
+
+In the following example, we see that the model builder does succeed
+in building a model of the assumptions together with the negation of
+the goal. That is, it succeeds in finding a model
+where there is a woman that every man loves; Adam is a man; Eve is a
+woman; but Adam does not love Eve.
+
+ >>> a4 = lp.parse('exists y. (woman(y) & all x. (man(x) -> love(x,y)))')
+ >>> a5 = lp.parse('man(adam)')
+ >>> a6 = lp.parse('woman(eve)')
+ >>> g = lp.parse('love(adam,eve)')
+ >>> print(mace.build_model(g, [a4, a5, a6]))
+ True
+
+The Model Builder will fail to find a model if the assumptions do entail
+the goal. Mace will continue to look for models of ever-increasing sizes
+until the end_size number is reached. By default, end_size is 500,
+but it can be set manually for quicker response time.
+
+ >>> a7 = lp.parse('all x.(man(x) -> mortal(x))')
+ >>> a8 = lp.parse('man(socrates)')
+ >>> g2 = lp.parse('mortal(socrates)')
+ >>> print(Mace(end_size=50).build_model(g2, [a7, a8]))
+ False
+
+There is also a ``ModelBuilderCommand`` class that, like ``ProverCommand``,
+stores a ``ModelBuilder``, a goal, assumptions, a result, and a model. The
+only implementation in NLTK is ``MaceCommand``.
+
+
+-----
+Mace4
+-----
+
+Mace4 Installation
+~~~~~~~~~~~~~~~~~~
+
+Mace4 is packaged with Prover9, and can be downloaded from the same
+source, namely http://www.cs.unm.edu/~mccune/prover9/. It is installed
+in the same manner as Prover9.
+
+Using Mace4
+~~~~~~~~~~~
+
+Check whether Mace4 can find a model.
+
+ >>> a = lp.parse('(see(mary,john) & -(mary = john))')
+ >>> mb = MaceCommand(assumptions=[a])
+ >>> mb.build_model()
+ True
+
+Show the model in 'tabular' format.
+
+ >>> print(mb.model(format='tabular'))
+ % number = 1
+ % seconds = 0
+ <BLANKLINE>
+ % Interpretation of size 2
+ <BLANKLINE>
+ john : 0
+ <BLANKLINE>
+ mary : 1
+ <BLANKLINE>
+ see :
+ | 0 1
+ ---+----
+ 0 | 0 0
+ 1 | 1 0
+ <BLANKLINE>
+
+Show the model in 'tabular' format.
+
+ >>> print(mb.model(format='cooked'))
+ % number = 1
+ % seconds = 0
+ <BLANKLINE>
+ % Interpretation of size 2
+ <BLANKLINE>
+ john = 0.
+ <BLANKLINE>
+ mary = 1.
+ <BLANKLINE>
+ - see(0,0).
+ - see(0,1).
+ see(1,0).
+ - see(1,1).
+ <BLANKLINE>
+
+The property ``valuation`` accesses the stored ``Valuation``.
+
+ >>> print(mb.valuation)
+ {'john': 'a', 'mary': 'b', 'see': {('b', 'a')}}
+
+We can return to our earlier example and inspect the model:
+
+ >>> mb = MaceCommand(g, assumptions=[a4, a5, a6])
+ >>> m = mb.build_model()
+ >>> print(mb.model(format='cooked'))
+ % number = 1
+ % seconds = 0
+ <BLANKLINE>
+ % Interpretation of size 2
+ <BLANKLINE>
+ adam = 0.
+ <BLANKLINE>
+ eve = 0.
+ <BLANKLINE>
+ c1 = 1.
+ <BLANKLINE>
+ man(0).
+ - man(1).
+ <BLANKLINE>
+ woman(0).
+ woman(1).
+ <BLANKLINE>
+ - love(0,0).
+ love(0,1).
+ - love(1,0).
+ - love(1,1).
+ <BLANKLINE>
+
+Here, we can see that ``adam`` and ``eve`` have been assigned the same
+individual, namely ``0`` as value; ``0`` is both a man and a woman; a second
+individual ``1`` is also a woman; and ``0`` loves ``1``. Thus, this is
+an interpretation in which there is a woman that every man loves but
+Adam doesn't love Eve.
+
+Mace can also be used with propositional logic.
+
+ >>> p = lp.parse('P')
+ >>> q = lp.parse('Q')
+ >>> mb = MaceCommand(q, [p, p>-q])
+ >>> mb.build_model()
+ True
+ >>> mb.valuation['P']
+ True
+ >>> mb.valuation['Q']
+ False
diff --git a/nltk/test/inference_fixt.py b/nltk/test/inference_fixt.py
new file mode 100644
index 0000000..e54f869
--- /dev/null
+++ b/nltk/test/inference_fixt.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+def setup_module(module):
+ from nose import SkipTest
+ from nltk.inference.mace import Mace
+ try:
+ m = Mace()
+ m._find_binary('mace4')
+ except LookupError:
+ raise SkipTest("Mace4/Prover9 is not available so inference.doctest was skipped")
diff --git a/nltk/test/internals.doctest b/nltk/test/internals.doctest
new file mode 100644
index 0000000..bc1d673
--- /dev/null
+++ b/nltk/test/internals.doctest
@@ -0,0 +1,140 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==========================================
+ Unit tests for the nltk.utilities module
+==========================================
+
+overridden()
+~~~~~~~~~~~~
+ >>> from nltk.internals import overridden
+
+The typical use case is in defining methods for an interface or
+abstract base class, in such a way that subclasses don't have to
+implement all of the methods:
+
+ >>> class EaterI(object):
+ ... '''Subclass must define eat() or batch_eat().'''
+ ... def eat(self, food):
+ ... if overridden(self.batch_eat):
+ ... return self.batch_eat([food])[0]
+ ... else:
+ ... raise NotImplementedError()
+ ... def batch_eat(self, foods):
+ ... return [self.eat(food) for food in foods]
+
+As long as a subclass implements one method, it will be used to
+perform the other method:
+
+ >>> class GoodEater1(EaterI):
+ ... def eat(self, food):
+ ... return 'yum'
+ >>> GoodEater1().eat('steak')
+ 'yum'
+ >>> GoodEater1().batch_eat(['steak', 'peas'])
+ ['yum', 'yum']
+
+ >>> class GoodEater2(EaterI):
+ ... def batch_eat(self, foods):
+ ... return ['yum' for food in foods]
+ >>> GoodEater2().eat('steak')
+ 'yum'
+ >>> GoodEater2().batch_eat(['steak', 'peas'])
+ ['yum', 'yum']
+
+But if a subclass doesn't implement either one, then they'll get an
+error when they try to call them. (nb this is better than infinite
+recursion):
+
+ >>> class BadEater1(EaterI):
+ ... pass
+ >>> BadEater1().eat('steak')
+ Traceback (most recent call last):
+ . . .
+ NotImplementedError
+ >>> BadEater1().batch_eat(['steak', 'peas'])
+ Traceback (most recent call last):
+ . . .
+ NotImplementedError
+
+Trying to use the abstract base class itself will also result in an
+error:
+
+ >>> class EaterI(EaterI):
+ ... pass
+ >>> EaterI().eat('steak')
+ Traceback (most recent call last):
+ . . .
+ NotImplementedError
+ >>> EaterI().batch_eat(['steak', 'peas'])
+ Traceback (most recent call last):
+ . . .
+ NotImplementedError
+
+It's ok to use intermediate abstract classes:
+
+ >>> class AbstractEater(EaterI):
+ ... pass
+
+ >>> class GoodEater3(AbstractEater):
+ ... def eat(self, food):
+ ... return 'yum'
+ ...
+ >>> GoodEater3().eat('steak')
+ 'yum'
+ >>> GoodEater3().batch_eat(['steak', 'peas'])
+ ['yum', 'yum']
+
+ >>> class GoodEater4(AbstractEater):
+ ... def batch_eat(self, foods):
+ ... return ['yum' for food in foods]
+ >>> GoodEater4().eat('steak')
+ 'yum'
+ >>> GoodEater4().batch_eat(['steak', 'peas'])
+ ['yum', 'yum']
+
+ >>> class BadEater2(AbstractEater):
+ ... pass
+ >>> BadEater2().eat('steak')
+ Traceback (most recent call last):
+ . . .
+ NotImplementedError
+ >>> BadEater2().batch_eat(['steak', 'peas'])
+ Traceback (most recent call last):
+ . . .
+ NotImplementedError
+
+Here's some extra tests:
+
+ >>> class A(object):
+ ... def f(x): pass
+ >>> class B(A):
+ ... def f(x): pass
+ >>> class C(A): pass
+ >>> class D(B): pass
+
+ >>> overridden(A().f)
+ False
+ >>> overridden(B().f)
+ True
+ >>> overridden(C().f)
+ False
+ >>> overridden(D().f)
+ True
+
+It works for classic classes, too:
+
+ >>> class A:
+ ... def f(x): pass
+ >>> class B(A):
+ ... def f(x): pass
+ >>> class C(A): pass
+ >>> class D(B): pass
+ >>> overridden(A().f)
+ False
+ >>> overridden(B().f)
+ True
+ >>> overridden(C().f)
+ False
+ >>> overridden(D().f)
+ True
diff --git a/nltk/test/japanese.doctest b/nltk/test/japanese.doctest
new file mode 100644
index 0000000..28c6f99
--- /dev/null
+++ b/nltk/test/japanese.doctest
@@ -0,0 +1,48 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+============================
+Japanese Language Processing
+============================
+
+ >>> from nltk import *
+
+-------------
+Corpus Access
+-------------
+
+KNB Corpus
+----------
+
+ >>> from nltk.corpus import knbc
+
+Access the words: this should produce a list of strings:
+
+ >>> type(knbc.words()[0]) is not bytes
+ True
+
+Access the sentences: this should produce a list of lists of strings:
+
+ >>> type(knbc.sents()[0][0]) is not bytes
+ True
+
+Access the tagged words: this should produce a list of word, tag pairs:
+
+ >>> type(knbc.tagged_words()[0])
+ <... 'tuple'>
+
+Access the tagged sentences: this should produce a list of lists of word, tag pairs:
+
+ >>> type(knbc.tagged_sents()[0][0])
+ <... 'tuple'>
+
+
+JEITA Corpus
+------------
+
+ >>> from nltk.corpus import jeita
+
+Access the tagged words: this should produce a list of word, tag pairs, where a tag is a string:
+
+ >>> type(jeita.tagged_words()[0][1]) is not bytes
+ True
diff --git a/nltk/test/logic.doctest b/nltk/test/logic.doctest
new file mode 100644
index 0000000..2e32a60
--- /dev/null
+++ b/nltk/test/logic.doctest
@@ -0,0 +1,1098 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=======================
+Logic & Lambda Calculus
+=======================
+
+The `nltk.logic` package allows expressions of First-Order Logic (FOL) to be
+parsed into ``Expression`` objects. In addition to FOL, the parser
+handles lambda-abstraction with variables of higher order.
+
+--------
+Overview
+--------
+
+ >>> from nltk.sem.logic import *
+
+The default inventory of logical constants is the following:
+
+ >>> boolean_ops() # doctest: +NORMALIZE_WHITESPACE
+ negation -
+ conjunction &
+ disjunction |
+ implication ->
+ equivalence <->
+ >>> equality_preds() # doctest: +NORMALIZE_WHITESPACE
+ equality =
+ inequality !=
+ >>> binding_ops() # doctest: +NORMALIZE_WHITESPACE
+ existential exists
+ universal all
+ lambda \
+
+----------------
+Regression Tests
+----------------
+
+
+Untyped Logic
++++++++++++++
+
+Process logical expressions conveniently:
+
+ >>> lexpr = Expression.fromstring
+
+Test for equality under alpha-conversion
+========================================
+
+ >>> e1 = lexpr('exists x.P(x)')
+ >>> print(e1)
+ exists x.P(x)
+ >>> e2 = e1.alpha_convert(Variable('z'))
+ >>> print(e2)
+ exists z.P(z)
+ >>> e1 == e2
+ True
+
+
+ >>> l = lexpr(r'\X.\X.X(X)(1)').simplify()
+ >>> id = lexpr(r'\X.X(X)')
+ >>> l == id
+ True
+
+Test numerals
+=============
+
+ >>> zero = lexpr(r'\F x.x')
+ >>> one = lexpr(r'\F x.F(x)')
+ >>> two = lexpr(r'\F x.F(F(x))')
+ >>> three = lexpr(r'\F x.F(F(F(x)))')
+ >>> four = lexpr(r'\F x.F(F(F(F(x))))')
+ >>> succ = lexpr(r'\N F x.F(N(F,x))')
+ >>> plus = lexpr(r'\M N F x.M(F,N(F,x))')
+ >>> mult = lexpr(r'\M N F.M(N(F))')
+ >>> pred = lexpr(r'\N F x.(N(\G H.H(G(F)))(\u.x)(\u.u))')
+ >>> v1 = ApplicationExpression(succ, zero).simplify()
+ >>> v1 == one
+ True
+ >>> v2 = ApplicationExpression(succ, v1).simplify()
+ >>> v2 == two
+ True
+ >>> v3 = ApplicationExpression(ApplicationExpression(plus, v1), v2).simplify()
+ >>> v3 == three
+ True
+ >>> v4 = ApplicationExpression(ApplicationExpression(mult, v2), v2).simplify()
+ >>> v4 == four
+ True
+ >>> v5 = ApplicationExpression(pred, ApplicationExpression(pred, v4)).simplify()
+ >>> v5 == two
+ True
+
+Overloaded operators also exist, for convenience.
+
+ >>> print(succ(zero).simplify() == one)
+ True
+ >>> print(plus(one,two).simplify() == three)
+ True
+ >>> print(mult(two,two).simplify() == four)
+ True
+ >>> print(pred(pred(four)).simplify() == two)
+ True
+
+ >>> john = lexpr(r'john')
+ >>> man = lexpr(r'\x.man(x)')
+ >>> walk = lexpr(r'\x.walk(x)')
+ >>> man(john).simplify()
+ <ApplicationExpression man(john)>
+ >>> print(-walk(john).simplify())
+ -walk(john)
+ >>> print((man(john) & walk(john)).simplify())
+ (man(john) & walk(john))
+ >>> print((man(john) | walk(john)).simplify())
+ (man(john) | walk(john))
+ >>> print((man(john) > walk(john)).simplify())
+ (man(john) -> walk(john))
+ >>> print((man(john) < walk(john)).simplify())
+ (man(john) <-> walk(john))
+
+Python's built-in lambda operator can also be used with Expressions
+
+ >>> john = VariableExpression(Variable('john'))
+ >>> run_var = VariableExpression(Variable('run'))
+ >>> run = lambda x: run_var(x)
+ >>> run(john)
+ <ApplicationExpression run(john)>
+
+
+``betaConversionTestSuite.pl``
+------------------------------
+
+Tests based on Blackburn & Bos' book, *Representation and Inference
+for Natural Language*.
+
+ >>> x1 = lexpr(r'\P.P(mia)(\x.walk(x))').simplify()
+ >>> x2 = lexpr(r'walk(mia)').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'exists x.(man(x) & ((\P.exists x.(woman(x) & P(x)))(\y.love(x,y))))').simplify()
+ >>> x2 = lexpr(r'exists x.(man(x) & exists y.(woman(y) & love(x,y)))').simplify()
+ >>> x1 == x2
+ True
+ >>> x1 = lexpr(r'\a.sleep(a)(mia)').simplify()
+ >>> x2 = lexpr(r'sleep(mia)').simplify()
+ >>> x1 == x2
+ True
+ >>> x1 = lexpr(r'\a.\b.like(b,a)(mia)').simplify()
+ >>> x2 = lexpr(r'\b.like(b,mia)').simplify()
+ >>> x1 == x2
+ True
+ >>> x1 = lexpr(r'\a.(\b.like(b,a)(vincent))').simplify()
+ >>> x2 = lexpr(r'\a.like(vincent,a)').simplify()
+ >>> x1 == x2
+ True
+ >>> x1 = lexpr(r'\a.((\b.like(b,a)(vincent)) & sleep(a))').simplify()
+ >>> x2 = lexpr(r'\a.(like(vincent,a) & sleep(a))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'(\a.\b.like(b,a)(mia)(vincent))').simplify()
+ >>> x2 = lexpr(r'like(vincent,mia)').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'P((\a.sleep(a)(vincent)))').simplify()
+ >>> x2 = lexpr(r'P(sleep(vincent))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'\A.A((\b.sleep(b)(vincent)))').simplify()
+ >>> x2 = lexpr(r'\A.A(sleep(vincent))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'\A.A(sleep(vincent))').simplify()
+ >>> x2 = lexpr(r'\A.A(sleep(vincent))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'(\A.A(vincent)(\b.sleep(b)))').simplify()
+ >>> x2 = lexpr(r'sleep(vincent)').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'\A.believe(mia,A(vincent))(\b.sleep(b))').simplify()
+ >>> x2 = lexpr(r'believe(mia,sleep(vincent))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'(\A.(A(vincent) & A(mia)))(\b.sleep(b))').simplify()
+ >>> x2 = lexpr(r'(sleep(vincent) & sleep(mia))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'\A.\B.(\C.C(A(vincent))(\d.probably(d)) & (\C.C(B(mia))(\d.improbably(d))))(\f.walk(f))(\f.talk(f))').simplify()
+ >>> x2 = lexpr(r'(probably(walk(vincent)) & improbably(talk(mia)))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'(\a.\b.(\C.C(a,b)(\d.\f.love(d,f))))(jules)(mia)').simplify()
+ >>> x2 = lexpr(r'love(jules,mia)').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'(\A.\B.exists c.(A(c) & B(c)))(\d.boxer(d),\d.sleep(d))').simplify()
+ >>> x2 = lexpr(r'exists c.(boxer(c) & sleep(c))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'\A.Z(A)(\c.\a.like(a,c))').simplify()
+ >>> x2 = lexpr(r'Z(\c.\a.like(a,c))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'\A.\b.A(b)(\c.\b.like(b,c))').simplify()
+ >>> x2 = lexpr(r'\b.(\c.\b.like(b,c)(b))').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'(\a.\b.(\C.C(a,b)(\b.\a.loves(b,a))))(jules)(mia)').simplify()
+ >>> x2 = lexpr(r'loves(jules,mia)').simplify()
+ >>> x1 == x2
+ True
+
+ >>> x1 = lexpr(r'(\A.\b.(exists b.A(b) & A(b)))(\c.boxer(c))(vincent)').simplify()
+ >>> x2 = lexpr(r'((exists b.boxer(b)) & boxer(vincent))').simplify()
+ >>> x1 == x2
+ True
+
+Test Parser
+===========
+
+ >>> print(lexpr(r'john'))
+ john
+ >>> print(lexpr(r'x'))
+ x
+ >>> print(lexpr(r'-man(x)'))
+ -man(x)
+ >>> print(lexpr(r'--man(x)'))
+ --man(x)
+ >>> print(lexpr(r'(man(x))'))
+ man(x)
+ >>> print(lexpr(r'((man(x)))'))
+ man(x)
+ >>> print(lexpr(r'man(x) <-> tall(x)'))
+ (man(x) <-> tall(x))
+ >>> print(lexpr(r'(man(x) <-> tall(x))'))
+ (man(x) <-> tall(x))
+ >>> print(lexpr(r'(man(x) & tall(x) & walks(x))'))
+ (man(x) & tall(x) & walks(x))
+ >>> print(lexpr(r'(man(x) & tall(x) & walks(x))').first)
+ (man(x) & tall(x))
+ >>> print(lexpr(r'man(x) | tall(x) & walks(x)'))
+ (man(x) | (tall(x) & walks(x)))
+ >>> print(lexpr(r'((man(x) & tall(x)) | walks(x))'))
+ ((man(x) & tall(x)) | walks(x))
+ >>> print(lexpr(r'man(x) & (tall(x) | walks(x))'))
+ (man(x) & (tall(x) | walks(x)))
+ >>> print(lexpr(r'(man(x) & (tall(x) | walks(x)))'))
+ (man(x) & (tall(x) | walks(x)))
+ >>> print(lexpr(r'P(x) -> Q(x) <-> R(x) | S(x) & T(x)'))
+ ((P(x) -> Q(x)) <-> (R(x) | (S(x) & T(x))))
+ >>> print(lexpr(r'exists x.man(x)'))
+ exists x.man(x)
+ >>> print(lexpr(r'exists x.(man(x) & tall(x))'))
+ exists x.(man(x) & tall(x))
+ >>> print(lexpr(r'exists x.(man(x) & tall(x) & walks(x))'))
+ exists x.(man(x) & tall(x) & walks(x))
+ >>> print(lexpr(r'-P(x) & Q(x)'))
+ (-P(x) & Q(x))
+ >>> lexpr(r'-P(x) & Q(x)') == lexpr(r'(-P(x)) & Q(x)')
+ True
+ >>> print(lexpr(r'\x.man(x)'))
+ \x.man(x)
+ >>> print(lexpr(r'\x.man(x)(john)'))
+ \x.man(x)(john)
+ >>> print(lexpr(r'\x.man(x)(john) & tall(x)'))
+ (\x.man(x)(john) & tall(x))
+ >>> print(lexpr(r'\x.\y.sees(x,y)'))
+ \x y.sees(x,y)
+ >>> print(lexpr(r'\x y.sees(x,y)'))
+ \x y.sees(x,y)
+ >>> print(lexpr(r'\x.\y.sees(x,y)(a)'))
+ (\x y.sees(x,y))(a)
+ >>> print(lexpr(r'\x y.sees(x,y)(a)'))
+ (\x y.sees(x,y))(a)
+ >>> print(lexpr(r'\x.\y.sees(x,y)(a)(b)'))
+ ((\x y.sees(x,y))(a))(b)
+ >>> print(lexpr(r'\x y.sees(x,y)(a)(b)'))
+ ((\x y.sees(x,y))(a))(b)
+ >>> print(lexpr(r'\x.\y.sees(x,y)(a,b)'))
+ ((\x y.sees(x,y))(a))(b)
+ >>> print(lexpr(r'\x y.sees(x,y)(a,b)'))
+ ((\x y.sees(x,y))(a))(b)
+ >>> print(lexpr(r'((\x.\y.sees(x,y))(a))(b)'))
+ ((\x y.sees(x,y))(a))(b)
+ >>> print(lexpr(r'P(x)(y)(z)'))
+ P(x,y,z)
+ >>> print(lexpr(r'P(Q)'))
+ P(Q)
+ >>> print(lexpr(r'P(Q(x))'))
+ P(Q(x))
+ >>> print(lexpr(r'(\x.exists y.walks(x,y))(x)'))
+ (\x.exists y.walks(x,y))(x)
+ >>> print(lexpr(r'exists x.(x = john)'))
+ exists x.(x = john)
+ >>> print(lexpr(r'((\P.\Q.exists x.(P(x) & Q(x)))(\x.dog(x)))(\x.bark(x))'))
+ ((\P Q.exists x.(P(x) & Q(x)))(\x.dog(x)))(\x.bark(x))
+ >>> a = lexpr(r'exists c.exists b.A(b,c) & A(b,c)')
+ >>> b = lexpr(r'(exists c.(exists b.A(b,c))) & A(b,c)')
+ >>> print(a == b)
+ True
+ >>> a = lexpr(r'exists c.(exists b.A(b,c) & A(b,c))')
+ >>> b = lexpr(r'exists c.((exists b.A(b,c)) & A(b,c))')
+ >>> print(a == b)
+ True
+ >>> print(lexpr(r'exists x.x = y'))
+ exists x.(x = y)
+ >>> print(lexpr('A(B)(C)'))
+ A(B,C)
+ >>> print(lexpr('(A(B))(C)'))
+ A(B,C)
+ >>> print(lexpr('A((B)(C))'))
+ A(B(C))
+ >>> print(lexpr('A(B(C))'))
+ A(B(C))
+ >>> print(lexpr('(A)(B(C))'))
+ A(B(C))
+ >>> print(lexpr('(((A)))(((B))(((C))))'))
+ A(B(C))
+ >>> print(lexpr(r'A != B'))
+ -(A = B)
+ >>> print(lexpr('P(x) & x=y & P(y)'))
+ (P(x) & (x = y) & P(y))
+ >>> try: print(lexpr(r'\walk.walk(x)'))
+ ... except LogicalExpressionException as e: print(e)
+ 'walk' is an illegal variable name. Constants may not be abstracted.
+ \walk.walk(x)
+ ^
+ >>> try: print(lexpr(r'all walk.walk(john)'))
+ ... except LogicalExpressionException as e: print(e)
+ 'walk' is an illegal variable name. Constants may not be quantified.
+ all walk.walk(john)
+ ^
+ >>> try: print(lexpr(r'x(john)'))
+ ... except LogicalExpressionException as e: print(e)
+ 'x' is an illegal predicate name. Individual variables may not be used as predicates.
+ x(john)
+ ^
+
+ >>> from nltk.sem.logic import _LogicParser # hack to give access to custom quote chars
+ >>> lpq = _LogicParser()
+ >>> lpq.quote_chars = [("'", "'", "\\", False)]
+ >>> print(lpq.parse(r"(man(x) & 'tall\'s,' (x) & walks (x) )"))
+ (man(x) & tall's,(x) & walks(x))
+ >>> lpq.quote_chars = [("'", "'", "\\", True)]
+ >>> print(lpq.parse(r"'tall\'s,'"))
+ 'tall\'s,'
+ >>> print(lpq.parse(r"'spaced name(x)'"))
+ 'spaced name(x)'
+ >>> print(lpq.parse(r"-'tall\'s,'(x)"))
+ -'tall\'s,'(x)
+ >>> print(lpq.parse(r"(man(x) & 'tall\'s,' (x) & walks (x) )"))
+ (man(x) & 'tall\'s,'(x) & walks(x))
+
+
+Simplify
+========
+
+ >>> print(lexpr(r'\x.man(x)(john)').simplify())
+ man(john)
+ >>> print(lexpr(r'\x.((man(x)))(john)').simplify())
+ man(john)
+ >>> print(lexpr(r'\x.\y.sees(x,y)(john, mary)').simplify())
+ sees(john,mary)
+ >>> print(lexpr(r'\x y.sees(x,y)(john, mary)').simplify())
+ sees(john,mary)
+ >>> print(lexpr(r'\x.\y.sees(x,y)(john)(mary)').simplify())
+ sees(john,mary)
+ >>> print(lexpr(r'\x y.sees(x,y)(john)(mary)').simplify())
+ sees(john,mary)
+ >>> print(lexpr(r'\x.\y.sees(x,y)(john)').simplify())
+ \y.sees(john,y)
+ >>> print(lexpr(r'\x y.sees(x,y)(john)').simplify())
+ \y.sees(john,y)
+ >>> print(lexpr(r'(\x.\y.sees(x,y)(john))(mary)').simplify())
+ sees(john,mary)
+ >>> print(lexpr(r'(\x y.sees(x,y)(john))(mary)').simplify())
+ sees(john,mary)
+ >>> print(lexpr(r'exists x.(man(x) & (\x.exists y.walks(x,y))(x))').simplify())
+ exists x.(man(x) & exists y.walks(x,y))
+ >>> e1 = lexpr(r'exists x.(man(x) & (\x.exists y.walks(x,y))(y))').simplify()
+ >>> e2 = lexpr(r'exists x.(man(x) & exists z1.walks(y,z1))')
+ >>> e1 == e2
+ True
+ >>> print(lexpr(r'(\P Q.exists x.(P(x) & Q(x)))(\x.dog(x))').simplify())
+ \Q.exists x.(dog(x) & Q(x))
+ >>> print(lexpr(r'((\P.\Q.exists x.(P(x) & Q(x)))(\x.dog(x)))(\x.bark(x))').simplify())
+ exists x.(dog(x) & bark(x))
+ >>> print(lexpr(r'\P.(P(x)(y))(\a b.Q(a,b))').simplify())
+ Q(x,y)
+
+Replace
+=======
+
+ >>> a = lexpr(r'a')
+ >>> x = lexpr(r'x')
+ >>> y = lexpr(r'y')
+ >>> z = lexpr(r'z')
+
+ >>> print(lexpr(r'man(x)').replace(x.variable, a, False))
+ man(a)
+ >>> print(lexpr(r'(man(x) & tall(x))').replace(x.variable, a, False))
+ (man(a) & tall(a))
+ >>> print(lexpr(r'exists x.man(x)').replace(x.variable, a, False))
+ exists x.man(x)
+ >>> print(lexpr(r'exists x.man(x)').replace(x.variable, a, True))
+ exists a.man(a)
+ >>> print(lexpr(r'exists x.give(x,y,z)').replace(y.variable, a, False))
+ exists x.give(x,a,z)
+ >>> print(lexpr(r'exists x.give(x,y,z)').replace(y.variable, a, True))
+ exists x.give(x,a,z)
+ >>> e1 = lexpr(r'exists x.give(x,y,z)').replace(y.variable, x, False)
+ >>> e2 = lexpr(r'exists z1.give(z1,x,z)')
+ >>> e1 == e2
+ True
+ >>> e1 = lexpr(r'exists x.give(x,y,z)').replace(y.variable, x, True)
+ >>> e2 = lexpr(r'exists z1.give(z1,x,z)')
+ >>> e1 == e2
+ True
+ >>> print(lexpr(r'\x y z.give(x,y,z)').replace(y.variable, a, False))
+ \x y z.give(x,y,z)
+ >>> print(lexpr(r'\x y z.give(x,y,z)').replace(y.variable, a, True))
+ \x a z.give(x,a,z)
+ >>> print(lexpr(r'\x.\y.give(x,y,z)').replace(z.variable, a, False))
+ \x y.give(x,y,a)
+ >>> print(lexpr(r'\x.\y.give(x,y,z)').replace(z.variable, a, True))
+ \x y.give(x,y,a)
+ >>> e1 = lexpr(r'\x.\y.give(x,y,z)').replace(z.variable, x, False)
+ >>> e2 = lexpr(r'\z1.\y.give(z1,y,x)')
+ >>> e1 == e2
+ True
+ >>> e1 = lexpr(r'\x.\y.give(x,y,z)').replace(z.variable, x, True)
+ >>> e2 = lexpr(r'\z1.\y.give(z1,y,x)')
+ >>> e1 == e2
+ True
+ >>> print(lexpr(r'\x.give(x,y,z)').replace(z.variable, y, False))
+ \x.give(x,y,y)
+ >>> print(lexpr(r'\x.give(x,y,z)').replace(z.variable, y, True))
+ \x.give(x,y,y)
+
+ >>> from nltk.sem import logic
+ >>> logic._counter._value = 0
+ >>> e1 = lexpr('e1')
+ >>> e2 = lexpr('e2')
+ >>> print(lexpr('exists e1 e2.(walk(e1) & talk(e2))').replace(e1.variable, e2, True))
+ exists e2 e01.(walk(e2) & talk(e01))
+
+
+Variables / Free
+================
+
+ >>> examples = [r'walk(john)',
+ ... r'walk(x)',
+ ... r'?vp(?np)',
+ ... r'see(john,mary)',
+ ... r'exists x.walk(x)',
+ ... r'\x.see(john,x)',
+ ... r'\x.see(john,x)(mary)',
+ ... r'P(x)',
+ ... r'\P.P(x)',
+ ... r'aa(x,bb(y),cc(z),P(w),u)',
+ ... r'bo(?det(?n), at x)']
+ >>> examples = [lexpr(e) for e in examples]
+
+ >>> for e in examples:
+ ... print('%-25s' % e, sorted(e.free()))
+ walk(john) []
+ walk(x) [Variable('x')]
+ ?vp(?np) []
+ see(john,mary) []
+ exists x.walk(x) []
+ \x.see(john,x) []
+ (\x.see(john,x))(mary) []
+ P(x) [Variable('P'), Variable('x')]
+ \P.P(x) [Variable('x')]
+ aa(x,bb(y),cc(z),P(w),u) [Variable('P'), Variable('u'), Variable('w'), Variable('x'), Variable('y'), Variable('z')]
+ bo(?det(?n), at x) []
+
+ >>> for e in examples:
+ ... print('%-25s' % e, sorted(e.constants()))
+ walk(john) [Variable('john')]
+ walk(x) []
+ ?vp(?np) [Variable('?np')]
+ see(john,mary) [Variable('john'), Variable('mary')]
+ exists x.walk(x) []
+ \x.see(john,x) [Variable('john')]
+ (\x.see(john,x))(mary) [Variable('john'), Variable('mary')]
+ P(x) []
+ \P.P(x) []
+ aa(x,bb(y),cc(z),P(w),u) []
+ bo(?det(?n), at x) [Variable('?n'), Variable('@x')]
+
+ >>> for e in examples:
+ ... print('%-25s' % e, sorted(e.predicates()))
+ walk(john) [Variable('walk')]
+ walk(x) [Variable('walk')]
+ ?vp(?np) [Variable('?vp')]
+ see(john,mary) [Variable('see')]
+ exists x.walk(x) [Variable('walk')]
+ \x.see(john,x) [Variable('see')]
+ (\x.see(john,x))(mary) [Variable('see')]
+ P(x) []
+ \P.P(x) []
+ aa(x,bb(y),cc(z),P(w),u) [Variable('aa'), Variable('bb'), Variable('cc')]
+ bo(?det(?n), at x) [Variable('?det'), Variable('bo')]
+
+ >>> for e in examples:
+ ... print('%-25s' % e, sorted(e.variables()))
+ walk(john) []
+ walk(x) [Variable('x')]
+ ?vp(?np) [Variable('?np'), Variable('?vp')]
+ see(john,mary) []
+ exists x.walk(x) []
+ \x.see(john,x) []
+ (\x.see(john,x))(mary) []
+ P(x) [Variable('P'), Variable('x')]
+ \P.P(x) [Variable('x')]
+ aa(x,bb(y),cc(z),P(w),u) [Variable('P'), Variable('u'), Variable('w'), Variable('x'), Variable('y'), Variable('z')]
+ bo(?det(?n), at x) [Variable('?det'), Variable('?n'), Variable('@x')]
+
+
+
+`normalize`
+ >>> print(lexpr(r'\e083.(walk(e083, z472) & talk(e092, z938))').normalize())
+ \e01.(walk(e01,z3) & talk(e02,z4))
+
+Typed Logic
++++++++++++
+
+ >>> from nltk.sem.logic import _LogicParser
+ >>> tlp = _LogicParser(True)
+ >>> print(tlp.parse(r'man(x)').type)
+ ?
+ >>> print(tlp.parse(r'walk(angus)').type)
+ ?
+ >>> print(tlp.parse(r'-man(x)').type)
+ t
+ >>> print(tlp.parse(r'(man(x) <-> tall(x))').type)
+ t
+ >>> print(tlp.parse(r'exists x.(man(x) & tall(x))').type)
+ t
+ >>> print(tlp.parse(r'\x.man(x)').type)
+ <e,?>
+ >>> print(tlp.parse(r'john').type)
+ e
+ >>> print(tlp.parse(r'\x y.sees(x,y)').type)
+ <e,<e,?>>
+ >>> print(tlp.parse(r'\x.man(x)(john)').type)
+ ?
+ >>> print(tlp.parse(r'\x.\y.sees(x,y)(john)').type)
+ <e,?>
+ >>> print(tlp.parse(r'\x.\y.sees(x,y)(john)(mary)').type)
+ ?
+ >>> print(tlp.parse(r'\P.\Q.exists x.(P(x) & Q(x))').type)
+ <<e,t>,<<e,t>,t>>
+ >>> print(tlp.parse(r'\x.y').type)
+ <?,e>
+ >>> print(tlp.parse(r'\P.P(x)').type)
+ <<e,?>,?>
+
+ >>> parsed = tlp.parse('see(john,mary)')
+ >>> print(parsed.type)
+ ?
+ >>> print(parsed.function)
+ see(john)
+ >>> print(parsed.function.type)
+ <e,?>
+ >>> print(parsed.function.function)
+ see
+ >>> print(parsed.function.function.type)
+ <e,<e,?>>
+
+ >>> parsed = tlp.parse('P(x,y)')
+ >>> print(parsed)
+ P(x,y)
+ >>> print(parsed.type)
+ ?
+ >>> print(parsed.function)
+ P(x)
+ >>> print(parsed.function.type)
+ <e,?>
+ >>> print(parsed.function.function)
+ P
+ >>> print(parsed.function.function.type)
+ <e,<e,?>>
+
+ >>> print(tlp.parse(r'P').type)
+ ?
+
+ >>> print(tlp.parse(r'P', {'P': 't'}).type)
+ t
+
+ >>> a = tlp.parse(r'P(x)')
+ >>> print(a.type)
+ ?
+ >>> print(a.function.type)
+ <e,?>
+ >>> print(a.argument.type)
+ e
+
+ >>> a = tlp.parse(r'-P(x)')
+ >>> print(a.type)
+ t
+ >>> print(a.term.type)
+ t
+ >>> print(a.term.function.type)
+ <e,t>
+ >>> print(a.term.argument.type)
+ e
+
+ >>> a = tlp.parse(r'P & Q')
+ >>> print(a.type)
+ t
+ >>> print(a.first.type)
+ t
+ >>> print(a.second.type)
+ t
+
+ >>> a = tlp.parse(r'(P(x) & Q(x))')
+ >>> print(a.type)
+ t
+ >>> print(a.first.type)
+ t
+ >>> print(a.first.function.type)
+ <e,t>
+ >>> print(a.first.argument.type)
+ e
+ >>> print(a.second.type)
+ t
+ >>> print(a.second.function.type)
+ <e,t>
+ >>> print(a.second.argument.type)
+ e
+
+ >>> a = tlp.parse(r'\x.P(x)')
+ >>> print(a.type)
+ <e,?>
+ >>> print(a.term.function.type)
+ <e,?>
+ >>> print(a.term.argument.type)
+ e
+
+ >>> a = tlp.parse(r'\P.P(x)')
+ >>> print(a.type)
+ <<e,?>,?>
+ >>> print(a.term.function.type)
+ <e,?>
+ >>> print(a.term.argument.type)
+ e
+
+ >>> a = tlp.parse(r'(\x.P(x)(john)) & Q(x)')
+ >>> print(a.type)
+ t
+ >>> print(a.first.type)
+ t
+ >>> print(a.first.function.type)
+ <e,t>
+ >>> print(a.first.function.term.function.type)
+ <e,t>
+ >>> print(a.first.function.term.argument.type)
+ e
+ >>> print(a.first.argument.type)
+ e
+
+ >>> a = tlp.parse(r'\x y.P(x,y)(john)(mary) & Q(x)')
+ >>> print(a.type)
+ t
+ >>> print(a.first.type)
+ t
+ >>> print(a.first.function.type)
+ <e,t>
+ >>> print(a.first.function.function.type)
+ <e,<e,t>>
+
+ >>> a = tlp.parse(r'--P')
+ >>> print(a.type)
+ t
+ >>> print(a.term.type)
+ t
+ >>> print(a.term.term.type)
+ t
+
+ >>> tlp.parse(r'\x y.P(x,y)').type
+ <e,<e,?>>
+ >>> tlp.parse(r'\x y.P(x,y)', {'P': '<e,<e,t>>'}).type
+ <e,<e,t>>
+
+ >>> a = tlp.parse(r'\P y.P(john,y)(\x y.see(x,y))')
+ >>> a.type
+ <e,?>
+ >>> a.function.type
+ <<e,<e,?>>,<e,?>>
+ >>> a.function.term.term.function.function.type
+ <e,<e,?>>
+ >>> a.argument.type
+ <e,<e,?>>
+
+ >>> a = tlp.parse(r'exists c f.(father(c) = f)')
+ >>> a.type
+ t
+ >>> a.term.term.type
+ t
+ >>> a.term.term.first.type
+ e
+ >>> a.term.term.first.function.type
+ <e,e>
+ >>> a.term.term.second.type
+ e
+
+typecheck()
+
+ >>> a = tlp.parse('P(x)')
+ >>> b = tlp.parse('Q(x)')
+ >>> a.type
+ ?
+ >>> c = a & b
+ >>> c.first.type
+ ?
+ >>> c.typecheck() # doctest: +ELLIPSIS
+ {...}
+ >>> c.first.type
+ t
+
+ >>> a = tlp.parse('P(x)')
+ >>> b = tlp.parse('P(x) & Q(x)')
+ >>> a.type
+ ?
+ >>> typecheck([a,b]) # doctest: +ELLIPSIS
+ {...}
+ >>> a.type
+ t
+
+ >>> e = tlp.parse(r'man(x)')
+ >>> print(dict((k,str(v)) for k,v in e.typecheck().items()) == {'x': 'e', 'man': '<e,?>'})
+ True
+ >>> sig = {'man': '<e, t>'}
+ >>> e = tlp.parse(r'man(x)', sig)
+ >>> print(e.function.type)
+ <e,t>
+ >>> print(dict((k,str(v)) for k,v in e.typecheck().items()) == {'x': 'e', 'man': '<e,t>'})
+ True
+ >>> print(e.function.type)
+ <e,t>
+ >>> print(dict((k,str(v)) for k,v in e.typecheck(sig).items()) == {'x': 'e', 'man': '<e,t>'})
+ True
+
+findtype()
+
+ >>> print(tlp.parse(r'man(x)').findtype(Variable('man')))
+ <e,?>
+ >>> print(tlp.parse(r'see(x,y)').findtype(Variable('see')))
+ <e,<e,?>>
+ >>> print(tlp.parse(r'P(Q(R(x)))').findtype(Variable('Q')))
+ ?
+
+parse_type()
+
+ >>> print(parse_type('e'))
+ e
+ >>> print(parse_type('<e,t>'))
+ <e,t>
+ >>> print(parse_type('<<e,t>,<e,t>>'))
+ <<e,t>,<e,t>>
+ >>> print(parse_type('<<e,?>,?>'))
+ <<e,?>,?>
+
+alternative type format
+
+ >>> print(parse_type('e').str())
+ IND
+ >>> print(parse_type('<e,?>').str())
+ (IND -> ANY)
+ >>> print(parse_type('<<e,t>,t>').str())
+ ((IND -> BOOL) -> BOOL)
+
+Type.__eq__()
+
+ >>> from nltk.sem.logic import *
+
+ >>> e = ENTITY_TYPE
+ >>> t = TRUTH_TYPE
+ >>> a = ANY_TYPE
+ >>> et = ComplexType(e,t)
+ >>> eet = ComplexType(e,ComplexType(e,t))
+ >>> at = ComplexType(a,t)
+ >>> ea = ComplexType(e,a)
+ >>> aa = ComplexType(a,a)
+
+ >>> e == e
+ True
+ >>> t == t
+ True
+ >>> e == t
+ False
+ >>> a == t
+ False
+ >>> t == a
+ False
+ >>> a == a
+ True
+ >>> et == et
+ True
+ >>> a == et
+ False
+ >>> et == a
+ False
+ >>> a == ComplexType(a,aa)
+ True
+ >>> ComplexType(a,aa) == a
+ True
+
+matches()
+
+ >>> e.matches(t)
+ False
+ >>> a.matches(t)
+ True
+ >>> t.matches(a)
+ True
+ >>> a.matches(et)
+ True
+ >>> et.matches(a)
+ True
+ >>> ea.matches(eet)
+ True
+ >>> eet.matches(ea)
+ True
+ >>> aa.matches(et)
+ True
+ >>> aa.matches(t)
+ True
+
+Type error during parsing
+=========================
+
+ >>> try: print(tlp.parse(r'exists x y.(P(x) & P(x,y))'))
+ ... except InconsistentTypeHierarchyException as e: print(e)
+ The variable 'P' was found in multiple places with different types.
+ >>> try: tlp.parse(r'\x y.see(x,y)(\x.man(x))')
+ ... except TypeException as e: print(e)
+ The function '\x y.see(x,y)' is of type '<e,<e,?>>' and cannot be applied to '\x.man(x)' of type '<e,?>'. Its argument must match type 'e'.
+ >>> try: tlp.parse(r'\P x y.-P(x,y)(\x.-man(x))')
+ ... except TypeException as e: print(e)
+ The function '\P x y.-P(x,y)' is of type '<<e,<e,t>>,<e,<e,t>>>' and cannot be applied to '\x.-man(x)' of type '<e,t>'. Its argument must match type '<e,<e,t>>'.
+
+ >>> a = tlp.parse(r'-talk(x)')
+ >>> signature = a.typecheck()
+ >>> try: print(tlp.parse(r'-talk(x,y)', signature))
+ ... except InconsistentTypeHierarchyException as e: print(e)
+ The variable 'talk' was found in multiple places with different types.
+
+ >>> a = tlp.parse(r'-P(x)')
+ >>> b = tlp.parse(r'-P(x,y)')
+ >>> a.typecheck() # doctest: +ELLIPSIS
+ {...}
+ >>> b.typecheck() # doctest: +ELLIPSIS
+ {...}
+ >>> try: typecheck([a,b])
+ ... except InconsistentTypeHierarchyException as e: print(e)
+ The variable 'P' was found in multiple places with different types.
+
+ >>> a = tlp.parse(r'P(x)')
+ >>> b = tlp.parse(r'P(x,y)')
+ >>> signature = {'P': '<e,t>'}
+ >>> a.typecheck(signature) # doctest: +ELLIPSIS
+ {...}
+ >>> try: typecheck([a,b], signature)
+ ... except InconsistentTypeHierarchyException as e: print(e)
+ The variable 'P' was found in multiple places with different types.
+
+Parse errors
+============
+
+ >>> try: lexpr(r'')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ <BLANKLINE>
+ ^
+ >>> try: lexpr(r'(')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ (
+ ^
+ >>> try: lexpr(r')')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ )
+ ^
+ >>> try: lexpr(r'()')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ ()
+ ^
+ >>> try: lexpr(r'(P(x) & Q(x)')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expected token ')'.
+ (P(x) & Q(x)
+ ^
+ >>> try: lexpr(r'(P(x) &')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ (P(x) &
+ ^
+ >>> try: lexpr(r'(P(x) | )')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ (P(x) | )
+ ^
+ >>> try: lexpr(r'P(x) ->')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ P(x) ->
+ ^
+ >>> try: lexpr(r'P(x')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expected token ')'.
+ P(x
+ ^
+ >>> try: lexpr(r'P(x,')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ P(x,
+ ^
+ >>> try: lexpr(r'P(x,)')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ P(x,)
+ ^
+ >>> try: lexpr(r'exists')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Variable and Expression expected following quantifier 'exists'.
+ exists
+ ^
+ >>> try: lexpr(r'exists x')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ exists x
+ ^
+ >>> try: lexpr(r'exists x.')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ exists x.
+ ^
+ >>> try: lexpr(r'\ ')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Variable and Expression expected following lambda operator.
+ \
+ ^
+ >>> try: lexpr(r'\ x')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ \ x
+ ^
+ >>> try: lexpr(r'\ x y')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ \ x y
+ ^
+ >>> try: lexpr(r'\ x.')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ \ x.
+ ^
+ >>> try: lexpr(r'P(x)Q(x)')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: 'Q'.
+ P(x)Q(x)
+ ^
+ >>> try: lexpr(r'(P(x)Q(x)')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: 'Q'. Expected token ')'.
+ (P(x)Q(x)
+ ^
+ >>> try: lexpr(r'exists x y')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ exists x y
+ ^
+ >>> try: lexpr(r'exists x y.')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expression expected.
+ exists x y.
+ ^
+ >>> try: lexpr(r'exists x -> y')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: '->'. Expression expected.
+ exists x -> y
+ ^
+
+
+ >>> try: lexpr(r'A -> ((P(x) & Q(x)) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expected token ')'.
+ A -> ((P(x) & Q(x)) -> Z
+ ^
+ >>> try: lexpr(r'A -> ((P(x) &) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> ((P(x) &) -> Z
+ ^
+ >>> try: lexpr(r'A -> ((P(x) | )) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> ((P(x) | )) -> Z
+ ^
+ >>> try: lexpr(r'A -> (P(x) ->) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (P(x) ->) -> Z
+ ^
+ >>> try: lexpr(r'A -> (P(x) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ End of input found. Expected token ')'.
+ A -> (P(x) -> Z
+ ^
+ >>> try: lexpr(r'A -> (P(x,) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (P(x,) -> Z
+ ^
+ >>> try: lexpr(r'A -> (P(x,)) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (P(x,)) -> Z
+ ^
+ >>> try: lexpr(r'A -> (exists) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ ')' is an illegal variable name. Constants may not be quantified.
+ A -> (exists) -> Z
+ ^
+ >>> try: lexpr(r'A -> (exists x) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (exists x) -> Z
+ ^
+ >>> try: lexpr(r'A -> (exists x.) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (exists x.) -> Z
+ ^
+ >>> try: lexpr(r'A -> (\ ) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ ')' is an illegal variable name. Constants may not be abstracted.
+ A -> (\ ) -> Z
+ ^
+ >>> try: lexpr(r'A -> (\ x) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (\ x) -> Z
+ ^
+ >>> try: lexpr(r'A -> (\ x y) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (\ x y) -> Z
+ ^
+ >>> try: lexpr(r'A -> (\ x.) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (\ x.) -> Z
+ ^
+ >>> try: lexpr(r'A -> (P(x)Q(x)) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: 'Q'. Expected token ')'.
+ A -> (P(x)Q(x)) -> Z
+ ^
+ >>> try: lexpr(r'A -> ((P(x)Q(x)) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: 'Q'. Expected token ')'.
+ A -> ((P(x)Q(x)) -> Z
+ ^
+ >>> try: lexpr(r'A -> (all x y) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (all x y) -> Z
+ ^
+ >>> try: lexpr(r'A -> (exists x y.) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: ')'. Expression expected.
+ A -> (exists x y.) -> Z
+ ^
+ >>> try: lexpr(r'A -> (exists x -> y) -> Z')
+ ... except LogicalExpressionException as e: print(e)
+ Unexpected token: '->'. Expression expected.
+ A -> (exists x -> y) -> Z
+ ^
+
+
diff --git a/nltk/test/metrics.doctest b/nltk/test/metrics.doctest
new file mode 100644
index 0000000..2736348
--- /dev/null
+++ b/nltk/test/metrics.doctest
@@ -0,0 +1,270 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=======
+Metrics
+=======
+
+The `nltk.metrics` package provides a variety of *evaluation measures*
+which can be used for a wide variety of NLP tasks.
+
+ >>> from __future__ import print_function
+ >>> from nltk.metrics import *
+
+------------------
+Standard IR Scores
+------------------
+
+We can use standard scores from information retrieval to test the
+performance of taggers, chunkers, etc.
+
+ >>> reference = 'DET NN VB DET JJ NN NN IN DET NN'.split()
+ >>> test = 'DET VB VB DET NN NN NN IN DET NN'.split()
+ >>> print(accuracy(reference, test))
+ 0.8
+
+
+The following measures apply to sets:
+
+ >>> reference_set = set(reference)
+ >>> test_set = set(test)
+ >>> precision(reference_set, test_set)
+ 1.0
+ >>> print(recall(reference_set, test_set))
+ 0.8
+ >>> print(f_measure(reference_set, test_set))
+ 0.88888888888...
+
+Measuring the likelihood of the data, given probability distributions:
+
+ >>> from nltk import FreqDist, MLEProbDist
+ >>> pdist1 = MLEProbDist(FreqDist("aldjfalskfjaldsf"))
+ >>> pdist2 = MLEProbDist(FreqDist("aldjfalssjjlldss"))
+ >>> print(log_likelihood(['a', 'd'], [pdist1, pdist2]))
+ -2.7075187496...
+
+
+----------------
+Distance Metrics
+----------------
+
+String edit distance (Levenshtein):
+
+ >>> edit_distance("rain", "shine")
+ 3
+
+Other distance measures:
+
+ >>> s1 = set([1,2,3,4])
+ >>> s2 = set([3,4,5])
+ >>> binary_distance(s1, s2)
+ 1.0
+ >>> print(jaccard_distance(s1, s2))
+ 0.6
+ >>> print(masi_distance(s1, s2))
+ 0.868...
+
+----------------------
+Miscellaneous Measures
+----------------------
+
+Rank Correlation works with two dictionaries mapping keys to ranks.
+The dictionaries should have the same set of keys.
+
+ >>> spearman_correlation({'e':1, 't':2, 'a':3}, {'e':1, 'a':2, 't':3})
+ 0.5
+
+Windowdiff uses a sliding window in comparing two segmentations of the same input (e.g. tokenizations, chunkings).
+Segmentations are represented using strings of zeros and ones.
+
+ >>> s1 = "000100000010"
+ >>> s2 = "000010000100"
+ >>> s3 = "100000010000"
+ >>> s4 = "000000000000"
+ >>> s5 = "111111111111"
+ >>> windowdiff(s1, s1, 3)
+ 0.0
+ >>> abs(windowdiff(s1, s2, 3) - 0.3) < 1e-6 # windowdiff(s1, s2, 3) == 0.3
+ True
+ >>> abs(windowdiff(s2, s3, 3) - 0.8) < 1e-6 # windowdiff(s2, s3, 3) == 0.8
+ True
+ >>> windowdiff(s1, s4, 3)
+ 0.5
+ >>> windowdiff(s1, s5, 3)
+ 1.0
+
+----------------
+Confusion Matrix
+----------------
+
+ >>> reference = 'This is the reference data. Testing 123. aoaeoeoe'
+ >>> test = 'Thos iz_the rifirenci data. Testeng 123. aoaeoeoe'
+ >>> print(ConfusionMatrix(reference, test))
+ | . 1 2 3 T _ a c d e f g h i n o r s t z |
+ --+-------------------------------------------+
+ |<8>. . . . . 1 . . . . . . . . . . . . . . |
+ . | .<2>. . . . . . . . . . . . . . . . . . . |
+ 1 | . .<1>. . . . . . . . . . . . . . . . . . |
+ 2 | . . .<1>. . . . . . . . . . . . . . . . . |
+ 3 | . . . .<1>. . . . . . . . . . . . . . . . |
+ T | . . . . .<2>. . . . . . . . . . . . . . . |
+ _ | . . . . . .<.>. . . . . . . . . . . . . . |
+ a | . . . . . . .<4>. . . . . . . . . . . . . |
+ c | . . . . . . . .<1>. . . . . . . . . . . . |
+ d | . . . . . . . . .<1>. . . . . . . . . . . |
+ e | . . . . . . . . . .<6>. . . 3 . . . . . . |
+ f | . . . . . . . . . . .<1>. . . . . . . . . |
+ g | . . . . . . . . . . . .<1>. . . . . . . . |
+ h | . . . . . . . . . . . . .<2>. . . . . . . |
+ i | . . . . . . . . . . 1 . . .<1>. 1 . . . . |
+ n | . . . . . . . . . . . . . . .<2>. . . . . |
+ o | . . . . . . . . . . . . . . . .<3>. . . . |
+ r | . . . . . . . . . . . . . . . . .<2>. . . |
+ s | . . . . . . . . . . . . . . . . . .<2>. 1 |
+ t | . . . . . . . . . . . . . . . . . . .<3>. |
+ z | . . . . . . . . . . . . . . . . . . . .<.>|
+ --+-------------------------------------------+
+ (row = reference; col = test)
+ <BLANKLINE>
+
+ >>> cm = ConfusionMatrix(reference, test)
+ >>> print(cm.pp(sort_by_count=True))
+ | e a i o s t . T h n r 1 2 3 c d f g _ z |
+ --+-------------------------------------------+
+ |<8>. . . . . . . . . . . . . . . . . . 1 . |
+ e | .<6>. 3 . . . . . . . . . . . . . . . . . |
+ a | . .<4>. . . . . . . . . . . . . . . . . . |
+ i | . 1 .<1>1 . . . . . . . . . . . . . . . . |
+ o | . . . .<3>. . . . . . . . . . . . . . . . |
+ s | . . . . .<2>. . . . . . . . . . . . . . 1 |
+ t | . . . . . .<3>. . . . . . . . . . . . . . |
+ . | . . . . . . .<2>. . . . . . . . . . . . . |
+ T | . . . . . . . .<2>. . . . . . . . . . . . |
+ h | . . . . . . . . .<2>. . . . . . . . . . . |
+ n | . . . . . . . . . .<2>. . . . . . . . . . |
+ r | . . . . . . . . . . .<2>. . . . . . . . . |
+ 1 | . . . . . . . . . . . .<1>. . . . . . . . |
+ 2 | . . . . . . . . . . . . .<1>. . . . . . . |
+ 3 | . . . . . . . . . . . . . .<1>. . . . . . |
+ c | . . . . . . . . . . . . . . .<1>. . . . . |
+ d | . . . . . . . . . . . . . . . .<1>. . . . |
+ f | . . . . . . . . . . . . . . . . .<1>. . . |
+ g | . . . . . . . . . . . . . . . . . .<1>. . |
+ _ | . . . . . . . . . . . . . . . . . . .<.>. |
+ z | . . . . . . . . . . . . . . . . . . . .<.>|
+ --+-------------------------------------------+
+ (row = reference; col = test)
+ <BLANKLINE>
+
+ >>> print(cm.pp(sort_by_count=True, truncate=10))
+ | e a i o s t . T h |
+ --+---------------------+
+ |<8>. . . . . . . . . |
+ e | .<6>. 3 . . . . . . |
+ a | . .<4>. . . . . . . |
+ i | . 1 .<1>1 . . . . . |
+ o | . . . .<3>. . . . . |
+ s | . . . . .<2>. . . . |
+ t | . . . . . .<3>. . . |
+ . | . . . . . . .<2>. . |
+ T | . . . . . . . .<2>. |
+ h | . . . . . . . . .<2>|
+ --+---------------------+
+ (row = reference; col = test)
+ <BLANKLINE>
+
+ >>> print(cm.pp(sort_by_count=True, truncate=10, values_in_chart=False))
+ | 1 |
+ | 1 2 3 4 5 6 7 8 9 0 |
+ ---+---------------------+
+ 1 |<8>. . . . . . . . . |
+ 2 | .<6>. 3 . . . . . . |
+ 3 | . .<4>. . . . . . . |
+ 4 | . 1 .<1>1 . . . . . |
+ 5 | . . . .<3>. . . . . |
+ 6 | . . . . .<2>. . . . |
+ 7 | . . . . . .<3>. . . |
+ 8 | . . . . . . .<2>. . |
+ 9 | . . . . . . . .<2>. |
+ 10 | . . . . . . . . .<2>|
+ ---+---------------------+
+ (row = reference; col = test)
+ Value key:
+ 1:
+ 2: e
+ 3: a
+ 4: i
+ 5: o
+ 6: s
+ 7: t
+ 8: .
+ 9: T
+ 10: h
+ <BLANKLINE>
+
+
+--------------------
+Association measures
+--------------------
+
+These measures are useful to determine whether the coocurrence of two random
+events is meaningful. They are used, for instance, to distinguish collocations
+from other pairs of adjacent words.
+
+We bring some examples of bigram association calculations from Manning and
+Schutze's SNLP, 2nd Ed. chapter 5.
+
+ >>> n_new_companies, n_new, n_companies, N = 8, 15828, 4675, 14307668
+ >>> bam = BigramAssocMeasures
+ >>> bam.raw_freq(20, (42, 20), N) == 20. / N
+ True
+ >>> bam.student_t(n_new_companies, (n_new, n_companies), N)
+ 0.999...
+ >>> bam.chi_sq(n_new_companies, (n_new, n_companies), N)
+ 1.54...
+ >>> bam.likelihood_ratio(150, (12593, 932), N)
+ 1291...
+
+For other associations, we ensure the ordering of the measures:
+
+ >>> bam.mi_like(20, (42, 20), N) > bam.mi_like(20, (41, 27), N)
+ True
+ >>> bam.pmi(20, (42, 20), N) > bam.pmi(20, (41, 27), N)
+ True
+ >>> bam.phi_sq(20, (42, 20), N) > bam.phi_sq(20, (41, 27), N)
+ True
+ >>> bam.poisson_stirling(20, (42, 20), N) > bam.poisson_stirling(20, (41, 27), N)
+ True
+ >>> bam.jaccard(20, (42, 20), N) > bam.jaccard(20, (41, 27), N)
+ True
+ >>> bam.dice(20, (42, 20), N) > bam.dice(20, (41, 27), N)
+ True
+ >>> bam.fisher(20, (42, 20), N) > bam.fisher(20, (41, 27), N)
+ False
+
+For trigrams, we have to provide more count information:
+
+ >>> n_w1_w2_w3 = 20
+ >>> n_w1_w2, n_w1_w3, n_w2_w3 = 35, 60, 40
+ >>> pair_counts = (n_w1_w2, n_w1_w3, n_w2_w3)
+ >>> n_w1, n_w2, n_w3 = 100, 200, 300
+ >>> uni_counts = (n_w1, n_w2, n_w3)
+ >>> N = 14307668
+ >>> tam = TrigramAssocMeasures
+ >>> tam.raw_freq(n_w1_w2_w3, pair_counts, uni_counts, N) == 1. * n_w1_w2_w3 / N
+ True
+ >>> uni_counts2 = (n_w1, n_w2, 100)
+ >>> tam.student_t(n_w1_w2_w3, pair_counts, uni_counts2, N) > tam.student_t(n_w1_w2_w3, pair_counts, uni_counts, N)
+ True
+ >>> tam.chi_sq(n_w1_w2_w3, pair_counts, uni_counts2, N) > tam.chi_sq(n_w1_w2_w3, pair_counts, uni_counts, N)
+ True
+ >>> tam.mi_like(n_w1_w2_w3, pair_counts, uni_counts2, N) > tam.mi_like(n_w1_w2_w3, pair_counts, uni_counts, N)
+ True
+ >>> tam.pmi(n_w1_w2_w3, pair_counts, uni_counts2, N) > tam.pmi(n_w1_w2_w3, pair_counts, uni_counts, N)
+ True
+ >>> tam.likelihood_ratio(n_w1_w2_w3, pair_counts, uni_counts2, N) > tam.likelihood_ratio(n_w1_w2_w3, pair_counts, uni_counts, N)
+ True
+ >>> tam.poisson_stirling(n_w1_w2_w3, pair_counts, uni_counts2, N) > tam.poisson_stirling(n_w1_w2_w3, pair_counts, uni_counts, N)
+ True
+ >>> tam.jaccard(n_w1_w2_w3, pair_counts, uni_counts2, N) > tam.jaccard(n_w1_w2_w3, pair_counts, uni_counts, N)
+ True
diff --git a/nltk/test/misc.doctest b/nltk/test/misc.doctest
new file mode 100644
index 0000000..b14e62d
--- /dev/null
+++ b/nltk/test/misc.doctest
@@ -0,0 +1,118 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+--------------------------------------------------------------------------------
+Unit tests for the miscellaneous sort functions.
+--------------------------------------------------------------------------------
+
+ >>> from copy import deepcopy
+ >>> from nltk.misc.sort import *
+
+A (very) small list of unsorted integers.
+
+ >>> test_data = [12, 67, 7, 28, 92, 56, 53, 720, 91, 57, 20, 20]
+
+Test each sorting method - each method returns the number of operations
+required to sort the data, and sorts in-place (desctructively - hence the need
+for multiple copies).
+
+ >>> sorted_data = deepcopy(test_data)
+ >>> selection(sorted_data)
+ 66
+
+ >>> sorted_data
+ [7, 12, 20, 20, 28, 53, 56, 57, 67, 91, 92, 720]
+
+ >>> sorted_data = deepcopy(test_data)
+ >>> bubble(sorted_data)
+ 30
+
+ >>> sorted_data
+ [7, 12, 20, 20, 28, 53, 56, 57, 67, 91, 92, 720]
+
+ >>> sorted_data = deepcopy(test_data)
+ >>> merge(sorted_data)
+ 30
+
+ >>> sorted_data
+ [7, 12, 20, 20, 28, 53, 56, 57, 67, 91, 92, 720]
+
+ >>> sorted_data = deepcopy(test_data)
+ >>> quick(sorted_data)
+ 13
+
+ >>> sorted_data
+ [7, 12, 20, 20, 28, 53, 56, 57, 67, 91, 92, 720]
+
+--------------------------------------------------------------------------------
+Unit tests for Wordfinder class
+--------------------------------------------------------------------------------
+
+ >>> import random
+
+ >>> # The following is not enough for reproducibility under Python 2/3
+ >>> # (see http://bugs.python.org/issue9025) so this test is skipped.
+ >>> random.seed(12345)
+
+ >>> from nltk.misc import wordfinder
+ >>> wordfinder.word_finder() # doctest: +SKIP
+ Word Finder
+ <BLANKLINE>
+ J V L A I R O T A T I S I V O D E R E T
+ H U U B E A R O E P O C S O R E T N E P
+ A D A U Z E E S R A P P A L L M E N T R
+ C X A D Q S Z T P E O R S N G P J A D E
+ I G Y K K T I A A R G F I D T E L C N S
+ R E C N B H T R L T N N B W N T A O A I
+ A Y I L O E I A M E I A A Y U R P L L D
+ G L T V S T S F E A D I P H D O O H N I
+ R L S E C I N I L R N N M E C G R U E A
+ A A Y G I C E N L L E O I G Q R T A E L
+ M R C E T I S T A E T L L E U A E N R L
+ O U O T A S E E C S O O N H Y P A T G Y
+ E M H O M M D R E S F P U L T H C F N V
+ L A C A I M A M A N L B R U T E D O M I
+ O R I L N E E E E E U A R S C R Y L I P
+ H T R K E S N N M S I L A S R E V I N U
+ T X T A A O U T K S E T A R R E S I B J
+ A E D L E L J I F O O R P E L K N I R W
+ K H A I D E Q O P R I C K T I M B E R P
+ Z K D O O H G N I H T U R V E Y D R O P
+ <BLANKLINE>
+ 1: INTERCHANGER
+ 2: TEARLESSNESS
+ 3: UNIVERSALISM
+ 4: DESENSITIZER
+ 5: INTERMENTION
+ 6: TRICHOCYSTIC
+ 7: EXTRAMURALLY
+ 8: VEGETOALKALI
+ 9: PALMELLACEAE
+ 10: AESTHETICISM
+ 11: PETROGRAPHER
+ 12: VISITATORIAL
+ 13: OLEOMARGARIC
+ 14: WRINKLEPROOF
+ 15: PRICKTIMBER
+ 16: PRESIDIALLY
+ 17: SCITAMINEAE
+ 18: ENTEROSCOPE
+ 19: APPALLMENT
+ 20: TURVEYDROP
+ 21: THINGHOOD
+ 22: BISERRATE
+ 23: GREENLAND
+ 24: BRUTEDOM
+ 25: POLONIAN
+ 26: ACOLHUAN
+ 27: LAPORTEA
+ 28: TENDING
+ 29: TEREDO
+ 30: MESOLE
+ 31: UNLIMP
+ 32: OSTARA
+ 33: PILY
+ 34: DUNT
+ 35: ONYX
+ 36: KATH
+ 37: JUNE
diff --git a/nltk/test/nonmonotonic.doctest b/nltk/test/nonmonotonic.doctest
new file mode 100644
index 0000000..7bc0904
--- /dev/null
+++ b/nltk/test/nonmonotonic.doctest
@@ -0,0 +1,286 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+======================
+Nonmonotonic Reasoning
+======================
+
+ >>> from nltk import *
+ >>> from nltk.inference.nonmonotonic import *
+ >>> from nltk.sem import logic
+ >>> logic._counter._value = 0
+ >>> lexpr = logic.Expression.fromstring
+
+------------------------
+Closed Domain Assumption
+------------------------
+
+The only entities in the domain are those found in the assumptions or goal.
+If the domain only contains "A" and "B", then the expression "exists x.P(x)" can
+be replaced with "P(A) | P(B)" and an expression "all x.P(x)" can be replaced
+with "P(A) & P(B)".
+
+ >>> p1 = lexpr(r'all x.(man(x) -> mortal(x))')
+ >>> p2 = lexpr(r'man(Socrates)')
+ >>> c = lexpr(r'mortal(Socrates)')
+ >>> prover = Prover9Command(c, [p1,p2])
+ >>> prover.prove()
+ True
+ >>> cdp = ClosedDomainProver(prover)
+ >>> for a in cdp.assumptions(): print(a) # doctest: +SKIP
+ (man(Socrates) -> mortal(Socrates))
+ man(Socrates)
+ >>> cdp.prove()
+ True
+
+ >>> p1 = lexpr(r'exists x.walk(x)')
+ >>> p2 = lexpr(r'man(Socrates)')
+ >>> c = lexpr(r'walk(Socrates)')
+ >>> prover = Prover9Command(c, [p1,p2])
+ >>> prover.prove()
+ False
+ >>> cdp = ClosedDomainProver(prover)
+ >>> for a in cdp.assumptions(): print(a) # doctest: +SKIP
+ walk(Socrates)
+ man(Socrates)
+ >>> cdp.prove()
+ True
+
+ >>> p1 = lexpr(r'exists x.walk(x)')
+ >>> p2 = lexpr(r'man(Socrates)')
+ >>> p3 = lexpr(r'-walk(Bill)')
+ >>> c = lexpr(r'walk(Socrates)')
+ >>> prover = Prover9Command(c, [p1,p2,p3])
+ >>> prover.prove()
+ False
+ >>> cdp = ClosedDomainProver(prover)
+ >>> for a in cdp.assumptions(): print(a) # doctest: +SKIP
+ (walk(Socrates) | walk(Bill))
+ man(Socrates)
+ -walk(Bill)
+ >>> cdp.prove()
+ True
+
+ >>> p1 = lexpr(r'walk(Socrates)')
+ >>> p2 = lexpr(r'walk(Bill)')
+ >>> c = lexpr(r'all x.walk(x)')
+ >>> prover = Prover9Command(c, [p1,p2])
+ >>> prover.prove()
+ False
+ >>> cdp = ClosedDomainProver(prover)
+ >>> for a in cdp.assumptions(): print(a) # doctest: +SKIP
+ walk(Socrates)
+ walk(Bill)
+ >>> print(cdp.goal()) # doctest: +SKIP
+ (walk(Socrates) & walk(Bill))
+ >>> cdp.prove()
+ True
+
+ >>> p1 = lexpr(r'girl(mary)')
+ >>> p2 = lexpr(r'dog(rover)')
+ >>> p3 = lexpr(r'all x.(girl(x) -> -dog(x))')
+ >>> p4 = lexpr(r'all x.(dog(x) -> -girl(x))')
+ >>> p5 = lexpr(r'chase(mary, rover)')
+ >>> c = lexpr(r'exists y.(dog(y) & all x.(girl(x) -> chase(x,y)))')
+ >>> prover = Prover9Command(c, [p1,p2,p3,p4,p5])
+ >>> print(prover.prove())
+ False
+ >>> cdp = ClosedDomainProver(prover)
+ >>> for a in cdp.assumptions(): print(a) # doctest: +SKIP
+ girl(mary)
+ dog(rover)
+ ((girl(rover) -> -dog(rover)) & (girl(mary) -> -dog(mary)))
+ ((dog(rover) -> -girl(rover)) & (dog(mary) -> -girl(mary)))
+ chase(mary,rover)
+ >>> print(cdp.goal()) # doctest: +SKIP
+ ((dog(rover) & (girl(rover) -> chase(rover,rover)) & (girl(mary) -> chase(mary,rover))) | (dog(mary) & (girl(rover) -> chase(rover,mary)) & (girl(mary) -> chase(mary,mary))))
+ >>> print(cdp.prove())
+ True
+
+-----------------------
+Unique Names Assumption
+-----------------------
+
+No two entities in the domain represent the same entity unless it can be
+explicitly proven that they do. Therefore, if the domain contains "A" and "B",
+then add the assumption "-(A = B)" if it is not the case that
+"<assumptions> \|- (A = B)".
+
+ >>> p1 = lexpr(r'man(Socrates)')
+ >>> p2 = lexpr(r'man(Bill)')
+ >>> c = lexpr(r'exists x.exists y.-(x = y)')
+ >>> prover = Prover9Command(c, [p1,p2])
+ >>> prover.prove()
+ False
+ >>> unp = UniqueNamesProver(prover)
+ >>> for a in unp.assumptions(): print(a) # doctest: +SKIP
+ man(Socrates)
+ man(Bill)
+ -(Socrates = Bill)
+ >>> unp.prove()
+ True
+
+ >>> p1 = lexpr(r'all x.(walk(x) -> (x = Socrates))')
+ >>> p2 = lexpr(r'Bill = William')
+ >>> p3 = lexpr(r'Bill = Billy')
+ >>> c = lexpr(r'-walk(William)')
+ >>> prover = Prover9Command(c, [p1,p2,p3])
+ >>> prover.prove()
+ False
+ >>> unp = UniqueNamesProver(prover)
+ >>> for a in unp.assumptions(): print(a) # doctest: +SKIP
+ all x.(walk(x) -> (x = Socrates))
+ (Bill = William)
+ (Bill = Billy)
+ -(William = Socrates)
+ -(Billy = Socrates)
+ -(Socrates = Bill)
+ >>> unp.prove()
+ True
+
+-----------------------
+Closed World Assumption
+-----------------------
+
+The only entities that have certain properties are those that is it stated
+have the properties. We accomplish this assumption by "completing" predicates.
+
+If the assumptions contain "P(A)", then "all x.(P(x) -> (x=A))" is the completion
+of "P". If the assumptions contain "all x.(ostrich(x) -> bird(x))", then
+"all x.(bird(x) -> ostrich(x))" is the completion of "bird". If the
+assumptions don't contain anything that are "P", then "all x.-P(x)" is the
+completion of "P".
+
+ >>> p1 = lexpr(r'walk(Socrates)')
+ >>> p2 = lexpr(r'-(Socrates = Bill)')
+ >>> c = lexpr(r'-walk(Bill)')
+ >>> prover = Prover9Command(c, [p1,p2])
+ >>> prover.prove()
+ False
+ >>> cwp = ClosedWorldProver(prover)
+ >>> for a in cwp.assumptions(): print(a) # doctest: +SKIP
+ walk(Socrates)
+ -(Socrates = Bill)
+ all z1.(walk(z1) -> (z1 = Socrates))
+ >>> cwp.prove()
+ True
+
+ >>> p1 = lexpr(r'see(Socrates, John)')
+ >>> p2 = lexpr(r'see(John, Mary)')
+ >>> p3 = lexpr(r'-(Socrates = John)')
+ >>> p4 = lexpr(r'-(John = Mary)')
+ >>> c = lexpr(r'-see(Socrates, Mary)')
+ >>> prover = Prover9Command(c, [p1,p2,p3,p4])
+ >>> prover.prove()
+ False
+ >>> cwp = ClosedWorldProver(prover)
+ >>> for a in cwp.assumptions(): print(a) # doctest: +SKIP
+ see(Socrates,John)
+ see(John,Mary)
+ -(Socrates = John)
+ -(John = Mary)
+ all z3 z4.(see(z3,z4) -> (((z3 = Socrates) & (z4 = John)) | ((z3 = John) & (z4 = Mary))))
+ >>> cwp.prove()
+ True
+
+ >>> p1 = lexpr(r'all x.(ostrich(x) -> bird(x))')
+ >>> p2 = lexpr(r'bird(Tweety)')
+ >>> p3 = lexpr(r'-ostrich(Sam)')
+ >>> p4 = lexpr(r'Sam != Tweety')
+ >>> c = lexpr(r'-bird(Sam)')
+ >>> prover = Prover9Command(c, [p1,p2,p3,p4])
+ >>> prover.prove()
+ False
+ >>> cwp = ClosedWorldProver(prover)
+ >>> for a in cwp.assumptions(): print(a) # doctest: +SKIP
+ all x.(ostrich(x) -> bird(x))
+ bird(Tweety)
+ -ostrich(Sam)
+ -(Sam = Tweety)
+ all z7.-ostrich(z7)
+ all z8.(bird(z8) -> ((z8 = Tweety) | ostrich(z8)))
+ >>> print(cwp.prove())
+ True
+
+-----------------------
+Multi-Decorator Example
+-----------------------
+
+Decorators can be nested to utilize multiple assumptions.
+
+ >>> p1 = lexpr(r'see(Socrates, John)')
+ >>> p2 = lexpr(r'see(John, Mary)')
+ >>> c = lexpr(r'-see(Socrates, Mary)')
+ >>> prover = Prover9Command(c, [p1,p2])
+ >>> print(prover.prove())
+ False
+ >>> cmd = ClosedDomainProver(UniqueNamesProver(ClosedWorldProver(prover)))
+ >>> print(cmd.prove())
+ True
+
+-----------------
+Default Reasoning
+-----------------
+ >>> logic._counter._value = 0
+ >>> premises = []
+
+define the taxonomy
+ >>> premises.append(lexpr(r'all x.(elephant(x) -> animal(x))'))
+ >>> premises.append(lexpr(r'all x.(bird(x) -> animal(x))'))
+ >>> premises.append(lexpr(r'all x.(dove(x) -> bird(x))'))
+ >>> premises.append(lexpr(r'all x.(ostrich(x) -> bird(x))'))
+ >>> premises.append(lexpr(r'all x.(flying_ostrich(x) -> ostrich(x))'))
+
+default the properties using abnormalities
+ >>> premises.append(lexpr(r'all x.((animal(x) & -Ab1(x)) -> -fly(x))')) #normal animals don't fly
+ >>> premises.append(lexpr(r'all x.((bird(x) & -Ab2(x)) -> fly(x))')) #normal birds fly
+ >>> premises.append(lexpr(r'all x.((ostrich(x) & -Ab3(x)) -> -fly(x))')) #normal ostriches don't fly
+
+specify abnormal entities
+ >>> premises.append(lexpr(r'all x.(bird(x) -> Ab1(x))')) #flight
+ >>> premises.append(lexpr(r'all x.(ostrich(x) -> Ab2(x))')) #non-flying bird
+ >>> premises.append(lexpr(r'all x.(flying_ostrich(x) -> Ab3(x))')) #flying ostrich
+
+define entities
+ >>> premises.append(lexpr(r'elephant(el)'))
+ >>> premises.append(lexpr(r'dove(do)'))
+ >>> premises.append(lexpr(r'ostrich(os)'))
+
+print the augmented assumptions list
+ >>> prover = Prover9Command(None, premises)
+ >>> command = UniqueNamesProver(ClosedWorldProver(prover))
+ >>> for a in command.assumptions(): print(a) # doctest: +SKIP
+ all x.(elephant(x) -> animal(x))
+ all x.(bird(x) -> animal(x))
+ all x.(dove(x) -> bird(x))
+ all x.(ostrich(x) -> bird(x))
+ all x.(flying_ostrich(x) -> ostrich(x))
+ all x.((animal(x) & -Ab1(x)) -> -fly(x))
+ all x.((bird(x) & -Ab2(x)) -> fly(x))
+ all x.((ostrich(x) & -Ab3(x)) -> -fly(x))
+ all x.(bird(x) -> Ab1(x))
+ all x.(ostrich(x) -> Ab2(x))
+ all x.(flying_ostrich(x) -> Ab3(x))
+ elephant(el)
+ dove(do)
+ ostrich(os)
+ all z1.(animal(z1) -> (elephant(z1) | bird(z1)))
+ all z2.(Ab1(z2) -> bird(z2))
+ all z3.(bird(z3) -> (dove(z3) | ostrich(z3)))
+ all z4.(dove(z4) -> (z4 = do))
+ all z5.(Ab2(z5) -> ostrich(z5))
+ all z6.(Ab3(z6) -> flying_ostrich(z6))
+ all z7.(ostrich(z7) -> ((z7 = os) | flying_ostrich(z7)))
+ all z8.-flying_ostrich(z8)
+ all z9.(elephant(z9) -> (z9 = el))
+ -(el = os)
+ -(el = do)
+ -(os = do)
+
+ >>> UniqueNamesProver(ClosedWorldProver(Prover9Command(lexpr('-fly(el)'), premises))).prove()
+ True
+ >>> UniqueNamesProver(ClosedWorldProver(Prover9Command(lexpr('fly(do)'), premises))).prove()
+ True
+ >>> UniqueNamesProver(ClosedWorldProver(Prover9Command(lexpr('-fly(os)'), premises))).prove()
+ True
+
diff --git a/nltk/test/nonmonotonic_fixt.py b/nltk/test/nonmonotonic_fixt.py
new file mode 100644
index 0000000..ddb2a48
--- /dev/null
+++ b/nltk/test/nonmonotonic_fixt.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+def setup_module(module):
+ from nose import SkipTest
+ from nltk.inference.mace import Mace
+ try:
+ m = Mace()
+ m._find_binary('mace4')
+ except LookupError:
+ raise SkipTest("Mace4/Prover9 is not available so nonmonotonic.doctest was skipped")
diff --git a/nltk/test/onto1.fol b/nltk/test/onto1.fol
new file mode 100644
index 0000000..5c526ee
--- /dev/null
+++ b/nltk/test/onto1.fol
@@ -0,0 +1,6 @@
+all x. ((boxer2 x) implies (dog x))
+all x. ((boxer1 x) implies (person x))
+all x. (not ((dog x) and (person x)))
+all x. (not ((kitchen x) and (garden x)))
+all x. ((kitchen x) implies (location x))
+all x. ((garden x) implies (location x))
\ No newline at end of file
diff --git a/nltk/test/parse.doctest b/nltk/test/parse.doctest
new file mode 100644
index 0000000..e47f251
--- /dev/null
+++ b/nltk/test/parse.doctest
@@ -0,0 +1,884 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=========
+ Parsing
+=========
+
+Unit tests for the Context Free Grammar class
+---------------------------------------------
+
+ >>> from nltk import Nonterminal, nonterminals, Production, CFG
+
+ >>> nt1 = Nonterminal('NP')
+ >>> nt2 = Nonterminal('VP')
+
+ >>> nt1.symbol()
+ 'NP'
+
+ >>> nt1 == Nonterminal('NP')
+ True
+
+ >>> nt1 == nt2
+ False
+
+ >>> S, NP, VP, PP = nonterminals('S, NP, VP, PP')
+ >>> N, V, P, DT = nonterminals('N, V, P, DT')
+
+ >>> prod1 = Production(S, [NP, VP])
+ >>> prod2 = Production(NP, [DT, NP])
+
+ >>> prod1.lhs()
+ S
+
+ >>> prod1.rhs()
+ (NP, VP)
+
+ >>> prod1 == Production(S, [NP, VP])
+ True
+
+ >>> prod1 == prod2
+ False
+
+ >>> grammar = CFG.fromstring("""
+ ... S -> NP VP
+ ... PP -> P NP
+ ... NP -> 'the' N | N PP | 'the' N PP
+ ... VP -> V NP | V PP | V NP PP
+ ... N -> 'cat'
+ ... N -> 'dog'
+ ... N -> 'rug'
+ ... V -> 'chased'
+ ... V -> 'sat'
+ ... P -> 'in'
+ ... P -> 'on'
+ ... """)
+
+Unit tests for the rd (Recursive Descent Parser) class
+------------------------------------------------------
+
+Create and run a recursive descent parser over both a syntactically ambiguous
+and unambiguous sentence.
+
+ >>> from nltk.parse import RecursiveDescentParser
+ >>> rd = RecursiveDescentParser(grammar)
+
+ >>> sentence1 = 'the cat chased the dog'.split()
+ >>> sentence2 = 'the cat chased the dog on the rug'.split()
+
+ >>> for t in rd.parse(sentence1):
+ ... print(t)
+ (S (NP the (N cat)) (VP (V chased) (NP the (N dog))))
+
+ >>> for t in rd.parse(sentence2):
+ ... print(t)
+ (S
+ (NP the (N cat))
+ (VP (V chased) (NP the (N dog) (PP (P on) (NP the (N rug))))))
+ (S
+ (NP the (N cat))
+ (VP (V chased) (NP the (N dog)) (PP (P on) (NP the (N rug)))))
+
+
+(dolist (expr doctest-font-lock-keywords)
+ (add-to-list 'font-lock-keywords expr))
+
+ font-lock-keywords
+(add-to-list 'font-lock-keywords
+ (car doctest-font-lock-keywords))
+
+
+Unit tests for the sr (Shift Reduce Parser) class
+-------------------------------------------------
+
+Create and run a shift reduce parser over both a syntactically ambiguous
+and unambiguous sentence. Note that unlike the recursive descent parser, one
+and only one parse is ever returned.
+
+ >>> from nltk.parse import ShiftReduceParser
+ >>> sr = ShiftReduceParser(grammar)
+
+ >>> sentence1 = 'the cat chased the dog'.split()
+ >>> sentence2 = 'the cat chased the dog on the rug'.split()
+
+ >>> for t in sr.parse(sentence1):
+ ... print(t)
+ (S (NP the (N cat)) (VP (V chased) (NP the (N dog))))
+
+
+The shift reduce parser uses heuristics to decide what to do when there are
+multiple possible shift or reduce operations available - for the supplied
+grammar clearly the wrong operation is selected.
+
+ >>> for t in sr.parse(sentence2):
+ ... print(t)
+
+
+Unit tests for the Chart Parser class
+-------------------------------------
+
+We use the demo() function for testing.
+We must turn off showing of times.
+
+ >>> import nltk
+
+First we test tracing with a short sentence
+
+ >>> nltk.parse.chart.demo(2, print_times=False, trace=1,
+ ... sent='I saw a dog', numparses=1)
+ * Sentence:
+ I saw a dog
+ ['I', 'saw', 'a', 'dog']
+ <BLANKLINE>
+ * Strategy: Bottom-up
+ <BLANKLINE>
+ |. I . saw . a . dog .|
+ |[---------] . . .| [0:1] 'I'
+ |. [---------] . .| [1:2] 'saw'
+ |. . [---------] .| [2:3] 'a'
+ |. . . [---------]| [3:4] 'dog'
+ |> . . . .| [0:0] NP -> * 'I'
+ |[---------] . . .| [0:1] NP -> 'I' *
+ |> . . . .| [0:0] S -> * NP VP
+ |> . . . .| [0:0] NP -> * NP PP
+ |[---------> . . .| [0:1] S -> NP * VP
+ |[---------> . . .| [0:1] NP -> NP * PP
+ |. > . . .| [1:1] Verb -> * 'saw'
+ |. [---------] . .| [1:2] Verb -> 'saw' *
+ |. > . . .| [1:1] VP -> * Verb NP
+ |. > . . .| [1:1] VP -> * Verb
+ |. [---------> . .| [1:2] VP -> Verb * NP
+ |. [---------] . .| [1:2] VP -> Verb *
+ |. > . . .| [1:1] VP -> * VP PP
+ |[-------------------] . .| [0:2] S -> NP VP *
+ |. [---------> . .| [1:2] VP -> VP * PP
+ |. . > . .| [2:2] Det -> * 'a'
+ |. . [---------] .| [2:3] Det -> 'a' *
+ |. . > . .| [2:2] NP -> * Det Noun
+ |. . [---------> .| [2:3] NP -> Det * Noun
+ |. . . > .| [3:3] Noun -> * 'dog'
+ |. . . [---------]| [3:4] Noun -> 'dog' *
+ |. . [-------------------]| [2:4] NP -> Det Noun *
+ |. . > . .| [2:2] S -> * NP VP
+ |. . > . .| [2:2] NP -> * NP PP
+ |. [-----------------------------]| [1:4] VP -> Verb NP *
+ |. . [------------------->| [2:4] S -> NP * VP
+ |. . [------------------->| [2:4] NP -> NP * PP
+ |[=======================================]| [0:4] S -> NP VP *
+ |. [----------------------------->| [1:4] VP -> VP * PP
+ Nr edges in chart: 33
+ (S (NP I) (VP (Verb saw) (NP (Det a) (Noun dog))))
+ <BLANKLINE>
+
+Then we test the different parsing Strategies.
+Note that the number of edges differ between the strategies.
+
+Top-down
+
+ >>> nltk.parse.chart.demo(1, print_times=False, trace=0,
+ ... sent='I saw John with a dog', numparses=2)
+ * Sentence:
+ I saw John with a dog
+ ['I', 'saw', 'John', 'with', 'a', 'dog']
+ <BLANKLINE>
+ * Strategy: Top-down
+ <BLANKLINE>
+ Nr edges in chart: 48
+ (S
+ (NP I)
+ (VP (Verb saw) (NP (NP John) (PP with (NP (Det a) (Noun dog))))))
+ (S
+ (NP I)
+ (VP (VP (Verb saw) (NP John)) (PP with (NP (Det a) (Noun dog)))))
+ <BLANKLINE>
+
+Bottom-up
+
+ >>> nltk.parse.chart.demo(2, print_times=False, trace=0,
+ ... sent='I saw John with a dog', numparses=2)
+ * Sentence:
+ I saw John with a dog
+ ['I', 'saw', 'John', 'with', 'a', 'dog']
+ <BLANKLINE>
+ * Strategy: Bottom-up
+ <BLANKLINE>
+ Nr edges in chart: 53
+ (S
+ (NP I)
+ (VP (VP (Verb saw) (NP John)) (PP with (NP (Det a) (Noun dog)))))
+ (S
+ (NP I)
+ (VP (Verb saw) (NP (NP John) (PP with (NP (Det a) (Noun dog))))))
+ <BLANKLINE>
+
+Bottom-up Left-Corner
+
+ >>> nltk.parse.chart.demo(3, print_times=False, trace=0,
+ ... sent='I saw John with a dog', numparses=2)
+ * Sentence:
+ I saw John with a dog
+ ['I', 'saw', 'John', 'with', 'a', 'dog']
+ <BLANKLINE>
+ * Strategy: Bottom-up left-corner
+ <BLANKLINE>
+ Nr edges in chart: 36
+ (S
+ (NP I)
+ (VP (VP (Verb saw) (NP John)) (PP with (NP (Det a) (Noun dog)))))
+ (S
+ (NP I)
+ (VP (Verb saw) (NP (NP John) (PP with (NP (Det a) (Noun dog))))))
+ <BLANKLINE>
+
+Left-Corner with Bottom-Up Filter
+
+ >>> nltk.parse.chart.demo(4, print_times=False, trace=0,
+ ... sent='I saw John with a dog', numparses=2)
+ * Sentence:
+ I saw John with a dog
+ ['I', 'saw', 'John', 'with', 'a', 'dog']
+ <BLANKLINE>
+ * Strategy: Filtered left-corner
+ <BLANKLINE>
+ Nr edges in chart: 28
+ (S
+ (NP I)
+ (VP (VP (Verb saw) (NP John)) (PP with (NP (Det a) (Noun dog)))))
+ (S
+ (NP I)
+ (VP (Verb saw) (NP (NP John) (PP with (NP (Det a) (Noun dog))))))
+ <BLANKLINE>
+
+The stepping chart parser
+
+ >>> nltk.parse.chart.demo(5, print_times=False, trace=1,
+ ... sent='I saw John with a dog', numparses=2)
+ * Sentence:
+ I saw John with a dog
+ ['I', 'saw', 'John', 'with', 'a', 'dog']
+ <BLANKLINE>
+ * Strategy: Stepping (top-down vs bottom-up)
+ <BLANKLINE>
+ *** SWITCH TO TOP DOWN
+ |[------] . . . . .| [0:1] 'I'
+ |. [------] . . . .| [1:2] 'saw'
+ |. . [------] . . .| [2:3] 'John'
+ |. . . [------] . .| [3:4] 'with'
+ |. . . . [------] .| [4:5] 'a'
+ |. . . . . [------]| [5:6] 'dog'
+ |> . . . . . .| [0:0] S -> * NP VP
+ |> . . . . . .| [0:0] NP -> * NP PP
+ |> . . . . . .| [0:0] NP -> * Det Noun
+ |> . . . . . .| [0:0] NP -> * 'I'
+ |[------] . . . . .| [0:1] NP -> 'I' *
+ |[------> . . . . .| [0:1] S -> NP * VP
+ |[------> . . . . .| [0:1] NP -> NP * PP
+ |. > . . . . .| [1:1] VP -> * VP PP
+ |. > . . . . .| [1:1] VP -> * Verb NP
+ |. > . . . . .| [1:1] VP -> * Verb
+ |. > . . . . .| [1:1] Verb -> * 'saw'
+ |. [------] . . . .| [1:2] Verb -> 'saw' *
+ |. [------> . . . .| [1:2] VP -> Verb * NP
+ |. [------] . . . .| [1:2] VP -> Verb *
+ |[-------------] . . . .| [0:2] S -> NP VP *
+ |. [------> . . . .| [1:2] VP -> VP * PP
+ *** SWITCH TO BOTTOM UP
+ |. . > . . . .| [2:2] NP -> * 'John'
+ |. . . > . . .| [3:3] PP -> * 'with' NP
+ |. . . > . . .| [3:3] Prep -> * 'with'
+ |. . . . > . .| [4:4] Det -> * 'a'
+ |. . . . . > .| [5:5] Noun -> * 'dog'
+ |. . [------] . . .| [2:3] NP -> 'John' *
+ |. . . [------> . .| [3:4] PP -> 'with' * NP
+ |. . . [------] . .| [3:4] Prep -> 'with' *
+ |. . . . [------] .| [4:5] Det -> 'a' *
+ |. . . . . [------]| [5:6] Noun -> 'dog' *
+ |. [-------------] . . .| [1:3] VP -> Verb NP *
+ |[--------------------] . . .| [0:3] S -> NP VP *
+ |. [-------------> . . .| [1:3] VP -> VP * PP
+ |. . > . . . .| [2:2] S -> * NP VP
+ |. . > . . . .| [2:2] NP -> * NP PP
+ |. . . . > . .| [4:4] NP -> * Det Noun
+ |. . [------> . . .| [2:3] S -> NP * VP
+ |. . [------> . . .| [2:3] NP -> NP * PP
+ |. . . . [------> .| [4:5] NP -> Det * Noun
+ |. . . . [-------------]| [4:6] NP -> Det Noun *
+ |. . . [--------------------]| [3:6] PP -> 'with' NP *
+ |. [----------------------------------]| [1:6] VP -> VP PP *
+ *** SWITCH TO TOP DOWN
+ |. . > . . . .| [2:2] NP -> * Det Noun
+ |. . . . > . .| [4:4] NP -> * NP PP
+ |. . . > . . .| [3:3] VP -> * VP PP
+ |. . . > . . .| [3:3] VP -> * Verb NP
+ |. . . > . . .| [3:3] VP -> * Verb
+ |[=========================================]| [0:6] S -> NP VP *
+ |. [---------------------------------->| [1:6] VP -> VP * PP
+ |. . [---------------------------]| [2:6] NP -> NP PP *
+ |. . . . [------------->| [4:6] NP -> NP * PP
+ |. [----------------------------------]| [1:6] VP -> Verb NP *
+ |. . [--------------------------->| [2:6] S -> NP * VP
+ |. . [--------------------------->| [2:6] NP -> NP * PP
+ |[=========================================]| [0:6] S -> NP VP *
+ |. [---------------------------------->| [1:6] VP -> VP * PP
+ |. . . . . . >| [6:6] VP -> * VP PP
+ |. . . . . . >| [6:6] VP -> * Verb NP
+ |. . . . . . >| [6:6] VP -> * Verb
+ *** SWITCH TO BOTTOM UP
+ |. . . . > . .| [4:4] S -> * NP VP
+ |. . . . [------------->| [4:6] S -> NP * VP
+ *** SWITCH TO TOP DOWN
+ *** SWITCH TO BOTTOM UP
+ *** SWITCH TO TOP DOWN
+ *** SWITCH TO BOTTOM UP
+ *** SWITCH TO TOP DOWN
+ *** SWITCH TO BOTTOM UP
+ Nr edges in chart: 61
+ (S
+ (NP I)
+ (VP (VP (Verb saw) (NP John)) (PP with (NP (Det a) (Noun dog)))))
+ (S
+ (NP I)
+ (VP (Verb saw) (NP (NP John) (PP with (NP (Det a) (Noun dog))))))
+ <BLANKLINE>
+
+
+Unit tests for the Incremental Chart Parser class
+-------------------------------------------------
+
+The incremental chart parsers are defined in earleychart.py.
+We use the demo() function for testing. We must turn off showing of times.
+
+ >>> import nltk
+
+Earley Chart Parser
+
+ >>> nltk.parse.earleychart.demo(print_times=False, trace=1,
+ ... sent='I saw John with a dog', numparses=2)
+ * Sentence:
+ I saw John with a dog
+ ['I', 'saw', 'John', 'with', 'a', 'dog']
+ <BLANKLINE>
+ |. I . saw . John . with . a . dog .|
+ |[------] . . . . .| [0:1] 'I'
+ |. [------] . . . .| [1:2] 'saw'
+ |. . [------] . . .| [2:3] 'John'
+ |. . . [------] . .| [3:4] 'with'
+ |. . . . [------] .| [4:5] 'a'
+ |. . . . . [------]| [5:6] 'dog'
+ |> . . . . . .| [0:0] S -> * NP VP
+ |> . . . . . .| [0:0] NP -> * NP PP
+ |> . . . . . .| [0:0] NP -> * Det Noun
+ |> . . . . . .| [0:0] NP -> * 'I'
+ |[------] . . . . .| [0:1] NP -> 'I' *
+ |[------> . . . . .| [0:1] S -> NP * VP
+ |[------> . . . . .| [0:1] NP -> NP * PP
+ |. > . . . . .| [1:1] VP -> * VP PP
+ |. > . . . . .| [1:1] VP -> * Verb NP
+ |. > . . . . .| [1:1] VP -> * Verb
+ |. > . . . . .| [1:1] Verb -> * 'saw'
+ |. [------] . . . .| [1:2] Verb -> 'saw' *
+ |. [------> . . . .| [1:2] VP -> Verb * NP
+ |. [------] . . . .| [1:2] VP -> Verb *
+ |[-------------] . . . .| [0:2] S -> NP VP *
+ |. [------> . . . .| [1:2] VP -> VP * PP
+ |. . > . . . .| [2:2] NP -> * NP PP
+ |. . > . . . .| [2:2] NP -> * Det Noun
+ |. . > . . . .| [2:2] NP -> * 'John'
+ |. . [------] . . .| [2:3] NP -> 'John' *
+ |. [-------------] . . .| [1:3] VP -> Verb NP *
+ |. . [------> . . .| [2:3] NP -> NP * PP
+ |. . . > . . .| [3:3] PP -> * 'with' NP
+ |[--------------------] . . .| [0:3] S -> NP VP *
+ |. [-------------> . . .| [1:3] VP -> VP * PP
+ |. . . [------> . .| [3:4] PP -> 'with' * NP
+ |. . . . > . .| [4:4] NP -> * NP PP
+ |. . . . > . .| [4:4] NP -> * Det Noun
+ |. . . . > . .| [4:4] Det -> * 'a'
+ |. . . . [------] .| [4:5] Det -> 'a' *
+ |. . . . [------> .| [4:5] NP -> Det * Noun
+ |. . . . . > .| [5:5] Noun -> * 'dog'
+ |. . . . . [------]| [5:6] Noun -> 'dog' *
+ |. . . . [-------------]| [4:6] NP -> Det Noun *
+ |. . . [--------------------]| [3:6] PP -> 'with' NP *
+ |. . . . [------------->| [4:6] NP -> NP * PP
+ |. . [---------------------------]| [2:6] NP -> NP PP *
+ |. [----------------------------------]| [1:6] VP -> VP PP *
+ |[=========================================]| [0:6] S -> NP VP *
+ |. [---------------------------------->| [1:6] VP -> VP * PP
+ |. [----------------------------------]| [1:6] VP -> Verb NP *
+ |. . [--------------------------->| [2:6] NP -> NP * PP
+ |[=========================================]| [0:6] S -> NP VP *
+ |. [---------------------------------->| [1:6] VP -> VP * PP
+ (S
+ (NP I)
+ (VP (VP (Verb saw) (NP John)) (PP with (NP (Det a) (Noun dog)))))
+ (S
+ (NP I)
+ (VP (Verb saw) (NP (NP John) (PP with (NP (Det a) (Noun dog))))))
+
+
+Unit tests for LARGE context-free grammars
+------------------------------------------
+
+Reading the ATIS grammar.
+
+ >>> grammar = nltk.data.load('grammars/large_grammars/atis.cfg')
+ >>> grammar
+ <Grammar with 5517 productions>
+
+Reading the test sentences.
+
+ >>> sentences = nltk.data.load('grammars/large_grammars/atis_sentences.txt')
+ >>> sentences = nltk.parse.util.extract_test_sentences(sentences)
+ >>> len(sentences)
+ 98
+ >>> testsentence = sentences[22]
+ >>> testsentence[0]
+ ['show', 'me', 'northwest', 'flights', 'to', 'detroit', '.']
+ >>> testsentence[1]
+ 17
+ >>> sentence = testsentence[0]
+
+Now we test all different parsing strategies.
+Note that the number of edges differ between the strategies.
+
+Bottom-up parsing.
+
+ >>> parser = nltk.parse.BottomUpChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 7661
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+Bottom-up Left-corner parsing.
+
+ >>> parser = nltk.parse.BottomUpLeftCornerChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 4986
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+Left-corner parsing with bottom-up filter.
+
+ >>> parser = nltk.parse.LeftCornerChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 1342
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+Top-down parsing.
+
+ >>> parser = nltk.parse.TopDownChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 28352
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+Incremental Bottom-up parsing.
+
+ >>> parser = nltk.parse.IncrementalBottomUpChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 7661
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+Incremental Bottom-up Left-corner parsing.
+
+ >>> parser = nltk.parse.IncrementalBottomUpLeftCornerChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 4986
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+Incremental Left-corner parsing with bottom-up filter.
+
+ >>> parser = nltk.parse.IncrementalLeftCornerChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 1342
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+Incremental Top-down parsing.
+
+ >>> parser = nltk.parse.IncrementalTopDownChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 28352
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+Earley parsing. This is similar to the incremental top-down algorithm.
+
+ >>> parser = nltk.parse.EarleyChartParser(grammar)
+ >>> chart = parser.chart_parse(sentence)
+ >>> print((chart.num_edges()))
+ 28352
+ >>> print((len(list(chart.parses(grammar.start())))))
+ 17
+
+
+Unit tests for the Probabilistic CFG class
+------------------------------------------
+
+ >>> from nltk.corpus import treebank
+ >>> from itertools import islice
+ >>> from nltk.grammar import PCFG, induce_pcfg, toy_pcfg1, toy_pcfg2
+
+Create a set of PCFG productions.
+
+ >>> grammar = PCFG.fromstring("""
+ ... A -> B B [.3] | C B C [.7]
+ ... B -> B D [.5] | C [.5]
+ ... C -> 'a' [.1] | 'b' [0.9]
+ ... D -> 'b' [1.0]
+ ... """)
+ >>> prod = grammar.productions()[0]
+ >>> prod
+ A -> B B [0.3]
+
+ >>> prod.lhs()
+ A
+
+ >>> prod.rhs()
+ (B, B)
+
+ >>> print((prod.prob()))
+ 0.3
+
+ >>> grammar.start()
+ A
+
+ >>> grammar.productions()
+ [A -> B B [0.3], A -> C B C [0.7], B -> B D [0.5], B -> C [0.5], C -> 'a' [0.1], C -> 'b' [0.9], D -> 'b' [1]]
+
+Induce some productions using parsed Treebank data.
+
+ >>> productions = []
+ >>> for fileid in treebank.fileids()[:2]:
+ ... for t in treebank.parsed_sents(fileid):
+ ... productions += t.productions()
+
+ >>> grammar = induce_pcfg(S, productions)
+ >>> grammar
+ <Grammar with 71 productions>
+
+ >>> sorted(grammar.productions(lhs=Nonterminal('PP')))[:2]
+ [PP -> IN NP [1]]
+ >>> sorted(grammar.productions(lhs=Nonterminal('NNP')))[:2]
+ [NNP -> 'Agnew' [0.0714286], NNP -> 'Consolidated' [0.0714286]]
+ >>> sorted(grammar.productions(lhs=Nonterminal('JJ')))[:2]
+ [JJ -> 'British' [0.142857], JJ -> 'former' [0.142857]]
+ >>> sorted(grammar.productions(lhs=Nonterminal('NP')))[:2]
+ [NP -> CD NNS [0.133333], NP -> DT JJ JJ NN [0.0666667]]
+
+Unit tests for the Probabilistic Chart Parse classes
+----------------------------------------------------
+
+ >>> tokens = "Jack saw Bob with my cookie".split()
+ >>> grammar = toy_pcfg2
+ >>> print(grammar)
+ Grammar with 23 productions (start state = S)
+ S -> NP VP [1]
+ VP -> V NP [0.59]
+ VP -> V [0.4]
+ VP -> VP PP [0.01]
+ NP -> Det N [0.41]
+ NP -> Name [0.28]
+ NP -> NP PP [0.31]
+ PP -> P NP [1]
+ V -> 'saw' [0.21]
+ V -> 'ate' [0.51]
+ V -> 'ran' [0.28]
+ N -> 'boy' [0.11]
+ N -> 'cookie' [0.12]
+ N -> 'table' [0.13]
+ N -> 'telescope' [0.14]
+ N -> 'hill' [0.5]
+ Name -> 'Jack' [0.52]
+ Name -> 'Bob' [0.48]
+ P -> 'with' [0.61]
+ P -> 'under' [0.39]
+ Det -> 'the' [0.41]
+ Det -> 'a' [0.31]
+ Det -> 'my' [0.28]
+
+Create several parsers using different queuing strategies and show the
+resulting parses.
+
+ >>> from nltk.parse import pchart
+
+ >>> parser = pchart.InsideChartParser(grammar)
+ >>> for t in parser.parse(tokens):
+ ... print(t)
+ (S
+ (NP (Name Jack))
+ (VP
+ (V saw)
+ (NP
+ (NP (Name Bob))
+ (PP (P with) (NP (Det my) (N cookie)))))) (p=6.31607e-06)
+ (S
+ (NP (Name Jack))
+ (VP
+ (VP (V saw) (NP (Name Bob)))
+ (PP (P with) (NP (Det my) (N cookie))))) (p=2.03744e-07)
+
+ >>> parser = pchart.RandomChartParser(grammar)
+ >>> for t in parser.parse(tokens):
+ ... print(t)
+ (S
+ (NP (Name Jack))
+ (VP
+ (V saw)
+ (NP
+ (NP (Name Bob))
+ (PP (P with) (NP (Det my) (N cookie)))))) (p=6.31607e-06)
+ (S
+ (NP (Name Jack))
+ (VP
+ (VP (V saw) (NP (Name Bob)))
+ (PP (P with) (NP (Det my) (N cookie))))) (p=2.03744e-07)
+
+ >>> parser = pchart.UnsortedChartParser(grammar)
+ >>> for t in parser.parse(tokens):
+ ... print(t)
+ (S
+ (NP (Name Jack))
+ (VP
+ (V saw)
+ (NP
+ (NP (Name Bob))
+ (PP (P with) (NP (Det my) (N cookie)))))) (p=6.31607e-06)
+ (S
+ (NP (Name Jack))
+ (VP
+ (VP (V saw) (NP (Name Bob)))
+ (PP (P with) (NP (Det my) (N cookie))))) (p=2.03744e-07)
+
+ >>> parser = pchart.LongestChartParser(grammar)
+ >>> for t in parser.parse(tokens):
+ ... print(t)
+ (S
+ (NP (Name Jack))
+ (VP
+ (V saw)
+ (NP
+ (NP (Name Bob))
+ (PP (P with) (NP (Det my) (N cookie)))))) (p=6.31607e-06)
+ (S
+ (NP (Name Jack))
+ (VP
+ (VP (V saw) (NP (Name Bob)))
+ (PP (P with) (NP (Det my) (N cookie))))) (p=2.03744e-07)
+
+ >>> parser = pchart.InsideChartParser(grammar, beam_size = len(tokens)+1)
+ >>> for t in parser.parse(tokens):
+ ... print(t)
+
+
+Unit tests for the Viterbi Parse classes
+----------------------------------------
+
+ >>> from nltk.parse import ViterbiParser
+ >>> tokens = "Jack saw Bob with my cookie".split()
+ >>> grammar = toy_pcfg2
+
+Parse the tokenized sentence.
+
+ >>> parser = ViterbiParser(grammar)
+ >>> for t in parser.parse(tokens):
+ ... print(t)
+ (S
+ (NP (Name Jack))
+ (VP
+ (V saw)
+ (NP
+ (NP (Name Bob))
+ (PP (P with) (NP (Det my) (N cookie)))))) (p=6.31607e-06)
+
+
+Unit tests for the FeatStructNonterminal class
+----------------------------------------------
+
+ >>> from nltk.grammar import FeatStructNonterminal
+ >>> FeatStructNonterminal(
+ ... pos='n', agr=FeatStructNonterminal(number='pl', gender='f'))
+ [agr=[gender='f', number='pl'], pos='n']
+
+ >>> FeatStructNonterminal('VP[+fin]/NP[+pl]')
+ VP[+fin]/NP[+pl]
+
+
+Tracing the Feature Chart Parser
+--------------------------------
+
+We use the featurechart.demo() function for tracing the Feature Chart Parser.
+
+ >>> nltk.parse.featurechart.demo(print_times=False,
+ ... print_grammar=True,
+ ... parser=nltk.parse.featurechart.FeatureChartParser,
+ ... sent='I saw John with a dog')
+ <BLANKLINE>
+ Grammar with 18 productions (start state = S[])
+ S[] -> NP[] VP[]
+ PP[] -> Prep[] NP[]
+ NP[] -> NP[] PP[]
+ VP[] -> VP[] PP[]
+ VP[] -> Verb[] NP[]
+ VP[] -> Verb[]
+ NP[] -> Det[pl=?x] Noun[pl=?x]
+ NP[] -> 'John'
+ NP[] -> 'I'
+ Det[] -> 'the'
+ Det[] -> 'my'
+ Det[-pl] -> 'a'
+ Noun[-pl] -> 'dog'
+ Noun[-pl] -> 'cookie'
+ Verb[] -> 'ate'
+ Verb[] -> 'saw'
+ Prep[] -> 'with'
+ Prep[] -> 'under'
+ <BLANKLINE>
+ * FeatureChartParser
+ Sentence: I saw John with a dog
+ |.I.s.J.w.a.d.|
+ |[-] . . . . .| [0:1] 'I'
+ |. [-] . . . .| [1:2] 'saw'
+ |. . [-] . . .| [2:3] 'John'
+ |. . . [-] . .| [3:4] 'with'
+ |. . . . [-] .| [4:5] 'a'
+ |. . . . . [-]| [5:6] 'dog'
+ |[-] . . . . .| [0:1] NP[] -> 'I' *
+ |[-> . . . . .| [0:1] S[] -> NP[] * VP[] {}
+ |[-> . . . . .| [0:1] NP[] -> NP[] * PP[] {}
+ |. [-] . . . .| [1:2] Verb[] -> 'saw' *
+ |. [-> . . . .| [1:2] VP[] -> Verb[] * NP[] {}
+ |. [-] . . . .| [1:2] VP[] -> Verb[] *
+ |. [-> . . . .| [1:2] VP[] -> VP[] * PP[] {}
+ |[---] . . . .| [0:2] S[] -> NP[] VP[] *
+ |. . [-] . . .| [2:3] NP[] -> 'John' *
+ |. . [-> . . .| [2:3] S[] -> NP[] * VP[] {}
+ |. . [-> . . .| [2:3] NP[] -> NP[] * PP[] {}
+ |. [---] . . .| [1:3] VP[] -> Verb[] NP[] *
+ |. [---> . . .| [1:3] VP[] -> VP[] * PP[] {}
+ |[-----] . . .| [0:3] S[] -> NP[] VP[] *
+ |. . . [-] . .| [3:4] Prep[] -> 'with' *
+ |. . . [-> . .| [3:4] PP[] -> Prep[] * NP[] {}
+ |. . . . [-] .| [4:5] Det[-pl] -> 'a' *
+ |. . . . [-> .| [4:5] NP[] -> Det[pl=?x] * Noun[pl=?x] {?x: False}
+ |. . . . . [-]| [5:6] Noun[-pl] -> 'dog' *
+ |. . . . [---]| [4:6] NP[] -> Det[-pl] Noun[-pl] *
+ |. . . . [--->| [4:6] S[] -> NP[] * VP[] {}
+ |. . . . [--->| [4:6] NP[] -> NP[] * PP[] {}
+ |. . . [-----]| [3:6] PP[] -> Prep[] NP[] *
+ |. . [-------]| [2:6] NP[] -> NP[] PP[] *
+ |. [---------]| [1:6] VP[] -> VP[] PP[] *
+ |. [--------->| [1:6] VP[] -> VP[] * PP[] {}
+ |[===========]| [0:6] S[] -> NP[] VP[] *
+ |. . [------->| [2:6] S[] -> NP[] * VP[] {}
+ |. . [------->| [2:6] NP[] -> NP[] * PP[] {}
+ |. [---------]| [1:6] VP[] -> Verb[] NP[] *
+ |. [--------->| [1:6] VP[] -> VP[] * PP[] {}
+ |[===========]| [0:6] S[] -> NP[] VP[] *
+ (S[]
+ (NP[] I)
+ (VP[]
+ (VP[] (Verb[] saw) (NP[] John))
+ (PP[] (Prep[] with) (NP[] (Det[-pl] a) (Noun[-pl] dog)))))
+ (S[]
+ (NP[] I)
+ (VP[]
+ (Verb[] saw)
+ (NP[]
+ (NP[] John)
+ (PP[] (Prep[] with) (NP[] (Det[-pl] a) (Noun[-pl] dog))))))
+
+
+Unit tests for the Feature Chart Parser classes
+-----------------------------------------------
+
+The list of parsers we want to test.
+
+ >>> parsers = [nltk.parse.featurechart.FeatureChartParser,
+ ... nltk.parse.featurechart.FeatureTopDownChartParser,
+ ... nltk.parse.featurechart.FeatureBottomUpChartParser,
+ ... nltk.parse.featurechart.FeatureBottomUpLeftCornerChartParser,
+ ... nltk.parse.earleychart.FeatureIncrementalChartParser,
+ ... nltk.parse.earleychart.FeatureEarleyChartParser,
+ ... nltk.parse.earleychart.FeatureIncrementalTopDownChartParser,
+ ... nltk.parse.earleychart.FeatureIncrementalBottomUpChartParser,
+ ... nltk.parse.earleychart.FeatureIncrementalBottomUpLeftCornerChartParser,
+ ... ]
+
+A helper function that tests each parser on the given grammar and sentence.
+We check that the number of trees are correct, and that all parsers
+return the same trees. Otherwise an error is printed.
+
+ >>> def unittest(grammar, sentence, nr_trees):
+ ... sentence = sentence.split()
+ ... trees = None
+ ... for P in parsers:
+ ... result = P(grammar).parse(sentence)
+ ... result = set(tree.freeze() for tree in result)
+ ... if len(result) != nr_trees:
+ ... print("Wrong nr of trees:", len(result))
+ ... elif trees is None:
+ ... trees = result
+ ... elif result != trees:
+ ... print("Trees differ for parser:", P.__name__)
+
+The demo grammar from before, with an ambiguous sentence.
+
+ >>> isawjohn = nltk.parse.featurechart.demo_grammar()
+ >>> unittest(isawjohn, "I saw John with a dog with my cookie", 5)
+
+This grammar tests that variables in different grammar rules are renamed
+before unification. (The problematic variable is in this case ?X).
+
+ >>> whatwasthat = nltk.grammar.FeatureGrammar.fromstring('''
+ ... S[] -> NP[num=?N] VP[num=?N, slash=?X]
+ ... NP[num=?X] -> "what"
+ ... NP[num=?X] -> "that"
+ ... VP[num=?P, slash=none] -> V[num=?P] NP[]
+ ... V[num=sg] -> "was"
+ ... ''')
+ >>> unittest(whatwasthat, "what was that", 1)
+
+This grammar tests that the same rule can be used in different places
+in another rule, and that the variables are properly renamed.
+
+ >>> thislovesthat = nltk.grammar.FeatureGrammar.fromstring('''
+ ... S[] -> NP[case=nom] V[] NP[case=acc]
+ ... NP[case=?X] -> Pron[case=?X]
+ ... Pron[] -> "this"
+ ... Pron[] -> "that"
+ ... V[] -> "loves"
+ ... ''')
+ >>> unittest(thislovesthat, "this loves that", 1)
+
+
+Tests for loading feature grammar files
+---------------------------------------
+
+Alternative 1: first load the grammar, then create the parser.
+
+ >>> fcfg = nltk.data.load('grammars/book_grammars/feat0.fcfg')
+ >>> fcp1 = nltk.parse.FeatureChartParser(fcfg)
+ >>> print((type(fcp1)))
+ <class 'nltk.parse.featurechart.FeatureChartParser'>
+
+Alternative 2: directly load the parser.
+
+ >>> fcp2 = nltk.parse.load_parser('grammars/book_grammars/feat0.fcfg')
+ >>> print((type(fcp2)))
+ <class 'nltk.parse.featurechart.FeatureChartParser'>
+
+
+
diff --git a/nltk/test/portuguese.doctest_latin1 b/nltk/test/portuguese.doctest_latin1
new file mode 100644
index 0000000..888f729
--- /dev/null
+++ b/nltk/test/portuguese.doctest_latin1
@@ -0,0 +1,300 @@
+==========================================
+Examplos para o processamento do portugu�s
+==========================================
+
+ >>> import nltk
+
+(NB. Este material parte do pressuposto de que o leitor esteja
+familiarizado com o livro do NLTK, dispon�vel em
+``http://nltk.org/index.php/Book``).
+
+Utilizando o Corpus MacMorpho Tagged
+------------------------------------
+
+O NLTK inclui o corpus de not�cias para o portugu�s brasileiro com tags de partes do discurso
+MAC-MORPHO, que conta com mais de um milh�o de palavras de textos jornal�sticos extra�dos
+de dez se��es do jornal di�rio *Folha de S�o Paulo*, do ano de 1994.
+
+Podemos utilizar este corpus como uma seq��ncia de palavras ou de palavras com tags da
+seguinte maneira:
+
+ >>> nltk.corpus.mac_morpho.words()
+ ['Jersei', 'atinge', 'm\xe9dia', 'de', 'Cr$', '1,4', ...]
+ >>> nltk.corpus.mac_morpho.sents()
+ [['Jersei', 'atinge', 'm\xe9dia', 'de', 'Cr$', '1,4', 'milh\xe3o',
+ 'em', 'a', 'venda', 'de', 'a', 'Pinhal', 'em', 'S\xe3o', 'Paulo'],
+ ['Programe', 'sua', 'viagem', 'a', 'a', 'Exposi\xe7\xe3o', 'Nacional',
+ 'do', 'Zebu', ',', 'que', 'come\xe7a', 'dia', '25'], ...]
+ >>> nltk.corpus.mac_morpho.tagged_words()
+ [('Jersei', 'N'), ('atinge', 'V'), ('m\xe9dia', 'N'), ...]
+
+Tamb�m � poss�vel utiliz�-lo em chunks de frases.
+
+ >>> nltk.corpus.mac_morpho.tagged_sents()
+ [[('Jersei', 'N'), ('atinge', 'V'), ('m\xe9dia', 'N'), ('de', 'PREP'),
+ ('Cr$', 'CUR'), ('1,4', 'NUM'), ('milh\xe3o', 'N'), ('em', 'PREP|+'),
+ ('a', 'ART'), ('venda', 'N'), ('de', 'PREP|+'), ('a', 'ART'),
+ ('Pinhal', 'NPROP'), ('em', 'PREP'), ('S\xe3o', 'NPROP'), ('Paulo', 'NPROP')],
+ [('Programe', 'V'), ('sua', 'PROADJ'), ('viagem', 'N'), ('a', 'PREP|+'),
+ ('a', 'ART'), ('Exposi\xe7\xe3o', 'NPROP'), ('Nacional', 'NPROP'),
+ ('do', 'NPROP'), ('Zebu', 'NPROP'), (',', ','), ('que', 'PRO-KS-REL'),
+ ('come\xe7a', 'V'), ('dia', 'N'), ('25', 'N|AP')], ...]
+
+Estes dados podem ser utilizados para efetuar o treinamento de taggers (como nos
+exemplos abaixo para o Floresta treebank).
+
+
+Utilizando o Floresta Portuguese Treebank
+-----------------------------------------
+
+A distribui��o de dados do NLTK inclui o
+"Floresta Sinta(c)tica Corpus" na vers�o 7.4, dispon�vel em
+``http://www.linguateca.pt/Floresta/``.
+
+Como para a amostra do Penn Treebank, � poss�vel
+utilizar o conte�do deste corpus como uma seq��ncia de palavras com
+informa��es de tags, da seguinte maneira:
+
+ >>> from nltk.corpus import floresta
+ >>> floresta.words()
+ ['Um', 'revivalismo', 'refrescante', 'O', '7_e_Meio', ...]
+ >>> floresta.tagged_words()
+ [('Um', '>N+art'), ('revivalismo', 'H+n'), ...]
+
+As tags s�o constitu�das por certas informa��es sint�ticas, seguidas por
+um sinal
+de mais, seguido por tag costumeira de parte do discurso
+(part-of-speech). Vamos
+remover o conte�do que antecede o sinal de mais:
+
+ >>> def simplify_tag(t):
+ ... if "+" in t:
+ ... return t[t.index("+")+1:]
+ ... else:
+ ... return t
+ >>> twords = nltk.corpus.floresta.tagged_words()
+ >>> twords = [(w.lower(),simplify_tag(t)) for (w,t) in twords]
+ >>> twords[:10] # doctest: +NORMALIZE_WHITESPACE
+ [('um', 'art'), ('revivalismo', 'n'), ('refrescante', 'adj'), ('o', 'art'), ('7_e_meio', 'prop'),
+ ('\xe9', 'v-fin'), ('um', 'art'), ('ex-libris', 'n'), ('de', 'prp'), ('a', 'art')]
+
+E exibir de maneira mais apropriada as palavras com informa��es de tags:
+
+ >>> print ' '.join(word + '/' + tag for (word, tag) in twords[:10])
+ um/art revivalismo/n refrescante/adj o/art 7_e_meio/prop ?/v-fin um/art ex-libris/n de/prp a/art
+
+Em seguida, vamos contar o n�mero de tokens de palavras e tipos, al�m de
+determinar qual a palavra mais comum:
+
+ >>> words = floresta.words()
+ >>> len(words)
+ 211870
+ >>> fd = nltk.FreqDist(words)
+ >>> len(fd)
+ 29425
+ >>> fd.max()
+ 'de'
+
+Podemos tamb�m listar as 20 tags mais freq�entes, em ordem decrescente de
+freq��ncia:
+
+ >>> tags = [simplify_tag(tag) for (word,tag) in floresta.tagged_words()]
+ >>> fd = nltk.FreqDist(tags)
+ >>> fd.sorted()[:20] # doctest: +NORMALIZE_WHITESPACE
+ ['n', 'prp', 'art', 'v-fin', ',', 'prop', 'adj', 'adv', '.', 'conj-c', 'v-inf',
+ 'pron-det', 'v-pcp', 'num', 'pron-indp', 'pron-pers', '\xab', '\xbb', 'conj-s', '}']
+
+Tamb�m podemos ler o corpus agrupado por enunciados:
+
+ >>> floresta.sents() # doctest: +NORMALIZE_WHITESPACE
+ [['Um', 'revivalismo', 'refrescante'], ['O', '7_e_Meio', '\xe9', 'um', 'ex-libris',
+ 'de', 'a', 'noite', 'algarvia', '.'], ...]
+ >>> floresta.tagged_sents() # doctest: +NORMALIZE_WHITESPACE
+ [[('Um', '>N+art'), ('revivalismo', 'H+n'), ('refrescante', 'N<+adj')],
+ [('O', '>N+art'), ('7_e_Meio', 'H+prop'), ('\xe9', 'P+v-fin'), ('um', '>N+art'),
+ ('ex-libris', 'H+n'), ('de', 'H+prp'), ('a', '>N+art'), ('noite', 'H+n'),
+ ('algarvia', 'N<+adj'), ('.', '.')], ...]
+ >>> floresta.parsed_sents() # doctest: +NORMALIZE_WHITESPACE
+ [Tree('UTT+np', [Tree('>N+art', ['Um']), Tree('H+n', ['revivalismo']),
+ Tree('N<+adj', ['refrescante'])]), Tree('STA+fcl', [Tree('SUBJ+np',
+ [Tree('>N+art', ['O']), Tree('H+prop', ['7_e_Meio'])]), Tree('P+v-fin', ['\xe9']),
+ Tree('SC+np', [Tree('>N+art', ['um']), Tree('H+n', ['ex-libris']),
+ Tree('N<+pp', [Tree('H+prp', ['de']), Tree('P<+np', [Tree('>N+art', ['a']),
+ Tree('H+n', ['noite']), Tree('N<+adj', ['algarvia'])])])]), Tree('.', ['.'])]), ...]
+
+Para ver uma �rvore de an�lise sint�tica, podemos utilizar o m�todo
+``draw()``, como no exemplo:
+
+ >>> psents = floresta.parsed_sents()
+ >>> psents[5].draw() # doctest: +SKIP
+
+
+Concord�ncia simples
+--------------------
+
+A seguir, apresentamos uma fun��o que recebe uma palavra e uma
+quantidade determinada
+de contexto (medido em caracteres) e gera uma concord�ncia para a mesma.
+
+ >>> def concordance(word, context=30):
+ ... for sent in floresta.sents():
+ ... if word in sent:
+ ... pos = sent.index(word)
+ ... left = ' '.join(sent[:pos])
+ ... right = ' '.join(sent[pos+1:])
+ ... print '%*s %s %-*s' %\
+ ... (context, left[-context:], word, context, right[:context])
+
+ >>> concordance("dar") # doctest: +SKIP
+ anduru , foi o suficiente para dar a volta a o resultado .
+ 1. O P?BLICO veio dar a a imprensa di?ria portuguesa
+ A fartura de pensamento pode dar maus resultados e n?s n?o quer
+ Come?a a dar resultados a pol?tica de a Uni
+ ial come?ar a incorporar- lo e dar forma a um ' site ' que tem se
+ r com Constantino para ele lhe dar tamb?m os pap?is assinados .
+ va a brincar , pois n?o lhe ia dar procura??o nenhuma enquanto n?
+ ?rica como o ant?doto capaz de dar sentido a o seu enorme poder .
+ . . .
+ >>> concordance("vender") # doctest: +SKIP
+ er recebido uma encomenda para vender 4000 blindados a o Iraque .
+ m?rico_Amorim caso conseguisse vender o lote de ac??es de o empres?r
+ mpre ter jovens simp?ticos a ? vender ? chega ! }
+ Disse que o governo vai vender ? desde autom?vel at? particip
+ ndiciou ontem duas pessoas por vender carro com ?gio .
+ A inten??o de Fleury ? vender as a??es para equilibrar as fi
+
+Tagging de partes do discurso
+-----------------------------
+
+Vamos come�ar obtendo os dados dos enunciados marcados com tags e
+simplificando
+estas �ltimas como descrito anteriormente.
+
+ >>> from nltk.corpus import floresta
+ >>> tsents = floresta.tagged_sents()
+ >>> tsents = [[(w.lower(),simplify_tag(t)) for (w,t) in sent] for sent in tsents if sent]
+ >>> train = tsents[100:]
+ >>> test = tsents[:100]
+
+J� sabemos que ``n`` � a tag mais comum; desta forma, podemos criar um
+tagger por default
+que marque toda palavra como substantivo e, em seguida, avaliar seu
+desempenho:
+
+ >>> tagger0 = nltk.DefaultTagger('n')
+ >>> nltk.tag.accuracy(tagger0, test)
+ 0.17690941385435169
+
+Como pode-se deduzir facilmente, uma em cada seis palavras � um
+substantivo. Vamos
+aperfei�oar estes resultados treinando um tagger unigrama:
+
+ >>> tagger1 = nltk.UnigramTagger(train, backoff=tagger0)
+ >>> nltk.tag.accuracy(tagger1, test)
+ 0.85115452930728241
+
+E, em seguida, um tagger bigrama:
+
+ >>> tagger2 = nltk.BigramTagger(train, backoff=tagger1)
+ >>> nltk.tag.accuracy(tagger2, test)
+ 0.86856127886323264
+
+Segmenta��o de frases
+---------------------
+
+O Punkt � uma ferramenta para segmenta��o de frases ling�isticamente independente, o qual
+requer um treinamento em texto puro.
+O texto de origem (obtido do Floresta Portuguese Treebank) cont�m uma frase por linha. Podemos
+ler o texto, dividi-lo em fun��o de suas linhas e ent�o agrupar estas linhas utilizando
+espa�os. Desta forma as informa��es sobre quebras de frases ter�o sido descartadas; podemos
+ent�o dividir este material em dados para treinamento e para verifica��o:
+
+ >>> text = open('floresta.txt').read()
+ >>> lines = text.split('\n')
+ >>> train = ' '.join(lines[10:])
+ >>> test = ' '.join(lines[:10])
+
+� agora poss�vel treinar o segmentador de frases (ou tokenizador de frases) e utiliz�-lo em
+nossas frases de verifica��o. (Para exibir o texto em uma forma leg�vel, pode ser necess�rio
+converter o texto para o UTF-8, utilizando ``print sent.decode('latin-1').encode('utf-8')``.)
+
+ >>> stok = nltk.PunktSentenceTokenizer(train)
+ >>> for sent in stok.tokenize(test):
+ ... print sent
+
+
+As vers�es do NLTK a partir da 0.9b1 incluem um modelo treinado para a segmenta��o de frases
+em portugu�s, o qual pode ser carregado pela maneira a seguir. � mais r�pido carregar um modelo
+j� treinado do que repetir o treinamento do mesmo.
+
+ >>> stok = nltk.data.load('tokenizers/punkt/portuguese.pickle')
+
+Stemming
+--------
+
+O NLTK inclui o stemmer para o portugu�s RSLP. Vamos demonstrar sua utiliza��o para algumas
+palavras em portugu�s:
+
+ >>> stemmer = nltk.stem.RSLPStemmer()
+ >>> stemmer.stem("copiar")
+ u'copi'
+ >>> stemmer.stem("paisagem")
+ u'pais'
+
+Stopwords
+---------
+
+O NLTK inclui stopword ("palavras limite") para o portugu�s:
+
+ >>> stopwords = nltk.corpus.stopwords.words('portuguese')
+ >>> stopwords[:10]
+ ['a', 'ao', 'aos', 'aquela', 'aquelas', 'aquele', 'aqueles', 'aquilo', 'as', 'at\xe9']
+
+A esta altura, � poss�vel utiliz�-las para filtrar textos. Vamos encontrar as palavras mais
+comuns (� exce��o das stopwords) e list�-las em ordem decrescente de freq��ncia:
+
+ >>> fd = nltk.FreqDist(w.lower() for w in floresta.words() if w not in stopwords)
+ >>> for word in fd.sorted()[:20]:
+ ... print word, fd[word]
+ , 13444
+ . 7725
+ ? 2369
+ ? 2310
+ ? 1137
+ o 1086
+ } 1047
+ { 1044
+ a 897
+ ; 633
+ em 516
+ ser 466
+ sobre 349
+ os 313
+ anos 301
+ ontem 292
+ ainda 279
+ segundo 256
+ ter 249
+ dois 231
+
+
+Codifica��es de caracteres
+--------------------------
+
+O Python � capaz de lidar com todas a codifica��es de caracteres mais utilizada para o portugu�s, a
+ISO 8859-1 (ISO Latin 1).
+
+ >>> text = open('floresta.txt').read()
+ >>> text[:60]
+ 'O 7 e Meio \xe9 um ex-libris da noite algarvia.\n\xc9 uma das mais '
+ >>> print text[:60]
+ O 7 e Meio ? um ex-libris da noite algarvia.
+ ? uma das mais
+ >>> text[:60].decode('latin-1')
+ u'O 7 e Meio \xe9 um ex-libris da noite algarvia.\n\xc9 uma das mais '
+ >>> text[:60].decode('latin-1').encode('utf-8')
+ 'O 7 e Meio \xc3\xa9 um ex-libris da noite algarvia.\n\xc3\x89 uma das mais '
+ >>> text[:60].decode('latin-1').encode('utf-8')
+ 'O 7 e Meio \xc3\xa9 um ex-libris da noite algarvia.\n\xc3\x89 uma das mais '
+ >>> text[:60].decode('latin-1').encode('utf-16')
+ '\xff\xfeO\x00 \x007\x00 \x00e\x00 \x00M\x00e\x00i\x00o\x00 \x00\xe9\x00 \x00u\x00m\x00 \x00e\x00x\x00-\x00l\x00i\x00b\x00r\x00i\x00s\x00 \x00d\x00a\x00 \x00n\x00o\x00i\x00t\x00e\x00 \x00a\x00l\x00g\x00a\x00r\x00v\x00i\x00a\x00.\x00\n\x00\xc9\x00 \x00u\x00m\x00a\x00 \x00d\x00a\x00s\x00 \x00m\x00a\x00i\x00s\x00 \x00'
diff --git a/nltk/test/portuguese_en.doctest b/nltk/test/portuguese_en.doctest
new file mode 100644
index 0000000..ab27cf2
--- /dev/null
+++ b/nltk/test/portuguese_en.doctest
@@ -0,0 +1,565 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==================================
+Examples for Portuguese Processing
+==================================
+
+This HOWTO contains a variety of examples relating to the Portuguese language.
+It is intended to be read in conjunction with the NLTK book
+(``http://nltk.org/book``). For instructions on running the Python
+interpreter, please see the section *Getting Started with Python*, in Chapter 1.
+
+--------------------------------------------
+Python Programming, with Portuguese Examples
+--------------------------------------------
+
+Chapter 1 of the NLTK book contains many elementary programming examples, all
+with English texts. In this section, we'll see some corresponding examples
+using Portuguese. Please refer to the chapter for full discussion. *Vamos!*
+
+ >>> from nltk.examples.pt import *
+ *** Introductory Examples for the NLTK Book ***
+ Loading ptext1, ... and psent1, ...
+ Type the name of the text or sentence to view it.
+ Type: 'texts()' or 'sents()' to list the materials.
+ ptext1: Memórias Póstumas de Brás Cubas (1881)
+ ptext2: Dom Casmurro (1899)
+ ptext3: Gênesis
+ ptext4: Folha de Sao Paulo (1994)
+
+
+Any time we want to find out about these texts, we just have
+to enter their names at the Python prompt:
+
+ >>> ptext2
+ <Text: Dom Casmurro (1899)>
+
+Searching Text
+--------------
+
+A concordance permits us to see words in context.
+
+ >>> ptext1.concordance('olhos')
+ Building index...
+ Displaying 25 of 138 matches:
+ De pé , à cabeceira da cama , com os olhos estúpidos , a boca entreaberta , a t
+ orelhas . Pela minha parte fechei os olhos e deixei - me ir à ventura . Já agor
+ xões de cérebro enfermo . Como ia de olhos fechados , não via o caminho ; lembr
+ gelos eternos . Com efeito , abri os olhos e vi que o meu animal galopava numa
+ me apareceu então , fitando - me uns olhos rutilantes como o sol . Tudo nessa f
+ mim mesmo . Então , encarei - a com olhos súplices , e pedi mais alguns anos .
+ ...
+
+For a given word, we can find words with a similar text distribution:
+
+ >>> ptext1.similar('chegar')
+ Building word-context index...
+ acabada acudir aludir avistar bramanismo casamento cheguei com contar
+ contrário corpo dali deixei desferirem dizer fazer filhos já leitor lhe
+ >>> ptext3.similar('chegar')
+ Building word-context index...
+ achar alumiar arrombar destruir governar guardar ir lavrar passar que
+ toda tomar ver vir
+
+We can search for the statistically significant collocations in a text:
+
+ >>> ptext1.collocations()
+ Building collocations list
+ Quincas Borba; Lobo Neves; alguma coisa; Brás Cubas; meu pai; dia
+ seguinte; não sei; Meu pai; alguns instantes; outra vez; outra coisa;
+ por exemplo; mim mesmo; coisa nenhuma; mesma coisa; não era; dias
+ depois; Passeio Público; olhar para; das coisas
+
+We can search for words in context, with the help of *regular expressions*, e.g.:
+
+ >>> ptext1.findall("<olhos> (<.*>)")
+ estúpidos; e; fechados; rutilantes; súplices; a; do; babavam;
+ na; moles; se; da; umas; espraiavam; chamejantes; espetados;
+ ...
+
+We can automatically generate random text based on a given text, e.g.:
+
+ >>> ptext3.generate() # doctest: +SKIP
+ No princípio , criou Deus os abençoou , dizendo : Onde { estão } e até
+ à ave dos céus , { que } será . Disse mais Abrão : Dá - me a mulher
+ que tomaste ; porque daquele poço Eseque , { tinha .} E disse : Não
+ poderemos descer ; mas , do campo ainda não estava na casa do teu
+ pescoço . E viveu Serugue , depois Simeão e Levi { são } estes ? E o
+ varão , porque habitava na terra de Node , da mão de Esaú : Jeús ,
+ Jalão e Corá
+
+Texts as List of Words
+----------------------
+
+A few sentences have been defined for you.
+
+ >>> psent1
+ ['o', 'amor', 'da', 'gl\xf3ria', 'era', 'a', 'coisa', 'mais',
+ 'verdadeiramente', 'humana', 'que', 'h\xe1', 'no', 'homem', ',',
+ 'e', ',', 'conseq\xfcentemente', ',', 'a', 'sua', 'mais',
+ 'genu\xedna', 'fei\xe7\xe3o', '.']
+ >>>
+
+Notice that the sentence has been *tokenized*. Each token is
+represented as a string, represented using quotes, e.g. ``'coisa'``.
+Some strings contain special characters, e.g. ``\xf3``,
+the internal representation for ó.
+The tokens are combined in the form of a *list*. How long is this list?
+
+ >>> len(psent1)
+ 25
+ >>>
+
+What is the vocabulary of this sentence?
+
+ >>> sorted(set(psent1))
+ [',', '.', 'a', 'amor', 'coisa', 'conseqüentemente', 'da', 'e', 'era',
+ 'feição', 'genuína', 'glória', 'homem', 'humana', 'há', 'mais', 'no',
+ 'o', 'que', 'sua', 'verdadeiramente']
+ >>>
+
+Let's iterate over each item in ``psent2``, and print information for each:
+
+ >>> for w in psent2:
+ ... print(w, len(w), w[-1])
+ ...
+ Não 3 o
+ consultes 9 s
+ dicionários 11 s
+ . 1 .
+
+Observe how we make a human-readable version of a string, using ``decode()``.
+Also notice that we accessed the last character of a string ``w`` using ``w[-1]``.
+
+We just saw a ``for`` loop above. Another useful control structure is a
+*list comprehension*.
+
+ >>> [w.upper() for w in psent2]
+ ['N\xc3O', 'CONSULTES', 'DICION\xc1RIOS', '.']
+ >>> [w for w in psent1 if w.endswith('a')]
+ ['da', 'gl\xf3ria', 'era', 'a', 'coisa', 'humana', 'a', 'sua', 'genu\xedna']
+ >>> [w for w in ptext4 if len(w) > 15]
+ [u'norte-irlandeses', u'pan-nacionalismo', u'predominatemente', u'primeiro-ministro',
+ u'primeiro-ministro', u'irlandesa-americana', u'responsabilidades', u'significativamente']
+
+We can examine the relative frequency of words in a text, using ``FreqDist``:
+
+ >>> fd1 = FreqDist(ptext1)
+ >>> fd1
+ <FreqDist with 10848 samples and 77098 outcomes>
+ >>> fd1['olhos']
+ 137
+ >>> fd1.max()
+ u','
+ >>> fd1.samples()[:100]
+ [u',', u'.', u'a', u'que', u'de', u'e', u'-', u'o', u';', u'me', u'um', u'n\xe3o',
+ u'\x97', u'se', u'do', u'da', u'uma', u'com', u'os', u'\xe9', u'era', u'as', u'eu',
+ u'lhe', u'ao', u'em', u'para', u'mas', u'...', u'!', u'\xe0', u'na', u'mais', u'?',
+ u'no', u'como', u'por', u'N\xe3o', u'dos', u'ou', u'ele', u':', u'Virg\xedlia',
+ u'meu', u'disse', u'minha', u'das', u'O', u'/', u'A', u'CAP\xcdTULO', u'muito',
+ u'depois', u'coisa', u'foi', u'sem', u'olhos', u'ela', u'nos', u'tinha', u'nem',
+ u'E', u'outro', u'vida', u'nada', u'tempo', u'menos', u'outra', u'casa', u'homem',
+ u'porque', u'quando', u'mim', u'mesmo', u'ser', u'pouco', u'estava', u'dia',
+ u't\xe3o', u'tudo', u'Mas', u'at\xe9', u'D', u'ainda', u's\xf3', u'alguma',
+ u'la', u'vez', u'anos', u'h\xe1', u'Era', u'pai', u'esse', u'lo', u'dizer', u'assim',
+ u'ent\xe3o', u'dizia', u'aos', u'Borba']
+
+---------------
+Reading Corpora
+---------------
+
+Accessing the Machado Text Corpus
+---------------------------------
+
+NLTK includes the complete works of Machado de Assis.
+
+ >>> from nltk.corpus import machado
+ >>> machado.fileids()
+ ['contos/macn001.txt', 'contos/macn002.txt', 'contos/macn003.txt', ...]
+
+Each file corresponds to one of the works of Machado de Assis. To see a complete
+list of works, you can look at the corpus README file: ``print machado.readme()``.
+Let's access the text of the *Posthumous Memories of Brás Cubas*.
+
+We can access the text as a list of characters, and access 200 characters starting
+from position 10,000.
+
+ >>> raw_text = machado.raw('romance/marm05.txt')
+ >>> raw_text[10000:10200]
+ u', primou no\nEstado, e foi um dos amigos particulares do vice-rei Conde
+ da Cunha.\n\nComo este apelido de Cubas lhe\ncheirasse excessivamente a
+ tanoaria, alegava meu pai, bisneto de Dami\xe3o, que o\ndito ape'
+
+However, this is not a very useful way to work with a text. We generally think
+of a text as a sequence of words and punctuation, not characters:
+
+ >>> text1 = machado.words('romance/marm05.txt')
+ >>> text1
+ ['Romance', ',', 'Mem\xf3rias', 'P\xf3stumas', 'de', ...]
+ >>> len(text1)
+ 77098
+ >>> len(set(text1))
+ 10848
+
+Here's a program that finds the most common ngrams that contain a
+particular target word.
+
+ >>> from nltk import ngrams, FreqDist
+ >>> target_word = 'olhos'
+ >>> fd = FreqDist(ng
+ ... for ng in ngrams(text1, 5)
+ ... if target_word in ng)
+ >>> for hit in fd.samples():
+ ... print(' '.join(hit))
+ ...
+ , com os olhos no
+ com os olhos no ar
+ com os olhos no chão
+ e todos com os olhos
+ me estar com os olhos
+ os olhos estúpidos , a
+ os olhos na costura ,
+ os olhos no ar ,
+ , com os olhos espetados
+ , com os olhos estúpidos
+ , com os olhos fitos
+ , com os olhos naquele
+ , com os olhos para
+
+
+Accessing the MacMorpho Tagged Corpus
+-------------------------------------
+
+NLTK includes the MAC-MORPHO Brazilian Portuguese POS-tagged news text,
+with over a million words of
+journalistic texts extracted from ten sections of
+the daily newspaper *Folha de Sao Paulo*, 1994.
+
+We can access this corpus as a sequence of words or tagged words as follows:
+ >>> import nltk.corpus
+ >>> nltk.corpus.mac_morpho.words()
+ ['Jersei', 'atinge', 'm\xe9dia', 'de', 'Cr$', '1,4', ...]
+ >>> nltk.corpus.mac_morpho.sents() # doctest: +NORMALIZE_WHITESPACE
+ [['Jersei', 'atinge', 'm\xe9dia', 'de', 'Cr$', '1,4', 'milh\xe3o',
+ 'em', 'a', 'venda', 'de', 'a', 'Pinhal', 'em', 'S\xe3o', 'Paulo'],
+ ['Programe', 'sua', 'viagem', 'a', 'a', 'Exposi\xe7\xe3o', 'Nacional',
+ 'do', 'Zebu', ',', 'que', 'come\xe7a', 'dia', '25'], ...]
+ >>> nltk.corpus.mac_morpho.tagged_words()
+ [('Jersei', 'N'), ('atinge', 'V'), ('m\xe9dia', 'N'), ...]
+
+We can also access it in sentence chunks.
+
+ >>> nltk.corpus.mac_morpho.tagged_sents() # doctest: +NORMALIZE_WHITESPACE
+ [[('Jersei', 'N'), ('atinge', 'V'), ('m\xe9dia', 'N'), ('de', 'PREP'),
+ ('Cr$', 'CUR'), ('1,4', 'NUM'), ('milh\xe3o', 'N'), ('em', 'PREP|+'),
+ ('a', 'ART'), ('venda', 'N'), ('de', 'PREP|+'), ('a', 'ART'),
+ ('Pinhal', 'NPROP'), ('em', 'PREP'), ('S\xe3o', 'NPROP'),
+ ('Paulo', 'NPROP')],
+ [('Programe', 'V'), ('sua', 'PROADJ'), ('viagem', 'N'), ('a', 'PREP|+'),
+ ('a', 'ART'), ('Exposi\xe7\xe3o', 'NPROP'), ('Nacional', 'NPROP'),
+ ('do', 'NPROP'), ('Zebu', 'NPROP'), (',', ','), ('que', 'PRO-KS-REL'),
+ ('come\xe7a', 'V'), ('dia', 'N'), ('25', 'N|AP')], ...]
+
+This data can be used to train taggers (examples below for the Floresta treebank).
+
+Accessing the Floresta Portuguese Treebank
+------------------------------------------
+
+The NLTK data distribution includes the
+"Floresta Sinta(c)tica Corpus" version 7.4, available from
+``http://www.linguateca.pt/Floresta/``.
+
+We can access this corpus as a sequence of words or tagged words as follows:
+
+ >>> from nltk.corpus import floresta
+ >>> floresta.words()
+ ['Um', 'revivalismo', 'refrescante', 'O', '7_e_Meio', ...]
+ >>> floresta.tagged_words()
+ [('Um', '>N+art'), ('revivalismo', 'H+n'), ...]
+
+The tags consist of some syntactic information, followed by a plus sign,
+followed by a conventional part-of-speech tag. Let's strip off the material before
+the plus sign:
+
+ >>> def simplify_tag(t):
+ ... if "+" in t:
+ ... return t[t.index("+")+1:]
+ ... else:
+ ... return t
+ >>> twords = floresta.tagged_words()
+ >>> twords = [(w.lower(), simplify_tag(t)) for (w,t) in twords]
+ >>> twords[:10]
+ [('um', 'art'), ('revivalismo', 'n'), ('refrescante', 'adj'), ('o', 'art'), ('7_e_meio', 'prop'),
+ ('\xe9', 'v-fin'), ('um', 'art'), ('ex-libris', 'n'), ('de', 'prp'), ('a', 'art')]
+
+Pretty printing the tagged words:
+
+ >>> print(' '.join(word + '/' + tag for (word, tag) in twords[:10]))
+ um/art revivalismo/n refrescante/adj o/art 7_e_meio/prop é/v-fin um/art ex-libris/n de/prp a/art
+
+Count the word tokens and types, and determine the most common word:
+
+ >>> words = floresta.words()
+ >>> len(words)
+ 211852
+ >>> fd = nltk.FreqDist(words)
+ >>> len(fd)
+ 29421
+ >>> fd.max()
+ 'de'
+
+List the 20 most frequent tags, in order of decreasing frequency:
+
+ >>> tags = [simplify_tag(tag) for (word,tag) in floresta.tagged_words()]
+ >>> fd = nltk.FreqDist(tags)
+ >>> fd.keys()[:20] # doctest: +NORMALIZE_WHITESPACE
+ ['n', 'prp', 'art', 'v-fin', ',', 'prop', 'adj', 'adv', '.',
+ 'conj-c', 'v-inf', 'pron-det', 'v-pcp', 'num', 'pron-indp',
+ 'pron-pers', '\xab', '\xbb', 'conj-s', '}']
+
+We can also access the corpus grouped by sentence:
+
+ >>> floresta.sents() # doctest: +NORMALIZE_WHITESPACE
+ [['Um', 'revivalismo', 'refrescante'],
+ ['O', '7_e_Meio', '\xe9', 'um', 'ex-libris', 'de', 'a', 'noite',
+ 'algarvia', '.'], ...]
+ >>> floresta.tagged_sents() # doctest: +NORMALIZE_WHITESPACE
+ [[('Um', '>N+art'), ('revivalismo', 'H+n'), ('refrescante', 'N<+adj')],
+ [('O', '>N+art'), ('7_e_Meio', 'H+prop'), ('\xe9', 'P+v-fin'),
+ ('um', '>N+art'), ('ex-libris', 'H+n'), ('de', 'H+prp'),
+ ('a', '>N+art'), ('noite', 'H+n'), ('algarvia', 'N<+adj'), ('.', '.')],
+ ...]
+ >>> floresta.parsed_sents() # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+ [Tree('UTT+np', [Tree('>N+art', ['Um']), Tree('H+n', ['revivalismo']),
+ Tree('N<+adj', ['refrescante'])]),
+ Tree('STA+fcl',
+ [Tree('SUBJ+np', [Tree('>N+art', ['O']),
+ Tree('H+prop', ['7_e_Meio'])]),
+ Tree('P+v-fin', ['\xe9']),
+ Tree('SC+np',
+ [Tree('>N+art', ['um']),
+ Tree('H+n', ['ex-libris']),
+ Tree('N<+pp', [Tree('H+prp', ['de']),
+ Tree('P<+np', [Tree('>N+art', ['a']),
+ Tree('H+n', ['noite']),
+ Tree('N<+adj', ['algarvia'])])])]),
+ Tree('.', ['.'])]), ...]
+
+To view a parse tree, use the ``draw()`` method, e.g.:
+
+ >>> psents = floresta.parsed_sents()
+ >>> psents[5].draw() # doctest: +SKIP
+
+Character Encodings
+-------------------
+
+Python understands the common character encoding used for Portuguese, ISO 8859-1 (ISO Latin 1).
+
+ >>> import os, nltk.test
+ >>> testdir = os.path.split(nltk.test.__file__)[0]
+ >>> text = open(os.path.join(testdir, 'floresta.txt'), 'rb').read().decode('ISO 8859-1')
+ >>> text[:60]
+ 'O 7 e Meio \xe9 um ex-libris da noite algarvia.\n\xc9 uma das mais '
+ >>> print(text[:60])
+ O 7 e Meio é um ex-libris da noite algarvia.
+ É uma das mais
+
+For more information about character encodings and Python, please see section 3.3 of the book.
+
+----------------
+Processing Tasks
+----------------
+
+
+Simple Concordancing
+--------------------
+
+Here's a function that takes a word and a specified amount of context (measured
+in characters), and generates a concordance for that word.
+
+ >>> def concordance(word, context=30):
+ ... for sent in floresta.sents():
+ ... if word in sent:
+ ... pos = sent.index(word)
+ ... left = ' '.join(sent[:pos])
+ ... right = ' '.join(sent[pos+1:])
+ ... print('%*s %s %-*s' %
+ ... (context, left[-context:], word, context, right[:context]))
+
+ >>> concordance("dar") # doctest: +SKIP
+ anduru , foi o suficiente para dar a volta a o resultado .
+ 1. O P?BLICO veio dar a a imprensa di?ria portuguesa
+ A fartura de pensamento pode dar maus resultados e n?s n?o quer
+ Come?a a dar resultados a pol?tica de a Uni
+ ial come?ar a incorporar- lo e dar forma a um ' site ' que tem se
+ r com Constantino para ele lhe dar tamb?m os pap?is assinados .
+ va a brincar , pois n?o lhe ia dar procura??o nenhuma enquanto n?
+ ?rica como o ant?doto capaz de dar sentido a o seu enorme poder .
+ . . .
+ >>> concordance("vender") # doctest: +SKIP
+ er recebido uma encomenda para vender 4000 blindados a o Iraque .
+ m?rico_Amorim caso conseguisse vender o lote de ac??es de o empres?r
+ mpre ter jovens simp?ticos a ? vender ? chega ! }
+ Disse que o governo vai vender ? desde autom?vel at? particip
+ ndiciou ontem duas pessoas por vender carro com ?gio .
+ A inten??o de Fleury ? vender as a??es para equilibrar as fi
+
+Part-of-Speech Tagging
+----------------------
+
+Let's begin by getting the tagged sentence data, and simplifying the tags
+as described earlier.
+
+ >>> from nltk.corpus import floresta
+ >>> tsents = floresta.tagged_sents()
+ >>> tsents = [[(w.lower(),simplify_tag(t)) for (w,t) in sent] for sent in tsents if sent]
+ >>> train = tsents[100:]
+ >>> test = tsents[:100]
+
+We already know that ``n`` is the most common tag, so we can set up a
+default tagger that tags every word as a noun, and see how well it does:
+
+ >>> tagger0 = nltk.DefaultTagger('n')
+ >>> nltk.tag.accuracy(tagger0, test)
+ 0.17697228144989338
+
+Evidently, about one in every six words is a noun. Let's improve on this by
+training a unigram tagger:
+
+ >>> tagger1 = nltk.UnigramTagger(train, backoff=tagger0)
+ >>> nltk.tag.accuracy(tagger1, test)
+ 0.87029140014214645
+
+Next a bigram tagger:
+
+ >>> tagger2 = nltk.BigramTagger(train, backoff=tagger1)
+ >>> nltk.tag.accuracy(tagger2, test)
+ 0.89019189765458417
+
+
+Sentence Segmentation
+---------------------
+
+Punkt is a language-neutral sentence segmentation tool. We
+
+ >>> sent_tokenizer=nltk.data.load('tokenizers/punkt/portuguese.pickle')
+ >>> raw_text = machado.raw('romance/marm05.txt')
+ >>> sentences = sent_tokenizer.tokenize(raw_text)
+ >>> for sent in sentences[1000:1005]:
+ ... print("<<", sent, ">>")
+ ...
+ << Em verdade, parecia ainda mais mulher do que era;
+ seria criança nos seus folgares de moça; mas assim quieta, impassível, tinha a
+ compostura da mulher casada. >>
+ << Talvez essa circunstância lhe diminuía um pouco da
+ graça virginal. >>
+ << Depressa nos familiarizamos; a mãe fazia-lhe grandes elogios, eu
+ escutava-os de boa sombra, e ela sorria com os olhos fúlgidos, como se lá dentro
+ do cérebro lhe estivesse a voar uma borboletinha de asas de ouro e olhos de
+ diamante... >>
+ << Digo lá dentro, porque cá fora o
+ que esvoaçou foi uma borboleta preta, que subitamente penetrou na varanda, e
+ começou a bater as asas em derredor de D. Eusébia. >>
+ << D. Eusébia deu um grito,
+ levantou-se, praguejou umas palavras soltas: - T'esconjuro!... >>
+
+The sentence tokenizer can be trained and evaluated on other text.
+The source text (from the Floresta Portuguese Treebank) contains one sentence per line.
+We read the text, split it into its lines, and then join these lines together using
+spaces. Now the information about sentence breaks has been discarded. We split this
+material into training and testing data:
+
+ >>> import os, nltk.test
+ >>> testdir = os.path.split(nltk.test.__file__)[0]
+ >>> text = open(os.path.join(testdir, 'floresta.txt'), 'rb').read().decode('ISO-8859-1')
+ >>> lines = text.split('\n')
+ >>> train = ' '.join(lines[10:])
+ >>> test = ' '.join(lines[:10])
+
+Now we train the sentence segmenter (or sentence tokenizer) and use it on our test sentences:
+
+ >>> stok = nltk.PunktSentenceTokenizer(train)
+ >>> print(stok.tokenize(test))
+ ['O 7 e Meio \xe9 um ex-libris da noite algarvia.',
+ '\xc9 uma das mais antigas discotecas do Algarve, situada em Albufeira,
+ que continua a manter os tra\xe7os decorativos e as clientelas de sempre.',
+ '\xc9 um pouco a vers\xe3o de uma esp\xe9cie de \xaboutro lado\xbb da noite,
+ a meio caminho entre os devaneios de uma fauna perif\xe9rica, seja de Lisboa,
+ Londres, Dublin ou Faro e Portim\xe3o, e a postura circunspecta dos fi\xe9is da casa,
+ que dela esperam a m\xfasica \xabgeracionista\xbb dos 60 ou dos 70.',
+ 'N\xe3o deixa de ser, nos tempos que correm, um certo \xabvery typical\xbb algarvio,
+ cabe\xe7a de cartaz para os que querem fugir a algumas movimenta\xe7\xf5es nocturnas
+ j\xe1 a caminho da ritualiza\xe7\xe3o de massas, do g\xe9nero \xabvamos todos ao
+ Calypso e encontramo-nos na Locomia\xbb.',
+ 'E assim, aos 2,5 milh\xf5es que o Minist\xe9rio do Planeamento e Administra\xe7\xe3o
+ do Territ\xf3rio j\xe1 gasta no pagamento do pessoal afecto a estes organismos,
+ v\xeam juntar-se os montantes das obras propriamente ditas, que os munic\xedpios,
+ j\xe1 com projectos na m\xe3o, v\xeam reivindicar junto do Executivo, como salienta
+ aquele membro do Governo.',
+ 'E o dinheiro \xabn\xe3o falta s\xf3 \xe0s c\xe2maras\xbb, lembra o secret\xe1rio de Estado,
+ que considera que a solu\xe7\xe3o para as autarquias \xe9 \xabespecializarem-se em
+ fundos comunit\xe1rios\xbb.',
+ 'Mas como, se muitas n\xe3o disp\xf5em, nos seus quadros, dos t\xe9cnicos necess\xe1rios?',
+ '\xabEncomendem-nos a projectistas de fora\xbb porque, se as obras vierem a ser financiadas,
+ eles at\xe9 saem de gra\xe7a, j\xe1 que, nesse caso, \xabos fundos comunit\xe1rios pagam
+ os projectos, o mesmo n\xe3o acontecendo quando eles s\xe3o feitos pelos GAT\xbb,
+ dado serem organismos do Estado.',
+ 'Essa poder\xe1 vir a ser uma hip\xf3tese, at\xe9 porque, no terreno, a capacidade dos GAT
+ est\xe1 cada vez mais enfraquecida.',
+ 'Alguns at\xe9 j\xe1 desapareceram, como o de Castro Verde, e outros t\xeam vindo a perder quadros.']
+
+NLTK's data collection includes a trained model for Portuguese sentence
+segmentation, which can be loaded as follows. It is faster to load a trained model than
+to retrain it.
+
+ >>> stok = nltk.data.load('tokenizers/punkt/portuguese.pickle')
+
+Stemming
+--------
+
+NLTK includes the RSLP Portuguese stemmer. Here we use it to stem some Portuguese text:
+
+ >>> stemmer = nltk.stem.RSLPStemmer()
+ >>> stemmer.stem("copiar")
+ 'copi'
+ >>> stemmer.stem("paisagem")
+ 'pais'
+
+
+Stopwords
+---------
+
+NLTK includes Portuguese stopwords:
+
+ >>> stopwords = nltk.corpus.stopwords.words('portuguese')
+ >>> stopwords[:10]
+ ['a', 'ao', 'aos', 'aquela', 'aquelas', 'aquele', 'aqueles', 'aquilo', 'as', 'at\xe9']
+
+Now we can use these to filter text. Let's find the most frequent words (other than stopwords)
+and print them in descending order of frequency:
+
+ >>> fd = nltk.FreqDist(w.lower() for w in floresta.words() if w not in stopwords)
+ >>> for word in list(fd.keys())[:20]:
+ ... print(word, fd[word])
+ , 13444
+ . 7725
+ « 2369
+ » 2310
+ é 1305
+ o 1086
+ } 1047
+ { 1044
+ a 897
+ ; 633
+ em 516
+ ser 466
+ sobre 349
+ os 313
+ anos 301
+ ontem 292
+ ainda 279
+ segundo 256
+ ter 249
+ dois 231
+
diff --git a/nltk/test/portuguese_en_fixt.py b/nltk/test/portuguese_en_fixt.py
new file mode 100644
index 0000000..a2e26a3
--- /dev/null
+++ b/nltk/test/portuguese_en_fixt.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from nltk.compat import PY3
+
+from nltk.corpus import teardown_module
+
+def setup_module(module):
+ from nose import SkipTest
+ if not PY3:
+ raise SkipTest("portuguese_en.doctest was skipped because "
+ "non-ascii doctests are not supported under Python 2.x")
\ No newline at end of file
diff --git a/nltk/test/probability.doctest b/nltk/test/probability.doctest
new file mode 100644
index 0000000..97f3b8b
--- /dev/null
+++ b/nltk/test/probability.doctest
@@ -0,0 +1,266 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+===========
+Probability
+===========
+
+ >>> import nltk
+ >>> from nltk.probability import *
+
+FreqDist
+--------
+
+ >>> text1 = ['no', 'good', 'fish', 'goes', 'anywhere', 'without', 'a', 'porpoise', '!']
+ >>> text2 = ['no', 'good', 'porpoise', 'likes', 'to', 'fish', 'fish', 'anywhere', '.']
+
+ >>> fd1 = nltk.FreqDist(text1)
+ >>> fd1 == nltk.FreqDist(text1)
+ True
+
+Note that items are sorted in order of decreasing frequency; two items of the same frequency appear in indeterminate order.
+
+ >>> import itertools
+ >>> both = nltk.FreqDist(text1 + text2)
+ >>> both_most_common = both.most_common()
+ >>> list(itertools.chain(*(sorted(ys) for k, ys in itertools.groupby(both_most_common, key=lambda t: t[1]))))
+ [('fish', 3), ('anywhere', 2), ('good', 2), ('no', 2), ('porpoise', 2), ('!', 1), ('.', 1), ('a', 1), ('goes', 1), ('likes', 1), ('to', 1), ('without', 1)]
+
+ >>> both == fd1 + nltk.FreqDist(text2)
+ True
+ >>> fd1 == nltk.FreqDist(text1) # But fd1 is unchanged
+ True
+
+ >>> fd2 = nltk.FreqDist(text2)
+ >>> fd1.update(fd2)
+ >>> fd1 == both
+ True
+
+ >>> fd1 = nltk.FreqDist(text1)
+ >>> fd1.update(text2)
+ >>> fd1 == both
+ True
+
+ >>> fd1 = nltk.FreqDist(text1)
+ >>> fd2 = nltk.FreqDist(fd1)
+ >>> fd2 == fd1
+ True
+
+``nltk.FreqDist`` can be pickled:
+
+ >>> import pickle
+ >>> fd1 = nltk.FreqDist(text1)
+ >>> pickled = pickle.dumps(fd1)
+ >>> fd1 == pickle.loads(pickled)
+ True
+
+Testing some HMM estimators
+---------------------------
+
+We extract a small part (500 sentences) of the Brown corpus
+
+ >>> corpus = nltk.corpus.brown.tagged_sents(categories='adventure')[:500]
+ >>> print(len(corpus))
+ 500
+
+We create a HMM trainer - note that we need the tags and symbols
+from the whole corpus, not just the training corpus
+
+ >>> from nltk.util import unique_list
+ >>> tag_set = unique_list(tag for sent in corpus for (word,tag) in sent)
+ >>> print(len(tag_set))
+ 92
+ >>> symbols = unique_list(word for sent in corpus for (word,tag) in sent)
+ >>> print(len(symbols))
+ 1464
+ >>> print(len(tag_set))
+ 92
+ >>> symbols = unique_list(word for sent in corpus for (word,tag) in sent)
+ >>> print(len(symbols))
+ 1464
+ >>> trainer = nltk.tag.HiddenMarkovModelTrainer(tag_set, symbols)
+
+We divide the corpus into 90% training and 10% testing
+
+ >>> train_corpus = []
+ >>> test_corpus = []
+ >>> for i in range(len(corpus)):
+ ... if i % 10:
+ ... train_corpus += [corpus[i]]
+ ... else:
+ ... test_corpus += [corpus[i]]
+ >>> print(len(train_corpus))
+ 450
+ >>> print(len(test_corpus))
+ 50
+
+And now we can test the estimators
+
+ >>> def train_and_test(est):
+ ... hmm = trainer.train_supervised(train_corpus, estimator=est)
+ ... print('%.2f%%' % (100 * hmm.evaluate(test_corpus)))
+
+Maximum Likelihood Estimation
+-----------------------------
+- this resulted in an initialization error before r7209
+
+ >>> mle = lambda fd, bins: MLEProbDist(fd)
+ >>> train_and_test(mle)
+ 22.75%
+
+Laplace (= Lidstone with gamma==1)
+
+ >>> train_and_test(LaplaceProbDist)
+ 66.04%
+
+Expected Likelihood Estimation (= Lidstone with gamma==0.5)
+
+ >>> train_and_test(ELEProbDist)
+ 73.01%
+
+Lidstone Estimation, for gamma==0.1, 0.5 and 1
+(the later two should be exactly equal to MLE and ELE above)
+
+ >>> def lidstone(gamma):
+ ... return lambda fd, bins: LidstoneProbDist(fd, gamma, bins)
+ >>> train_and_test(lidstone(0.1))
+ 82.51%
+ >>> train_and_test(lidstone(0.5))
+ 73.01%
+ >>> train_and_test(lidstone(1.0))
+ 66.04%
+
+Witten Bell Estimation
+----------------------
+- This resulted in ZeroDivisionError before r7209
+
+ >>> train_and_test(WittenBellProbDist)
+ 88.12%
+
+Good Turing Estimation
+
+ >>> gt = lambda fd, bins: SimpleGoodTuringProbDist(fd, bins=1e5)
+ >>> train_and_test(gt)
+ 86.93%
+
+Kneser Ney Estimation
+---------------------
+Since the Kneser-Ney distribution is best suited for trigrams, we must adjust
+our testing accordingly.
+
+ >>> corpus = [[((x[0],y[0],z[0]),(x[1],y[1],z[1]))
+ ... for x, y, z in nltk.trigrams(sent)]
+ ... for sent in corpus[:100]]
+
+We will then need to redefine the rest of the training/testing variables
+ >>> tag_set = unique_list(tag for sent in corpus for (word,tag) in sent)
+ >>> len(tag_set)
+ 906
+
+ >>> symbols = unique_list(word for sent in corpus for (word,tag) in sent)
+ >>> len(symbols)
+ 1341
+
+ >>> trainer = nltk.tag.HiddenMarkovModelTrainer(tag_set, symbols)
+ >>> train_corpus = []
+ >>> test_corpus = []
+
+ >>> for i in range(len(corpus)):
+ ... if i % 10:
+ ... train_corpus += [corpus[i]]
+ ... else:
+ ... test_corpus += [corpus[i]]
+
+ >>> len(train_corpus)
+ 90
+ >>> len(test_corpus)
+ 10
+
+ >>> kn = lambda fd, bins: KneserNeyProbDist(fd)
+ >>> train_and_test(kn)
+ 0.86%
+
+Remains to be added:
+- Tests for HeldoutProbDist, CrossValidationProbDist and MutableProbDist
+
+Squashed bugs
+-------------
+
+Issue 511: override pop and popitem to invalidate the cache
+
+ >>> fd = nltk.FreqDist('a')
+ >>> list(fd.keys())
+ ['a']
+ >>> fd.pop('a')
+ 1
+ >>> list(fd.keys())
+ []
+
+Issue 533: access cumulative frequencies with no arguments
+
+ >>> fd = nltk.FreqDist('aab')
+ >>> list(fd._cumulative_frequencies(['a']))
+ [2.0]
+ >>> list(fd._cumulative_frequencies(['a', 'b']))
+ [2.0, 3.0]
+
+Issue 579: override clear to reset some variables
+
+ >>> fd = FreqDist('aab')
+ >>> fd.clear()
+ >>> fd.N()
+ 0
+
+Issue 351: fix fileids method of CategorizedCorpusReader to inadvertently
+add errant categories
+
+ >>> from nltk.corpus import brown
+ >>> brown.fileids('blah')
+ Traceback (most recent call last):
+ ...
+ ValueError: Category blah not found
+ >>> brown.categories()
+ ['adventure', 'belles_lettres', 'editorial', 'fiction', 'government', 'hobbies', 'humor', 'learned', 'lore', 'mystery', 'news', 'religion', 'reviews', 'romance', 'science_fiction']
+
+Issue 175: add the unseen bin to SimpleGoodTuringProbDist by default
+otherwise any unseen events get a probability of zero, i.e.,
+they don't get smoothed
+
+ >>> from nltk import SimpleGoodTuringProbDist, FreqDist
+ >>> fd = FreqDist({'a':1, 'b':1, 'c': 2, 'd': 3, 'e': 4, 'f': 4, 'g': 4, 'h': 5, 'i': 5, 'j': 6, 'k': 6, 'l': 6, 'm': 7, 'n': 7, 'o': 8, 'p': 9, 'q': 10})
+ >>> p = SimpleGoodTuringProbDist(fd)
+ >>> p.prob('a')
+ 0.017649766667026317...
+ >>> p.prob('o')
+ 0.08433050215340411...
+ >>> p.prob('z')
+ 0.022727272727272728...
+ >>> p.prob('foobar')
+ 0.022727272727272728...
+
+``MLEProbDist``, ``ConditionalProbDist'', ``DictionaryConditionalProbDist`` and
+``ConditionalFreqDist`` can be pickled:
+
+ >>> import pickle
+ >>> pd = MLEProbDist(fd)
+ >>> sorted(pd.samples()) == sorted(pickle.loads(pickle.dumps(pd)).samples())
+ True
+ >>> dpd = DictionaryConditionalProbDist({'x': pd})
+ >>> unpickled = pickle.loads(pickle.dumps(dpd))
+ >>> dpd['x'].prob('a')
+ 0.011363636...
+ >>> dpd['x'].prob('a') == unpickled['x'].prob('a')
+ True
+ >>> cfd = nltk.probability.ConditionalFreqDist()
+ >>> cfd['foo']['hello'] += 1
+ >>> cfd['foo']['hello'] += 1
+ >>> cfd['bar']['hello'] += 1
+ >>> cfd2 = pickle.loads(pickle.dumps(cfd))
+ >>> cfd2 == cfd
+ True
+ >>> cpd = ConditionalProbDist(cfd, SimpleGoodTuringProbDist)
+ >>> cpd2 = pickle.loads(pickle.dumps(cpd))
+ >>> cpd['foo'].prob('hello') == cpd2['foo'].prob('hello')
+ True
+
+
diff --git a/nltk/test/probability_fixt.py b/nltk/test/probability_fixt.py
new file mode 100644
index 0000000..6daf2d5
--- /dev/null
+++ b/nltk/test/probability_fixt.py
@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+# probability.doctest uses HMM which requires numpy;
+# skip probability.doctest if numpy is not available
+
+def setup_module(module):
+ from nose import SkipTest
+ try:
+ import numpy
+ except ImportError:
+ raise SkipTest("probability.doctest requires numpy")
\ No newline at end of file
diff --git a/nltk/test/propbank.doctest b/nltk/test/propbank.doctest
new file mode 100644
index 0000000..fceea1e
--- /dev/null
+++ b/nltk/test/propbank.doctest
@@ -0,0 +1,176 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+========
+PropBank
+========
+
+The PropBank Corpus provides predicate-argument annotation for the
+entire Penn Treebank. Each verb in the treebank is annotated by a single
+instance in PropBank, containing information about the location of
+the verb, and the location and identity of its arguments:
+
+ >>> from nltk.corpus import propbank
+ >>> pb_instances = propbank.instances()
+ >>> print(pb_instances) # doctest: +NORMALIZE_WHITESPACE
+ [<PropbankInstance: wsj_0001.mrg, sent 0, word 8>,
+ <PropbankInstance: wsj_0001.mrg, sent 1, word 10>, ...]
+
+Each propbank instance defines the following member variables:
+
+ - Location information: `fileid`, `sentnum`, `wordnum`
+ - Annotator information: `tagger`
+ - Inflection information: `inflection`
+ - Roleset identifier: `roleset`
+ - Verb (aka predicate) location: `predicate`
+ - Argument locations and types: `arguments`
+
+The following examples show the types of these arguments:
+
+ >>> inst = pb_instances[103]
+ >>> (inst.fileid, inst.sentnum, inst.wordnum)
+ ('wsj_0004.mrg', 8, 16)
+ >>> inst.tagger
+ 'gold'
+ >>> inst.inflection
+ <PropbankInflection: vp--a>
+ >>> infl = inst.inflection
+ >>> infl.form, infl.tense, infl.aspect, infl.person, infl.voice
+ ('v', 'p', '-', '-', 'a')
+ >>> inst.roleset
+ 'rise.01'
+ >>> inst.predicate
+ PropbankTreePointer(16, 0)
+ >>> inst.arguments # doctest: +NORMALIZE_WHITESPACE
+ ((PropbankTreePointer(0, 2), 'ARG1'),
+ (PropbankTreePointer(13, 1), 'ARGM-DIS'),
+ (PropbankTreePointer(17, 1), 'ARG4-to'),
+ (PropbankTreePointer(20, 1), 'ARG3-from'))
+
+The location of the predicate and of the arguments are encoded using
+`PropbankTreePointer` objects, as well as `PropbankChainTreePointer`
+objects and `PropbankSplitTreePointer` objects. A
+`PropbankTreePointer` consists of a `wordnum` and a `height`:
+
+ >>> print(inst.predicate.wordnum, inst.predicate.height)
+ 16 0
+
+This identifies the tree constituent that is headed by the word that
+is the `wordnum`\ 'th token in the sentence, and whose span is found
+by going `height` nodes up in the tree. This type of pointer is only
+useful if we also have the corresponding tree structure, since it
+includes empty elements such as traces in the word number count. The
+trees for 10% of the standard PropBank Corpus are contained in the
+`treebank` corpus:
+
+ >>> tree = inst.tree
+
+ >>> from nltk.corpus import treebank
+ >>> assert tree == treebank.parsed_sents(inst.fileid)[inst.sentnum]
+
+ >>> inst.predicate.select(tree)
+ Tree('VBD', ['rose'])
+ >>> for (argloc, argid) in inst.arguments:
+ ... print('%-10s %s' % (argid, argloc.select(tree).pprint(500)[:50]))
+ ARG1 (NP-SBJ (NP (DT The) (NN yield)) (PP (IN on) (NP (
+ ARGM-DIS (PP (IN for) (NP (NN example)))
+ ARG4-to (PP-DIR (TO to) (NP (CD 8.04) (NN %)))
+ ARG3-from (PP-DIR (IN from) (NP (CD 7.90) (NN %)))
+
+Propbank tree pointers can be converted to standard tree locations,
+which are usually easier to work with, using the `treepos()` method:
+
+ >>> treepos = inst.predicate.treepos(tree)
+ >>> print (treepos, tree[treepos])
+ (4, 0) (VBD rose)
+
+In some cases, argument locations will be encoded using
+`PropbankChainTreePointer`\ s (for trace chains) or
+`PropbankSplitTreePointer`\ s (for discontinuous constituents). Both
+of these objects contain a single member variable, `pieces`,
+containing a list of the constituent pieces. They also define the
+method `select()`, which will return a tree containing all the
+elements of the argument. (A new head node is created, labeled
+"*CHAIN*" or "*SPLIT*", since the argument is not a single constituent
+in the original tree). Sentence #6 contains an example of an argument
+that is both discontinuous and contains a chain:
+
+ >>> inst = pb_instances[6]
+ >>> inst.roleset
+ 'expose.01'
+ >>> argloc, argid = inst.arguments[2]
+ >>> argloc
+ <PropbankChainTreePointer: 22:1,24:0,25:1*27:0>
+ >>> argloc.pieces
+ [<PropbankSplitTreePointer: 22:1,24:0,25:1>, PropbankTreePointer(27, 0)]
+ >>> argloc.pieces[0].pieces
+ ... # doctest: +NORMALIZE_WHITESPACE
+ [PropbankTreePointer(22, 1), PropbankTreePointer(24, 0),
+ PropbankTreePointer(25, 1)]
+ >>> print(argloc.select(inst.tree))
+ (*CHAIN*
+ (*SPLIT* (NP (DT a) (NN group)) (IN of) (NP (NNS workers)))
+ (-NONE- *))
+
+The PropBank Corpus also provides access to the frameset files, which
+define the argument labels used by the annotations, on a per-verb
+basis. Each frameset file contains one or more predicates, such as
+'turn' or 'turn_on', each of which is divided into coarse-grained word
+senses called rolesets. For each roleset, the frameset file provides
+descriptions of the argument roles, along with examples.
+
+ >>> expose_01 = propbank.roleset('expose.01')
+ >>> turn_01 = propbank.roleset('turn.01')
+ >>> print(turn_01) # doctest: +ELLIPSIS
+ <Element 'roleset' at ...>
+ >>> for role in turn_01.findall("roles/role"):
+ ... print(role.attrib['n'], role.attrib['descr'])
+ 0 turner
+ 1 thing turning
+ m direction, location
+
+ >>> from xml.etree import ElementTree
+ >>> print(ElementTree.tostring(turn_01.find('example')).decode('utf8').strip())
+ <example name="transitive agentive">
+ <text>
+ John turned the key in the lock.
+ </text>
+ <arg n="0">John</arg>
+ <rel>turned</rel>
+ <arg n="1">the key</arg>
+ <arg f="LOC" n="m">in the lock</arg>
+ </example>
+
+Note that the standard corpus distribution only contains 10% of the
+treebank, so the parse trees are not available for instances starting
+at 9353:
+
+ >>> inst = pb_instances[9352]
+ >>> inst.fileid
+ 'wsj_0199.mrg'
+ >>> print(inst.tree) # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS
+ (S (NP-SBJ (NNP Trinity)) (VP (VBD said) (SBAR (-NONE- 0) ...))
+ >>> print(inst.predicate.select(inst.tree))
+ (VB begin)
+
+ >>> inst = pb_instances[9353]
+ >>> inst.fileid
+ 'wsj_0200.mrg'
+ >>> print(inst.tree)
+ None
+ >>> print(inst.predicate.select(inst.tree))
+ Traceback (most recent call last):
+ . . .
+ ValueError: Parse tree not avaialable
+
+However, if you supply your own version of the treebank corpus (by
+putting it before the nltk-provided version on `nltk.data.path`, or
+by creating a `ptb` directory as described above and using the
+`propbank_ptb` module), then you can access the trees for all
+instances.
+
+A list of the verb lemmas contained in PropBank is returned by the
+`propbank.verbs()` method:
+
+ >>> propbank.verbs()
+ ['abandon', 'abate', 'abdicate', 'abet', 'abide', ...]
diff --git a/nltk/test/relextract.doctest b/nltk/test/relextract.doctest
new file mode 100644
index 0000000..588551e
--- /dev/null
+++ b/nltk/test/relextract.doctest
@@ -0,0 +1,263 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+======================
+Information Extraction
+======================
+
+Information Extraction standardly consists of three subtasks:
+
+#. Named Entity Recognition
+
+#. Relation Extraction
+
+#. Template Filling
+
+Named Entities
+~~~~~~~~~~~~~~
+
+The IEER corpus is marked up for a variety of Named Entities. A `Named
+Entity`:dt: (more strictly, a Named Entity mention) is a name of an
+entity belonging to a specified class. For example, the Named Entity
+classes in IEER include PERSON, LOCATION, ORGANIZATION, DATE and so
+on. Within NLTK, Named Entities are represented as subtrees within a
+chunk structure: the class name is treated as node label, while the
+entity mention itself appears as the leaves of the subtree. This is
+illustrated below, where we have show an extract of the chunk
+representation of document NYT_19980315.064:
+
+ >>> from nltk.corpus import ieer
+ >>> docs = ieer.parsed_docs('NYT_19980315')
+ >>> tree = docs[1].text
+ >>> print(tree) # doctest: +ELLIPSIS
+ (DOCUMENT
+ ...
+ ``It's
+ a
+ chance
+ to
+ think
+ about
+ first-level
+ questions,''
+ said
+ Ms.
+ (PERSON Cohn)
+ ,
+ a
+ partner
+ in
+ the
+ (ORGANIZATION McGlashan & Sarrail)
+ firm
+ in
+ (LOCATION San Mateo)
+ ,
+ (LOCATION Calif.)
+ ...)
+
+Thus, the Named Entity mentions in this example are *Cohn*, *McGlashan &
+Sarrail*, *San Mateo* and *Calif.*.
+
+The CoNLL2002 Dutch and Spanish data is treated similarly, although in
+this case, the strings are also POS tagged.
+
+ >>> from nltk.corpus import conll2002
+ >>> for doc in conll2002.chunked_sents('ned.train')[27]:
+ ... print(doc)
+ (u'Het', u'Art')
+ (ORG Hof/N van/Prep Cassatie/N)
+ (u'verbrak', u'V')
+ (u'het', u'Art')
+ (u'arrest', u'N')
+ (u'zodat', u'Conj')
+ (u'het', u'Pron')
+ (u'moest', u'V')
+ (u'worden', u'V')
+ (u'overgedaan', u'V')
+ (u'door', u'Prep')
+ (u'het', u'Art')
+ (u'hof', u'N')
+ (u'van', u'Prep')
+ (u'beroep', u'N')
+ (u'van', u'Prep')
+ (LOC Antwerpen/N)
+ (u'.', u'Punc')
+
+Relation Extraction
+~~~~~~~~~~~~~~~~~~~
+
+Relation Extraction standardly consists of identifying specified
+relations between Named Entities. For example, assuming that we can
+recognize ORGANIZATIONs and LOCATIONs in text, we might want to also
+recognize pairs *(o, l)* of these kinds of entities such that *o* is
+located in *l*.
+
+The `sem.relextract` module provides some tools to help carry out a
+simple version of this task. The `tree2semi_rel()` function splits a chunk
+document into a list of two-member lists, each of which consists of a
+(possibly empty) string followed by a `Tree` (i.e., a Named Entity):
+
+ >>> from nltk.sem import relextract
+ >>> pairs = relextract.tree2semi_rel(tree)
+ >>> for s, tree in pairs[18:22]:
+ ... print('("...%s", %s)' % (" ".join(s[-5:]),tree))
+ ("...about first-level questions,'' said Ms.", (PERSON Cohn))
+ ("..., a partner in the", (ORGANIZATION McGlashan & Sarrail))
+ ("...firm in", (LOCATION San Mateo))
+ ("...,", (LOCATION Calif.))
+
+The function `semi_rel2reldict()` processes triples of these pairs, i.e.,
+pairs of the form ``((string1, Tree1), (string2, Tree2), (string3,
+Tree3))`` and outputs a dictionary (a `reldict`) in which ``Tree1`` is
+the subject of the relation, ``string2`` is the filler
+and ``Tree3`` is the object of the relation. ``string1`` and ``string3`` are
+stored as left and right context respectively.
+
+ >>> reldicts = relextract.semi_rel2reldict(pairs)
+ >>> for k, v in sorted(reldicts[0].items()):
+ ... print(k, '=>', v) # doctest: +ELLIPSIS
+ filler => of messages to their own ``Cyberia'' ...
+ lcon => transactions.'' Each week, they post
+ objclass => ORGANIZATION
+ objsym => white_house
+ objtext => White House
+ rcon => for access to its planned
+ subjclass => CARDINAL
+ subjsym => hundreds
+ subjtext => hundreds
+ untagged_filler => of messages to their own ``Cyberia'' ...
+
+The next example shows some of the values for two `reldict`\ s
+corresponding to the ``'NYT_19980315'`` text extract shown earlier.
+
+ >>> for r in reldicts[18:20]:
+ ... print('=' * 20)
+ ... print(r['subjtext'])
+ ... print(r['filler'])
+ ... print(r['objtext'])
+ ====================
+ Cohn
+ , a partner in the
+ McGlashan & Sarrail
+ ====================
+ McGlashan & Sarrail
+ firm in
+ San Mateo
+
+The function `relextract()` allows us to filter the `reldict`\ s
+according to the classes of the subject and object named entities. In
+addition, we can specify that the filler text has to match a given
+regular expression, as illustrated in the next example. Here, we are
+looking for pairs of entities in the IN relation, where IN has
+signature <ORG, LOC>.
+
+ >>> import re
+ >>> IN = re.compile(r'.*\bin\b(?!\b.+ing\b)')
+ >>> for fileid in ieer.fileids():
+ ... for doc in ieer.parsed_docs(fileid):
+ ... for rel in relextract.extract_rels('ORG', 'LOC', doc, corpus='ieer', pattern = IN):
+ ... print(relextract.rtuple(rel)) # doctest: +ELLIPSIS
+ [ORG: 'Christian Democrats'] ', the leading political forces in' [LOC: 'Italy']
+ [ORG: 'AP'] ') _ Lebanese guerrillas attacked Israeli forces in southern' [LOC: 'Lebanon']
+ [ORG: 'Security Council'] 'adopted Resolution 425. Huge yellow banners hung across intersections in' [LOC: 'Beirut']
+ [ORG: 'U.N.'] 'failures in' [LOC: 'Africa']
+ [ORG: 'U.N.'] 'peacekeeping operation in' [LOC: 'Somalia']
+ [ORG: 'U.N.'] 'partners on a more effective role in' [LOC: 'Africa']
+ [ORG: 'AP'] ') _ A bomb exploded in a mosque in central' [LOC: 'San`a']
+ [ORG: 'Krasnoye Sormovo'] 'shipyard in the Soviet city of' [LOC: 'Gorky']
+ [ORG: 'Kelab Golf Darul Ridzuan'] 'in' [LOC: 'Perak']
+ [ORG: 'U.N.'] 'peacekeeping operation in' [LOC: 'Somalia']
+ [ORG: 'WHYY'] 'in' [LOC: 'Philadelphia']
+ [ORG: 'McGlashan & Sarrail'] 'firm in' [LOC: 'San Mateo']
+ [ORG: 'Freedom Forum'] 'in' [LOC: 'Arlington']
+ [ORG: 'Brookings Institution'] ', the research group in' [LOC: 'Washington']
+ [ORG: 'Idealab'] ', a self-described business incubator based in' [LOC: 'Los Angeles']
+ [ORG: 'Open Text'] ', based in' [LOC: 'Waterloo']
+ ...
+
+The next example illustrates a case where the patter is a disjunction
+of roles that a PERSON can occupy in an ORGANIZATION.
+
+ >>> roles = """
+ ... (.*(
+ ... analyst|
+ ... chair(wo)?man|
+ ... commissioner|
+ ... counsel|
+ ... director|
+ ... economist|
+ ... editor|
+ ... executive|
+ ... foreman|
+ ... governor|
+ ... head|
+ ... lawyer|
+ ... leader|
+ ... librarian).*)|
+ ... manager|
+ ... partner|
+ ... president|
+ ... producer|
+ ... professor|
+ ... researcher|
+ ... spokes(wo)?man|
+ ... writer|
+ ... ,\sof\sthe?\s* # "X, of (the) Y"
+ ... """
+ >>> ROLES = re.compile(roles, re.VERBOSE)
+ >>> for fileid in ieer.fileids():
+ ... for doc in ieer.parsed_docs(fileid):
+ ... for rel in relextract.extract_rels('PER', 'ORG', doc, corpus='ieer', pattern=ROLES):
+ ... print(relextract.rtuple(rel)) # doctest: +ELLIPSIS
+ [PER: 'Kivutha Kibwana'] ', of the' [ORG: 'National Convention Assembly']
+ [PER: 'Boban Boskovic'] ', chief executive of the' [ORG: 'Plastika']
+ [PER: 'Annan'] ', the first sub-Saharan African to head the' [ORG: 'United Nations']
+ [PER: 'Kiriyenko'] 'became a foreman at the' [ORG: 'Krasnoye Sormovo']
+ [PER: 'Annan'] ', the first sub-Saharan African to head the' [ORG: 'United Nations']
+ [PER: 'Mike Godwin'] ', chief counsel for the' [ORG: 'Electronic Frontier Foundation']
+ ...
+
+In the case of the CoNLL2002 data, we can include POS tags in the
+query pattern. This example also illustrates how the output can be
+presented as something that looks more like a clause in a logical language.
+
+ >>> de = """
+ ... .*
+ ... (
+ ... de/SP|
+ ... del/SP
+ ... )
+ ... """
+ >>> DE = re.compile(de, re.VERBOSE)
+ >>> rels = [rel for doc in conll2002.chunked_sents('esp.train')
+ ... for rel in relextract.extract_rels('ORG', 'LOC', doc, corpus='conll2002', pattern = DE)]
+ >>> for r in rels[:10]:
+ ... print(relextract.clause(r, relsym='DE')) # doctest: +NORMALIZE_WHITESPACE
+ DE(u'tribunal_supremo', u'victoria')
+ DE(u'museo_de_arte', u'alcorc\xf3n')
+ DE(u'museo_de_bellas_artes', u'a_coru\xf1a')
+ DE(u'siria', u'l\xedbano')
+ DE(u'uni\xf3n_europea', u'pek\xedn')
+ DE(u'ej\xe9rcito', u'rogberi')
+ DE(u'juzgado_de_instrucci\xf3n_n\xfamero_1', u'san_sebasti\xe1n')
+ DE(u'psoe', u'villanueva_de_la_serena')
+ DE(u'ej\xe9rcito', u'l\xedbano')
+ DE(u'juzgado_de_lo_penal_n\xfamero_2', u'ceuta')
+ >>> vnv = """
+ ... (
+ ... is/V|
+ ... was/V|
+ ... werd/V|
+ ... wordt/V
+ ... )
+ ... .*
+ ... van/Prep
+ ... """
+ >>> VAN = re.compile(vnv, re.VERBOSE)
+ >>> for doc in conll2002.chunked_sents('ned.train'):
+ ... for r in relextract.extract_rels('PER', 'ORG', doc, corpus='conll2002', pattern=VAN):
+ ... print(relextract.clause(r, relsym="VAN"))
+ VAN(u"cornet_d'elzius", u'buitenlandse_handel')
+ VAN(u'johan_rottiers', u'kardinaal_van_roey_instituut')
+ VAN(u'annie_lennox', u'eurythmics')
diff --git a/nltk/test/resolution.doctest b/nltk/test/resolution.doctest
new file mode 100644
index 0000000..e182c7c
--- /dev/null
+++ b/nltk/test/resolution.doctest
@@ -0,0 +1,221 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=========================
+Resolution Theorem Prover
+=========================
+
+ >>> from nltk.inference.resolution import *
+ >>> from nltk.sem import logic
+ >>> from nltk.sem.logic import *
+ >>> logic._counter._value = 0
+ >>> lexpr = logic.Expression.fromstring
+
+ >>> P = lexpr('P')
+ >>> Q = lexpr('Q')
+ >>> R = lexpr('R')
+ >>> A = lexpr('A')
+ >>> B = lexpr('B')
+ >>> x = lexpr('x')
+ >>> y = lexpr('y')
+ >>> z = lexpr('z')
+
+-------------------------------
+Test most_general_unification()
+-------------------------------
+ >>> print(most_general_unification(x, x))
+ {}
+ >>> print(most_general_unification(A, A))
+ {}
+ >>> print(most_general_unification(A, x))
+ {x: A}
+ >>> print(most_general_unification(x, A))
+ {x: A}
+ >>> print(most_general_unification(x, y))
+ {x: y}
+ >>> print(most_general_unification(P(x), P(A)))
+ {x: A}
+ >>> print(most_general_unification(P(x,B), P(A,y)))
+ {x: A, y: B}
+ >>> print(most_general_unification(P(x,B), P(B,x)))
+ {x: B}
+ >>> print(most_general_unification(P(x,y), P(A,x)))
+ {x: A, y: x}
+ >>> print(most_general_unification(P(Q(x)), P(y)))
+ {y: Q(x)}
+
+------------
+Test unify()
+------------
+ >>> print(Clause([]).unify(Clause([])))
+ []
+ >>> print(Clause([P(x)]).unify(Clause([-P(A)])))
+ [{}]
+ >>> print(Clause([P(A), Q(x)]).unify(Clause([-P(x), R(x)])))
+ [{R(A), Q(A)}]
+ >>> print(Clause([P(A), Q(x), R(x,y)]).unify(Clause([-P(x), Q(y)])))
+ [{Q(y), Q(A), R(A,y)}]
+ >>> print(Clause([P(A), -Q(y)]).unify(Clause([-P(x), Q(B)])))
+ [{}]
+ >>> print(Clause([P(x), Q(x)]).unify(Clause([-P(A), -Q(B)])))
+ [{-Q(B), Q(A)}, {-P(A), P(B)}]
+ >>> print(Clause([P(x,x), Q(x), R(x)]).unify(Clause([-P(A,z), -Q(B)])))
+ [{-Q(B), Q(A), R(A)}, {-P(A,z), R(B), P(B,B)}]
+
+ >>> a = clausify(lexpr('P(A)'))
+ >>> b = clausify(lexpr('A=B'))
+ >>> print(a[0].unify(b[0]))
+ [{P(B)}]
+
+-------------------------
+Test is_tautology()
+-------------------------
+ >>> print(Clause([P(A), -P(A)]).is_tautology())
+ True
+ >>> print(Clause([-P(A), P(A)]).is_tautology())
+ True
+ >>> print(Clause([P(x), -P(A)]).is_tautology())
+ False
+ >>> print(Clause([Q(B), -P(A), P(A)]).is_tautology())
+ True
+ >>> print(Clause([-Q(A), P(R(A)), -P(R(A)), Q(x), -R(y)]).is_tautology())
+ True
+ >>> print(Clause([P(x), -Q(A)]).is_tautology())
+ False
+
+-------------------------
+Test subsumes()
+-------------------------
+ >>> print(Clause([P(A), Q(B)]).subsumes(Clause([P(A), Q(B)])))
+ True
+ >>> print(Clause([-P(A)]).subsumes(Clause([P(A)])))
+ False
+ >>> print(Clause([P(A), Q(B)]).subsumes(Clause([Q(B), P(A)])))
+ True
+ >>> print(Clause([P(A), Q(B)]).subsumes(Clause([Q(B), R(A), P(A)])))
+ True
+ >>> print(Clause([P(A), R(A), Q(B)]).subsumes(Clause([Q(B), P(A)])))
+ False
+ >>> print(Clause([P(x)]).subsumes(Clause([P(A)])))
+ True
+ >>> print(Clause([P(A)]).subsumes(Clause([P(x)])))
+ True
+
+------------
+Test prove()
+------------
+ >>> print(ResolutionProverCommand(lexpr('man(x)')).prove())
+ False
+ >>> print(ResolutionProverCommand(lexpr('(man(x) -> man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('(man(x) -> --man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('-(man(x) & -man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('(man(x) | -man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('(man(x) -> man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('-(man(x) & -man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('(man(x) | -man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('(man(x) -> man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('(man(x) <-> man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('-(man(x) <-> -man(x))')).prove())
+ True
+ >>> print(ResolutionProverCommand(lexpr('all x.man(x)')).prove())
+ False
+ >>> print(ResolutionProverCommand(lexpr('-all x.some y.F(x,y) & some x.all y.(-F(x,y))')).prove())
+ False
+ >>> print(ResolutionProverCommand(lexpr('some x.all y.sees(x,y)')).prove())
+ False
+
+ >>> p1 = lexpr('all x.(man(x) -> mortal(x))')
+ >>> p2 = lexpr('man(Socrates)')
+ >>> c = lexpr('mortal(Socrates)')
+ >>> ResolutionProverCommand(c, [p1,p2]).prove()
+ True
+
+ >>> p1 = lexpr('all x.(man(x) -> walks(x))')
+ >>> p2 = lexpr('man(John)')
+ >>> c = lexpr('some y.walks(y)')
+ >>> ResolutionProverCommand(c, [p1,p2]).prove()
+ True
+
+ >>> p = lexpr('some e1.some e2.(believe(e1,john,e2) & walk(e2,mary))')
+ >>> c = lexpr('some e0.walk(e0,mary)')
+ >>> ResolutionProverCommand(c, [p]).prove()
+ True
+
+------------
+Test proof()
+------------
+ >>> p1 = lexpr('all x.(man(x) -> mortal(x))')
+ >>> p2 = lexpr('man(Socrates)')
+ >>> c = lexpr('mortal(Socrates)')
+ >>> logic._counter._value = 0
+ >>> tp = ResolutionProverCommand(c, [p1,p2])
+ >>> tp.prove()
+ True
+ >>> print(tp.proof())
+ [1] {-mortal(Socrates)} A
+ [2] {-man(z2), mortal(z2)} A
+ [3] {man(Socrates)} A
+ [4] {-man(Socrates)} (1, 2)
+ [5] {mortal(Socrates)} (2, 3)
+ [6] {} (1, 5)
+ <BLANKLINE>
+
+------------------
+Question Answering
+------------------
+One answer
+ >>> p1 = lexpr('father_of(art,john)')
+ >>> p2 = lexpr('father_of(bob,kim)')
+ >>> p3 = lexpr('all x.all y.(father_of(x,y) -> parent_of(x,y))')
+ >>> c = lexpr('all x.(parent_of(x,john) -> ANSWER(x))')
+ >>> logic._counter._value = 0
+ >>> tp = ResolutionProverCommand(None, [p1,p2,p3,c])
+ >>> sorted(tp.find_answers())
+ [<ConstantExpression art>]
+ >>> print(tp.proof()) # doctest: +SKIP
+ [1] {father_of(art,john)} A
+ [2] {father_of(bob,kim)} A
+ [3] {-father_of(z3,z4), parent_of(z3,z4)} A
+ [4] {-parent_of(z6,john), ANSWER(z6)} A
+ [5] {parent_of(art,john)} (1, 3)
+ [6] {parent_of(bob,kim)} (2, 3)
+ [7] {ANSWER(z6), -father_of(z6,john)} (3, 4)
+ [8] {ANSWER(art)} (1, 7)
+ [9] {ANSWER(art)} (4, 5)
+ <BLANKLINE>
+
+Multiple answers
+ >>> p1 = lexpr('father_of(art,john)')
+ >>> p2 = lexpr('mother_of(ann,john)')
+ >>> p3 = lexpr('all x.all y.(father_of(x,y) -> parent_of(x,y))')
+ >>> p4 = lexpr('all x.all y.(mother_of(x,y) -> parent_of(x,y))')
+ >>> c = lexpr('all x.(parent_of(x,john) -> ANSWER(x))')
+ >>> logic._counter._value = 0
+ >>> tp = ResolutionProverCommand(None, [p1,p2,p3,p4,c])
+ >>> sorted(tp.find_answers())
+ [<ConstantExpression ann>, <ConstantExpression art>]
+ >>> print(tp.proof()) # doctest: +SKIP
+ [ 1] {father_of(art,john)} A
+ [ 2] {mother_of(ann,john)} A
+ [ 3] {-father_of(z3,z4), parent_of(z3,z4)} A
+ [ 4] {-mother_of(z7,z8), parent_of(z7,z8)} A
+ [ 5] {-parent_of(z10,john), ANSWER(z10)} A
+ [ 6] {parent_of(art,john)} (1, 3)
+ [ 7] {parent_of(ann,john)} (2, 4)
+ [ 8] {ANSWER(z10), -father_of(z10,john)} (3, 5)
+ [ 9] {ANSWER(art)} (1, 8)
+ [10] {ANSWER(z10), -mother_of(z10,john)} (4, 5)
+ [11] {ANSWER(ann)} (2, 10)
+ [12] {ANSWER(art)} (5, 6)
+ [13] {ANSWER(ann)} (5, 7)
+ <BLANKLINE>
+
diff --git a/nltk/test/runtests.py b/nltk/test/runtests.py
new file mode 100755
index 0000000..13d8089
--- /dev/null
+++ b/nltk/test/runtests.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, print_function
+import sys
+import os
+import nose
+from nose.plugins.manager import PluginManager
+from nose.plugins.doctests import Doctest
+from nose.plugins import builtin
+
+NLTK_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
+sys.path.insert(0, NLTK_ROOT)
+
+NLTK_TEST_DIR = os.path.join(NLTK_ROOT, 'nltk')
+
+if __name__ == '__main__':
+ # there shouldn't be import from NLTK for coverage to work properly
+ from doctest_nose_plugin import DoctestFix
+
+ class NltkPluginManager(PluginManager):
+ """
+ Nose plugin manager that replaces standard doctest plugin
+ with a patched version.
+ """
+ def loadPlugins(self):
+ for plug in builtin.plugins:
+ if plug != Doctest:
+ self.addPlugin(plug())
+ self.addPlugin(DoctestFix())
+ super(NltkPluginManager, self).loadPlugins()
+
+ manager = NltkPluginManager()
+ manager.loadPlugins()
+
+ # allow passing extra options and running individual tests
+ # Examples:
+ #
+ # python runtests.py semantics.doctest
+ # python runtests.py --with-id -v
+ # python runtests.py --with-id -v nltk.featstruct
+
+ args = sys.argv[1:]
+ if not args:
+ args = [NLTK_TEST_DIR]
+
+ if all(arg.startswith('-') for arg in args):
+ # only extra options were passed
+ args += [NLTK_TEST_DIR]
+
+ arguments = [
+ '--exclude=', # why is this needed?
+ #'--with-xunit',
+ #'--xunit-file=$WORKSPACE/nosetests.xml',
+ #'--nocapture',
+ '--with-doctest',
+ #'--doctest-tests',
+ #'--debug=nose,nose.importer,nose.inspector,nose.plugins,nose.result,nose.selector',
+ '--doctest-extension=.doctest',
+ '--doctest-fixtures=_fixt',
+ '--doctest-options=+ELLIPSIS,+NORMALIZE_WHITESPACE,+IGNORE_EXCEPTION_DETAIL,+ALLOW_UNICODE,doctestencoding=utf-8',
+ #'--verbosity=3',
+ ] + args
+
+ nose.main(argv=arguments, plugins=manager.plugins)
diff --git a/nltk/test/segmentation_fixt.py b/nltk/test/segmentation_fixt.py
new file mode 100644
index 0000000..b4a72be
--- /dev/null
+++ b/nltk/test/segmentation_fixt.py
@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+
+# skip segmentation.doctest if numpy is not available
+def setup_module(module):
+ from nose import SkipTest
+ try:
+ import numpy
+ except ImportError:
+ raise SkipTest("segmentation.doctest requires numpy")
\ No newline at end of file
diff --git a/nltk/test/sem3.cfg b/nltk/test/sem3.cfg
new file mode 100644
index 0000000..948f1fd
--- /dev/null
+++ b/nltk/test/sem3.cfg
@@ -0,0 +1,14 @@
+#######################################
+# sem1.cfg
+#######################################
+# Minimal feature-based grammar with determiner semantics.
+
+
+% start S
+
+S[sem=?vp] -> NP[sem=?np] VP[subj=?np, sem=?vp]
+VP[sem=?v, subj=?np] -> IV[sem=?v, subj=?np]
+NP[sem=[index='k',name='kim']] -> 'Kim'
+IV[sem=[rel='bark', arg=?i], subj=[sem=[index=?i]]] -> 'barks'
+#IV[fsem=[rel='bark', arg=(1)[]], subj=[fsem=[index->(1)]]] -> 'barks'
+
diff --git a/nltk/test/semantics.doctest b/nltk/test/semantics.doctest
new file mode 100644
index 0000000..f22c4a5
--- /dev/null
+++ b/nltk/test/semantics.doctest
@@ -0,0 +1,665 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=========
+Semantics
+=========
+
+ >>> import nltk
+ >>> from nltk.sem import Valuation, Model
+ >>> v = [('adam', 'b1'), ('betty', 'g1'), ('fido', 'd1'),
+ ... ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])),
+ ... ('dog', set(['d1'])),
+ ... ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]))]
+ >>> val = Valuation(v)
+ >>> dom = val.domain
+ >>> m = Model(dom, val)
+
+Evaluation
+----------
+
+The top-level method of a ``Model`` instance is ``evaluate()``, which
+assigns a semantic value to expressions of the ``logic`` module, under
+an assignment ``g``:
+
+ >>> dom = val.domain
+ >>> g = nltk.sem.Assignment(dom)
+ >>> m.evaluate('all x.(boy(x) -> - girl(x))', g)
+ True
+
+
+``evaluate()`` calls a recursive function ``satisfy()``, which in turn
+calls a function ``i()`` to interpret non-logical constants and
+individual variables. ``i()`` delegates the interpretation of these to
+the the model's ``Valuation`` and the variable assignment ``g``
+respectively. Any atomic expression which cannot be assigned a value
+by ``i`` raises an ``Undefined`` exception; this is caught by
+``evaluate``, which returns the string ``'Undefined'``.
+
+ >>> m.evaluate('walk(adam)', g, trace=2)
+ <BLANKLINE>
+ 'walk(adam)' is undefined under M, g
+ 'Undefined'
+
+Batch Processing
+----------------
+
+The utility functions ``interpret_sents()`` and ``evaluate_sents()`` are intended to
+help with processing multiple sentences. Here's an example of the first of these:
+
+ >>> sents = ['Mary walks']
+ >>> results = nltk.sem.util.interpret_sents(sents, 'grammars/sample_grammars/sem2.fcfg')
+ >>> for result in results:
+ ... for (synrep, semrep) in result:
+ ... print(synrep)
+ (S[SEM=<walk(mary)>]
+ (NP[-LOC, NUM='sg', SEM=<\P.P(mary)>]
+ (PropN[-LOC, NUM='sg', SEM=<\P.P(mary)>] Mary))
+ (VP[NUM='sg', SEM=<\x.walk(x)>]
+ (IV[NUM='sg', SEM=<\x.walk(x)>, TNS='pres'] walks)))
+
+In order to provide backwards compatibility with 'legacy' grammars where the semantics value
+is specified with a lowercase
+``sem`` feature, the relevant feature name can be passed to the function using the
+``semkey`` parameter, as shown here:
+
+ >>> sents = ['raining']
+ >>> g = nltk.grammar.FeatureGrammar.fromstring("""
+ ... % start S
+ ... S[sem=<raining>] -> 'raining'
+ ... """)
+ >>> results = nltk.sem.util.interpret_sents(sents, g, semkey='sem')
+ >>> for result in results:
+ ... for (synrep, semrep) in result:
+ ... print(semrep)
+ raining
+
+The function ``evaluate_sents()`` works in a similar manner, but also needs to be
+passed a ``Model`` against which the semantic representations are evaluated.
+
+Unit Tests
+==========
+
+
+Unit tests for relations and valuations
+---------------------------------------
+
+ >>> from nltk.sem import *
+
+Relations are sets of tuples, all of the same length.
+
+ >>> s1 = set([('d1', 'd2'), ('d1', 'd1'), ('d2', 'd1')])
+ >>> is_rel(s1)
+ True
+ >>> s2 = set([('d1', 'd2'), ('d1', 'd2'), ('d1',)])
+ >>> is_rel(s2)
+ Traceback (most recent call last):
+ . . .
+ ValueError: Set set([('d1', 'd2'), ('d1',)]) contains sequences of different lengths
+ >>> s3 = set(['d1', 'd2'])
+ >>> is_rel(s3)
+ Traceback (most recent call last):
+ . . .
+ ValueError: Set set(['d2', 'd1']) contains sequences of different lengths
+ >>> s4 = set2rel(s3)
+ >>> is_rel(s4)
+ True
+ >>> is_rel(set())
+ True
+ >>> null_binary_rel = set([(None, None)])
+ >>> is_rel(null_binary_rel)
+ True
+
+Sets of entities are converted into sets of singleton tuples
+(containing strings).
+
+ >>> sorted(set2rel(s3))
+ [('d1',), ('d2',)]
+ >>> sorted(set2rel(set([1,3,5,])))
+ ['1', '3', '5']
+ >>> set2rel(set()) == set()
+ True
+ >>> set2rel(set2rel(s3)) == set2rel(s3)
+ True
+
+Predication is evaluated by set membership.
+
+ >>> ('d1', 'd2') in s1
+ True
+ >>> ('d2', 'd2') in s1
+ False
+ >>> ('d1',) in s1
+ False
+ >>> 'd2' in s1
+ False
+ >>> ('d1',) in s4
+ True
+ >>> ('d1',) in set()
+ False
+ >>> 'd1' in null_binary_rel
+ False
+
+
+ >>> val = Valuation([('Fido', 'd1'), ('dog', set(['d1', 'd2'])), ('walk', set())])
+ >>> sorted(val['dog'])
+ [('d1',), ('d2',)]
+ >>> val.domain == set(['d1', 'd2'])
+ True
+ >>> print(val.symbols)
+ ['Fido', 'dog', 'walk']
+
+
+Parse a valuation from a string.
+
+ >>> v = """
+ ... john => b1
+ ... mary => g1
+ ... suzie => g2
+ ... fido => d1
+ ... tess => d2
+ ... noosa => n
+ ... girl => {g1, g2}
+ ... boy => {b1, b2}
+ ... dog => {d1, d2}
+ ... bark => {d1, d2}
+ ... walk => {b1, g2, d1}
+ ... chase => {(b1, g1), (b2, g1), (g1, d1), (g2, d2)}
+ ... see => {(b1, g1), (b2, d2), (g1, b1),(d2, b1), (g2, n)}
+ ... in => {(b1, n), (b2, n), (d2, n)}
+ ... with => {(b1, g1), (g1, b1), (d1, b1), (b1, d1)}
+ ... """
+ >>> val = parse_valuation(v)
+
+ >>> print(val) # doctest: +SKIP
+ {'bark': set([('d1',), ('d2',)]),
+ 'boy': set([('b1',), ('b2',)]),
+ 'chase': set([('b1', 'g1'), ('g2', 'd2'), ('g1', 'd1'), ('b2', 'g1')]),
+ 'dog': set([('d1',), ('d2',)]),
+ 'fido': 'd1',
+ 'girl': set([('g2',), ('g1',)]),
+ 'in': set([('d2', 'n'), ('b1', 'n'), ('b2', 'n')]),
+ 'john': 'b1',
+ 'mary': 'g1',
+ 'noosa': 'n',
+ 'see': set([('b1', 'g1'), ('b2', 'd2'), ('d2', 'b1'), ('g2', 'n'), ('g1', 'b1')]),
+ 'suzie': 'g2',
+ 'tess': 'd2',
+ 'walk': set([('d1',), ('b1',), ('g2',)]),
+ 'with': set([('b1', 'g1'), ('d1', 'b1'), ('b1', 'd1'), ('g1', 'b1')])}
+
+
+Unit tests for function argument application in a Model
+-------------------------------------------------------
+
+ >>> v = [('adam', 'b1'), ('betty', 'g1'), ('fido', 'd1'),\
+ ... ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])), ('dog', set(['d1'])),
+ ... ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')])),
+ ... ('kiss', null_binary_rel)]
+ >>> val = Valuation(v)
+ >>> dom = val.domain
+ >>> m = Model(dom, val)
+ >>> g = Assignment(dom)
+ >>> sorted(val['boy'])
+ [('b1',), ('b2',)]
+ >>> ('b1',) in val['boy']
+ True
+ >>> ('g1',) in val['boy']
+ False
+ >>> ('foo',) in val['boy']
+ False
+ >>> ('b1', 'g1') in val['love']
+ True
+ >>> ('b1', 'b1') in val['kiss']
+ False
+ >>> sorted(val.domain)
+ ['b1', 'b2', 'd1', 'g1', 'g2']
+
+
+Model Tests
+===========
+
+Extension of Lambda expressions
+
+ >>> v0 = [('adam', 'b1'), ('betty', 'g1'), ('fido', 'd1'),\
+ ... ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])),
+ ... ('dog', set(['d1'])),
+ ... ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]))]
+
+ >>> val0 = Valuation(v0)
+ >>> dom0 = val0.domain
+ >>> m0 = Model(dom0, val0)
+ >>> g0 = Assignment(dom0)
+
+ >>> print(m0.evaluate(r'\x. \y. love(x, y)', g0) == {'g2': {'g2': False, 'b2': False, 'b1': True, 'g1': False, 'd1': False}, 'b2': {'g2': True, 'b2': False, 'b1': False, 'g1': False, 'd1': False}, 'b1': {'g2': False, 'b2': False, 'b1': False, 'g1': True, 'd1': False}, 'g1': {'g2': False, 'b2': False, 'b1': True, 'g1': False, 'd1': False}, 'd1': {'g2': False, 'b2': False, 'b1': False, 'g1': False, 'd1': False}})
+ True
+ >>> print(m0.evaluate(r'\x. dog(x) (adam)', g0))
+ False
+ >>> print(m0.evaluate(r'\x. (dog(x) | boy(x)) (adam)', g0))
+ True
+ >>> print(m0.evaluate(r'\x. \y. love(x, y)(fido)', g0) == {'g2': False, 'b2': False, 'b1': False, 'g1': False, 'd1': False})
+ True
+ >>> print(m0.evaluate(r'\x. \y. love(x, y)(adam)', g0) == {'g2': False, 'b2': False, 'b1': False, 'g1': True, 'd1': False})
+ True
+ >>> print(m0.evaluate(r'\x. \y. love(x, y)(betty)', g0) == {'g2': False, 'b2': False, 'b1': True, 'g1': False, 'd1': False})
+ True
+ >>> print(m0.evaluate(r'\x. \y. love(x, y)(betty)(adam)', g0))
+ True
+ >>> print(m0.evaluate(r'\x. \y. love(x, y)(betty, adam)', g0))
+ True
+ >>> print(m0.evaluate(r'\y. \x. love(x, y)(fido)(adam)', g0))
+ False
+ >>> print(m0.evaluate(r'\y. \x. love(x, y)(betty, adam)', g0))
+ True
+ >>> print(m0.evaluate(r'\x. exists y. love(x, y)', g0) == {'g2': True, 'b2': True, 'b1': True, 'g1': True, 'd1': False})
+ True
+ >>> print(m0.evaluate(r'\z. adam', g0) == {'g2': 'b1', 'b2': 'b1', 'b1': 'b1', 'g1': 'b1', 'd1': 'b1'})
+ True
+ >>> print(m0.evaluate(r'\z. love(x, y)', g0) == {'g2': False, 'b2': False, 'b1': False, 'g1': False, 'd1': False})
+ True
+
+
+Propositional Model Test
+------------------------
+
+ >>> tests = [
+ ... ('P & Q', True),
+ ... ('P & R', False),
+ ... ('- P', False),
+ ... ('- R', True),
+ ... ('- - P', True),
+ ... ('- (P & R)', True),
+ ... ('P | R', True),
+ ... ('R | P', True),
+ ... ('R | R', False),
+ ... ('- P | R', False),
+ ... ('P | - P', True),
+ ... ('P -> Q', True),
+ ... ('P -> R', False),
+ ... ('R -> P', True),
+ ... ('P <-> P', True),
+ ... ('R <-> R', True),
+ ... ('P <-> R', False),
+ ... ]
+ >>> val1 = Valuation([('P', True), ('Q', True), ('R', False)])
+ >>> dom = set([])
+ >>> m = Model(dom, val1)
+ >>> g = Assignment(dom)
+ >>> for (sent, testvalue) in tests:
+ ... semvalue = m.evaluate(sent, g)
+ ... if semvalue == testvalue:
+ ... print('*', end=' ')
+ * * * * * * * * * * * * * * * * *
+
+
+Test of i Function
+------------------
+
+ >>> from nltk.sem import Expression
+ >>> v = [('adam', 'b1'), ('betty', 'g1'), ('fido', 'd1'),
+ ... ('girl', set(['g1', 'g2'])), ('boy', set(['b1', 'b2'])), ('dog', set(['d1'])),
+ ... ('love', set([('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]))]
+ >>> val = Valuation(v)
+ >>> dom = val.domain
+ >>> m = Model(dom, val)
+ >>> g = Assignment(dom, [('x', 'b1'), ('y', 'g2')])
+ >>> exprs = ['adam', 'girl', 'love', 'walks', 'x', 'y', 'z']
+ >>> parsed_exprs = [Expression.fromstring(e) for e in exprs]
+ >>> sorted_set = lambda x: sorted(x) if isinstance(x, set) else x
+ >>> for parsed in parsed_exprs:
+ ... try:
+ ... print("'%s' gets value %s" % (parsed, sorted_set(m.i(parsed, g))))
+ ... except Undefined:
+ ... print("'%s' is Undefined" % parsed)
+ 'adam' gets value b1
+ 'girl' gets value [('g1',), ('g2',)]
+ 'love' gets value [('b1', 'g1'), ('b2', 'g2'), ('g1', 'b1'), ('g2', 'b1')]
+ 'walks' is Undefined
+ 'x' gets value b1
+ 'y' gets value g2
+ 'z' is Undefined
+
+Test for formulas in Model
+--------------------------
+
+ >>> tests = [
+ ... ('love(adam, betty)', True),
+ ... ('love(adam, sue)', 'Undefined'),
+ ... ('dog(fido)', True),
+ ... ('- dog(fido)', False),
+ ... ('- - dog(fido)', True),
+ ... ('- dog(sue)', 'Undefined'),
+ ... ('dog(fido) & boy(adam)', True),
+ ... ('- (dog(fido) & boy(adam))', False),
+ ... ('- dog(fido) & boy(adam)', False),
+ ... ('dog(fido) | boy(adam)', True),
+ ... ('- (dog(fido) | boy(adam))', False),
+ ... ('- dog(fido) | boy(adam)', True),
+ ... ('- dog(fido) | - boy(adam)', False),
+ ... ('dog(fido) -> boy(adam)', True),
+ ... ('- (dog(fido) -> boy(adam))', False),
+ ... ('- dog(fido) -> boy(adam)', True),
+ ... ('exists x . love(adam, x)', True),
+ ... ('all x . love(adam, x)', False),
+ ... ('fido = fido', True),
+ ... ('exists x . all y. love(x, y)', False),
+ ... ('exists x . (x = fido)', True),
+ ... ('all x . (dog(x) | - dog(x))', True),
+ ... ('adam = mia', 'Undefined'),
+ ... ('\\x. (boy(x) | girl(x))', {'g2': True, 'b2': True, 'b1': True, 'g1': True, 'd1': False}),
+ ... ('\\x. exists y. (boy(x) & love(x, y))', {'g2': False, 'b2': True, 'b1': True, 'g1': False, 'd1': False}),
+ ... ('exists z1. boy(z1)', True),
+ ... ('exists x. (boy(x) & - (x = adam))', True),
+ ... ('exists x. (boy(x) & all y. love(y, x))', False),
+ ... ('all x. (boy(x) | girl(x))', False),
+ ... ('all x. (girl(x) -> exists y. boy(y) & love(x, y))', False),
+ ... ('exists x. (boy(x) & all y. (girl(y) -> love(y, x)))', True),
+ ... ('exists x. (boy(x) & all y. (girl(y) -> love(x, y)))', False),
+ ... ('all x. (dog(x) -> - girl(x))', True),
+ ... ('exists x. exists y. (love(x, y) & love(x, y))', True),
+ ... ]
+ >>> for (sent, testvalue) in tests:
+ ... semvalue = m.evaluate(sent, g)
+ ... if semvalue == testvalue:
+ ... print('*', end=' ')
+ ... else:
+ ... print(sent, semvalue)
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+
+
+
+Satisfier Tests
+---------------
+
+ >>> formulas = [
+ ... 'boy(x)',
+ ... '(x = x)',
+ ... '(boy(x) | girl(x))',
+ ... '(boy(x) & girl(x))',
+ ... 'love(adam, x)',
+ ... 'love(x, adam)',
+ ... '- (x = adam)',
+ ... 'exists z22. love(x, z22)',
+ ... 'exists y. love(y, x)',
+ ... 'all y. (girl(y) -> love(x, y))',
+ ... 'all y. (girl(y) -> love(y, x))',
+ ... 'all y. (girl(y) -> (boy(x) & love(y, x)))',
+ ... 'boy(x) & all y. (girl(y) -> love(x, y))',
+ ... 'boy(x) & all y. (girl(y) -> love(y, x))',
+ ... 'boy(x) & exists y. (girl(y) & love(y, x))',
+ ... 'girl(x) -> dog(x)',
+ ... 'all y. (dog(y) -> (x = y))',
+ ... '- exists y. love(y, x)',
+ ... 'exists y. (love(adam, y) & love(y, x))'
+ ... ]
+ >>> g.purge()
+ >>> g.add('x', 'b1')
+ {'x': 'b1'}
+ >>> for f in formulas: # doctest: +NORMALIZE_WHITESPACE
+ ... try:
+ ... print("'%s' gets value: %s" % (f, m.evaluate(f, g)))
+ ... except Undefined:
+ ... print("'%s' is Undefined" % f)
+ 'boy(x)' gets value: True
+ '(x = x)' gets value: True
+ '(boy(x) | girl(x))' gets value: True
+ '(boy(x) & girl(x))' gets value: False
+ 'love(adam, x)' gets value: False
+ 'love(x, adam)' gets value: False
+ '- (x = adam)' gets value: False
+ 'exists z22. love(x, z22)' gets value: True
+ 'exists y. love(y, x)' gets value: True
+ 'all y. (girl(y) -> love(x, y))' gets value: False
+ 'all y. (girl(y) -> love(y, x))' gets value: True
+ 'all y. (girl(y) -> (boy(x) & love(y, x)))' gets value: True
+ 'boy(x) & all y. (girl(y) -> love(x, y))' gets value: False
+ 'boy(x) & all y. (girl(y) -> love(y, x))' gets value: True
+ 'boy(x) & exists y. (girl(y) & love(y, x))' gets value: True
+ 'girl(x) -> dog(x)' gets value: True
+ 'all y. (dog(y) -> (x = y))' gets value: False
+ '- exists y. love(y, x)' gets value: False
+ 'exists y. (love(adam, y) & love(y, x))' gets value: True
+
+ >>> from nltk.sem import Expression
+ >>> for fmla in formulas: # doctest: +NORMALIZE_WHITESPACE
+ ... p = Expression.fromstring(fmla)
+ ... g.purge()
+ ... print("Satisfiers of '%s':\n\t%s" % (p, sorted(m.satisfiers(p, 'x', g))))
+ Satisfiers of 'boy(x)':
+ ['b1', 'b2']
+ Satisfiers of '(x = x)':
+ ['b1', 'b2', 'd1', 'g1', 'g2']
+ Satisfiers of '(boy(x) | girl(x))':
+ ['b1', 'b2', 'g1', 'g2']
+ Satisfiers of '(boy(x) & girl(x))':
+ []
+ Satisfiers of 'love(adam,x)':
+ ['g1']
+ Satisfiers of 'love(x,adam)':
+ ['g1', 'g2']
+ Satisfiers of '-(x = adam)':
+ ['b2', 'd1', 'g1', 'g2']
+ Satisfiers of 'exists z22.love(x,z22)':
+ ['b1', 'b2', 'g1', 'g2']
+ Satisfiers of 'exists y.love(y,x)':
+ ['b1', 'g1', 'g2']
+ Satisfiers of 'all y.(girl(y) -> love(x,y))':
+ []
+ Satisfiers of 'all y.(girl(y) -> love(y,x))':
+ ['b1']
+ Satisfiers of 'all y.(girl(y) -> (boy(x) & love(y,x)))':
+ ['b1']
+ Satisfiers of '(boy(x) & all y.(girl(y) -> love(x,y)))':
+ []
+ Satisfiers of '(boy(x) & all y.(girl(y) -> love(y,x)))':
+ ['b1']
+ Satisfiers of '(boy(x) & exists y.(girl(y) & love(y,x)))':
+ ['b1']
+ Satisfiers of '(girl(x) -> dog(x))':
+ ['b1', 'b2', 'd1']
+ Satisfiers of 'all y.(dog(y) -> (x = y))':
+ ['d1']
+ Satisfiers of '-exists y.love(y,x)':
+ ['b2', 'd1']
+ Satisfiers of 'exists y.(love(adam,y) & love(y,x))':
+ ['b1']
+
+
+Tests based on the Blackburn & Bos testsuite
+--------------------------------------------
+
+ >>> v1 = [('jules', 'd1'), ('vincent', 'd2'), ('pumpkin', 'd3'),
+ ... ('honey_bunny', 'd4'), ('yolanda', 'd5'),
+ ... ('customer', set(['d1', 'd2'])),
+ ... ('robber', set(['d3', 'd4'])),
+ ... ('love', set([('d3', 'd4')]))]
+ >>> val1 = Valuation(v1)
+ >>> dom1 = val1.domain
+ >>> m1 = Model(dom1, val1)
+ >>> g1 = Assignment(dom1)
+
+ >>> v2 = [('jules', 'd1'), ('vincent', 'd2'), ('pumpkin', 'd3'),
+ ... ('honey_bunny', 'd4'), ('yolanda', 'd4'),
+ ... ('customer', set(['d1', 'd2', 'd5', 'd6'])),
+ ... ('robber', set(['d3', 'd4'])),
+ ... ('love', set([(None, None)]))]
+ >>> val2 = Valuation(v2)
+ >>> dom2 = set(['d1', 'd2', 'd3', 'd4', 'd5', 'd6'])
+ >>> m2 = Model(dom2, val2)
+ >>> g2 = Assignment(dom2)
+ >>> g21 = Assignment(dom2)
+ >>> g21.add('y', 'd3')
+ {'y': 'd3'}
+
+ >>> v3 = [('mia', 'd1'), ('jody', 'd2'), ('jules', 'd3'),
+ ... ('vincent', 'd4'),
+ ... ('woman', set(['d1', 'd2'])), ('man', set(['d3', 'd4'])),
+ ... ('joke', set(['d5', 'd6'])), ('episode', set(['d7', 'd8'])),
+ ... ('in', set([('d5', 'd7'), ('d5', 'd8')])),
+ ... ('tell', set([('d1', 'd5'), ('d2', 'd6')]))]
+ >>> val3 = Valuation(v3)
+ >>> dom3 = set(['d1', 'd2', 'd3', 'd4', 'd5', 'd6', 'd7', 'd8'])
+ >>> m3 = Model(dom3, val3)
+ >>> g3 = Assignment(dom3)
+
+ >>> tests = [
+ ... ('exists x. robber(x)', m1, g1, True),
+ ... ('exists x. exists y. love(y, x)', m1, g1, True),
+ ... ('exists x0. exists x1. love(x1, x0)', m2, g2, False),
+ ... ('all x. all y. love(y, x)', m2, g2, False),
+ ... ('- (all x. all y. love(y, x))', m2, g2, True),
+ ... ('all x. all y. - love(y, x)', m2, g2, True),
+ ... ('yolanda = honey_bunny', m2, g2, True),
+ ... ('mia = honey_bunny', m2, g2, 'Undefined'),
+ ... ('- (yolanda = honey_bunny)', m2, g2, False),
+ ... ('- (mia = honey_bunny)', m2, g2, 'Undefined'),
+ ... ('all x. (robber(x) | customer(x))', m2, g2, True),
+ ... ('- (all x. (robber(x) | customer(x)))', m2, g2, False),
+ ... ('(robber(x) | customer(x))', m2, g2, 'Undefined'),
+ ... ('(robber(y) | customer(y))', m2, g21, True),
+ ... ('exists x. (man(x) & exists x. woman(x))', m3, g3, True),
+ ... ('exists x. (man(x) & exists x. woman(x))', m3, g3, True),
+ ... ('- exists x. woman(x)', m3, g3, False),
+ ... ('exists x. (tasty(x) & burger(x))', m3, g3, 'Undefined'),
+ ... ('- exists x. (tasty(x) & burger(x))', m3, g3, 'Undefined'),
+ ... ('exists x. (man(x) & - exists y. woman(y))', m3, g3, False),
+ ... ('exists x. (man(x) & - exists x. woman(x))', m3, g3, False),
+ ... ('exists x. (woman(x) & - exists x. customer(x))', m2, g2, 'Undefined'),
+ ... ]
+
+ >>> for item in tests:
+ ... sentence, model, g, testvalue = item
+ ... semvalue = model.evaluate(sentence, g)
+ ... if semvalue == testvalue:
+ ... print('*', end=' ')
+ ... g.purge()
+ * * * * * * * * * * * * * * * * * * * * * *
+
+
+Tests for mapping from syntax to semantics
+------------------------------------------
+
+Load a valuation from a file.
+
+ >>> import nltk.data
+ >>> from nltk.sem.util import parse_sents
+ >>> val = nltk.data.load('grammars/sample_grammars/valuation1.val')
+ >>> dom = val.domain
+ >>> m = Model(dom, val)
+ >>> g = Assignment(dom)
+ >>> gramfile = 'grammars/sample_grammars/sem2.fcfg'
+ >>> inputs = ['John sees a girl', 'every dog barks']
+ >>> parses = parse_sents(inputs, gramfile)
+ >>> for sent, trees in zip(inputs, parses):
+ ... print()
+ ... print("Sentence: %s" % sent)
+ ... for tree in trees:
+ ... print("Parse:\n %s" %tree)
+ ... print("Semantics: %s" % root_semrep(tree))
+ <BLANKLINE>
+ Sentence: John sees a girl
+ Parse:
+ (S[SEM=<exists x.(girl(x) & see(john,x))>]
+ (NP[-LOC, NUM='sg', SEM=<\P.P(john)>]
+ (PropN[-LOC, NUM='sg', SEM=<\P.P(john)>] John))
+ (VP[NUM='sg', SEM=<\y.exists x.(girl(x) & see(y,x))>]
+ (TV[NUM='sg', SEM=<\X y.X(\x.see(y,x))>, TNS='pres'] sees)
+ (NP[NUM='sg', SEM=<\Q.exists x.(girl(x) & Q(x))>]
+ (Det[NUM='sg', SEM=<\P Q.exists x.(P(x) & Q(x))>] a)
+ (Nom[NUM='sg', SEM=<\x.girl(x)>]
+ (N[NUM='sg', SEM=<\x.girl(x)>] girl)))))
+ Semantics: exists x.(girl(x) & see(john,x))
+ <BLANKLINE>
+ Sentence: every dog barks
+ Parse:
+ (S[SEM=<all x.(dog(x) -> bark(x))>]
+ (NP[NUM='sg', SEM=<\Q.all x.(dog(x) -> Q(x))>]
+ (Det[NUM='sg', SEM=<\P Q.all x.(P(x) -> Q(x))>] every)
+ (Nom[NUM='sg', SEM=<\x.dog(x)>]
+ (N[NUM='sg', SEM=<\x.dog(x)>] dog)))
+ (VP[NUM='sg', SEM=<\x.bark(x)>]
+ (IV[NUM='sg', SEM=<\x.bark(x)>, TNS='pres'] barks)))
+ Semantics: all x.(dog(x) -> bark(x))
+
+ >>> sent = "every dog barks"
+ >>> result = nltk.sem.util.interpret_sents([sent], gramfile)[0]
+ >>> for (syntree, semrep) in result:
+ ... print(syntree)
+ ... print()
+ ... print(semrep)
+ (S[SEM=<all x.(dog(x) -> bark(x))>]
+ (NP[NUM='sg', SEM=<\Q.all x.(dog(x) -> Q(x))>]
+ (Det[NUM='sg', SEM=<\P Q.all x.(P(x) -> Q(x))>] every)
+ (Nom[NUM='sg', SEM=<\x.dog(x)>]
+ (N[NUM='sg', SEM=<\x.dog(x)>] dog)))
+ (VP[NUM='sg', SEM=<\x.bark(x)>]
+ (IV[NUM='sg', SEM=<\x.bark(x)>, TNS='pres'] barks)))
+ <BLANKLINE>
+ all x.(dog(x) -> bark(x))
+
+ >>> result = nltk.sem.util.evaluate_sents([sent], gramfile, m, g)[0]
+ >>> for (syntree, semrel, value) in result:
+ ... print(syntree)
+ ... print()
+ ... print(semrep)
+ ... print()
+ ... print(value)
+ (S[SEM=<all x.(dog(x) -> bark(x))>]
+ (NP[NUM='sg', SEM=<\Q.all x.(dog(x) -> Q(x))>]
+ (Det[NUM='sg', SEM=<\P Q.all x.(P(x) -> Q(x))>] every)
+ (Nom[NUM='sg', SEM=<\x.dog(x)>]
+ (N[NUM='sg', SEM=<\x.dog(x)>] dog)))
+ (VP[NUM='sg', SEM=<\x.bark(x)>]
+ (IV[NUM='sg', SEM=<\x.bark(x)>, TNS='pres'] barks)))
+ <BLANKLINE>
+ all x.(dog(x) -> bark(x))
+ <BLANKLINE>
+ True
+
+ >>> sents = ['Mary walks', 'John sees a dog']
+ >>> results = nltk.sem.util.interpret_sents(sents, 'grammars/sample_grammars/sem2.fcfg')
+ >>> for result in results:
+ ... for (synrep, semrep) in result:
+ ... print(synrep)
+ (S[SEM=<walk(mary)>]
+ (NP[-LOC, NUM='sg', SEM=<\P.P(mary)>]
+ (PropN[-LOC, NUM='sg', SEM=<\P.P(mary)>] Mary))
+ (VP[NUM='sg', SEM=<\x.walk(x)>]
+ (IV[NUM='sg', SEM=<\x.walk(x)>, TNS='pres'] walks)))
+ (S[SEM=<exists x.(dog(x) & see(john,x))>]
+ (NP[-LOC, NUM='sg', SEM=<\P.P(john)>]
+ (PropN[-LOC, NUM='sg', SEM=<\P.P(john)>] John))
+ (VP[NUM='sg', SEM=<\y.exists x.(dog(x) & see(y,x))>]
+ (TV[NUM='sg', SEM=<\X y.X(\x.see(y,x))>, TNS='pres'] sees)
+ (NP[NUM='sg', SEM=<\Q.exists x.(dog(x) & Q(x))>]
+ (Det[NUM='sg', SEM=<\P Q.exists x.(P(x) & Q(x))>] a)
+ (Nom[NUM='sg', SEM=<\x.dog(x)>]
+ (N[NUM='sg', SEM=<\x.dog(x)>] dog)))))
+
+Cooper Storage
+--------------
+
+ >>> from nltk.sem import cooper_storage as cs
+ >>> sentence = 'every girl chases a dog'
+ >>> trees = cs.parse_with_bindops(sentence, grammar='grammars/book_grammars/storage.fcfg')
+ >>> semrep = trees[0].label()['SEM']
+ >>> cs_semrep = cs.CooperStore(semrep)
+ >>> print(cs_semrep.core)
+ chase(z2,z4)
+ >>> for bo in cs_semrep.store:
+ ... print(bo)
+ bo(\P.all x.(girl(x) -> P(x)),z2)
+ bo(\P.exists x.(dog(x) & P(x)),z4)
+ >>> cs_semrep.s_retrieve(trace=True)
+ Permutation 1
+ (\P.all x.(girl(x) -> P(x)))(\z2.chase(z2,z4))
+ (\P.exists x.(dog(x) & P(x)))(\z4.all x.(girl(x) -> chase(x,z4)))
+ Permutation 2
+ (\P.exists x.(dog(x) & P(x)))(\z4.chase(z2,z4))
+ (\P.all x.(girl(x) -> P(x)))(\z2.exists x.(dog(x) & chase(z2,x)))
+
+ >>> for reading in cs_semrep.readings:
+ ... print(reading)
+ exists x.(dog(x) & all z3.(girl(z3) -> chase(z3,x)))
+ all x.(girl(x) -> exists z4.(dog(z4) & chase(x,z4)))
+
+
diff --git a/nltk/test/semantics_fixt.py b/nltk/test/semantics_fixt.py
new file mode 100644
index 0000000..8d96da7
--- /dev/null
+++ b/nltk/test/semantics_fixt.py
@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+# reset the variables counter before running tests
+def setup_module(module):
+ from nltk.sem import logic
+ logic._counter._value = 0
diff --git a/nltk/test/sentiwordnet.doctest b/nltk/test/sentiwordnet.doctest
new file mode 100644
index 0000000..8e49b8d
--- /dev/null
+++ b/nltk/test/sentiwordnet.doctest
@@ -0,0 +1,39 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+======================
+SentiWordNet Interface
+======================
+
+SentiWordNet can be imported like this:
+
+ >>> from nltk.corpus import sentiwordnet as swn
+
+------------
+SentiSynsets
+------------
+
+ >>> breakdown = swn.senti_synset('breakdown.n.03')
+ >>> print(breakdown)
+ <breakdown.n.03: PosScore=0.0 NegScore=0.25>
+ >>> breakdown.pos_score()
+ 0.0
+ >>> breakdown.neg_score()
+ 0.25
+ >>> breakdown.obj_score()
+ 0.75
+
+
+------
+Lookup
+------
+
+ >>> list(swn.senti_synsets('slow')) # doctest: +NORMALIZE_WHITESPACE
+ [SentiSynset('decelerate.v.01'), SentiSynset('slow.v.02'),
+ SentiSynset('slow.v.03'), SentiSynset('slow.a.01'),
+ SentiSynset('slow.a.02'), SentiSynset('slow.a.04'),
+ SentiSynset('slowly.r.01'), SentiSynset('behind.r.03')]
+
+ >>> happy = swn.senti_synsets('happy', 'a')
+
+ >>> all = swn.all_senti_synsets()
diff --git a/nltk/test/simple.doctest b/nltk/test/simple.doctest
new file mode 100644
index 0000000..71e8c40
--- /dev/null
+++ b/nltk/test/simple.doctest
@@ -0,0 +1,85 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=================
+EasyInstall Tests
+=================
+
+This file contains some simple tests that will be run by EasyInstall in
+order to test the installation when NLTK-Data is absent.
+
+ >>> from __future__ import print_function
+
+------------
+Tokenization
+------------
+
+ >>> from nltk.tokenize import wordpunct_tokenize
+ >>> s = ("Good muffins cost $3.88\nin New York. Please buy me\n"
+ ... "two of them.\n\nThanks.")
+ >>> wordpunct_tokenize(s) # doctest: +NORMALIZE_WHITESPACE
+ ['Good', 'muffins', 'cost', '$', '3', '.', '88', 'in', 'New', 'York', '.',
+ 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+
+-------
+Metrics
+-------
+
+ >>> from nltk.metrics import precision, recall, f_measure
+ >>> reference = 'DET NN VB DET JJ NN NN IN DET NN'.split()
+ >>> test = 'DET VB VB DET NN NN NN IN DET NN'.split()
+ >>> reference_set = set(reference)
+ >>> test_set = set(test)
+ >>> precision(reference_set, test_set)
+ 1.0
+ >>> print(recall(reference_set, test_set))
+ 0.8
+ >>> print(f_measure(reference_set, test_set))
+ 0.88888888888...
+
+------------------
+Feature Structures
+------------------
+
+ >>> from nltk import FeatStruct
+ >>> fs1 = FeatStruct(PER=3, NUM='pl', GND='fem')
+ >>> fs2 = FeatStruct(POS='N', AGR=fs1)
+ >>> print(fs2)
+ [ [ GND = 'fem' ] ]
+ [ AGR = [ NUM = 'pl' ] ]
+ [ [ PER = 3 ] ]
+ [ ]
+ [ POS = 'N' ]
+ >>> print(fs2['AGR'])
+ [ GND = 'fem' ]
+ [ NUM = 'pl' ]
+ [ PER = 3 ]
+ >>> print(fs2['AGR']['PER'])
+ 3
+
+-------
+Parsing
+-------
+
+ >>> from nltk.parse.recursivedescent import RecursiveDescentParser
+ >>> from nltk.grammar import CFG
+ >>> grammar = CFG.fromstring("""
+ ... S -> NP VP
+ ... PP -> P NP
+ ... NP -> 'the' N | N PP | 'the' N PP
+ ... VP -> V NP | V PP | V NP PP
+ ... N -> 'cat' | 'dog' | 'rug'
+ ... V -> 'chased'
+ ... P -> 'on'
+ ... """)
+ >>> rd = RecursiveDescentParser(grammar)
+ >>> sent = 'the cat chased the dog on the rug'.split()
+ >>> for t in rd.parse(sent):
+ ... print(t)
+ (S
+ (NP the (N cat))
+ (VP (V chased) (NP the (N dog) (PP (P on) (NP the (N rug))))))
+ (S
+ (NP the (N cat))
+ (VP (V chased) (NP the (N dog)) (PP (P on) (NP the (N rug)))))
+
diff --git a/nltk/test/stem.doctest b/nltk/test/stem.doctest
new file mode 100644
index 0000000..fdca6dc
--- /dev/null
+++ b/nltk/test/stem.doctest
@@ -0,0 +1,81 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+==========
+ Stemmers
+==========
+
+Overview
+~~~~~~~~
+
+Stemmers remove morphological affixes from words, leaving only the
+word stem.
+
+ >>> from __future__ import print_function
+ >>> from nltk.stem import *
+
+Unit tests for the Porter stemmer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ >>> from nltk.stem.porter import *
+
+Create a new Porter stemmer.
+
+ >>> stemmer = PorterStemmer()
+
+Test the stemmer on various pluralised words.
+
+ >>> plurals = ['caresses', 'flies', 'dies', 'mules', 'denied',
+ ... 'died', 'agreed', 'owned', 'humbled', 'sized',
+ ... 'meeting', 'stating', 'siezing', 'itemization',
+ ... 'sensational', 'traditional', 'reference', 'colonizer',
+ ... 'plotted']
+ >>> singles = []
+
+ >>> for plural in plurals:
+ ... singles.append(stemmer.stem(plural))
+
+ >>> singles # doctest: +NORMALIZE_WHITESPACE
+ [u'caress', u'fli', u'die', u'mule', u'deni', u'die', u'agre', u'own',
+ u'humbl', u'size', u'meet', u'state', u'siez', u'item', u'sensat',
+ u'tradit', u'refer', u'colon', u'plot']
+
+
+Unit tests for Snowball stemmer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+ >>> from nltk.stem.snowball import SnowballStemmer
+
+See which languages are supported.
+
+ >>> print(" ".join(SnowballStemmer.languages))
+ danish dutch english finnish french german hungarian italian
+ norwegian porter portuguese romanian russian spanish swedish
+
+Create a new instance of a language specific subclass.
+
+ >>> stemmer = SnowballStemmer("english")
+
+Stem a word.
+
+ >>> print(stemmer.stem("running"))
+ run
+
+Decide not to stem stopwords.
+
+ >>> stemmer2 = SnowballStemmer("english", ignore_stopwords=True)
+ >>> print(stemmer.stem("having"))
+ have
+ >>> print(stemmer2.stem("having"))
+ having
+
+The 'english' stemmer is better than the original 'porter' stemmer.
+
+ >>> print(SnowballStemmer("english").stem("generously"))
+ generous
+ >>> print(SnowballStemmer("porter").stem("generously"))
+ gener
+
+.. note::
+
+ Extra stemmer tests can be found in `nltk.test.unit.test_stem`.
diff --git a/nltk/test/tag.doctest b/nltk/test/tag.doctest
new file mode 100644
index 0000000..b365c3c
--- /dev/null
+++ b/nltk/test/tag.doctest
@@ -0,0 +1,22 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+Regression Tests
+~~~~~~~~~~~~~~~~
+
+Sequential Taggers
+------------------
+
+Add tests for:
+ - make sure backoff is being done correctly.
+ - make sure ngram taggers don't use previous sentences for context.
+ - make sure ngram taggers see 'beginning of the sentence' as a
+ unique context
+ - make sure regexp tagger's regexps are tried in order
+ - train on some simple examples, & make sure that the size & the
+ generated models are correct.
+ - make sure cutoff works as intended
+ - make sure that ngram models only exclude contexts covered by the
+ backoff tagger if the backoff tagger gets that context correct at
+ *all* locations.
+
diff --git a/nltk/test/tokenize.doctest b/nltk/test/tokenize.doctest
new file mode 100644
index 0000000..c554e5f
--- /dev/null
+++ b/nltk/test/tokenize.doctest
@@ -0,0 +1,102 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+ >>> from __future__ import print_function
+ >>> from nltk.tokenize import *
+
+Regression Tests: Treebank Tokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some test strings.
+
+ >>> s1 = "On a $50,000 mortgage of 30 years at 8 percent, the monthly payment would be $366.88."
+ >>> word_tokenize(s1)
+ ['On', 'a', '$', '50,000', 'mortgage', 'of', '30', 'years', 'at', '8', 'percent', ',', 'the', 'monthly', 'payment', 'would', 'be', '$', '366.88', '.']
+ >>> s2 = "\"We beat some pretty good teams to get here,\" Slocum said."
+ >>> word_tokenize(s2)
+ ['``', 'We', 'beat', 'some', 'pretty', 'good', 'teams', 'to', 'get', 'here', ',', "''", 'Slocum', 'said', '.']
+ >>> s3 = "Well, we couldn't have this predictable, cliche-ridden, \"Touched by an Angel\" (a show creator John Masius worked on) wanna-be if she didn't."
+ >>> word_tokenize(s3)
+ ['Well', ',', 'we', 'could', "n't", 'have', 'this', 'predictable', ',', 'cliche-ridden', ',', '``', 'Touched', 'by', 'an', 'Angel', "''", '(', 'a', 'show', 'creator', 'John', 'Masius', 'worked', 'on', ')', 'wanna-be', 'if', 'she', 'did', "n't", '.']
+ >>> s4 = "I cannot cannot work under these conditions!"
+ >>> word_tokenize(s4)
+ ['I', 'can', 'not', 'can', 'not', 'work', 'under', 'these', 'conditions', '!']
+ >>> s5 = "The company spent $30,000,000 last year."
+ >>> word_tokenize(s5)
+ ['The', 'company', 'spent', '$', '30,000,000', 'last', 'year', '.']
+ >>> s6 = "The company spent 40.75% of its income last year."
+ >>> word_tokenize(s6)
+ ['The', 'company', 'spent', '40.75', '%', 'of', 'its', 'income', 'last', 'year', '.']
+ >>> s7 = "He arrived at 3:00 pm."
+ >>> word_tokenize(s7)
+ ['He', 'arrived', 'at', '3:00', 'pm', '.']
+ >>> s8 = "I bought these items: books, pencils, and pens."
+ >>> word_tokenize(s8)
+ ['I', 'bought', 'these', 'items', ':', 'books', ',', 'pencils', ',', 'and', 'pens', '.']
+ >>> s9 = "Though there were 150, 100 of them were old."
+ >>> word_tokenize(s9)
+ ['Though', 'there', 'were', '150', ',', '100', 'of', 'them', 'were', 'old', '.']
+ >>> s10 = "There were 300,000, but that wasn't enough."
+ >>> word_tokenize(s10)
+ ['There', 'were', '300,000', ',', 'but', 'that', 'was', "n't", 'enough', '.']
+
+Regression Tests: Regexp Tokenizer
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some additional test strings.
+
+ >>> s = ("Good muffins cost $3.88\nin New York. Please buy me\n"
+ ... "two of them.\n\nThanks.")
+ >>> s2 = ("Alas, it has not rained today. When, do you think, "
+ ... "will it rain again?")
+ >>> s3 = ("<p>Although this is <b>not</b> the case here, we must "
+ ... "not relax our vigilance!</p>")
+
+ >>> regexp_tokenize(s2, r'[,\.\?!"]\s*', gaps=False)
+ [', ', '. ', ', ', ', ', '?']
+ >>> regexp_tokenize(s2, r'[,\.\?!"]\s*', gaps=True)
+ ['Alas', 'it has not rained today', 'When', 'do you think',
+ 'will it rain again']
+
+Make sure that grouping parentheses don't confuse the tokenizer:
+
+ >>> regexp_tokenize(s3, r'</?(b|p)>', gaps=False)
+ ['<p>', '<b>', '</b>', '</p>']
+ >>> regexp_tokenize(s3, r'</?(b|p)>', gaps=True)
+ ['Although this is ', 'not',
+ ' the case here, we must not relax our vigilance!']
+
+Make sure that named groups don't confuse the tokenizer:
+
+ >>> regexp_tokenize(s3, r'</?(?P<named>b|p)>', gaps=False)
+ ['<p>', '<b>', '</b>', '</p>']
+ >>> regexp_tokenize(s3, r'</?(?P<named>b|p)>', gaps=True)
+ ['Although this is ', 'not',
+ ' the case here, we must not relax our vigilance!']
+
+Make sure that nested groups don't confuse the tokenizer:
+
+ >>> regexp_tokenize(s2, r'(h|r|l)a(s|(i|n0))', gaps=False)
+ ['las', 'has', 'rai', 'rai']
+ >>> regexp_tokenize(s2, r'(h|r|l)a(s|(i|n0))', gaps=True)
+ ['A', ', it ', ' not ', 'ned today. When, do you think, will it ',
+ 'n again?']
+
+The tokenizer should reject any patterns with backreferences:
+
+ >>> regexp_tokenize(s2, r'(.)\1')
+ Traceback (most recent call last):
+ ...
+ ValueError: Regular expressions with back-references are
+ not supported: '(.)\\1'
+ >>> regexp_tokenize(s2, r'(?P<foo>)(?P=foo)')
+ Traceback (most recent call last):
+ ...
+ ValueError: Regular expressions with back-references are
+ not supported: '(?P<foo>)(?P=foo)'
+
+A simple sentence tokenizer '\.(\s+|$)'
+
+ >>> regexp_tokenize(s, pattern=r'\.(\s+|$)', gaps=True)
+ ['Good muffins cost $3.88\nin New York',
+ 'Please buy me\ntwo of them', 'Thanks']
diff --git a/nltk/test/toolbox.doctest b/nltk/test/toolbox.doctest
new file mode 100644
index 0000000..4cf27f5
--- /dev/null
+++ b/nltk/test/toolbox.doctest
@@ -0,0 +1,307 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+===============================
+Unit test cases for ``toolbox``
+===============================
+
+ >>> from nltk import toolbox
+
+--------------------------
+``toolbox.StandardFormat``
+--------------------------
+
+ >>> f = toolbox.StandardFormat()
+
+``toolbox.StandardFormat.open()``
+---------------------------------
+ >>> import os, tempfile
+ >>> (fd, fname) = tempfile.mkstemp()
+ >>> tf = os.fdopen(fd, "w")
+ >>> _ = tf.write('\\lx a value\n\\lx another value\n')
+ >>> tf.close()
+ >>> f = toolbox.StandardFormat()
+ >>> f.open(fname)
+ >>> list(f.fields())
+ [('lx', 'a value'), ('lx', 'another value')]
+ >>> f.close()
+ >>> os.unlink(fname)
+
+``toolbox.StandardFormat.open_string()``
+----------------------------------------
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\n\\lx another value\n')
+ >>> list(f.fields())
+ [('lx', 'a value'), ('lx', 'another value')]
+ >>> f.close()
+
+``toolbox.StandardFormat.close()``
+----------------------------------
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\n\\lx another value\n')
+ >>> list(f.fields())
+ [('lx', 'a value'), ('lx', 'another value')]
+ >>> f.close()
+
+``toolbox.StandardFormat.line_num``
+---------------------------------------
+
+``StandardFormat.line_num`` contains the line number of the last line returned:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\n\\lx another value\n\\lx a third value\n')
+ >>> line_nums = []
+ >>> for l in f.raw_fields():
+ ... line_nums.append(f.line_num)
+ >>> line_nums
+ [1, 2, 3]
+
+``StandardFormat.line_num`` contains the line number of the last line returned:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx two\nlines\n\\lx three\nlines\n\n\\lx two\nlines\n')
+ >>> line_nums = []
+ >>> for l in f.raw_fields():
+ ... line_nums.append(f.line_num)
+ >>> line_nums
+ [2, 5, 7]
+
+``StandardFormat.line_num`` doesn't exist before openning or after closing
+a file or string:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.line_num
+ Traceback (most recent call last):
+ ...
+ AttributeError: 'StandardFormat' object has no attribute 'line_num'
+ >>> f.open_string('\\lx two\nlines\n\\lx three\nlines\n\n\\lx two\nlines\n')
+ >>> line_nums = []
+ >>> for l in f.raw_fields():
+ ... line_nums.append(f.line_num)
+ >>> line_nums
+ [2, 5, 7]
+ >>> f.close()
+ >>> f.line_num
+ Traceback (most recent call last):
+ ...
+ AttributeError: 'StandardFormat' object has no attribute 'line_num'
+
+``toolbox.StandardFormat.raw_fields()``
+---------------------------------------
+``raw_fields()`` returns an iterator over tuples of two strings representing the
+marker and its value. The marker is given without the backslash and the value
+without its trailing newline:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\n\\lx another value\n')
+ >>> list(f.raw_fields())
+ [('lx', 'a value'), ('lx', 'another value')]
+
+an empty file returns nothing:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('')
+ >>> list(f.raw_fields())
+ []
+
+file with only a newline returns WHAT SHOULD IT RETURN???:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\n')
+ >>> list(f.raw_fields())
+ [(None, '')]
+
+file with only one field should be parsed ok:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx one value\n')
+ >>> list(f.raw_fields())
+ [('lx', 'one value')]
+
+file without a trailing newline should be parsed ok:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\n\\lx another value')
+ >>> list(f.raw_fields())
+ [('lx', 'a value'), ('lx', 'another value')]
+
+trailing white space is preserved except for the final newline:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx trailing space \n\\lx trailing tab\t\n\\lx extra newline\n\n')
+ >>> list(f.raw_fields())
+ [('lx', 'trailing space '), ('lx', 'trailing tab\t'), ('lx', 'extra newline\n')]
+
+line wrapping is preserved:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\nmore of the value\nand still more\n\\lc another val\n')
+ >>> list(f.raw_fields())
+ [('lx', 'a value\nmore of the value\nand still more'), ('lc', 'another val')]
+
+file beginning with a multiline record should be parsed ok:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\nmore of the value\nand still more\n\\lc another val\n')
+ >>> list(f.raw_fields())
+ [('lx', 'a value\nmore of the value\nand still more'), ('lc', 'another val')]
+
+file ending with a multiline record should be parsed ok:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lc a value\n\\lx another value\nmore of the value\nand still more\n')
+ >>> list(f.raw_fields())
+ [('lc', 'a value'), ('lx', 'another value\nmore of the value\nand still more')]
+
+file beginning with a BOM should be parsed ok:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\xef\xbb\xbf\\lx a value\n\\lx another value\n')
+ >>> list(f.raw_fields())
+ [('lx', 'a value'), ('lx', 'another value')]
+
+file beginning with two BOMs should ignore only the first one:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\xef\xbb\xbf\xef\xbb\xbf\\lx a value\n\\lx another value\n')
+ >>> list(f.raw_fields())
+ [(None, '\xef\xbb\xbf\\lx a value'), ('lx', 'another value')]
+
+should not ignore a BOM not at the beginning of the file:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\n\xef\xbb\xbf\\lx another value\n')
+ >>> list(f.raw_fields())
+ [('lx', 'a value\n\xef\xbb\xbf\\lx another value')]
+
+``toolbox.StandardFormat.fields()``
+-----------------------------------
+trailing white space is not preserved:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx trailing space \n\\lx trailing tab\t\n\\lx extra newline\n\n')
+ >>> list(f.fields())
+ [('lx', 'trailing space'), ('lx', 'trailing tab'), ('lx', 'extra newline')]
+
+multiline fields are unwrapped:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\lx a value\nmore of the value\nand still more\n\\lc another val\n')
+ >>> list(f.fields())
+ [('lx', 'a value more of the value and still more'), ('lc', 'another val')]
+
+markers
+-------
+A backslash in the first position on a new line indicates the start of a
+marker. The backslash is not part of the marker:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\mk a value\n')
+ >>> list(f.fields())
+ [('mk', 'a value')]
+
+If the backslash occurs later in the line it does not indicate the start
+of a marker:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\mk a value\n \\mk another one\n')
+ >>> list(f.raw_fields())
+ [('mk', 'a value\n \\mk another one')]
+
+There is no specific limit to the length of a marker:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\this_is_an_extremely_long_marker value\n')
+ >>> list(f.fields())
+ [('this_is_an_extremely_long_marker', 'value')]
+
+A marker can contain any non white space character:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\`~!@#$%^&*()_-=+[{]}\|,<.>/?;:"0123456789 value\n')
+ >>> list(f.fields())
+ [('`~!@#$%^&*()_-=+[{]}\\|,<.>/?;:"0123456789', 'value')]
+
+A marker is terminated by any white space character:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\mk a value\n\\mk\tanother one\n\\mk\rthird one\n\\mk\ffourth one')
+ >>> list(f.fields())
+ [('mk', 'a value'), ('mk', 'another one'), ('mk', 'third one'), ('mk', 'fourth one')]
+
+Consecutive whitespace characters (except newline) are treated the same as one:
+
+ >>> f = toolbox.StandardFormat()
+ >>> f.open_string('\\mk \t\r\fa value\n')
+ >>> list(f.fields())
+ [('mk', 'a value')]
+
+-----------------------
+``toolbox.ToolboxData``
+-----------------------
+
+ >>> db = toolbox.ToolboxData()
+
+``toolbox.ToolboxData.parse()``
+-------------------------------
+check that normal parsing works:
+
+ >>> from xml.etree import ElementTree
+ >>> td = toolbox.ToolboxData()
+ >>> s = """\\_sh v3.0 400 Rotokas Dictionary
+ ... \\_DateStampHasFourDigitYear
+ ...
+ ... \\lx kaa
+ ... \\ps V.A
+ ... \\ge gag
+ ... \\gp nek i pas
+ ...
+ ... \\lx kaa
+ ... \\ps V.B
+ ... \\ge strangle
+ ... \\gp pasim nek
+ ... """
+ >>> td.open_string(s)
+ >>> tree = td.parse(key='lx')
+ >>> tree.tag
+ 'toolbox_data'
+ >>> ElementTree.tostring(list(tree)[0]).decode('utf8')
+ '<header><_sh>v3.0 400 Rotokas Dictionary</_sh><_DateStampHasFourDigitYear /></header>'
+ >>> ElementTree.tostring(list(tree)[1]).decode('utf8')
+ '<record><lx>kaa</lx><ps>V.A</ps><ge>gag</ge><gp>nek i pas</gp></record>'
+ >>> ElementTree.tostring(list(tree)[2]).decode('utf8')
+ '<record><lx>kaa</lx><ps>V.B</ps><ge>strangle</ge><gp>pasim nek</gp></record>'
+
+check that guessing the key marker works:
+
+ >>> from xml.etree import ElementTree
+ >>> td = toolbox.ToolboxData()
+ >>> s = """\\_sh v3.0 400 Rotokas Dictionary
+ ... \\_DateStampHasFourDigitYear
+ ...
+ ... \\lx kaa
+ ... \\ps V.A
+ ... \\ge gag
+ ... \\gp nek i pas
+ ...
+ ... \\lx kaa
+ ... \\ps V.B
+ ... \\ge strangle
+ ... \\gp pasim nek
+ ... """
+ >>> td.open_string(s)
+ >>> tree = td.parse()
+ >>> ElementTree.tostring(list(tree)[0]).decode('utf8')
+ '<header><_sh>v3.0 400 Rotokas Dictionary</_sh><_DateStampHasFourDigitYear /></header>'
+ >>> ElementTree.tostring(list(tree)[1]).decode('utf8')
+ '<record><lx>kaa</lx><ps>V.A</ps><ge>gag</ge><gp>nek i pas</gp></record>'
+ >>> ElementTree.tostring(list(tree)[2]).decode('utf8')
+ '<record><lx>kaa</lx><ps>V.B</ps><ge>strangle</ge><gp>pasim nek</gp></record>'
+
+-----------------------
+``toolbox`` functions
+-----------------------
+
+``toolbox.to_sfm_string()``
+-------------------------------
+
diff --git a/nltk/test/toy.cfg b/nltk/test/toy.cfg
new file mode 100644
index 0000000..0977292
--- /dev/null
+++ b/nltk/test/toy.cfg
@@ -0,0 +1,9 @@
+S -> NP VP
+PP -> P NP
+NP -> Det N | NP PP
+VP -> V NP | VP PP
+Det -> 'a' | 'the'
+N -> 'dog' | 'cat'
+V -> 'chased' | 'sat'
+P -> 'on' | 'in'
+
diff --git a/nltk/test/tree.doctest b/nltk/test/tree.doctest
new file mode 100644
index 0000000..bcfbffb
--- /dev/null
+++ b/nltk/test/tree.doctest
@@ -0,0 +1,1077 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+===============================
+ Unit tests for nltk.tree.Tree
+===============================
+
+ >>> from nltk.tree import *
+
+Some trees to run tests on:
+
+ >>> dp1 = Tree('dp', [Tree('d', ['the']), Tree('np', ['dog'])])
+ >>> dp2 = Tree('dp', [Tree('d', ['the']), Tree('np', ['cat'])])
+ >>> vp = Tree('vp', [Tree('v', ['chased']), dp2])
+ >>> tree = Tree('s', [dp1, vp])
+ >>> print(tree)
+ (s (dp (d the) (np dog)) (vp (v chased) (dp (d the) (np cat))))
+
+The node label is accessed using the `label()` method:
+
+ >>> dp1.label(), dp2.label(), vp.label(), tree.label()
+ ('dp', 'dp', 'vp', 's')
+
+ >>> print(tree[1,1,1,0])
+ cat
+
+The `treepositions` method returns a list of the tree positions of
+subtrees and leaves in a tree. By default, it gives the position of
+every tree, subtree, and leaf, in prefix order:
+
+ >>> print(tree.treepositions())
+ [(), (0,), (0, 0), (0, 0, 0), (0, 1), (0, 1, 0), (1,), (1, 0), (1, 0, 0), (1, 1), (1, 1, 0), (1, 1, 0, 0), (1, 1, 1), (1, 1, 1, 0)]
+
+In addition to `str` and `repr`, several methods exist to convert a
+tree object to one of several standard tree encodings:
+
+ >>> print(tree.pprint_latex_qtree())
+ \Tree [.s
+ [.dp [.d the ] [.np dog ] ]
+ [.vp [.v chased ] [.dp [.d the ] [.np cat ] ] ] ]
+
+Trees can be initialized from treebank strings:
+
+ >>> tree2 = Tree.fromstring('(S (NP I) (VP (V enjoyed) (NP my cookie)))')
+ >>> print(tree2)
+ (S (NP I) (VP (V enjoyed) (NP my cookie)))
+
+Trees can be compared for equality:
+
+ >>> tree == Tree.fromstring(str(tree))
+ True
+ >>> tree2 == Tree.fromstring(str(tree2))
+ True
+ >>> tree == tree2
+ False
+ >>> tree == Tree.fromstring(str(tree2))
+ False
+ >>> tree2 == Tree.fromstring(str(tree))
+ False
+
+ >>> tree != Tree.fromstring(str(tree))
+ False
+ >>> tree2 != Tree.fromstring(str(tree2))
+ False
+ >>> tree != tree2
+ True
+ >>> tree != Tree.fromstring(str(tree2))
+ True
+ >>> tree2 != Tree.fromstring(str(tree))
+ True
+
+ >>> tree < tree2 or tree > tree2
+ True
+
+Tree Parsing
+============
+
+The class method `Tree.fromstring()` can be used to parse trees, and it
+provides some additional options.
+
+ >>> tree = Tree.fromstring('(S (NP I) (VP (V enjoyed) (NP my cookie)))')
+ >>> print(tree)
+ (S (NP I) (VP (V enjoyed) (NP my cookie)))
+
+When called on a subclass of `Tree`, it will create trees of that
+type:
+
+ >>> tree = ImmutableTree.fromstring('(VP (V enjoyed) (NP my cookie))')
+ >>> print(tree)
+ (VP (V enjoyed) (NP my cookie))
+ >>> print(type(tree))
+ <class 'nltk.tree.ImmutableTree'>
+ >>> tree[1] = 'x'
+ Traceback (most recent call last):
+ . . .
+ ValueError: ImmutableTree may not be modified
+ >>> del tree[0]
+ Traceback (most recent call last):
+ . . .
+ ValueError: ImmutableTree may not be modified
+
+The ``brackets`` parameter can be used to specify two characters that
+should be used as brackets:
+
+ >>> print(Tree.fromstring('[S [NP I] [VP [V enjoyed] [NP my cookie]]]',
+ ... brackets='[]'))
+ (S (NP I) (VP (V enjoyed) (NP my cookie)))
+ >>> print(Tree.fromstring('<S <NP I> <VP <V enjoyed> <NP my cookie>>>',
+ ... brackets='<>'))
+ (S (NP I) (VP (V enjoyed) (NP my cookie)))
+
+If ``brackets`` is not a string, or is not exactly two characters,
+then `Tree.fromstring` raises an exception:
+
+ >>> Tree.fromstring('<VP <V enjoyed> <NP my cookie>>', brackets='')
+ Traceback (most recent call last):
+ . . .
+ TypeError: brackets must be a length-2 string
+ >>> Tree.fromstring('<VP <V enjoyed> <NP my cookie>>', brackets='<<>>')
+ Traceback (most recent call last):
+ . . .
+ TypeError: brackets must be a length-2 string
+ >>> Tree.fromstring('<VP <V enjoyed> <NP my cookie>>', brackets=12)
+ Traceback (most recent call last):
+ . . .
+ TypeError: brackets must be a length-2 string
+ >>> Tree.fromstring('<<NP my cookie>>', brackets=('<<','>>'))
+ Traceback (most recent call last):
+ . . .
+ TypeError: brackets must be a length-2 string
+
+(We may add support for multi-character brackets in the future, in
+which case the ``brackets=('<<','>>')`` example would start working.)
+
+Whitespace brackets are not permitted:
+
+ >>> Tree.fromstring('(NP my cookie\n', brackets='(\n')
+ Traceback (most recent call last):
+ . . .
+ TypeError: whitespace brackets not allowed
+
+If an invalid tree is given to Tree.fromstring, then it raises a
+ValueError, with a description of the problem:
+
+ >>> Tree.fromstring('(NP my cookie) (NP my milk)')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Tree.fromstring(): expected 'end-of-string' but got '(NP'
+ at index 15.
+ "...y cookie) (NP my mil..."
+ ^
+ >>> Tree.fromstring(')NP my cookie(')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Tree.fromstring(): expected '(' but got ')'
+ at index 0.
+ ")NP my coo..."
+ ^
+ >>> Tree.fromstring('(NP my cookie))')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Tree.fromstring(): expected 'end-of-string' but got ')'
+ at index 14.
+ "...my cookie))"
+ ^
+ >>> Tree.fromstring('my cookie)')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Tree.fromstring(): expected '(' but got 'my'
+ at index 0.
+ "my cookie)"
+ ^
+ >>> Tree.fromstring('(NP my cookie')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Tree.fromstring(): expected ')' but got 'end-of-string'
+ at index 13.
+ "... my cookie"
+ ^
+ >>> Tree.fromstring('')
+ Traceback (most recent call last):
+ . . .
+ ValueError: Tree.fromstring(): expected '(' but got 'end-of-string'
+ at index 0.
+ ""
+ ^
+
+Trees with no children are supported:
+
+ >>> print(Tree.fromstring('(S)'))
+ (S )
+ >>> print(Tree.fromstring('(X (Y) (Z))'))
+ (X (Y ) (Z ))
+
+Trees with an empty node label and no children are supported:
+
+ >>> print(Tree.fromstring('()'))
+ ( )
+ >>> print(Tree.fromstring('(X () ())'))
+ (X ( ) ( ))
+
+Trees with an empty node label and children are supported, but only if the
+first child is not a leaf (otherwise, it will be treated as the node label).
+
+ >>> print(Tree.fromstring('((A) (B) (C))'))
+ ( (A ) (B ) (C ))
+ >>> print(Tree.fromstring('((A) leaf)'))
+ ( (A ) leaf)
+ >>> print(Tree.fromstring('(((())))'))
+ ( ( ( ( ))))
+
+The optional arguments `read_node` and `read_leaf` may be used to
+transform the string values of nodes or leaves.
+
+ >>> print(Tree.fromstring('(A b (C d e) (F (G h i)))',
+ ... read_node=lambda s: '<%s>' % s,
+ ... read_leaf=lambda s: '"%s"' % s))
+ (<A> "b" (<C> "d" "e") (<F> (<G> "h" "i")))
+
+These transformation functions are typically used when the node or
+leaf labels should be parsed to a non-string value (such as a feature
+structure). If node and leaf labels need to be able to include
+whitespace, then you must also use the optional `node_pattern` and
+`leaf_pattern` arguments.
+
+ >>> from nltk.featstruct import FeatStruct
+ >>> tree = Tree.fromstring('([cat=NP] [lex=the] [lex=dog])',
+ ... read_node=FeatStruct, read_leaf=FeatStruct)
+ >>> tree.set_label(tree.label().unify(FeatStruct('[num=singular]')))
+ >>> print(tree)
+ ([cat='NP', num='singular'] [lex='the'] [lex='dog'])
+
+The optional argument ``remove_empty_top_bracketing`` can be used to
+remove any top-level empty bracketing that occurs.
+
+ >>> print(Tree.fromstring('((S (NP I) (VP (V enjoyed) (NP my cookie))))',
+ ... remove_empty_top_bracketing=True))
+ (S (NP I) (VP (V enjoyed) (NP my cookie)))
+
+It will not remove a top-level empty bracketing with multiple children:
+
+ >>> print(Tree.fromstring('((A a) (B b))'))
+ ( (A a) (B b))
+
+Parented Trees
+==============
+`ParentedTree` is a subclass of `Tree` that automatically maintains
+parent pointers for single-parented trees. Parented trees can be
+created directly from a node label and a list of children:
+
+ >>> ptree = (
+ ... ParentedTree('VP', [
+ ... ParentedTree('VERB', ['saw']),
+ ... ParentedTree('NP', [
+ ... ParentedTree('DET', ['the']),
+ ... ParentedTree('NOUN', ['dog'])])]))
+ >>> print(ptree)
+ (VP (VERB saw) (NP (DET the) (NOUN dog)))
+
+Parented trees can be created from strings using the classmethod
+`ParentedTree.fromstring`:
+
+ >>> ptree = ParentedTree.fromstring('(VP (VERB saw) (NP (DET the) (NOUN dog)))')
+ >>> print(ptree)
+ (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ >>> print(type(ptree))
+ <class 'nltk.tree.ParentedTree'>
+
+Parented trees can also be created by using the classmethod
+`ParentedTree.convert` to convert another type of tree to a parented
+tree:
+
+ >>> tree = Tree.fromstring('(VP (VERB saw) (NP (DET the) (NOUN dog)))')
+ >>> ptree = ParentedTree.convert(tree)
+ >>> print(ptree)
+ (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ >>> print(type(ptree))
+ <class 'nltk.tree.ParentedTree'>
+
+.. clean-up:
+
+ >>> del tree
+
+`ParentedTree`\ s should never be used in the same tree as `Tree`\ s
+or `MultiParentedTree`\ s. Mixing tree implementations may result in
+incorrect parent pointers and in `TypeError` exceptions:
+
+ >>> # Inserting a Tree in a ParentedTree gives an exception:
+ >>> ParentedTree('NP', [
+ ... Tree('DET', ['the']), Tree('NOUN', ['dog'])])
+ Traceback (most recent call last):
+ . . .
+ TypeError: Can not insert a non-ParentedTree into a ParentedTree
+
+ >>> # inserting a ParentedTree in a Tree gives incorrect parent pointers:
+ >>> broken_tree = Tree('NP', [
+ ... ParentedTree('DET', ['the']), ParentedTree('NOUN', ['dog'])])
+ >>> print(broken_tree[0].parent())
+ None
+
+Parented Tree Methods
+------------------------
+In addition to all the methods defined by the `Tree` class, the
+`ParentedTree` class adds six new methods whose values are
+automatically updated whenver a parented tree is modified: `parent()`,
+`parent_index()`, `left_sibling()`, `right_sibling()`, `root()`, and
+`treeposition()`.
+
+The `parent()` method contains a `ParentedTree`\ 's parent, if it has
+one; and ``None`` otherwise. `ParentedTree`\ s that do not have
+parents are known as "root trees."
+
+ >>> for subtree in ptree.subtrees():
+ ... print(subtree)
+ ... print(' Parent = %s' % subtree.parent())
+ (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ Parent = None
+ (VERB saw)
+ Parent = (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ (NP (DET the) (NOUN dog))
+ Parent = (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ (DET the)
+ Parent = (NP (DET the) (NOUN dog))
+ (NOUN dog)
+ Parent = (NP (DET the) (NOUN dog))
+
+The `parent_index()` method stores the index of a tree in its parent's
+child list. If a tree does not have a parent, then its `parent_index`
+is ``None``.
+
+ >>> for subtree in ptree.subtrees():
+ ... print(subtree)
+ ... print(' Parent Index = %s' % subtree.parent_index())
+ ... assert (subtree.parent() is None or
+ ... subtree.parent()[subtree.parent_index()] is subtree)
+ (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ Parent Index = None
+ (VERB saw)
+ Parent Index = 0
+ (NP (DET the) (NOUN dog))
+ Parent Index = 1
+ (DET the)
+ Parent Index = 0
+ (NOUN dog)
+ Parent Index = 1
+
+Note that ``ptree.parent().index(ptree)`` is *not* equivalent to
+``ptree.parent_index()``. In particular, ``ptree.parent().index(ptree)``
+will return the index of the first child of ``ptree.parent()`` that is
+equal to ``ptree`` (using ``==``); and that child may not be
+``ptree``:
+
+ >>> on_and_on = ParentedTree('CONJP', [
+ ... ParentedTree('PREP', ['on']),
+ ... ParentedTree('COJN', ['and']),
+ ... ParentedTree('PREP', ['on'])])
+ >>> second_on = on_and_on[2]
+ >>> print(second_on.parent_index())
+ 2
+ >>> print(second_on.parent().index(second_on))
+ 0
+
+The methods `left_sibling()` and `right_sibling()` can be used to get a
+parented tree's siblings. If a tree does not have a left or right
+sibling, then the corresponding method's value is ``None``:
+
+ >>> for subtree in ptree.subtrees():
+ ... print(subtree)
+ ... print(' Left Sibling = %s' % subtree.left_sibling())
+ ... print(' Right Sibling = %s' % subtree.right_sibling())
+ (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ Left Sibling = None
+ Right Sibling = None
+ (VERB saw)
+ Left Sibling = None
+ Right Sibling = (NP (DET the) (NOUN dog))
+ (NP (DET the) (NOUN dog))
+ Left Sibling = (VERB saw)
+ Right Sibling = None
+ (DET the)
+ Left Sibling = None
+ Right Sibling = (NOUN dog)
+ (NOUN dog)
+ Left Sibling = (DET the)
+ Right Sibling = None
+
+A parented tree's root tree can be accessed using the `root()`
+method. This method follows the tree's parent pointers until it
+finds a tree without a parent. If a tree does not have a parent, then
+it is its own root:
+
+ >>> for subtree in ptree.subtrees():
+ ... print(subtree)
+ ... print(' Root = %s' % subtree.root())
+ (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ Root = (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ (VERB saw)
+ Root = (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ (NP (DET the) (NOUN dog))
+ Root = (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ (DET the)
+ Root = (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ (NOUN dog)
+ Root = (VP (VERB saw) (NP (DET the) (NOUN dog)))
+
+The `treeposition()` method can be used to find a tree's treeposition
+relative to its root:
+
+ >>> for subtree in ptree.subtrees():
+ ... print(subtree)
+ ... print(' Tree Position = %s' % (subtree.treeposition(),))
+ ... assert subtree.root()[subtree.treeposition()] is subtree
+ (VP (VERB saw) (NP (DET the) (NOUN dog)))
+ Tree Position = ()
+ (VERB saw)
+ Tree Position = (0,)
+ (NP (DET the) (NOUN dog))
+ Tree Position = (1,)
+ (DET the)
+ Tree Position = (1, 0)
+ (NOUN dog)
+ Tree Position = (1, 1)
+
+Whenever a parented tree is modified, all of the methods described
+above (`parent()`, `parent_index()`, `left_sibling()`, `right_sibling()`,
+`root()`, and `treeposition()`) are automatically updated. For example,
+if we replace ``ptree``\ 's subtree for the word "dog" with a new
+subtree for "cat," the method values for both the "dog" subtree and the
+"cat" subtree get automatically updated:
+
+ >>> # Replace the dog with a cat
+ >>> dog = ptree[1,1]
+ >>> cat = ParentedTree('NOUN', ['cat'])
+ >>> ptree[1,1] = cat
+
+ >>> # the noun phrase is no longer the dog's parent:
+ >>> print(dog.parent(), dog.parent_index(), dog.left_sibling())
+ None None None
+ >>> # dog is now its own root.
+ >>> print(dog.root())
+ (NOUN dog)
+ >>> print(dog.treeposition())
+ ()
+
+ >>> # the cat's parent is now the noun phrase:
+ >>> print(cat.parent())
+ (NP (DET the) (NOUN cat))
+ >>> print(cat.parent_index())
+ 1
+ >>> print(cat.left_sibling())
+ (DET the)
+ >>> print(cat.root())
+ (VP (VERB saw) (NP (DET the) (NOUN cat)))
+ >>> print(cat.treeposition())
+ (1, 1)
+
+ParentedTree Regression Tests
+-----------------------------
+Keep track of all trees that we create (including subtrees) using this
+variable:
+
+ >>> all_ptrees = []
+
+Define a helper funciton to create new parented trees:
+
+ >>> def make_ptree(s):
+ ... ptree = ParentedTree.convert(Tree.fromstring(s))
+ ... all_ptrees.extend(t for t in ptree.subtrees()
+ ... if isinstance(t, Tree))
+ ... return ptree
+
+Define a test function that examines every subtree in all_ptrees; and
+checks that all six of its methods are defined correctly. If any
+ptrees are passed as arguments, then they are printed.
+
+ >>> def pcheck(*print_ptrees):
+ ... for ptree in all_ptrees:
+ ... # Check ptree's methods.
+ ... if ptree.parent() is not None:
+ ... i = ptree.parent_index()
+ ... assert ptree.parent()[i] is ptree
+ ... if i > 0:
+ ... assert ptree.left_sibling() is ptree.parent()[i-1]
+ ... if i < (len(ptree.parent())-1):
+ ... assert ptree.right_sibling() is ptree.parent()[i+1]
+ ... assert len(ptree.treeposition()) > 0
+ ... assert (ptree.treeposition() ==
+ ... ptree.parent().treeposition() + (ptree.parent_index(),))
+ ... assert ptree.root() is not ptree
+ ... assert ptree.root() is not None
+ ... assert ptree.root() is ptree.parent().root()
+ ... assert ptree.root()[ptree.treeposition()] is ptree
+ ... else:
+ ... assert ptree.parent_index() is None
+ ... assert ptree.left_sibling() is None
+ ... assert ptree.right_sibling() is None
+ ... assert ptree.root() is ptree
+ ... assert ptree.treeposition() == ()
+ ... # Check ptree's children's methods:
+ ... for i, child in enumerate(ptree):
+ ... if isinstance(child, Tree):
+ ... # pcheck parent() & parent_index() methods
+ ... assert child.parent() is ptree
+ ... assert child.parent_index() == i
+ ... # pcheck sibling methods
+ ... if i == 0:
+ ... assert child.left_sibling() is None
+ ... else:
+ ... assert child.left_sibling() is ptree[i-1]
+ ... if i == len(ptree)-1:
+ ... assert child.right_sibling() is None
+ ... else:
+ ... assert child.right_sibling() is ptree[i+1]
+ ... if print_ptrees:
+ ... print('ok!', end=' ')
+ ... for ptree in print_ptrees: print(ptree)
+ ... else:
+ ... print('ok!')
+
+Run our test function on a variety of newly-created trees:
+
+ >>> pcheck(make_ptree('(A)'))
+ ok! (A )
+ >>> pcheck(make_ptree('(A (B (C (D) (E f)) g) h)'))
+ ok! (A (B (C (D ) (E f)) g) h)
+ >>> pcheck(make_ptree('(A (B) (C c) (D d d) (E e e e))'))
+ ok! (A (B ) (C c) (D d d) (E e e e))
+ >>> pcheck(make_ptree('(A (B) (C (c)) (D (d) (d)) (E (e) (e) (e)))'))
+ ok! (A (B ) (C (c )) (D (d ) (d )) (E (e ) (e ) (e )))
+
+Run our test function after performing various tree-modification
+operations:
+
+**__delitem__()**
+
+ >>> ptree = make_ptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> e = ptree[0,0,1]
+ >>> del ptree[0,0,1]; pcheck(ptree); pcheck(e)
+ ok! (A (B (C (D ) (Q p)) g) h)
+ ok! (E f)
+ >>> del ptree[0,0,0]; pcheck(ptree)
+ ok! (A (B (C (Q p)) g) h)
+ >>> del ptree[0,1]; pcheck(ptree)
+ ok! (A (B (C (Q p))) h)
+ >>> del ptree[-1]; pcheck(ptree)
+ ok! (A (B (C (Q p))))
+ >>> del ptree[-100]
+ Traceback (most recent call last):
+ . . .
+ IndexError: index out of range
+ >>> del ptree[()]
+ Traceback (most recent call last):
+ . . .
+ IndexError: The tree position () may not be deleted.
+
+ >>> # With slices:
+ >>> ptree = make_ptree('(A (B c) (D e) f g (H i) j (K l))')
+ >>> b = ptree[0]
+ >>> del ptree[0:0]; pcheck(ptree)
+ ok! (A (B c) (D e) f g (H i) j (K l))
+ >>> del ptree[:1]; pcheck(ptree); pcheck(b)
+ ok! (A (D e) f g (H i) j (K l))
+ ok! (B c)
+ >>> del ptree[-2:]; pcheck(ptree)
+ ok! (A (D e) f g (H i))
+ >>> del ptree[1:3]; pcheck(ptree)
+ ok! (A (D e) (H i))
+ >>> ptree = make_ptree('(A (B c) (D e) f g (H i) j (K l))')
+ >>> del ptree[5:1000]; pcheck(ptree)
+ ok! (A (B c) (D e) f g (H i))
+ >>> del ptree[-2:1000]; pcheck(ptree)
+ ok! (A (B c) (D e) f)
+ >>> del ptree[-100:1]; pcheck(ptree)
+ ok! (A (D e) f)
+ >>> ptree = make_ptree('(A (B c) (D e) f g (H i) j (K l))')
+ >>> del ptree[1:-2:2]; pcheck(ptree)
+ ok! (A (B c) f (H i) j (K l))
+
+**__setitem__()**
+
+ >>> ptree = make_ptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> d, e, q = ptree[0,0]
+ >>> ptree[0,0,0] = 'x'; pcheck(ptree); pcheck(d)
+ ok! (A (B (C x (E f) (Q p)) g) h)
+ ok! (D )
+ >>> ptree[0,0,1] = make_ptree('(X (Y z))'); pcheck(ptree); pcheck(e)
+ ok! (A (B (C x (X (Y z)) (Q p)) g) h)
+ ok! (E f)
+ >>> ptree[1] = d; pcheck(ptree)
+ ok! (A (B (C x (X (Y z)) (Q p)) g) (D ))
+ >>> ptree[-1] = 'x'; pcheck(ptree)
+ ok! (A (B (C x (X (Y z)) (Q p)) g) x)
+ >>> ptree[-100] = 'y'
+ Traceback (most recent call last):
+ . . .
+ IndexError: index out of range
+ >>> ptree[()] = make_ptree('(X y)')
+ Traceback (most recent call last):
+ . . .
+ IndexError: The tree position () may not be assigned to.
+
+ >>> # With slices:
+ >>> ptree = make_ptree('(A (B c) (D e) f g (H i) j (K l))')
+ >>> b = ptree[0]
+ >>> ptree[0:0] = ('x', make_ptree('(Y)')); pcheck(ptree)
+ ok! (A x (Y ) (B c) (D e) f g (H i) j (K l))
+ >>> ptree[2:6] = (); pcheck(ptree); pcheck(b)
+ ok! (A x (Y ) (H i) j (K l))
+ ok! (B c)
+ >>> ptree[-2:] = ('z', 'p'); pcheck(ptree)
+ ok! (A x (Y ) (H i) z p)
+ >>> ptree[1:3] = [make_ptree('(X)') for x in range(10)]; pcheck(ptree)
+ ok! (A x (X ) (X ) (X ) (X ) (X ) (X ) (X ) (X ) (X ) (X ) z p)
+ >>> ptree[5:1000] = []; pcheck(ptree)
+ ok! (A x (X ) (X ) (X ) (X ))
+ >>> ptree[-2:1000] = ['n']; pcheck(ptree)
+ ok! (A x (X ) (X ) n)
+ >>> ptree[-100:1] = [make_ptree('(U v)')]; pcheck(ptree)
+ ok! (A (U v) (X ) (X ) n)
+ >>> ptree[-1:] = (make_ptree('(X)') for x in range(3)); pcheck(ptree)
+ ok! (A (U v) (X ) (X ) (X ) (X ) (X ))
+ >>> ptree[1:-2:2] = ['x', 'y']; pcheck(ptree)
+ ok! (A (U v) x (X ) y (X ) (X ))
+
+**append()**
+
+ >>> ptree = make_ptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> ptree.append('x'); pcheck(ptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x)
+ >>> ptree.append(make_ptree('(X (Y z))')); pcheck(ptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x (X (Y z)))
+
+**extend()**
+
+ >>> ptree = make_ptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> ptree.extend(['x', 'y', make_ptree('(X (Y z))')]); pcheck(ptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x y (X (Y z)))
+ >>> ptree.extend([]); pcheck(ptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x y (X (Y z)))
+ >>> ptree.extend(make_ptree('(X)') for x in range(3)); pcheck(ptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x y (X (Y z)) (X ) (X ) (X ))
+
+**insert()**
+
+ >>> ptree = make_ptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> ptree.insert(0, make_ptree('(X (Y z))')); pcheck(ptree)
+ ok! (A (X (Y z)) (B (C (D ) (E f) (Q p)) g) h)
+ >>> ptree.insert(-1, make_ptree('(X (Y z))')); pcheck(ptree)
+ ok! (A (X (Y z)) (B (C (D ) (E f) (Q p)) g) (X (Y z)) h)
+ >>> ptree.insert(-4, make_ptree('(X (Y z))')); pcheck(ptree)
+ ok! (A (X (Y z)) (X (Y z)) (B (C (D ) (E f) (Q p)) g) (X (Y z)) h)
+ >>> # Note: as with ``list``, inserting at a negative index that
+ >>> # gives a position before the start of the list does *not*
+ >>> # raise an IndexError exception; it just inserts at 0.
+ >>> ptree.insert(-400, make_ptree('(X (Y z))')); pcheck(ptree)
+ ok! (A
+ (X (Y z))
+ (X (Y z))
+ (X (Y z))
+ (B (C (D ) (E f) (Q p)) g)
+ (X (Y z))
+ h)
+
+**pop()**
+
+ >>> ptree = make_ptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> ptree[0,0].pop(1); pcheck(ptree)
+ ParentedTree('E', ['f'])
+ ok! (A (B (C (D ) (Q p)) g) h)
+ >>> ptree[0].pop(-1); pcheck(ptree)
+ 'g'
+ ok! (A (B (C (D ) (Q p))) h)
+ >>> ptree.pop(); pcheck(ptree)
+ 'h'
+ ok! (A (B (C (D ) (Q p))))
+ >>> ptree.pop(-100)
+ Traceback (most recent call last):
+ . . .
+ IndexError: index out of range
+
+**remove()**
+
+ >>> ptree = make_ptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> e = ptree[0,0,1]
+ >>> ptree[0,0].remove(ptree[0,0,1]); pcheck(ptree); pcheck(e)
+ ok! (A (B (C (D ) (Q p)) g) h)
+ ok! (E f)
+ >>> ptree[0,0].remove(make_ptree('(Q p)')); pcheck(ptree)
+ ok! (A (B (C (D )) g) h)
+ >>> ptree[0,0].remove(make_ptree('(Q p)'))
+ Traceback (most recent call last):
+ . . .
+ ValueError: ParentedTree('Q', ['p']) is not in list
+ >>> ptree.remove('h'); pcheck(ptree)
+ ok! (A (B (C (D )) g))
+ >>> ptree.remove('h');
+ Traceback (most recent call last):
+ . . .
+ ValueError: 'h' is not in list
+ >>> # remove() removes the first subtree that is equal (==) to the
+ >>> # given tree, which may not be the identical tree we give it:
+ >>> ptree = make_ptree('(A (X x) (Y y) (X x))')
+ >>> x1, y, x2 = ptree
+ >>> ptree.remove(ptree[-1]); pcheck(ptree)
+ ok! (A (Y y) (X x))
+ >>> print(x1.parent()); pcheck(x1)
+ None
+ ok! (X x)
+ >>> print(x2.parent())
+ (A (Y y) (X x))
+
+Test that a tree can not be given multiple parents:
+
+ >>> ptree = make_ptree('(A (X x) (Y y) (Z z))')
+ >>> ptree[0] = ptree[1]
+ Traceback (most recent call last):
+ . . .
+ ValueError: Can not insert a subtree that already has a parent.
+ >>> pcheck()
+ ok!
+
+[more to be written]
+
+
+ImmutableParentedTree Regression Tests
+--------------------------------------
+
+ >>> iptree = ImmutableParentedTree.convert(ptree)
+ >>> type(iptree)
+ <class 'nltk.tree.ImmutableParentedTree'>
+ >>> del iptree[0]
+ Traceback (most recent call last):
+ . . .
+ ValueError: ImmutableParentedTree may not be modified
+ >>> iptree.set_label('newnode')
+ Traceback (most recent call last):
+ . . .
+ ValueError: ImmutableParentedTree may not be modified
+
+
+MultiParentedTree Regression Tests
+----------------------------------
+Keep track of all trees that we create (including subtrees) using this
+variable:
+
+ >>> all_mptrees = []
+
+Define a helper funciton to create new parented trees:
+
+ >>> def make_mptree(s):
+ ... mptree = MultiParentedTree.convert(Tree.fromstring(s))
+ ... all_mptrees.extend(t for t in mptree.subtrees()
+ ... if isinstance(t, Tree))
+ ... return mptree
+
+Define a test function that examines every subtree in all_mptrees; and
+checks that all six of its methods are defined correctly. If any
+mptrees are passed as arguments, then they are printed.
+
+ >>> def mpcheck(*print_mptrees):
+ ... def has(seq, val): # uses identity comparison
+ ... for item in seq:
+ ... if item is val: return True
+ ... return False
+ ... for mptree in all_mptrees:
+ ... # Check mptree's methods.
+ ... if len(mptree.parents()) == 0:
+ ... assert len(mptree.left_siblings()) == 0
+ ... assert len(mptree.right_siblings()) == 0
+ ... assert len(mptree.roots()) == 1
+ ... assert mptree.roots()[0] is mptree
+ ... assert mptree.treepositions(mptree) == [()]
+ ... left_siblings = right_siblings = ()
+ ... roots = {id(mptree): 1}
+ ... else:
+ ... roots = dict((id(r), 0) for r in mptree.roots())
+ ... left_siblings = mptree.left_siblings()
+ ... right_siblings = mptree.right_siblings()
+ ... for parent in mptree.parents():
+ ... for i in mptree.parent_indices(parent):
+ ... assert parent[i] is mptree
+ ... # check left siblings
+ ... if i > 0:
+ ... for j in range(len(left_siblings)):
+ ... if left_siblings[j] is parent[i-1]:
+ ... del left_siblings[j]
+ ... break
+ ... else:
+ ... assert 0, 'sibling not found!'
+ ... # check ight siblings
+ ... if i < (len(parent)-1):
+ ... for j in range(len(right_siblings)):
+ ... if right_siblings[j] is parent[i+1]:
+ ... del right_siblings[j]
+ ... break
+ ... else:
+ ... assert 0, 'sibling not found!'
+ ... # check roots
+ ... for root in parent.roots():
+ ... assert id(root) in roots, 'missing root'
+ ... roots[id(root)] += 1
+ ... # check that we don't have any unexplained values
+ ... assert len(left_siblings)==0, 'unexpected sibling'
+ ... assert len(right_siblings)==0, 'unexpected sibling'
+ ... for v in roots.values(): assert v>0, roots #'unexpected root'
+ ... # check treepositions
+ ... for root in mptree.roots():
+ ... for treepos in mptree.treepositions(root):
+ ... assert root[treepos] is mptree
+ ... # Check mptree's children's methods:
+ ... for i, child in enumerate(mptree):
+ ... if isinstance(child, Tree):
+ ... # mpcheck parent() & parent_index() methods
+ ... assert has(child.parents(), mptree)
+ ... assert i in child.parent_indices(mptree)
+ ... # mpcheck sibling methods
+ ... if i > 0:
+ ... assert has(child.left_siblings(), mptree[i-1])
+ ... if i < len(mptree)-1:
+ ... assert has(child.right_siblings(), mptree[i+1])
+ ... if print_mptrees:
+ ... print('ok!', end=' ')
+ ... for mptree in print_mptrees: print(mptree)
+ ... else:
+ ... print('ok!')
+
+Run our test function on a variety of newly-created trees:
+
+ >>> mpcheck(make_mptree('(A)'))
+ ok! (A )
+ >>> mpcheck(make_mptree('(A (B (C (D) (E f)) g) h)'))
+ ok! (A (B (C (D ) (E f)) g) h)
+ >>> mpcheck(make_mptree('(A (B) (C c) (D d d) (E e e e))'))
+ ok! (A (B ) (C c) (D d d) (E e e e))
+ >>> mpcheck(make_mptree('(A (B) (C (c)) (D (d) (d)) (E (e) (e) (e)))'))
+ ok! (A (B ) (C (c )) (D (d ) (d )) (E (e ) (e ) (e )))
+ >>> subtree = make_mptree('(A (B (C (D) (E f)) g) h)')
+
+Including some trees that contain multiple parents:
+
+ >>> mpcheck(MultiParentedTree('Z', [subtree, subtree]))
+ ok! (Z (A (B (C (D ) (E f)) g) h) (A (B (C (D ) (E f)) g) h))
+
+Run our test function after performing various tree-modification
+operations (n.b., these are the same tests that we ran for
+`ParentedTree`, above; thus, none of these trees actually *uses*
+multiple parents.)
+
+**__delitem__()**
+
+ >>> mptree = make_mptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> e = mptree[0,0,1]
+ >>> del mptree[0,0,1]; mpcheck(mptree); mpcheck(e)
+ ok! (A (B (C (D ) (Q p)) g) h)
+ ok! (E f)
+ >>> del mptree[0,0,0]; mpcheck(mptree)
+ ok! (A (B (C (Q p)) g) h)
+ >>> del mptree[0,1]; mpcheck(mptree)
+ ok! (A (B (C (Q p))) h)
+ >>> del mptree[-1]; mpcheck(mptree)
+ ok! (A (B (C (Q p))))
+ >>> del mptree[-100]
+ Traceback (most recent call last):
+ . . .
+ IndexError: index out of range
+ >>> del mptree[()]
+ Traceback (most recent call last):
+ . . .
+ IndexError: The tree position () may not be deleted.
+
+ >>> # With slices:
+ >>> mptree = make_mptree('(A (B c) (D e) f g (H i) j (K l))')
+ >>> b = mptree[0]
+ >>> del mptree[0:0]; mpcheck(mptree)
+ ok! (A (B c) (D e) f g (H i) j (K l))
+ >>> del mptree[:1]; mpcheck(mptree); mpcheck(b)
+ ok! (A (D e) f g (H i) j (K l))
+ ok! (B c)
+ >>> del mptree[-2:]; mpcheck(mptree)
+ ok! (A (D e) f g (H i))
+ >>> del mptree[1:3]; mpcheck(mptree)
+ ok! (A (D e) (H i))
+ >>> mptree = make_mptree('(A (B c) (D e) f g (H i) j (K l))')
+ >>> del mptree[5:1000]; mpcheck(mptree)
+ ok! (A (B c) (D e) f g (H i))
+ >>> del mptree[-2:1000]; mpcheck(mptree)
+ ok! (A (B c) (D e) f)
+ >>> del mptree[-100:1]; mpcheck(mptree)
+ ok! (A (D e) f)
+ >>> mptree = make_mptree('(A (B c) (D e) f g (H i) j (K l))')
+ >>> del mptree[1:-2:2]; mpcheck(mptree)
+ ok! (A (B c) f (H i) j (K l))
+
+**__setitem__()**
+
+ >>> mptree = make_mptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> d, e, q = mptree[0,0]
+ >>> mptree[0,0,0] = 'x'; mpcheck(mptree); mpcheck(d)
+ ok! (A (B (C x (E f) (Q p)) g) h)
+ ok! (D )
+ >>> mptree[0,0,1] = make_mptree('(X (Y z))'); mpcheck(mptree); mpcheck(e)
+ ok! (A (B (C x (X (Y z)) (Q p)) g) h)
+ ok! (E f)
+ >>> mptree[1] = d; mpcheck(mptree)
+ ok! (A (B (C x (X (Y z)) (Q p)) g) (D ))
+ >>> mptree[-1] = 'x'; mpcheck(mptree)
+ ok! (A (B (C x (X (Y z)) (Q p)) g) x)
+ >>> mptree[-100] = 'y'
+ Traceback (most recent call last):
+ . . .
+ IndexError: index out of range
+ >>> mptree[()] = make_mptree('(X y)')
+ Traceback (most recent call last):
+ . . .
+ IndexError: The tree position () may not be assigned to.
+
+ >>> # With slices:
+ >>> mptree = make_mptree('(A (B c) (D e) f g (H i) j (K l))')
+ >>> b = mptree[0]
+ >>> mptree[0:0] = ('x', make_mptree('(Y)')); mpcheck(mptree)
+ ok! (A x (Y ) (B c) (D e) f g (H i) j (K l))
+ >>> mptree[2:6] = (); mpcheck(mptree); mpcheck(b)
+ ok! (A x (Y ) (H i) j (K l))
+ ok! (B c)
+ >>> mptree[-2:] = ('z', 'p'); mpcheck(mptree)
+ ok! (A x (Y ) (H i) z p)
+ >>> mptree[1:3] = [make_mptree('(X)') for x in range(10)]; mpcheck(mptree)
+ ok! (A x (X ) (X ) (X ) (X ) (X ) (X ) (X ) (X ) (X ) (X ) z p)
+ >>> mptree[5:1000] = []; mpcheck(mptree)
+ ok! (A x (X ) (X ) (X ) (X ))
+ >>> mptree[-2:1000] = ['n']; mpcheck(mptree)
+ ok! (A x (X ) (X ) n)
+ >>> mptree[-100:1] = [make_mptree('(U v)')]; mpcheck(mptree)
+ ok! (A (U v) (X ) (X ) n)
+ >>> mptree[-1:] = (make_mptree('(X)') for x in range(3)); mpcheck(mptree)
+ ok! (A (U v) (X ) (X ) (X ) (X ) (X ))
+ >>> mptree[1:-2:2] = ['x', 'y']; mpcheck(mptree)
+ ok! (A (U v) x (X ) y (X ) (X ))
+
+**append()**
+
+ >>> mptree = make_mptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> mptree.append('x'); mpcheck(mptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x)
+ >>> mptree.append(make_mptree('(X (Y z))')); mpcheck(mptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x (X (Y z)))
+
+**extend()**
+
+ >>> mptree = make_mptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> mptree.extend(['x', 'y', make_mptree('(X (Y z))')]); mpcheck(mptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x y (X (Y z)))
+ >>> mptree.extend([]); mpcheck(mptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x y (X (Y z)))
+ >>> mptree.extend(make_mptree('(X)') for x in range(3)); mpcheck(mptree)
+ ok! (A (B (C (D ) (E f) (Q p)) g) h x y (X (Y z)) (X ) (X ) (X ))
+
+**insert()**
+
+ >>> mptree = make_mptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> mptree.insert(0, make_mptree('(X (Y z))')); mpcheck(mptree)
+ ok! (A (X (Y z)) (B (C (D ) (E f) (Q p)) g) h)
+ >>> mptree.insert(-1, make_mptree('(X (Y z))')); mpcheck(mptree)
+ ok! (A (X (Y z)) (B (C (D ) (E f) (Q p)) g) (X (Y z)) h)
+ >>> mptree.insert(-4, make_mptree('(X (Y z))')); mpcheck(mptree)
+ ok! (A (X (Y z)) (X (Y z)) (B (C (D ) (E f) (Q p)) g) (X (Y z)) h)
+ >>> # Note: as with ``list``, inserting at a negative index that
+ >>> # gives a position before the start of the list does *not*
+ >>> # raise an IndexError exception; it just inserts at 0.
+ >>> mptree.insert(-400, make_mptree('(X (Y z))')); mpcheck(mptree)
+ ok! (A
+ (X (Y z))
+ (X (Y z))
+ (X (Y z))
+ (B (C (D ) (E f) (Q p)) g)
+ (X (Y z))
+ h)
+
+**pop()**
+
+ >>> mptree = make_mptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> mptree[0,0].pop(1); mpcheck(mptree)
+ MultiParentedTree('E', ['f'])
+ ok! (A (B (C (D ) (Q p)) g) h)
+ >>> mptree[0].pop(-1); mpcheck(mptree)
+ 'g'
+ ok! (A (B (C (D ) (Q p))) h)
+ >>> mptree.pop(); mpcheck(mptree)
+ 'h'
+ ok! (A (B (C (D ) (Q p))))
+ >>> mptree.pop(-100)
+ Traceback (most recent call last):
+ . . .
+ IndexError: index out of range
+
+**remove()**
+
+ >>> mptree = make_mptree('(A (B (C (D) (E f) (Q p)) g) h)')
+ >>> e = mptree[0,0,1]
+ >>> mptree[0,0].remove(mptree[0,0,1]); mpcheck(mptree); mpcheck(e)
+ ok! (A (B (C (D ) (Q p)) g) h)
+ ok! (E f)
+ >>> mptree[0,0].remove(make_mptree('(Q p)')); mpcheck(mptree)
+ ok! (A (B (C (D )) g) h)
+ >>> mptree[0,0].remove(make_mptree('(Q p)'))
+ Traceback (most recent call last):
+ . . .
+ ValueError: MultiParentedTree('Q', ['p']) is not in list
+ >>> mptree.remove('h'); mpcheck(mptree)
+ ok! (A (B (C (D )) g))
+ >>> mptree.remove('h');
+ Traceback (most recent call last):
+ . . .
+ ValueError: 'h' is not in list
+ >>> # remove() removes the first subtree that is equal (==) to the
+ >>> # given tree, which may not be the identical tree we give it:
+ >>> mptree = make_mptree('(A (X x) (Y y) (X x))')
+ >>> x1, y, x2 = mptree
+ >>> mptree.remove(mptree[-1]); mpcheck(mptree)
+ ok! (A (Y y) (X x))
+ >>> print([str(p) for p in x1.parents()])
+ []
+ >>> print([str(p) for p in x2.parents()])
+ ['(A (Y y) (X x))']
+
+
+ImmutableMultiParentedTree Regression Tests
+-------------------------------------------
+
+ >>> imptree = ImmutableMultiParentedTree.convert(mptree)
+ >>> type(imptree)
+ <class 'nltk.tree.ImmutableMultiParentedTree'>
+ >>> del imptree[0]
+ Traceback (most recent call last):
+ . . .
+ ValueError: ImmutableMultiParentedTree may not be modified
+ >>> imptree.set_label('newnode')
+ Traceback (most recent call last):
+ . . .
+ ValueError: ImmutableMultiParentedTree may not be modified
+
+
+ProbabilisticTree Regression Tests
+----------------------------------
+
+ >>> prtree = ProbabilisticTree("S", [ProbabilisticTree("NP", ["N"], prob=0.3)], prob=0.6)
+ >>> print(prtree)
+ (S (NP N)) (p=0.6)
+ >>> import copy
+ >>> prtree == copy.deepcopy(prtree) == prtree.copy(deep=True) == prtree.copy()
+ True
+ >>> prtree[0] is prtree.copy()[0]
+ True
+ >>> prtree[0] is prtree.copy(deep=True)[0]
+ False
+
+ >>> imprtree = ImmutableProbabilisticTree.convert(prtree)
+ >>> type(imprtree)
+ <class 'nltk.tree.ImmutableProbabilisticTree'>
+ >>> del imprtree[0]
+ Traceback (most recent call last):
+ . . .
+ ValueError: ImmutableProbabilisticTree may not be modified
+ >>> imprtree.set_label('newnode')
+ Traceback (most recent call last):
+ . . .
+ ValueError: ImmutableProbabilisticTree may not be modified
+
+
+Squashed Bugs
+=============
+
+This used to discard the ``(B b)`` subtree (fixed in svn 6270):
+
+ >>> print(Tree.fromstring('((A a) (B b))'))
+ ( (A a) (B b))
+
diff --git a/nltk/test/treetransforms.doctest b/nltk/test/treetransforms.doctest
new file mode 100644
index 0000000..6cc4764
--- /dev/null
+++ b/nltk/test/treetransforms.doctest
@@ -0,0 +1,156 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+-------------------------------------------
+Unit tests for the TreeTransformation class
+-------------------------------------------
+
+ >>> from copy import deepcopy
+ >>> from nltk.tree import *
+ >>> from nltk.treetransforms import *
+
+ >>> tree_string = "(TOP (S (S (VP (VBN Turned) (ADVP (RB loose)) (PP (IN in) (NP (NP (NNP Shane) (NNP Longman) (POS 's)) (NN trading) (NN room))))) (, ,) (NP (DT the) (NN yuppie) (NNS dealers)) (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right)))) (. .)))"
+
+ >>> tree = Tree.fromstring(tree_string)
+ >>> print(tree)
+ (TOP
+ (S
+ (S
+ (VP
+ (VBN Turned)
+ (ADVP (RB loose))
+ (PP
+ (IN in)
+ (NP
+ (NP (NNP Shane) (NNP Longman) (POS 's))
+ (NN trading)
+ (NN room)))))
+ (, ,)
+ (NP (DT the) (NN yuppie) (NNS dealers))
+ (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right))))
+ (. .)))
+
+Make a copy of the original tree and collapse the subtrees with only one child
+
+ >>> collapsedTree = deepcopy(tree)
+ >>> collapse_unary(collapsedTree)
+ >>> print(collapsedTree)
+ (TOP
+ (S
+ (S+VP
+ (VBN Turned)
+ (ADVP (RB loose))
+ (PP
+ (IN in)
+ (NP
+ (NP (NNP Shane) (NNP Longman) (POS 's))
+ (NN trading)
+ (NN room))))
+ (, ,)
+ (NP (DT the) (NN yuppie) (NNS dealers))
+ (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right))))
+ (. .)))
+
+ >>> collapsedTree2 = deepcopy(tree)
+ >>> collapse_unary(collapsedTree2, collapsePOS=True, collapseRoot=True)
+ >>> print(collapsedTree2)
+ (TOP+S
+ (S+VP
+ (VBN Turned)
+ (ADVP+RB loose)
+ (PP
+ (IN in)
+ (NP
+ (NP (NNP Shane) (NNP Longman) (POS 's))
+ (NN trading)
+ (NN room))))
+ (, ,)
+ (NP (DT the) (NN yuppie) (NNS dealers))
+ (VP (AUX do) (NP (NP+RB little) (ADJP+RB right)))
+ (. .))
+
+Convert the tree to Chomsky Normal Form i.e. each subtree has either two
+subtree children or a single leaf value. This conversion can be performed
+using either left- or right-factoring.
+
+ >>> cnfTree = deepcopy(collapsedTree)
+ >>> chomsky_normal_form(cnfTree, factor='left')
+ >>> print(cnfTree)
+ (TOP
+ (S
+ (S|<S+VP-,-NP-VP>
+ (S|<S+VP-,-NP>
+ (S|<S+VP-,>
+ (S+VP
+ (S+VP|<VBN-ADVP> (VBN Turned) (ADVP (RB loose)))
+ (PP
+ (IN in)
+ (NP
+ (NP|<NP-NN>
+ (NP
+ (NP|<NNP-NNP> (NNP Shane) (NNP Longman))
+ (POS 's))
+ (NN trading))
+ (NN room))))
+ (, ,))
+ (NP (NP|<DT-NN> (DT the) (NN yuppie)) (NNS dealers)))
+ (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right)))))
+ (. .)))
+
+ >>> cnfTree = deepcopy(collapsedTree)
+ >>> chomsky_normal_form(cnfTree, factor='right')
+ >>> print(cnfTree)
+ (TOP
+ (S
+ (S+VP
+ (VBN Turned)
+ (S+VP|<ADVP-PP>
+ (ADVP (RB loose))
+ (PP
+ (IN in)
+ (NP
+ (NP (NNP Shane) (NP|<NNP-POS> (NNP Longman) (POS 's)))
+ (NP|<NN-NN> (NN trading) (NN room))))))
+ (S|<,-NP-VP-.>
+ (, ,)
+ (S|<NP-VP-.>
+ (NP (DT the) (NP|<NN-NNS> (NN yuppie) (NNS dealers)))
+ (S|<VP-.>
+ (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right))))
+ (. .))))))
+
+Employ some Markov smoothing to make the artificial node labels a bit more
+readable. See the treetransforms.py documentation for more details.
+
+ >>> markovTree = deepcopy(collapsedTree)
+ >>> chomsky_normal_form(markovTree, horzMarkov=2, vertMarkov=1)
+ >>> print(markovTree)
+ (TOP
+ (S^<TOP>
+ (S+VP^<S>
+ (VBN Turned)
+ (S+VP|<ADVP-PP>^<S>
+ (ADVP^<S+VP> (RB loose))
+ (PP^<S+VP>
+ (IN in)
+ (NP^<PP>
+ (NP^<NP>
+ (NNP Shane)
+ (NP|<NNP-POS>^<NP> (NNP Longman) (POS 's)))
+ (NP|<NN-NN>^<PP> (NN trading) (NN room))))))
+ (S|<,-NP>^<TOP>
+ (, ,)
+ (S|<NP-VP>^<TOP>
+ (NP^<S> (DT the) (NP|<NN-NNS>^<S> (NN yuppie) (NNS dealers)))
+ (S|<VP-.>^<TOP>
+ (VP^<S>
+ (AUX do)
+ (NP^<VP> (NP^<NP> (RB little)) (ADJP^<NP> (RB right))))
+ (. .))))))
+
+Convert the transformed tree back to its original form
+
+ >>> un_chomsky_normal_form(markovTree)
+ >>> tree == markovTree
+ True
+
diff --git a/nltk/test/unit/__init__.py b/nltk/test/unit/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/nltk/test/unit/test_2x_compat.py b/nltk/test/unit/test_2x_compat.py
new file mode 100644
index 0000000..78329bf
--- /dev/null
+++ b/nltk/test/unit/test_2x_compat.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for nltk.compat.
+See also nltk/test/compat.doctest.
+"""
+from __future__ import absolute_import, unicode_literals
+import unittest
+
+from nltk.text import Text
+from nltk.compat import PY3, python_2_unicode_compatible
+
+def setup_module(module):
+ from nose import SkipTest
+ if PY3:
+ raise SkipTest("test_2x_compat is for testing nltk.compat under Python 2.x")
+
+
+class TestTextTransliteration(unittest.TestCase):
+ txt = Text(["São", "Tomé", "and", "Príncipe"])
+
+ def test_repr(self):
+ self.assertEqual(repr(self.txt), br"<Text: S\xe3o Tom\xe9 and Pr\xedncipe...>")
+
+ def test_str(self):
+ self.assertEqual(str(self.txt), b"<Text: Sao Tome and Principe...>")
+
diff --git a/nltk/test/unit/test_classify.py b/nltk/test/unit/test_classify.py
new file mode 100644
index 0000000..1b7f2e1
--- /dev/null
+++ b/nltk/test/unit/test_classify.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for nltk.classify. See also: nltk/test/classify.doctest
+"""
+from __future__ import absolute_import
+from nose import SkipTest
+from nltk import classify
+
+TRAIN = [
+ (dict(a=1,b=1,c=1), 'y'),
+ (dict(a=1,b=1,c=1), 'x'),
+ (dict(a=1,b=1,c=0), 'y'),
+ (dict(a=0,b=1,c=1), 'x'),
+ (dict(a=0,b=1,c=1), 'y'),
+ (dict(a=0,b=0,c=1), 'y'),
+ (dict(a=0,b=1,c=0), 'x'),
+ (dict(a=0,b=0,c=0), 'x'),
+ (dict(a=0,b=1,c=1), 'y'),
+ ]
+
+TEST = [
+ (dict(a=1,b=0,c=1)), # unseen
+ (dict(a=1,b=0,c=0)), # unseen
+ (dict(a=0,b=1,c=1)), # seen 3 times, labels=y,y,x
+ (dict(a=0,b=1,c=0)), # seen 1 time, label=x
+]
+
+RESULTS = [
+ (0.16, 0.84),
+ (0.46, 0.54),
+ (0.41, 0.59),
+ (0.76, 0.24),
+]
+
+def assert_classifier_correct(algorithm):
+ try:
+ classifier = classify.MaxentClassifier.train(
+ TRAIN, algorithm, trace=0, max_iter=1000
+ )
+ except (LookupError, AttributeError) as e:
+ raise SkipTest(str(e))
+
+ for (px, py), featureset in zip(RESULTS, TEST):
+ pdist = classifier.prob_classify(featureset)
+ assert abs(pdist.prob('x') - px) < 1e-2, (pdist.prob('x'), px)
+ assert abs(pdist.prob('y') - py) < 1e-2, (pdist.prob('y'), py)
+
+
+def test_megam():
+ assert_classifier_correct('MEGAM')
+
+def test_tadm():
+ assert_classifier_correct('TADM')
diff --git a/nltk/test/unit/test_collocations.py b/nltk/test/unit/test_collocations.py
new file mode 100644
index 0000000..d1c7950
--- /dev/null
+++ b/nltk/test/unit/test_collocations.py
@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+import unittest
+
+from nltk.collocations import BigramCollocationFinder
+from nltk.metrics import BigramAssocMeasures
+
+## Test bigram counters with discontinuous bigrams and repeated words
+
+_EPSILON = 1e-8
+
+
+def close_enough(x, y):
+ """Verify that two sequences of n-gram association values are within
+ _EPSILON of each other.
+ """
+
+ for (x1, y1) in zip(x, y):
+ if x1[0] != y1[0] or abs(x1[1] - y1[1]) > _EPSILON:
+ return False
+ return True
+
+
+class TestBigram(unittest.TestCase):
+ def test_bigram2(self):
+ sent = 'this this is is a a test test'.split()
+
+ b = BigramCollocationFinder.from_words(sent)
+
+ #python 2.6 does not have assertItemsEqual or assertListEqual
+ self.assertEqual(
+ sorted(b.ngram_fd.items()),
+ sorted([(('a', 'a'), 1), (('a', 'test'), 1), (('is', 'a'), 1), (('is', 'is'), 1), (('test', 'test'), 1), (('this', 'is'), 1), (('this', 'this'), 1)])
+ )
+ self.assertEqual(
+ sorted(b.word_fd.items()),
+ sorted([('a', 2), ('is', 2), ('test', 2), ('this', 2)])
+ )
+ self.assertTrue(len(sent) == sum(b.word_fd.values()) == sum(b.ngram_fd.values()) + 1)
+ self.assertTrue(close_enough(
+ sorted(b.score_ngrams(BigramAssocMeasures.pmi)),
+ sorted([(('a', 'a'), 1.0), (('a', 'test'), 1.0), (('is', 'a'), 1.0), (('is', 'is'), 1.0), (('test', 'test'), 1.0), (('this', 'is'), 1.0), (('this', 'this'), 1.0)])
+ ))
+
+ def test_bigram3(self):
+ sent = 'this this is is a a test test'.split()
+
+ b = BigramCollocationFinder.from_words(sent, window_size=3)
+ self.assertEqual(
+ sorted(b.ngram_fd.items()),
+ sorted([(('a', 'test'), 3), (('is', 'a'), 3), (('this', 'is'), 3), (('a', 'a'), 1), (('is', 'is'), 1), (('test', 'test'), 1), (('this', 'this'), 1)])
+ )
+ self.assertEqual(
+ sorted(b.word_fd.items()),
+ sorted([('a', 2), ('is', 2), ('test', 2), ('this', 2)])
+ )
+ self.assertTrue(len(sent) == sum(b.word_fd.values()) == (sum(b.ngram_fd.values()) + 2 + 1) / 2.0)
+ self.assertTrue(close_enough(
+ sorted(b.score_ngrams(BigramAssocMeasures.pmi)),
+ sorted([(('a', 'test'), 1.584962500721156), (('is', 'a'), 1.584962500721156), (('this', 'is'), 1.584962500721156), (('a', 'a'), 0.0), (('is', 'is'), 0.0), (('test', 'test'), 0.0), (('this', 'this'), 0.0)])
+ ))
+
+ def test_bigram5(self):
+ sent = 'this this is is a a test test'.split()
+
+ b = BigramCollocationFinder.from_words(sent, window_size=5)
+ self.assertEqual(
+ sorted(b.ngram_fd.items()),
+ sorted([(('a', 'test'), 4), (('is', 'a'), 4), (('this', 'is'), 4), (('is', 'test'), 3), (('this', 'a'), 3), (('a', 'a'), 1), (('is', 'is'), 1), (('test', 'test'), 1), (('this', 'this'), 1)])
+ )
+ self.assertEqual(
+ sorted(b.word_fd.items()),
+ sorted([('a', 2), ('is', 2), ('test', 2), ('this', 2)])
+ )
+ self.assertTrue(len(sent) == sum(b.word_fd.values()) == (sum(b.ngram_fd.values()) + 4 + 3 + 2 + 1) / 4.0)
+ self.assertTrue(close_enough(
+ sorted(b.score_ngrams(BigramAssocMeasures.pmi)),
+ sorted([(('a', 'test'), 1.0), (('is', 'a'), 1.0), (('this', 'is'), 1.0), (('is', 'test'), 0.5849625007211562), (('this', 'a'), 0.5849625007211562), (('a', 'a'), -1.0), (('is', 'is'), -1.0), (('test', 'test'), -1.0), (('this', 'this'), -1.0)])
+ ))
diff --git a/nltk/test/unit/test_corpora.py b/nltk/test/unit/test_corpora.py
new file mode 100644
index 0000000..8b39b45
--- /dev/null
+++ b/nltk/test/unit/test_corpora.py
@@ -0,0 +1,181 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+import unittest
+from nltk.corpus import (sinica_treebank, conll2007, indian, cess_cat, cess_esp,
+ floresta, ptb, udhr)
+from nltk.tree import Tree
+from nltk.test.unit.utils import skipIf
+
+
+class TestUdhr(unittest.TestCase):
+
+ def test_words(self):
+ for name in udhr.fileids():
+ try:
+ words = list(udhr.words(name))
+ except AssertionError:
+ print(name)
+ raise
+ self.assertTrue(words)
+
+ def test_raw_unicode(self):
+ for name in udhr.fileids():
+ txt = udhr.raw(name)
+ assert not isinstance(txt, bytes), name
+
+
+class TestIndian(unittest.TestCase):
+
+ def test_words(self):
+ words = indian.words()[:3]
+ self.assertEqual(words, ['মহিষের', 'সন্তান', ':'])
+
+ def test_tagged_words(self):
+ tagged_words = indian.tagged_words()[:3]
+ self.assertEqual(tagged_words, [('মহিষের', 'NN'), ('সন্তান', 'NN'), (':', 'SYM')])
+
+
+class TestCess(unittest.TestCase):
+ def test_catalan(self):
+ words = cess_cat.words()[:15]
+ txt = "El Tribunal_Suprem -Fpa- TS -Fpt- ha confirmat la condemna a quatre anys d' inhabilitació especial"
+ self.assertEqual(words, txt.split())
+
+ def test_esp(self):
+ words = cess_esp.words()[:15]
+ txt = "El grupo estatal Electricité_de_France -Fpa- EDF -Fpt- anunció hoy , jueves , la compra del"
+ self.assertEqual(words, txt.split())
+
+
+class TestFloresta(unittest.TestCase):
+ def test_words(self):
+ words = floresta.words()[:10]
+ txt = "Um revivalismo refrescante O 7_e_Meio é um ex-libris de a"
+ self.assertEqual(words, txt.split())
+
+class TestSinicaTreebank(unittest.TestCase):
+
+ def test_sents(self):
+ first_3_sents = sinica_treebank.sents()[:3]
+ self.assertEqual(
+ first_3_sents,
+ [['一'], ['友情'], ['嘉珍', '和', '我', '住在', '同一條', '巷子']]
+ )
+
+ def test_parsed_sents(self):
+ parsed_sents = sinica_treebank.parsed_sents()[25]
+ self.assertEqual(parsed_sents,
+ Tree('S', [
+ Tree('NP', [
+ Tree('Nba', ['嘉珍'])
+ ]),
+ Tree('V‧地', [
+ Tree('VA11', ['不停']),
+ Tree('DE', ['的'])
+ ]),
+ Tree('VA4', ['哭泣'])
+ ]))
+
+
+class TestCoNLL2007(unittest.TestCase):
+ # Reading the CoNLL 2007 Dependency Treebanks
+
+ def test_sents(self):
+ sents = conll2007.sents('esp.train')[0]
+ self.assertEqual(
+ sents[:6],
+ ['El', 'aumento', 'del', 'índice', 'de', 'desempleo']
+ )
+
+ def test_parsed_sents(self):
+
+ parsed_sents = conll2007.parsed_sents('esp.train')[0]
+
+ self.assertEqual(parsed_sents.tree(),
+ Tree('fortaleció', [
+ Tree('aumento', [
+ 'El',
+ Tree('del', [
+ Tree('índice', [
+ Tree('de', [
+ Tree('desempleo', ['estadounidense'])
+ ])
+ ])
+ ])
+ ]),
+ 'hoy',
+ 'considerablemente',
+ Tree('al', [
+ Tree('euro', [
+ Tree('cotizaba', [
+ ',',
+ 'que',
+ Tree('a', [
+ Tree('15.35', ['las', 'GMT'])
+ ]),
+ 'se',
+ Tree('en', [
+ Tree('mercado', [
+ 'el',
+ Tree('de', ['divisas']),
+ Tree('de', ['Fráncfort'])
+ ])
+ ]),
+ Tree('a', ['0,9452_dólares']),
+ Tree('frente_a', [
+ ',',
+ Tree('0,9349_dólares', [
+ 'los',
+ Tree('de', [
+ Tree('mañana', ['esta'])
+ ])
+ ])
+ ])
+ ])
+ ])
+ ]),
+ '.'
+ ])
+ )
+
+
+ at skipIf(not ptb.fileids(), "A full installation of the Penn Treebank is not available")
+class TestPTB(unittest.TestCase):
+ def test_fileids(self):
+ self.assertEqual(
+ ptb.fileids()[:4],
+ ['BROWN/CF/CF01.MRG', 'BROWN/CF/CF02.MRG', 'BROWN/CF/CF03.MRG', 'BROWN/CF/CF04.MRG']
+ )
+
+ def test_words(self):
+ self.assertEqual(
+ ptb.words('WSJ/00/WSJ_0003.MRG')[:7],
+ ['A', 'form', 'of', 'asbestos', 'once', 'used', '*']
+ )
+
+ def test_tagged_words(self):
+ self.assertEqual(
+ ptb.tagged_words('WSJ/00/WSJ_0003.MRG')[:3],
+ [('A', 'DT'), ('form', 'NN'), ('of', 'IN')]
+ )
+
+ def test_categories(self):
+ self.assertEqual(
+ ptb.categories(),
+ ['adventure', 'belles_lettres', 'fiction', 'humor', 'lore', 'mystery', 'news', 'romance', 'science_fiction']
+ )
+
+ def test_news_fileids(self):
+ self.assertEqual(
+ ptb.fileids('news')[:3],
+ ['WSJ/00/WSJ_0001.MRG', 'WSJ/00/WSJ_0002.MRG', 'WSJ/00/WSJ_0003.MRG']
+ )
+
+ def test_category_words(self):
+ self.assertEqual(
+ ptb.words(categories=['humor','fiction'])[:6],
+ ['Thirty-three', 'Scotty', 'did', 'not', 'go', 'back']
+ )
+
+# unload corpora
+from nltk.corpus import teardown_module
diff --git a/nltk/test/unit/test_corpus_views.py b/nltk/test/unit/test_corpus_views.py
new file mode 100644
index 0000000..8c17f0d
--- /dev/null
+++ b/nltk/test/unit/test_corpus_views.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""
+Corpus View Regression Tests
+"""
+from __future__ import absolute_import, unicode_literals
+import unittest
+import nltk.data
+from nltk.corpus.reader.util import (StreamBackedCorpusView,
+ read_whitespace_block, read_line_block)
+
+class TestCorpusViews(unittest.TestCase):
+
+ linetok = nltk.LineTokenizer(blanklines='keep')
+ names = [
+ 'corpora/inaugural/README', # A very short file (160 chars)
+ 'corpora/inaugural/1793-Washington.txt', # A relatively short file (791 chars)
+ 'corpora/inaugural/1909-Taft.txt', # A longer file (32k chars)
+ ]
+
+ def data(self):
+ for name in self.names:
+ f = nltk.data.find(name)
+ with f.open() as fp:
+ file_data = fp.read().decode('utf8')
+ yield f, file_data
+
+ def test_correct_values(self):
+ # Check that corpus views produce the correct sequence of values.
+
+ for f, file_data in self.data():
+ v = StreamBackedCorpusView(f, read_whitespace_block)
+ self.assertEqual(list(v), file_data.split())
+
+ v = StreamBackedCorpusView(f, read_line_block)
+ self.assertEqual(list(v), self.linetok.tokenize(file_data))
+
+ def test_correct_length(self):
+ # Check that the corpus views report the correct lengths:
+
+ for f, file_data in self.data():
+ v = StreamBackedCorpusView(f, read_whitespace_block)
+ self.assertEqual(len(v), len(file_data.split()))
+
+ v = StreamBackedCorpusView(f, read_line_block)
+ self.assertEqual(len(v), len(self.linetok.tokenize(file_data)))
diff --git a/nltk/test/unit/test_hmm.py b/nltk/test/unit/test_hmm.py
new file mode 100644
index 0000000..ab73f37
--- /dev/null
+++ b/nltk/test/unit/test_hmm.py
@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+from nltk.tag import hmm
+
+def _wikipedia_example_hmm():
+ # Example from wikipedia
+ # (http://en.wikipedia.org/wiki/Forward%E2%80%93backward_algorithm)
+
+ states = ['rain', 'no rain']
+ symbols = ['umbrella', 'no umbrella']
+
+ A = [[0.7, 0.3], [0.3, 0.7]] # transition probabilities
+ B = [[0.9, 0.1], [0.2, 0.8]] # emission probabilities
+ pi = [0.5, 0.5] # initial probabilities
+
+ seq = ['umbrella', 'umbrella', 'no umbrella', 'umbrella', 'umbrella']
+ seq = list(zip(seq, [None]*len(seq)))
+
+ model = hmm._create_hmm_tagger(states, symbols, A, B, pi)
+ return model, states, symbols, seq
+
+
+def test_forward_probability():
+ from numpy.testing import assert_array_almost_equal
+
+ # example from p. 385, Huang et al
+ model, states, symbols = hmm._market_hmm_example()
+ seq = [('up', None), ('up', None)]
+ expected = [
+ [0.35, 0.02, 0.09],
+ [0.1792, 0.0085, 0.0357]
+ ]
+
+ fp = 2**model._forward_probability(seq)
+
+ assert_array_almost_equal(fp, expected)
+
+
+def test_forward_probability2():
+ from numpy.testing import assert_array_almost_equal
+
+ model, states, symbols, seq = _wikipedia_example_hmm()
+ fp = 2**model._forward_probability(seq)
+
+ # examples in wikipedia are normalized
+ fp = (fp.T / fp.sum(axis=1)).T
+
+ wikipedia_results = [
+ [0.8182, 0.1818],
+ [0.8834, 0.1166],
+ [0.1907, 0.8093],
+ [0.7308, 0.2692],
+ [0.8673, 0.1327],
+ ]
+
+ assert_array_almost_equal(wikipedia_results, fp, 4)
+
+
+def test_backward_probability():
+ from numpy.testing import assert_array_almost_equal
+
+ model, states, symbols, seq = _wikipedia_example_hmm()
+
+ bp = 2**model._backward_probability(seq)
+ # examples in wikipedia are normalized
+
+ bp = (bp.T / bp.sum(axis=1)).T
+
+ wikipedia_results = [
+ # Forward-backward algorithm doesn't need b0_5,
+ # so .backward_probability doesn't compute it.
+ # [0.6469, 0.3531],
+ [0.5923, 0.4077],
+ [0.3763, 0.6237],
+ [0.6533, 0.3467],
+ [0.6273, 0.3727],
+ [0.5, 0.5],
+ ]
+
+ assert_array_almost_equal(wikipedia_results, bp, 4)
+
+
+def setup_module(module):
+ from nose import SkipTest
+ try:
+ import numpy
+ except ImportError:
+ raise SkipTest("numpy is required for nltk.test.test_hmm")
diff --git a/nltk/test/unit/test_naivebayes.py b/nltk/test/unit/test_naivebayes.py
new file mode 100644
index 0000000..706eccf
--- /dev/null
+++ b/nltk/test/unit/test_naivebayes.py
@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+
+
+import unittest
+from nltk.classify.naivebayes import NaiveBayesClassifier
+
+
+class NaiveBayesClassifierTest(unittest.TestCase):
+
+ def test_simple(self):
+ training_features = [
+ ({'nice': True, 'good': True}, 'positive'),
+ ({'bad': True, 'mean': True}, 'negative')
+ ]
+
+ classifier = NaiveBayesClassifier.train(training_features)
+
+ result = classifier.prob_classify({'nice': True})
+ self.assertTrue(result.prob('positive') >
+ result.prob('negative'))
+ self.assertEqual(result.max(), 'positive')
+
+ result = classifier.prob_classify({'bad': True})
+ self.assertTrue(result.prob('positive') < result.prob('negative'))
+ self.assertEqual(result.max(), 'negative')
diff --git a/nltk/test/unit/test_seekable_unicode_stream_reader.py b/nltk/test/unit/test_seekable_unicode_stream_reader.py
new file mode 100644
index 0000000..e65b47b
--- /dev/null
+++ b/nltk/test/unit/test_seekable_unicode_stream_reader.py
@@ -0,0 +1,124 @@
+# -*- coding: utf-8 -*-
+"""
+The following test performs a random series of reads, seeks, and
+tells, and checks that the results are consistent.
+"""
+from __future__ import absolute_import, unicode_literals
+import random
+import functools
+from io import BytesIO
+from nltk.corpus.reader import SeekableUnicodeStreamReader
+
+def check_reader(unicode_string, encoding, n=1000):
+ bytestr = unicode_string.encode(encoding)
+ strlen = len(unicode_string)
+ stream = BytesIO(bytestr)
+ reader = SeekableUnicodeStreamReader(stream, encoding)
+ # Find all character positions
+ chars = []
+ while True:
+ pos = reader.tell()
+ chars.append( (pos, reader.read(1)) )
+ if chars[-1][1] == '': break
+ # Find all strings
+ strings = dict( (pos,'') for (pos,c) in chars )
+ for pos1, char in chars:
+ for pos2, _ in chars:
+ if pos2 <= pos1:
+ strings[pos2] += char
+ while True:
+ op = random.choice('tsrr')
+ # Check our position?
+ if op == 't': # tell
+ reader.tell()
+ # Perform a seek?
+ if op == 's': # seek
+ new_pos = random.choice([p for (p,c) in chars])
+ reader.seek(new_pos)
+ # Perform a read?
+ if op == 'r': # read
+ if random.random() < .3: pos = reader.tell()
+ else: pos = None
+ if random.random() < .2: size = None
+ elif random.random() < .8:
+ size = random.randint(0, int(strlen/6))
+ else: size = random.randint(0, strlen+20)
+ if random.random() < .8:
+ s = reader.read(size)
+ else:
+ s = reader.readline(size)
+ # check that everything's consistent
+ if pos is not None:
+ assert pos in strings
+ assert strings[pos].startswith(s)
+ n -= 1
+ if n == 0:
+ return 'passed'
+
+
+#Call the randomized test function `check_reader` with a variety of
+#input strings and encodings.
+
+ENCODINGS = ['ascii', 'latin1', 'greek', 'hebrew', 'utf-16', 'utf-8']
+
+STRINGS = [
+ """
+ This is a test file.
+ It is fairly short.
+ """,
+
+ "This file can be encoded with latin1. \x83",
+
+ """\
+ This is a test file.
+ Here's a blank line:
+
+ And here's some unicode: \xee \u0123 \uffe3
+ """,
+
+ """\
+ This is a test file.
+ Unicode characters: \xf3 \u2222 \u3333\u4444 \u5555
+ """,
+
+]
+
+def test_reader():
+ for string in STRINGS:
+ for encoding in ENCODINGS:
+ try:
+ # skip strings that can't be encoded with the current encoding
+ string.encode(encoding)
+ yield check_reader, string, encoding
+ except UnicodeEncodeError:
+ pass
+
+
+
+# nose shows the whole string arguments in a verbose mode; this is annoying,
+# so large string test is separated.
+
+LARGE_STRING = """\
+This is a larger file. It has some lines that are longer \
+than 72 characters. It's got lots of repetition. Here's \
+some unicode chars: \xee \u0123 \uffe3 \ueeee \u2345
+
+How fun! Let's repeat it twenty times.
+"""*10
+
+def test_reader_on_large_string():
+ for encoding in ENCODINGS:
+ try:
+ # skip strings that can't be encoded with the current encoding
+ LARGE_STRING.encode(encoding)
+ def _check(encoding, n=1000):
+ check_reader(LARGE_STRING, encoding, n)
+
+ yield _check, encoding
+
+ except UnicodeEncodeError:
+ pass
+
+def teardown_module(module=None):
+ import gc
+ gc.collect()
diff --git a/nltk/test/unit/test_stem.py b/nltk/test/unit/test_stem.py
new file mode 100644
index 0000000..8e78576
--- /dev/null
+++ b/nltk/test/unit/test_stem.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function, unicode_literals
+import unittest
+from nltk.stem.snowball import SnowballStemmer
+
+class SnowballTest(unittest.TestCase):
+
+ def test_russian(self):
+ # Russian words both consisting of Cyrillic
+ # and Roman letters can be stemmed.
+ stemmer_russian = SnowballStemmer("russian")
+ assert stemmer_russian.stem("авантненькая") == "авантненьк"
+ assert stemmer_russian.stem("avenantnen'kai^a") == "avenantnen'k"
+
+ def test_german(self):
+ stemmer_german = SnowballStemmer("german")
+ stemmer_german2 = SnowballStemmer("german", ignore_stopwords=True)
+
+ assert stemmer_german.stem("Schr\xe4nke") == 'schrank'
+ assert stemmer_german2.stem("Schr\xe4nke") == 'schrank'
+
+ assert stemmer_german.stem("keinen") == 'kein'
+ assert stemmer_german2.stem("keinen") == 'keinen'
+
+ def test_short_strings_bug(self):
+ stemmer = SnowballStemmer('english')
+ assert stemmer.stem("y's") == 'y'
diff --git a/nltk/test/unit/test_tag.py b/nltk/test/unit/test_tag.py
new file mode 100644
index 0000000..99d6113
--- /dev/null
+++ b/nltk/test/unit/test_tag.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, unicode_literals
+
+def test_basic():
+ from nltk.tag import pos_tag
+ from nltk.tokenize import word_tokenize
+
+ result = pos_tag(word_tokenize("John's big idea isn't all that bad."))
+ assert result == [('John', 'NNP'), ("'s", 'POS'), ('big', 'JJ'),
+ ('idea', 'NN'), ('is', 'VBZ'), ("n't", 'RB'),
+ ('all', 'DT'), ('that', 'DT'), ('bad', 'JJ'),
+ ('.', '.')]
+
+
+def setup_module(module):
+ from nose import SkipTest
+ try:
+ import numpy
+ except ImportError:
+ raise SkipTest("numpy is required for nltk.test.test_tag")
diff --git a/nltk/test/unit/utils.py b/nltk/test/unit/utils.py
new file mode 100644
index 0000000..f86cb6b
--- /dev/null
+++ b/nltk/test/unit/utils.py
@@ -0,0 +1,42 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from unittest import TestCase
+from functools import wraps
+from nose.plugins.skip import SkipTest
+from nltk.util import py26
+
+def skip(reason):
+ """
+ Unconditionally skip a test.
+ """
+ def decorator(test_item):
+ is_test_class = isinstance(test_item, type) and issubclass(test_item, TestCase)
+
+ if is_test_class and py26():
+ # Patch all test_ methods to raise SkipText exception.
+ # This is necessary for Python 2.6 because its unittest
+ # doesn't understand __unittest_skip__.
+ for meth_name in (m for m in dir(test_item) if m.startswith('test_')):
+ patched_method = skip(reason)(getattr(test_item, meth_name))
+ setattr(test_item, meth_name, patched_method)
+
+ if not is_test_class:
+ @wraps(test_item)
+ def skip_wrapper(*args, **kwargs):
+ raise SkipTest(reason)
+ skip_wrapper.__name__ = test_item.__name__
+ test_item = skip_wrapper
+
+ test_item.__unittest_skip__ = True
+ test_item.__unittest_skip_why__ = reason
+ return test_item
+ return decorator
+
+
+def skipIf(condition, reason):
+ """
+ Skip a test if the condition is true.
+ """
+ if condition:
+ return skip(reason)
+ return lambda obj: obj
\ No newline at end of file
diff --git a/nltk/test/util.doctest b/nltk/test/util.doctest
new file mode 100644
index 0000000..0aafce2
--- /dev/null
+++ b/nltk/test/util.doctest
@@ -0,0 +1,48 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=================
+Utility functions
+=================
+
+ >>> from __future__ import print_function
+ >>> from nltk.util import *
+ >>> from nltk.tree import Tree
+
+ >>> print_string("This is a long string, therefore it should break", 25)
+ This is a long string,
+ therefore it should break
+
+ >>> re_show("[a-z]+", "sdf123")
+ {sdf}123
+
+ >>> tree = Tree(5,
+ ... [Tree(4, [Tree(2, [1, 3])]),
+ ... Tree(8, [Tree(6, [7]), 9])])
+ >>> for x in breadth_first(tree):
+ ... if isinstance(x, int): print(x)
+ ... else: print(x.label())
+ 5
+ 4
+ 8
+ 2
+ 6
+ 9
+ 1
+ 3
+ 7
+ >>> for x in breadth_first(tree, maxdepth=2):
+ ... if isinstance(x, int): print(x)
+ ... else: print(x.label())
+ 5
+ 4
+ 8
+ 2
+ 6
+ 9
+
+ >>> invert_dict({1: 2})
+ defaultdict(<... 'list'>, {2: 1})
+
+ >>> invert_dict({1: [3, 4, 5]})
+ defaultdict(<... 'list'>, {3: [1], 4: [1], 5: [1]})
diff --git a/nltk/test/wordnet.doctest b/nltk/test/wordnet.doctest
new file mode 100644
index 0000000..f2c614c
--- /dev/null
+++ b/nltk/test/wordnet.doctest
@@ -0,0 +1,580 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+=================
+WordNet Interface
+=================
+
+WordNet is just another NLTK corpus reader, and can be imported like this:
+
+ >>> from nltk.corpus import wordnet
+
+For more compact code, we recommend:
+
+ >>> from nltk.corpus import wordnet as wn
+
+-----
+Words
+-----
+
+Look up a word using ``synsets()``; this function has an optional ``pos`` argument
+which lets you constrain the part of speech of the word:
+
+ >>> wn.synsets('dog') # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
+ [Synset('dog.n.01'), Synset('frump.n.01'), Synset('dog.n.03'), Synset('cad.n.01'),
+ Synset('frank.n.02'), Synset('pawl.n.01'), Synset('andiron.n.01'), Synset('chase.v.01')]
+ >>> wn.synsets('dog', pos=wn.VERB)
+ [Synset('chase.v.01')]
+
+The other parts of speech are ``NOUN``, ``ADJ`` and ``ADV``.
+A synset is identified with a 3-part name of the form: word.pos.nn:
+
+ >>> wn.synset('dog.n.01')
+ Synset('dog.n.01')
+ >>> print(wn.synset('dog.n.01').definition())
+ a member of the genus Canis (probably descended from the common wolf) that has been domesticated by man since prehistoric times; occurs in many breeds
+ >>> len(wn.synset('dog.n.01').examples())
+ 1
+ >>> print(wn.synset('dog.n.01').examples()[0])
+ the dog barked all night
+ >>> wn.synset('dog.n.01').lemmas()
+ [Lemma('dog.n.01.dog'), Lemma('dog.n.01.domestic_dog'), Lemma('dog.n.01.Canis_familiaris')]
+ >>> [str(lemma.name()) for lemma in wn.synset('dog.n.01').lemmas()]
+ ['dog', 'domestic_dog', 'Canis_familiaris']
+ >>> wn.lemma('dog.n.01.dog').synset()
+ Synset('dog.n.01')
+
+The WordNet corpus reader gives access to the Open Multilingual
+WordNet, using ISO-639 language codes.
+
+ >>> sorted(wn.langs())
+ ['als', 'arb', 'cat', 'cmn', 'dan', 'eng', 'eus', 'fas',
+ 'fin', 'fra', 'fre', 'glg', 'heb', 'ind', 'ita', 'jpn', 'nno',
+ 'nob', 'pol', 'por', 'spa', 'tha', 'zsm']
+ >>> wn.synsets(b'\xe7\x8a\xac'.decode('utf-8'), lang='jpn')
+ [Synset('dog.n.01'), Synset('spy.n.01')]
+ >>> wn.synset('spy.n.01').lemma_names('jpn')
+ ['\u3044\u306c', '\u307e\u308f\u3057\u8005', '\u30b9\u30d1\u30a4', '\u56de\u3057\u8005',
+ '\u56de\u8005', '\u5bc6\u5075', '\u5de5\u4f5c\u54e1', '\u5efb\u3057\u8005',
+ '\u5efb\u8005', '\u63a2', '\u63a2\u308a', '\u72ac', '\u79d8\u5bc6\u635c\u67fb\u54e1',
+ '\u8adc\u5831\u54e1', '\u8adc\u8005', '\u9593\u8005', '\u9593\u8adc', '\u96a0\u5bc6']
+ >>> wn.synset('dog.n.01').lemma_names('ita')
+ ['cane', 'Canis_familiaris']
+ >>> wn.lemmas('cane', lang='ita')
+ [Lemma('dog.n.01.cane'), Lemma('hammer.n.01.cane'), Lemma('cramp.n.02.cane'),
+ Lemma('bad_person.n.01.cane'), Lemma('incompetent.n.01.cane')]
+ >>> sorted(wn.synset('dog.n.01').lemmas('dan'))
+ [Lemma('dog.n.01.hund'), Lemma('dog.n.01.k\xf8ter'),
+ Lemma('dog.n.01.vovhund'), Lemma('dog.n.01.vovse')]
+ >>> sorted(wn.synset('dog.n.01').lemmas('por'))
+ [Lemma('dog.n.01.cachorro'), Lemma('dog.n.01.c\xe3es'),
+ Lemma('dog.n.01.c\xe3o'), Lemma('dog.n.01.c\xe3o')]
+ >>> dog_lemma = wn.lemma(b'dog.n.01.c\xc3\xa3o'.decode('utf-8'), lang='por')
+ >>> dog_lemma
+ Lemma('dog.n.01.c\xe3o')
+ >>> dog_lemma.lang()
+ 'por'
+ >>> len(wordnet.all_lemma_names(pos='n', lang='jpn'))
+ 66027
+
+-------
+Synsets
+-------
+
+`Synset`: a set of synonyms that share a common meaning.
+
+ >>> dog = wn.synset('dog.n.01')
+ >>> dog.hypernyms()
+ [Synset('canine.n.02'), Synset('domestic_animal.n.01')]
+ >>> dog.hyponyms() # doctest: +ELLIPSIS
+ [Synset('basenji.n.01'), Synset('corgi.n.01'), Synset('cur.n.01'), Synset('dalmatian.n.02'), ...]
+ >>> dog.member_holonyms()
+ [Synset('canis.n.01'), Synset('pack.n.06')]
+ >>> dog.root_hypernyms()
+ [Synset('entity.n.01')]
+ >>> wn.synset('dog.n.01').lowest_common_hypernyms(wn.synset('cat.n.01'))
+ [Synset('carnivore.n.01')]
+
+Each synset contains one or more lemmas, which represent a specific
+sense of a specific word.
+
+Note that some relations are defined by WordNet only over Lemmas:
+
+ >>> good = wn.synset('good.a.01')
+ >>> good.antonyms()
+ Traceback (most recent call last):
+ File "<stdin>", line 1, in <module>
+ AttributeError: 'Synset' object has no attribute 'antonyms'
+ >>> good.lemmas()[0].antonyms()
+ [Lemma('bad.a.01.bad')]
+
+The relations that are currently defined in this way are `antonyms`,
+`derivationally_related_forms` and `pertainyms`.
+
+
+------
+Lemmas
+------
+
+ >>> eat = wn.lemma('eat.v.03.eat')
+ >>> eat
+ Lemma('feed.v.06.eat')
+ >>> print(eat.key())
+ eat%2:34:02::
+ >>> eat.count()
+ 4
+ >>> wn.lemma_from_key(eat.key())
+ Lemma('feed.v.06.eat')
+ >>> wn.lemma_from_key(eat.key()).synset()
+ Synset('feed.v.06')
+ >>> wn.lemma_from_key('feebleminded%5:00:00:retarded:00')
+ Lemma('backward.s.03.feebleminded')
+ >>> for lemma in wn.synset('eat.v.03').lemmas():
+ ... print(lemma, lemma.count())
+ ...
+ Lemma('feed.v.06.feed') 3
+ Lemma('feed.v.06.eat') 4
+ >>> for lemma in wn.lemmas('eat', 'v'):
+ ... print(lemma, lemma.count())
+ ...
+ Lemma('eat.v.01.eat') 61
+ Lemma('eat.v.02.eat') 13
+ Lemma('feed.v.06.eat') 4
+ Lemma('eat.v.04.eat') 0
+ Lemma('consume.v.05.eat') 0
+ Lemma('corrode.v.01.eat') 0
+
+Lemmas can also have relations between them:
+
+ >>> vocal = wn.lemma('vocal.a.01.vocal')
+ >>> vocal.derivationally_related_forms()
+ [Lemma('vocalize.v.02.vocalize')]
+ >>> vocal.pertainyms()
+ [Lemma('voice.n.02.voice')]
+ >>> vocal.antonyms()
+ [Lemma('instrumental.a.01.instrumental')]
+
+The three relations above exist only on lemmas, not on synsets.
+
+-----------
+Verb Frames
+-----------
+
+ >>> wn.synset('think.v.01').frame_ids()
+ [5, 9]
+ >>> for lemma in wn.synset('think.v.01').lemmas():
+ ... print(lemma, lemma.frame_ids())
+ ... print(" | ".join(lemma.frame_strings()))
+ ...
+ Lemma('think.v.01.think') [5, 9]
+ Something think something Adjective/Noun | Somebody think somebody
+ Lemma('think.v.01.believe') [5, 9]
+ Something believe something Adjective/Noun | Somebody believe somebody
+ Lemma('think.v.01.consider') [5, 9]
+ Something consider something Adjective/Noun | Somebody consider somebody
+ Lemma('think.v.01.conceive') [5, 9]
+ Something conceive something Adjective/Noun | Somebody conceive somebody
+ >>> wn.synset('stretch.v.02').frame_ids()
+ [8]
+ >>> for lemma in wn.synset('stretch.v.02').lemmas():
+ ... print(lemma, lemma.frame_ids())
+ ... print(" | ".join(lemma.frame_strings()))
+ ...
+ Lemma('stretch.v.02.stretch') [8, 2]
+ Somebody stretch something | Somebody stretch
+ Lemma('stretch.v.02.extend') [8]
+ Somebody extend something
+
+
+----------
+Similarity
+----------
+
+ >>> dog = wn.synset('dog.n.01')
+ >>> cat = wn.synset('cat.n.01')
+
+ >>> hit = wn.synset('hit.v.01')
+ >>> slap = wn.synset('slap.v.01')
+
+
+``synset1.path_similarity(synset2):``
+Return a score denoting how similar two word senses are, based on the
+shortest path that connects the senses in the is-a (hypernym/hypnoym)
+taxonomy. The score is in the range 0 to 1. By default, there is now
+a fake root node added to verbs so for cases where previously a path
+could not be found---and None was returned---it should return a value.
+The old behavior can be achieved by setting simulate_root to be False.
+A score of 1 represents identity i.e. comparing a sense with itself
+will return 1.
+
+ >>> dog.path_similarity(cat) # doctest: +ELLIPSIS
+ 0.2...
+
+ >>> hit.path_similarity(slap) # doctest: +ELLIPSIS
+ 0.142...
+
+ >>> wn.path_similarity(hit, slap) # doctest: +ELLIPSIS
+ 0.142...
+
+ >>> print(hit.path_similarity(slap, simulate_root=False))
+ None
+
+ >>> print(wn.path_similarity(hit, slap, simulate_root=False))
+ None
+
+``synset1.lch_similarity(synset2):``
+Leacock-Chodorow Similarity:
+Return a score denoting how similar two word senses are, based on the
+shortest path that connects the senses (as above) and the maximum depth
+of the taxonomy in which the senses occur. The relationship is given
+as -log(p/2d) where p is the shortest path length and d the taxonomy
+depth.
+
+ >>> dog.lch_similarity(cat) # doctest: +ELLIPSIS
+ 2.028...
+
+ >>> hit.lch_similarity(slap) # doctest: +ELLIPSIS
+ 1.312...
+
+ >>> wn.lch_similarity(hit, slap) # doctest: +ELLIPSIS
+ 1.312...
+
+ >>> print(hit.lch_similarity(slap, simulate_root=False))
+ None
+
+ >>> print(wn.lch_similarity(hit, slap, simulate_root=False))
+ None
+
+``synset1.wup_similarity(synset2):``
+Wu-Palmer Similarity:
+Return a score denoting how similar two word senses are, based on the
+depth of the two senses in the taxonomy and that of their Least Common
+Subsumer (most specific ancestor node). Note that at this time the
+scores given do _not_ always agree with those given by Pedersen's Perl
+implementation of Wordnet Similarity.
+
+The LCS does not necessarily feature in the shortest path connecting the
+two senses, as it is by definition the common ancestor deepest in the
+taxonomy, not closest to the two senses. Typically, however, it will so
+feature. Where multiple candidates for the LCS exist, that whose
+shortest path to the root node is the longest will be selected. Where
+the LCS has multiple paths to the root, the longer path is used for
+the purposes of the calculation.
+
+ >>> dog.wup_similarity(cat) # doctest: +ELLIPSIS
+ 0.857...
+
+ >>> hit.wup_similarity(slap)
+ 0.25
+
+ >>> wn.wup_similarity(hit, slap)
+ 0.25
+
+ >>> print(hit.wup_similarity(slap, simulate_root=False))
+ None
+
+ >>> print(wn.wup_similarity(hit, slap, simulate_root=False))
+ None
+
+``wordnet_ic``
+Information Content:
+Load an information content file from the wordnet_ic corpus.
+
+ >>> from nltk.corpus import wordnet_ic
+ >>> brown_ic = wordnet_ic.ic('ic-brown.dat')
+ >>> semcor_ic = wordnet_ic.ic('ic-semcor.dat')
+
+Or you can create an information content dictionary from a corpus (or
+anything that has a words() method).
+
+ >>> from nltk.corpus import genesis
+ >>> genesis_ic = wn.ic(genesis, False, 0.0)
+
+``synset1.res_similarity(synset2, ic):``
+Resnik Similarity:
+Return a score denoting how similar two word senses are, based on the
+Information Content (IC) of the Least Common Subsumer (most specific
+ancestor node). Note that for any similarity measure that uses
+information content, the result is dependent on the corpus used to
+generate the information content and the specifics of how the
+information content was created.
+
+ >>> dog.res_similarity(cat, brown_ic) # doctest: +ELLIPSIS
+ 7.911...
+ >>> dog.res_similarity(cat, genesis_ic) # doctest: +ELLIPSIS
+ 7.204...
+
+``synset1.jcn_similarity(synset2, ic):``
+Jiang-Conrath Similarity
+Return a score denoting how similar two word senses are, based on the
+Information Content (IC) of the Least Common Subsumer (most specific
+ancestor node) and that of the two input Synsets. The relationship is
+given by the equation 1 / (IC(s1) + IC(s2) - 2 * IC(lcs)).
+
+ >>> dog.jcn_similarity(cat, brown_ic) # doctest: +ELLIPSIS
+ 0.449...
+ >>> dog.jcn_similarity(cat, genesis_ic) # doctest: +ELLIPSIS
+ 0.285...
+
+``synset1.lin_similarity(synset2, ic):``
+Lin Similarity:
+Return a score denoting how similar two word senses are, based on the
+Information Content (IC) of the Least Common Subsumer (most specific
+ancestor node) and that of the two input Synsets. The relationship is
+given by the equation 2 * IC(lcs) / (IC(s1) + IC(s2)).
+
+ >>> dog.lin_similarity(cat, semcor_ic) # doctest: +ELLIPSIS
+ 0.886...
+
+
+---------------------
+Access to all Synsets
+---------------------
+
+Iterate over all the noun synsets:
+
+ >>> for synset in list(wn.all_synsets('n'))[:10]:
+ ... print(synset)
+ ...
+ Synset('entity.n.01')
+ Synset('physical_entity.n.01')
+ Synset('abstraction.n.06')
+ Synset('thing.n.12')
+ Synset('object.n.01')
+ Synset('whole.n.02')
+ Synset('congener.n.03')
+ Synset('living_thing.n.01')
+ Synset('organism.n.01')
+ Synset('benthos.n.02')
+
+Get all synsets for this word, possibly restricted by POS:
+
+ >>> wn.synsets('dog') # doctest: +ELLIPSIS
+ [Synset('dog.n.01'), Synset('frump.n.01'), Synset('dog.n.03'), Synset('cad.n.01'), ...]
+ >>> wn.synsets('dog', pos='v')
+ [Synset('chase.v.01')]
+
+Walk through the noun synsets looking at their hypernyms:
+
+ >>> from itertools import islice
+ >>> for synset in islice(wn.all_synsets('n'), 5):
+ ... print(synset, synset.hypernyms())
+ ...
+ Synset('entity.n.01') []
+ Synset('physical_entity.n.01') [Synset('entity.n.01')]
+ Synset('abstraction.n.06') [Synset('entity.n.01')]
+ Synset('thing.n.12') [Synset('physical_entity.n.01')]
+ Synset('object.n.01') [Synset('physical_entity.n.01')]
+
+
+------
+Morphy
+------
+
+Look up forms not in WordNet, with the help of Morphy:
+
+ >>> wn.morphy('denied', wn.NOUN)
+ >>> print(wn.morphy('denied', wn.VERB))
+ deny
+ >>> wn.synsets('denied', wn.NOUN)
+ []
+ >>> wn.synsets('denied', wn.VERB) # doctest: +NORMALIZE_WHITESPACE
+ [Synset('deny.v.01'), Synset('deny.v.02'), Synset('deny.v.03'), Synset('deny.v.04'),
+ Synset('deny.v.05'), Synset('traverse.v.03'), Synset('deny.v.07')]
+
+Morphy uses a combination of inflectional ending rules and exception
+lists to handle a variety of different possibilities:
+
+ >>> print(wn.morphy('dogs'))
+ dog
+ >>> print(wn.morphy('churches'))
+ church
+ >>> print(wn.morphy('aardwolves'))
+ aardwolf
+ >>> print(wn.morphy('abaci'))
+ abacus
+ >>> print(wn.morphy('book', wn.NOUN))
+ book
+ >>> wn.morphy('hardrock', wn.ADV)
+ >>> wn.morphy('book', wn.ADJ)
+ >>> wn.morphy('his', wn.NOUN)
+ >>>
+
+---------------
+Synset Closures
+---------------
+
+Compute transitive closures of synsets
+
+ >>> dog = wn.synset('dog.n.01')
+ >>> hypo = lambda s: s.hyponyms()
+ >>> hyper = lambda s: s.hypernyms()
+ >>> list(dog.closure(hypo, depth=1)) == dog.hyponyms()
+ True
+ >>> list(dog.closure(hyper, depth=1)) == dog.hypernyms()
+ True
+ >>> list(dog.closure(hypo))
+ [Synset('basenji.n.01'), Synset('corgi.n.01'), Synset('cur.n.01'),
+ Synset('dalmatian.n.02'), Synset('great_pyrenees.n.01'),
+ Synset('griffon.n.02'), Synset('hunting_dog.n.01'), Synset('lapdog.n.01'),
+ Synset('leonberg.n.01'), Synset('mexican_hairless.n.01'),
+ Synset('newfoundland.n.01'), Synset('pooch.n.01'), Synset('poodle.n.01'), ...]
+ >>> list(dog.closure(hyper))
+ [Synset('canine.n.02'), Synset('domestic_animal.n.01'), Synset('carnivore.n.01'),
+ Synset('animal.n.01'), Synset('placental.n.01'), Synset('organism.n.01'),
+ Synset('mammal.n.01'), Synset('living_thing.n.01'), Synset('vertebrate.n.01'),
+ Synset('whole.n.02'), Synset('chordate.n.01'), Synset('object.n.01'),
+ Synset('physical_entity.n.01'), Synset('entity.n.01')]
+
+
+----------------
+Regression Tests
+----------------
+
+Bug 85: morphy returns the base form of a word, if it's input is given
+as a base form for a POS for which that word is not defined:
+
+ >>> wn.synsets('book', wn.NOUN)
+ [Synset('book.n.01'), Synset('book.n.02'), Synset('record.n.05'), Synset('script.n.01'), Synset('ledger.n.01'), Synset('book.n.06'), Synset('book.n.07'), Synset('koran.n.01'), Synset('bible.n.01'), Synset('book.n.10'), Synset('book.n.11')]
+ >>> wn.synsets('book', wn.ADJ)
+ []
+ >>> wn.morphy('book', wn.NOUN)
+ 'book'
+ >>> wn.morphy('book', wn.ADJ)
+
+Bug 160: wup_similarity breaks when the two synsets have no common hypernym
+
+ >>> t = wn.synsets('picasso')[0]
+ >>> m = wn.synsets('male')[1]
+ >>> t.wup_similarity(m) # doctest: +ELLIPSIS
+ 0.631...
+
+ >>> t = wn.synsets('titan')[1]
+ >>> s = wn.synsets('say', wn.VERB)[0]
+ >>> print(t.wup_similarity(s))
+ None
+
+Bug 21: "instance of" not included in LCS (very similar to bug 160)
+
+ >>> a = wn.synsets("writings")[0]
+ >>> b = wn.synsets("scripture")[0]
+ >>> brown_ic = wordnet_ic.ic('ic-brown.dat')
+ >>> a.jcn_similarity(b, brown_ic) # doctest: +ELLIPSIS
+ 0.175...
+
+Bug 221: Verb root IC is zero
+
+ >>> from nltk.corpus.reader.wordnet import information_content
+ >>> s = wn.synsets('say', wn.VERB)[0]
+ >>> information_content(s, brown_ic) # doctest: +ELLIPSIS
+ 4.623...
+
+Bug 161: Comparison between WN keys/lemmas should not be case sensitive
+
+ >>> k = wn.synsets("jefferson")[0].lemmas()[0].key()
+ >>> wn.lemma_from_key(k)
+ Lemma('jefferson.n.01.Jefferson')
+ >>> wn.lemma_from_key(k.upper())
+ Lemma('jefferson.n.01.Jefferson')
+
+Bug 99: WordNet root_hypernyms gives incorrect results
+
+ >>> from nltk.corpus import wordnet as wn
+ >>> for s in wn.all_synsets(wn.NOUN):
+ ... if s.root_hypernyms()[0] != wn.synset('entity.n.01'):
+ ... print(s, s.root_hypernyms())
+ ...
+ >>>
+
+Bug 382: JCN Division by zero error
+
+ >>> tow = wn.synset('tow.v.01')
+ >>> shlep = wn.synset('shlep.v.02')
+ >>> from nltk.corpus import wordnet_ic
+ >>> brown_ic = wordnet_ic.ic('ic-brown.dat')
+ >>> tow.jcn_similarity(shlep, brown_ic) # doctest: +ELLIPSIS
+ 1...e+300
+
+Bug 428: Depth is zero for instance nouns
+
+ >>> s = wn.synset("lincoln.n.01")
+ >>> s.max_depth() > 0
+ True
+
+Bug 429: Information content smoothing used old reference to all_synsets
+
+ >>> genesis_ic = wn.ic(genesis, True, 1.0)
+
+Bug 430: all_synsets used wrong pos lookup when synsets were cached
+
+ >>> for ii in wn.all_synsets(): pass
+ >>> for ii in wn.all_synsets(): pass
+
+Bug 470: shortest_path_distance ignored instance hypernyms
+
+ >>> google = wordnet.synsets("google")[0]
+ >>> earth = wordnet.synsets("earth")[0]
+ >>> google.wup_similarity(earth) # doctest: +ELLIPSIS
+ 0.1...
+
+Bug 484: similarity metrics returned -1 instead of None for no LCS
+
+ >>> t = wn.synsets('fly', wn.VERB)[0]
+ >>> s = wn.synsets('say', wn.VERB)[0]
+ >>> print(s.shortest_path_distance(t))
+ None
+ >>> print(s.path_similarity(t, simulate_root=False))
+ None
+ >>> print(s.lch_similarity(t, simulate_root=False))
+ None
+ >>> print(s.wup_similarity(t, simulate_root=False))
+ None
+
+Bug 427: "pants" does not return all the senses it should
+
+ >>> from nltk.corpus import wordnet
+ >>> wordnet.synsets("pants",'n')
+ [Synset('bloomers.n.01'), Synset('pant.n.01'), Synset('trouser.n.01'), Synset('gasp.n.01')]
+
+Bug 482: Some nouns not being lemmatised by WordNetLemmatizer().lemmatize
+
+ >>> from nltk.stem.wordnet import WordNetLemmatizer
+ >>> WordNetLemmatizer().lemmatize("eggs", pos="n")
+ 'egg'
+ >>> WordNetLemmatizer().lemmatize("legs", pos="n")
+ 'leg'
+
+Bug 284: instance hypernyms not used in similarity calculations
+
+ >>> wn.synset('john.n.02').lch_similarity(wn.synset('dog.n.01')) # doctest: +ELLIPSIS
+ 1.335...
+ >>> wn.synset('john.n.02').wup_similarity(wn.synset('dog.n.01')) # doctest: +ELLIPSIS
+ 0.571...
+ >>> wn.synset('john.n.02').res_similarity(wn.synset('dog.n.01'), brown_ic) # doctest: +ELLIPSIS
+ 2.224...
+ >>> wn.synset('john.n.02').jcn_similarity(wn.synset('dog.n.01'), brown_ic) # doctest: +ELLIPSIS
+ 0.075...
+ >>> wn.synset('john.n.02').lin_similarity(wn.synset('dog.n.01'), brown_ic) # doctest: +ELLIPSIS
+ 0.252...
+ >>> wn.synset('john.n.02').hypernym_paths() # doctest: +ELLIPSIS
+ [[Synset('entity.n.01'), ..., Synset('john.n.02')]]
+
+Issue 541: add domains to wordnet
+
+ >>> wn.synset('code.n.03').topic_domains()
+ [Synset('computer_science.n.01')]
+ >>> wn.synset('pukka.a.01').region_domains()
+ [Synset('india.n.01')]
+ >>> wn.synset('freaky.a.01').usage_domains()
+ [Synset('slang.n.02')]
+
+Issue 629: wordnet failures when python run with -O optimizations
+
+ >>> # Run the test suite with python -O to check this
+ >>> wn.synsets("brunch")
+ [Synset('brunch.n.01'), Synset('brunch.v.01')]
+
+Issue 395: wordnet returns incorrect result for lowest_common_hypernyms of chef and policeman
+
+ >>> wn.synset('policeman.n.01').lowest_common_hypernyms(wn.synset('chef.n.01'))
+ [Synset('person.n.01')]
diff --git a/nltk/test/wordnet_fixt.py b/nltk/test/wordnet_fixt.py
new file mode 100644
index 0000000..c42c8b6
--- /dev/null
+++ b/nltk/test/wordnet_fixt.py
@@ -0,0 +1,6 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+
+def teardown_module(module=None):
+ from nltk.corpus import wordnet
+ wordnet._unload()
diff --git a/nltk/test/wordnet_lch.doctest b/nltk/test/wordnet_lch.doctest
new file mode 100644
index 0000000..80cae25
--- /dev/null
+++ b/nltk/test/wordnet_lch.doctest
@@ -0,0 +1,53 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+===============================
+WordNet Lowest Common Hypernyms
+===============================
+
+Wordnet's lowest_common_hypernyms() method is based used to locate the
+lowest single hypernym that is shared by two given words:
+
+ >>> from nltk.corpus import wordnet as wn
+ >>> wn.synset('kin.n.01').lowest_common_hypernyms(wn.synset('mother.n.01'))
+ [Synset('relative.n.01')]
+
+ >>> wn.synset('policeman.n.01').lowest_common_hypernyms(wn.synset('chef.n.01'))
+ [Synset('person.n.01')]
+
+This method generally returns a single result, but in some cases, more than one
+valid LCH is possible:
+
+ >>> wn.synset('body.n.09').lowest_common_hypernyms(wn.synset('sidereal_day.n.01'))
+ [Synset('attribute.n.02'), Synset('measure.n.02')]
+
+In some cases, lowest_common_hypernyms() can return one of the synsets which was
+passed to it as an argument:
+
+ >>> wn.synset('woman.n.01').lowest_common_hypernyms(wn.synset('girlfriend.n.02'))
+ [Synset('woman.n.01')]
+
+In NLTK 3.0a2 the behavior of lowest_common_hypernyms() was changed to give more
+accurate results in a small set of cases, generally when dealing with nouns describing
+social roles or jobs. To emulate the pre v3.0a2 behavior, you can set the use_min_depth=True
+flag:
+
+ >>> wn.synset('policeman.n.01').lowest_common_hypernyms(wn.synset('chef.n.01'))
+ [Synset('person.n.01')]
+ >>> wn.synset('policeman.n.01').lowest_common_hypernyms(wn.synset('chef.n.01'), use_min_depth=True)
+ [Synset('organism.n.01')]
+
+In some cases use_min_depth=True may return more or fewer results than the default
+behavior:
+
+ >>> wn.synset('woman.n.01').lowest_common_hypernyms(wn.synset('girlfriend.n.02'))
+ [Synset('woman.n.01')]
+ >>> wn.synset('woman.n.01').lowest_common_hypernyms(wn.synset('girlfriend.n.02'), use_min_depth=True)
+ [Synset('organism.n.01'), Synset('woman.n.01')]
+
+In the general case, however, they tend to return the same results:
+
+ >>> wn.synset('body.n.09').lowest_common_hypernyms(wn.synset('sidereal_day.n.01'))
+ [Synset('attribute.n.02'), Synset('measure.n.02')]
+ >>> wn.synset('body.n.09').lowest_common_hypernyms(wn.synset('sidereal_day.n.01'), use_min_depth=True)
+ [Synset('attribute.n.02'), Synset('measure.n.02')]
diff --git a/nltk/test/wsd.doctest b/nltk/test/wsd.doctest
new file mode 100644
index 0000000..e4d9eda
--- /dev/null
+++ b/nltk/test/wsd.doctest
@@ -0,0 +1,52 @@
+.. Copyright (C) 2001-2014 NLTK Project
+.. For license information, see LICENSE.TXT
+
+.. -*- coding: utf-8 -*-
+
+=========================
+Word Sense Disambiguation
+=========================
+
+
+Lesk Algorithm
+--------------
+
+
+Performs the classic Lesk algorithm for Word Sense Disambiguation (WSD) using
+a the definitions of the ambiguous word.
+
+Given an ambiguous word and the context in which the word occurs, Lesk returns
+a Synset with the highest number of overlapping words between the context
+sentence and different definitions from each Synset.
+
+ >>> from nltk.wsd import lesk
+ >>> from nltk import word_tokenize
+ >>> sent = word_tokenize("I went to the bank to deposit money.")
+ >>> word = "bank"
+ >>> pos = "n"
+ >>> print(lesk(sent, word, pos))
+ Synset('depository_financial_institution.n.01')
+
+The definitions for "bank" are:
+
+ >>> from nltk.corpus import wordnet as wn
+ >>> for ss in wn.synsets('bank'):
+ ... print(ss, ss.definition())
+ Synset('bank.n.01') sloping land (especially the slope beside a body of water)
+ Synset('depository_financial_institution.n.01') a financial institution that accepts deposits and channels the money into lending activities
+ Synset('bank.n.03') a long ridge or pile
+ Synset('bank.n.04') an arrangement of similar objects in a row or in tiers
+ Synset('bank.n.05') a supply or stock held in reserve for future use (especially in emergencies)
+ Synset('bank.n.06') the funds held by a gambling house or the dealer in some gambling games
+ Synset('bank.n.07') a slope in the turn of a road or track; the outside is higher than the inside in order to reduce the effects of centrifugal force
+ Synset('savings_bank.n.02') a container (usually with a slot in the top) for keeping money at home
+ Synset('bank.n.09') a building in which the business of banking transacted
+ Synset('bank.n.10') a flight maneuver; aircraft tips laterally about its longitudinal axis (especially in turning)
+ Synset('bank.v.01') tip laterally
+ Synset('bank.v.02') enclose with a bank
+ Synset('bank.v.03') do business with a bank or keep an account at a bank
+ Synset('bank.v.04') act as the banker in a game or in gambling
+ Synset('bank.v.05') be in the banking business
+ Synset('deposit.v.02') put into a bank account
+ Synset('bank.v.07') cover with ashes so to control the rate of burning
+ Synset('trust.v.01') have confidence or faith in
diff --git a/nltk/text.py b/nltk/text.py
new file mode 100644
index 0000000..cc8067e
--- /dev/null
+++ b/nltk/text.py
@@ -0,0 +1,610 @@
+# Natural Language Toolkit: Texts
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+This module brings together a variety of NLTK functionality for
+text analysis, and provides simple, interactive interfaces.
+Functionality includes: concordancing, collocation discovery,
+regular expression search over tokenized strings, and
+distributional similarity.
+"""
+from __future__ import print_function, division, unicode_literals
+
+from math import log
+from collections import defaultdict
+from functools import reduce
+from itertools import islice
+import re
+
+from nltk.probability import FreqDist, LidstoneProbDist
+from nltk.probability import ConditionalFreqDist as CFD
+from nltk.util import tokenwrap, LazyConcatenation
+from nltk.metrics import f_measure, BigramAssocMeasures
+from nltk.collocations import BigramCollocationFinder
+from nltk.compat import python_2_unicode_compatible, text_type
+
+
+class ContextIndex(object):
+ """
+ A bidirectional index between words and their 'contexts' in a text.
+ The context of a word is usually defined to be the words that occur
+ in a fixed window around the word; but other definitions may also
+ be used by providing a custom context function.
+ """
+ @staticmethod
+ def _default_context(tokens, i):
+ """One left token and one right token, normalized to lowercase"""
+ left = (tokens[i-1].lower() if i != 0 else '*START*')
+ right = (tokens[i+1].lower() if i != len(tokens) - 1 else '*END*')
+ return (left, right)
+
+ def __init__(self, tokens, context_func=None, filter=None, key=lambda x:x):
+ self._key = key
+ self._tokens = tokens
+ if context_func:
+ self._context_func = context_func
+ else:
+ self._context_func = self._default_context
+ if filter:
+ tokens = [t for t in tokens if filter(t)]
+ self._word_to_contexts = CFD((self._key(w), self._context_func(tokens, i))
+ for i, w in enumerate(tokens))
+ self._context_to_words = CFD((self._context_func(tokens, i), self._key(w))
+ for i, w in enumerate(tokens))
+
+ def tokens(self):
+ """
+ :rtype: list(str)
+ :return: The document that this context index was
+ created from.
+ """
+ return self._tokens
+
+ def word_similarity_dict(self, word):
+ """
+ Return a dictionary mapping from words to 'similarity scores,'
+ indicating how often these two words occur in the same
+ context.
+ """
+ word = self._key(word)
+ word_contexts = set(self._word_to_contexts[word])
+
+ scores = {}
+ for w, w_contexts in self._word_to_contexts.items():
+ scores[w] = f_measure(word_contexts, set(w_contexts))
+
+ return scores
+
+ def similar_words(self, word, n=20):
+ scores = defaultdict(int)
+ for c in self._word_to_contexts[self._key(word)]:
+ for w in self._context_to_words[c]:
+ if w != word:
+ scores[w] += self._context_to_words[c][word] * self._context_to_words[c][w]
+ return sorted(scores, key=scores.get, reverse=True)[:n]
+
+ def common_contexts(self, words, fail_on_unknown=False):
+ """
+ Find contexts where the specified words can all appear; and
+ return a frequency distribution mapping each context to the
+ number of times that context was used.
+
+ :param words: The words used to seed the similarity search
+ :type words: str
+ :param fail_on_unknown: If true, then raise a value error if
+ any of the given words do not occur at all in the index.
+ """
+ words = [self._key(w) for w in words]
+ contexts = [set(self._word_to_contexts[w]) for w in words]
+ empty = [words[i] for i in range(len(words)) if not contexts[i]]
+ common = reduce(set.intersection, contexts)
+ if empty and fail_on_unknown:
+ raise ValueError("The following word(s) were not found:",
+ " ".join(words))
+ elif not common:
+ # nothing in common -- just return an empty freqdist.
+ return FreqDist()
+ else:
+ fd = FreqDist(c for w in words
+ for c in self._word_to_contexts[w]
+ if c in common)
+ return fd
+
+ at python_2_unicode_compatible
+class ConcordanceIndex(object):
+ """
+ An index that can be used to look up the offset locations at which
+ a given word occurs in a document.
+ """
+ def __init__(self, tokens, key=lambda x:x):
+ """
+ Construct a new concordance index.
+
+ :param tokens: The document (list of tokens) that this
+ concordance index was created from. This list can be used
+ to access the context of a given word occurrence.
+ :param key: A function that maps each token to a normalized
+ version that will be used as a key in the index. E.g., if
+ you use ``key=lambda s:s.lower()``, then the index will be
+ case-insensitive.
+ """
+ self._tokens = tokens
+ """The document (list of tokens) that this concordance index
+ was created from."""
+
+ self._key = key
+ """Function mapping each token to an index key (or None)."""
+
+ self._offsets = defaultdict(list)
+ """Dictionary mapping words (or keys) to lists of offset
+ indices."""
+
+ # Initialize the index (self._offsets)
+ for index, word in enumerate(tokens):
+ word = self._key(word)
+ self._offsets[word].append(index)
+
+ def tokens(self):
+ """
+ :rtype: list(str)
+ :return: The document that this concordance index was
+ created from.
+ """
+ return self._tokens
+
+ def offsets(self, word):
+ """
+ :rtype: list(int)
+ :return: A list of the offset positions at which the given
+ word occurs. If a key function was specified for the
+ index, then given word's key will be looked up.
+ """
+ word = self._key(word)
+ return self._offsets[word]
+
+ def __repr__(self):
+ return '<ConcordanceIndex for %d tokens (%d types)>' % (
+ len(self._tokens), len(self._offsets))
+
+ def print_concordance(self, word, width=75, lines=25):
+ """
+ Print a concordance for ``word`` with the specified context window.
+
+ :param word: The target word
+ :type word: str
+ :param width: The width of each line, in characters (default=80)
+ :type width: int
+ :param lines: The number of lines to display (default=25)
+ :type lines: int
+ """
+ half_width = (width - len(word) - 2) // 2
+ context = width // 4 # approx number of words of context
+
+ offsets = self.offsets(word)
+ if offsets:
+ lines = min(lines, len(offsets))
+ print("Displaying %s of %s matches:" % (lines, len(offsets)))
+ for i in offsets:
+ if lines <= 0:
+ break
+ left = (' ' * half_width +
+ ' '.join(self._tokens[i-context:i]))
+ right = ' '.join(self._tokens[i+1:i+context])
+ left = left[-half_width:]
+ right = right[:half_width]
+ print(left, self._tokens[i], right)
+ lines -= 1
+ else:
+ print("No matches")
+
+class TokenSearcher(object):
+ """
+ A class that makes it easier to use regular expressions to search
+ over tokenized strings. The tokenized string is converted to a
+ string where tokens are marked with angle brackets -- e.g.,
+ ``'<the><window><is><still><open>'``. The regular expression
+ passed to the ``findall()`` method is modified to treat angle
+ brackets as non-capturing parentheses, in addition to matching the
+ token boundaries; and to have ``'.'`` not match the angle brackets.
+ """
+ def __init__(self, tokens):
+ self._raw = ''.join('<'+w+'>' for w in tokens)
+
+ def findall(self, regexp):
+ """
+ Find instances of the regular expression in the text.
+ The text is a list of tokens, and a regexp pattern to match
+ a single token must be surrounded by angle brackets. E.g.
+
+ >>> from nltk.text import TokenSearcher
+ >>> print('hack'); from nltk.book import text1, text5, text9
+ hack...
+ >>> text5.findall("<.*><.*><bro>")
+ you rule bro; telling you bro; u twizted bro
+ >>> text1.findall("<a>(<.*>)<man>")
+ monied; nervous; dangerous; white; white; white; pious; queer; good;
+ mature; white; Cape; great; wise; wise; butterless; white; fiendish;
+ pale; furious; better; certain; complete; dismasted; younger; brave;
+ brave; brave; brave
+ >>> text9.findall("<th.*>{3,}")
+ thread through those; the thought that; that the thing; the thing
+ that; that that thing; through these than through; them that the;
+ through the thick; them that they; thought that the
+
+ :param regexp: A regular expression
+ :type regexp: str
+ """
+ # preprocess the regular expression
+ regexp = re.sub(r'\s', '', regexp)
+ regexp = re.sub(r'<', '(?:<(?:', regexp)
+ regexp = re.sub(r'>', ')>)', regexp)
+ regexp = re.sub(r'(?<!\\)\.', '[^>]', regexp)
+
+ # perform the search
+ hits = re.findall(regexp, self._raw)
+
+ # Sanity check
+ for h in hits:
+ if not h.startswith('<') and h.endswith('>'):
+ raise ValueError('Bad regexp for TokenSearcher.findall')
+
+ # postprocess the output
+ hits = [h[1:-1].split('><') for h in hits]
+ return hits
+
+
+ at python_2_unicode_compatible
+class Text(object):
+ """
+ A wrapper around a sequence of simple (string) tokens, which is
+ intended to support initial exploration of texts (via the
+ interactive console). Its methods perform a variety of analyses
+ on the text's contexts (e.g., counting, concordancing, collocation
+ discovery), and display the results. If you wish to write a
+ program which makes use of these analyses, then you should bypass
+ the ``Text`` class, and use the appropriate analysis function or
+ class directly instead.
+
+ A ``Text`` is typically initialized from a given document or
+ corpus. E.g.:
+
+ >>> import nltk.corpus
+ >>> from nltk.text import Text
+ >>> moby = Text(nltk.corpus.gutenberg.words('melville-moby_dick.txt'))
+
+ """
+ # This defeats lazy loading, but makes things faster. This
+ # *shouldn't* be necessary because the corpus view *should* be
+ # doing intelligent caching, but without this it's running slow.
+ # Look into whether the caching is working correctly.
+ _COPY_TOKENS = True
+
+ def __init__(self, tokens, name=None):
+ """
+ Create a Text object.
+
+ :param tokens: The source text.
+ :type tokens: sequence of str
+ """
+ if self._COPY_TOKENS:
+ tokens = list(tokens)
+ self.tokens = tokens
+
+ if name:
+ self.name = name
+ elif ']' in tokens[:20]:
+ end = tokens[:20].index(']')
+ self.name = " ".join(text_type(tok) for tok in tokens[1:end])
+ else:
+ self.name = " ".join(text_type(tok) for tok in tokens[:8]) + "..."
+
+ #////////////////////////////////////////////////////////////
+ # Support item & slice access
+ #////////////////////////////////////////////////////////////
+
+ def __getitem__(self, i):
+ if isinstance(i, slice):
+ return self.tokens[i.start:i.stop]
+ else:
+ return self.tokens[i]
+
+ def __len__(self):
+ return len(self.tokens)
+
+ #////////////////////////////////////////////////////////////
+ # Interactive console methods
+ #////////////////////////////////////////////////////////////
+
+ def concordance(self, word, width=79, lines=25):
+ """
+ Print a concordance for ``word`` with the specified context window.
+ Word matching is not case-sensitive.
+ :seealso: ``ConcordanceIndex``
+ """
+ if '_concordance_index' not in self.__dict__:
+ print("Building index...")
+ self._concordance_index = ConcordanceIndex(self.tokens,
+ key=lambda s:s.lower())
+
+ self._concordance_index.print_concordance(word, width, lines)
+
+ def collocations(self, num=20, window_size=2):
+ """
+ Print collocations derived from the text, ignoring stopwords.
+
+ :seealso: find_collocations
+ :param num: The maximum number of collocations to print.
+ :type num: int
+ :param window_size: The number of tokens spanned by a collocation (default=2)
+ :type window_size: int
+ """
+ if not ('_collocations' in self.__dict__ and self._num == num and self._window_size == window_size):
+ self._num = num
+ self._window_size = window_size
+
+ print("Building collocations list")
+ from nltk.corpus import stopwords
+ ignored_words = stopwords.words('english')
+ finder = BigramCollocationFinder.from_words(self.tokens, window_size)
+ finder.apply_freq_filter(2)
+ finder.apply_word_filter(lambda w: len(w) < 3 or w.lower() in ignored_words)
+ bigram_measures = BigramAssocMeasures()
+ self._collocations = finder.nbest(bigram_measures.likelihood_ratio, num)
+ colloc_strings = [w1+' '+w2 for w1, w2 in self._collocations]
+ print(tokenwrap(colloc_strings, separator="; "))
+
+ def count(self, word):
+ """
+ Count the number of times this word appears in the text.
+ """
+ return self.tokens.count(word)
+
+ def index(self, word):
+ """
+ Find the index of the first occurrence of the word in the text.
+ """
+ return self.tokens.index(word)
+
+ def readability(self, method):
+ # code from nltk_contrib.readability
+ raise NotImplementedError
+
+ def similar(self, word, num=20):
+ """
+ Distributional similarity: find other words which appear in the
+ same contexts as the specified word; list most similar words first.
+
+ :param word: The word used to seed the similarity search
+ :type word: str
+ :param num: The number of words to generate (default=20)
+ :type num: int
+ :seealso: ContextIndex.similar_words()
+ """
+ if '_word_context_index' not in self.__dict__:
+ print('Building word-context index...')
+ self._word_context_index = ContextIndex(self.tokens,
+ filter=lambda x:x.isalpha(),
+ key=lambda s:s.lower())
+
+# words = self._word_context_index.similar_words(word, num)
+
+ word = word.lower()
+ wci = self._word_context_index._word_to_contexts
+ if word in wci.conditions():
+ contexts = set(wci[word])
+ fd = FreqDist(w for w in wci.conditions() for c in wci[w]
+ if c in contexts and not w == word)
+ words = islice(fd.keys(), num)
+ print(tokenwrap(words))
+ else:
+ print("No matches")
+
+
+ def common_contexts(self, words, num=20):
+ """
+ Find contexts where the specified words appear; list
+ most frequent common contexts first.
+
+ :param word: The word used to seed the similarity search
+ :type word: str
+ :param num: The number of words to generate (default=20)
+ :type num: int
+ :seealso: ContextIndex.common_contexts()
+ """
+ if '_word_context_index' not in self.__dict__:
+ print('Building word-context index...')
+ self._word_context_index = ContextIndex(self.tokens,
+ key=lambda s:s.lower())
+
+ try:
+ fd = self._word_context_index.common_contexts(words, True)
+ if not fd:
+ print("No common contexts were found")
+ else:
+ ranked_contexts = islice(fd.keys(), num)
+ print(tokenwrap(w1+"_"+w2 for w1,w2 in ranked_contexts))
+
+ except ValueError as e:
+ print(e)
+
+ def dispersion_plot(self, words):
+ """
+ Produce a plot showing the distribution of the words through the text.
+ Requires pylab to be installed.
+
+ :param words: The words to be plotted
+ :type word: str
+ :seealso: nltk.draw.dispersion_plot()
+ """
+ from nltk.draw import dispersion_plot
+ dispersion_plot(self, words)
+
+ def plot(self, *args):
+ """
+ See documentation for FreqDist.plot()
+ :seealso: nltk.prob.FreqDist.plot()
+ """
+ self.vocab().plot(*args)
+
+ def vocab(self):
+ """
+ :seealso: nltk.prob.FreqDist
+ """
+ if "_vocab" not in self.__dict__:
+ print("Building vocabulary index...")
+ self._vocab = FreqDist(self)
+ return self._vocab
+
+ def findall(self, regexp):
+ """
+ Find instances of the regular expression in the text.
+ The text is a list of tokens, and a regexp pattern to match
+ a single token must be surrounded by angle brackets. E.g.
+
+ >>> print('hack'); from nltk.book import text1, text5, text9
+ hack...
+ >>> text5.findall("<.*><.*><bro>")
+ you rule bro; telling you bro; u twizted bro
+ >>> text1.findall("<a>(<.*>)<man>")
+ monied; nervous; dangerous; white; white; white; pious; queer; good;
+ mature; white; Cape; great; wise; wise; butterless; white; fiendish;
+ pale; furious; better; certain; complete; dismasted; younger; brave;
+ brave; brave; brave
+ >>> text9.findall("<th.*>{3,}")
+ thread through those; the thought that; that the thing; the thing
+ that; that that thing; through these than through; them that the;
+ through the thick; them that they; thought that the
+
+ :param regexp: A regular expression
+ :type regexp: str
+ """
+
+ if "_token_searcher" not in self.__dict__:
+ self._token_searcher = TokenSearcher(self)
+
+ hits = self._token_searcher.findall(regexp)
+ hits = [' '.join(h) for h in hits]
+ print(tokenwrap(hits, "; "))
+
+ #////////////////////////////////////////////////////////////
+ # Helper Methods
+ #////////////////////////////////////////////////////////////
+
+ _CONTEXT_RE = re.compile('\w+|[\.\!\?]')
+ def _context(self, tokens, i):
+ """
+ One left & one right token, both case-normalized. Skip over
+ non-sentence-final punctuation. Used by the ``ContextIndex``
+ that is created for ``similar()`` and ``common_contexts()``.
+ """
+ # Left context
+ j = i-1
+ while j>=0 and not self._CONTEXT_RE.match(tokens[j]):
+ j -= 1
+ left = (tokens[j] if j != 0 else '*START*')
+
+ # Right context
+ j = i+1
+ while j<len(tokens) and not self._CONTEXT_RE.match(tokens[j]):
+ j += 1
+ right = (tokens[j] if j != len(tokens) else '*END*')
+
+ return (left, right)
+
+ #////////////////////////////////////////////////////////////
+ # String Display
+ #////////////////////////////////////////////////////////////
+
+ def __str__(self):
+ return '<Text: %s>' % self.name
+
+ def __repr__(self):
+ return '<Text: %s>' % self.name
+
+
+# Prototype only; this approach will be slow to load
+class TextCollection(Text):
+ """A collection of texts, which can be loaded with list of texts, or
+ with a corpus consisting of one or more texts, and which supports
+ counting, concordancing, collocation discovery, etc. Initialize a
+ TextCollection as follows:
+
+ >>> import nltk.corpus
+ >>> from nltk.text import TextCollection
+ >>> print('hack'); from nltk.book import text1, text2, text3
+ hack...
+ >>> gutenberg = TextCollection(nltk.corpus.gutenberg)
+ >>> mytexts = TextCollection([text1, text2, text3])
+
+ Iterating over a TextCollection produces all the tokens of all the
+ texts in order.
+ """
+ def __init__(self, source, name=None):
+ if hasattr(source, 'words'): # bridge to the text corpus reader
+ source = [source.words(f) for f in source.fileids()]
+
+ self._texts = source
+ Text.__init__(self, LazyConcatenation(source))
+ self._idf_cache = {}
+
+ def tf(self, term, text, method=None):
+ """ The frequency of the term in text. """
+ return text.count(term) / len(text)
+
+ def idf(self, term, method=None):
+ """ The number of texts in the corpus divided by the
+ number of texts that the term appears in.
+ If a term does not appear in the corpus, 0.0 is returned. """
+ # idf values are cached for performance.
+ idf = self._idf_cache.get(term)
+ if idf is None:
+ matches = len([True for text in self._texts if term in text])
+ # FIXME Should this raise some kind of error instead?
+ idf = (log(float(len(self._texts)) / matches) if matches else 0.0)
+ self._idf_cache[term] = idf
+ return idf
+
+ def tf_idf(self, term, text):
+ return self.tf(term, text) * self.idf(term)
+
+def demo():
+ from nltk.corpus import brown
+ text = Text(brown.words(categories='news'))
+ print(text)
+ print()
+ print("Concordance:")
+ text.concordance('news')
+ print()
+ print("Distributionally similar words:")
+ text.similar('news')
+ print()
+ print("Collocations:")
+ text.collocations()
+ print()
+ print("Automatically generated text:")
+ text.generate()
+ print()
+ print("Dispersion plot:")
+ text.dispersion_plot(['news', 'report', 'said', 'announced'])
+ print()
+ print("Vocabulary plot:")
+ text.plot(50)
+ print()
+ print("Indexing:")
+ print("text[3]:", text[3])
+ print("text[3:5]:", text[3:5])
+ print("text.vocab()['news']:", text.vocab()['news'])
+
+if __name__ == '__main__':
+ demo()
+
+__all__ = ["ContextIndex",
+ "ConcordanceIndex",
+ "TokenSearcher",
+ "Text",
+ "TextCollection"]
diff --git a/nltk/tokenize/__init__.py b/nltk/tokenize/__init__.py
new file mode 100644
index 0000000..b360bc4
--- /dev/null
+++ b/nltk/tokenize/__init__.py
@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Tokenizers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com> (minor additions)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+r"""
+NLTK Tokenizer Package
+
+Tokenizers divide strings into lists of substrings. For example,
+tokenizers can be used to find the words and punctuation in a string:
+
+ >>> from nltk.tokenize import word_tokenize
+ >>> s = '''Good muffins cost $3.88\nin New York. Please buy me
+ ... two of them.\n\nThanks.'''
+ >>> word_tokenize(s)
+ ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.',
+ 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+
+This particular tokenizer requires the Punkt sentence tokenization
+models to be installed. NLTK also provides a simpler,
+regular-expression based tokenizer, which splits text on whitespace
+and punctuation:
+
+ >>> from nltk.tokenize import wordpunct_tokenize
+ >>> wordpunct_tokenize(s)
+ ['Good', 'muffins', 'cost', '$', '3', '.', '88', 'in', 'New', 'York', '.',
+ 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+
+We can also operate at the level of sentences, using the sentence
+tokenizer directly as follows:
+
+ >>> from nltk.tokenize import sent_tokenize, word_tokenize
+ >>> sent_tokenize(s)
+ ['Good muffins cost $3.88\nin New York.', 'Please buy me\ntwo of them.', 'Thanks.']
+ >>> [word_tokenize(t) for t in sent_tokenize(s)]
+ [['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.'],
+ ['Please', 'buy', 'me', 'two', 'of', 'them', '.'], ['Thanks', '.']]
+
+Caution: when tokenizing a Unicode string, make sure you are not
+using an encoded version of the string (it may be necessary to
+decode it first, e.g. with ``s.decode("utf8")``.
+
+NLTK tokenizers can produce token-spans, represented as tuples of integers
+having the same semantics as string slices, to support efficient comparison
+of tokenizers. (These methods are implemented as generators.)
+
+ >>> from nltk.tokenize import WhitespaceTokenizer
+ >>> list(WhitespaceTokenizer().span_tokenize(s))
+ [(0, 4), (5, 12), (13, 17), (18, 23), (24, 26), (27, 30), (31, 36), (38, 44),
+ (45, 48), (49, 51), (52, 55), (56, 58), (59, 64), (66, 73)]
+
+There are numerous ways to tokenize text. If you need more control over
+tokenization, see the other methods provided in this package.
+
+For further information, please see Chapter 3 of the NLTK book.
+"""
+
+from nltk.data import load
+from nltk.tokenize.simple import (SpaceTokenizer, TabTokenizer, LineTokenizer,
+ line_tokenize)
+from nltk.tokenize.regexp import (RegexpTokenizer, WhitespaceTokenizer,
+ BlanklineTokenizer, WordPunctTokenizer,
+ wordpunct_tokenize, regexp_tokenize,
+ blankline_tokenize)
+from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktWordTokenizer
+from nltk.tokenize.sexpr import SExprTokenizer, sexpr_tokenize
+from nltk.tokenize.treebank import TreebankWordTokenizer
+from nltk.tokenize.texttiling import TextTilingTokenizer
+
+# Standard sentence tokenizer.
+def sent_tokenize(text):
+ """
+ Return a sentence-tokenized copy of *text*,
+ using NLTK's recommended sentence tokenizer
+ (currently :class:`.PunktSentenceTokenizer`).
+ """
+ tokenizer = load('tokenizers/punkt/english.pickle')
+ return tokenizer.tokenize(text)
+
+# Standard word tokenizer.
+_treebank_word_tokenize = TreebankWordTokenizer().tokenize
+def word_tokenize(text):
+ """
+ Return a tokenized copy of *text*,
+ using NLTK's recommended word tokenizer
+ (currently :class:`.TreebankWordTokenizer`
+ along with :class:`.PunktSentenceTokenizer`).
+ """
+ return [token for sent in sent_tokenize(text)
+ for token in _treebank_word_tokenize(sent)]
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tokenize/api.py b/nltk/tokenize/api.py
new file mode 100644
index 0000000..5f4d965
--- /dev/null
+++ b/nltk/tokenize/api.py
@@ -0,0 +1,78 @@
+# Natural Language Toolkit: Tokenizer Interface
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Tokenizer Interface
+"""
+
+from nltk.internals import overridden
+from nltk.tokenize.util import string_span_tokenize
+
+class TokenizerI(object):
+ """
+ A processing interface for tokenizing a string.
+ Subclasses must define ``tokenize()`` or ``tokenize_sents()`` (or both).
+ """
+ def tokenize(self, s):
+ """
+ Return a tokenized copy of *s*.
+
+ :rtype: list of str
+ """
+ if overridden(self.tokenize_sents):
+ return self.tokenize_sents([s])[0]
+ else:
+ raise NotImplementedError()
+
+ def span_tokenize(self, s):
+ """
+ Identify the tokens using integer offsets ``(start_i, end_i)``,
+ where ``s[start_i:end_i]`` is the corresponding token.
+
+ :rtype: iter(tuple(int, int))
+ """
+ raise NotImplementedError()
+
+ def tokenize_sents(self, strings):
+ """
+ Apply ``self.tokenize()`` to each element of ``strings``. I.e.:
+
+ return [self.tokenize(s) for s in strings]
+
+ :rtype: list(list(str))
+ """
+ return [self.tokenize(s) for s in strings]
+
+ def span_tokenize_sents(self, strings):
+ """
+ Apply ``self.span_tokenize()`` to each element of ``strings``. I.e.:
+
+ return [self.span_tokenize(s) for s in strings]
+
+ :rtype: iter(list(tuple(int, int)))
+ """
+ for s in strings:
+ yield list(self.span_tokenize(s))
+
+
+class StringTokenizer(TokenizerI):
+ """A tokenizer that divides a string into substrings by splitting
+ on the specified string (defined in subclasses).
+ """
+
+ def tokenize(self, s):
+ return s.split(self._string)
+
+ def span_tokenize(self, s):
+ for span in string_span_tokenize(s, self._string):
+ yield span
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tokenize/punkt.py b/nltk/tokenize/punkt.py
new file mode 100644
index 0000000..211a67e
--- /dev/null
+++ b/nltk/tokenize/punkt.py
@@ -0,0 +1,1648 @@
+# Natural Language Toolkit: Punkt sentence tokenizer
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Algorithm: Kiss & Strunk (2006)
+# Author: Willy <willy at csse.unimelb.edu.au> (original Python port)
+# Steven Bird <stevenbird1 at gmail.com> (additions)
+# Edward Loper <edloper at gmail.com> (rewrite)
+# Joel Nothman <jnothman at student.usyd.edu.au> (almost rewrite)
+# Arthur Darcet <arthur at darcet.fr> (fixes)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+r"""
+Punkt Sentence Tokenizer
+
+This tokenizer divides a text into a list of sentences,
+by using an unsupervised algorithm to build a model for abbreviation
+words, collocations, and words that start sentences. It must be
+trained on a large collection of plaintext in the target language
+before it can be used.
+
+The NLTK data package includes a pre-trained Punkt tokenizer for
+English.
+
+ >>> import nltk.data
+ >>> text = '''
+ ... Punkt knows that the periods in Mr. Smith and Johann S. Bach
+ ... do not mark sentence boundaries. And sometimes sentences
+ ... can start with non-capitalized words. i is a good variable
+ ... name.
+ ... '''
+ >>> sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')
+ >>> print('\n-----\n'.join(sent_detector.tokenize(text.strip())))
+ Punkt knows that the periods in Mr. Smith and Johann S. Bach
+ do not mark sentence boundaries.
+ -----
+ And sometimes sentences
+ can start with non-capitalized words.
+ -----
+ i is a good variable
+ name.
+
+(Note that whitespace from the original text, including newlines, is
+retained in the output.)
+
+Punctuation following sentences is also included by default
+(from NLTK 3.0 onwards). It can be excluded with the realign_boundaries
+flag.
+
+ >>> text = '''
+ ... (How does it deal with this parenthesis?) "It should be part of the
+ ... previous sentence." "(And the same with this one.)" ('And this one!')
+ ... "('(And (this)) '?)" [(and this. )]
+ ... '''
+ >>> print('\n-----\n'.join(
+ ... sent_detector.tokenize(text.strip())))
+ (How does it deal with this parenthesis?)
+ -----
+ "It should be part of the
+ previous sentence."
+ -----
+ "(And the same with this one.)"
+ -----
+ ('And this one!')
+ -----
+ "('(And (this)) '?)"
+ -----
+ [(and this. )]
+ >>> print('\n-----\n'.join(
+ ... sent_detector.tokenize(text.strip(), realign_boundaries=False)))
+ (How does it deal with this parenthesis?
+ -----
+ ) "It should be part of the
+ previous sentence.
+ -----
+ " "(And the same with this one.
+ -----
+ )" ('And this one!
+ -----
+ ')
+ "('(And (this)) '?
+ -----
+ )" [(and this.
+ -----
+ )]
+
+However, Punkt is designed to learn parameters (a list of abbreviations, etc.)
+unsupervised from a corpus similar to the target domain. The pre-packaged models
+may therefore be unsuitable: use ``PunktSentenceTokenizer(text)`` to learn
+parameters from the given text.
+
+:class:`.PunktTrainer` learns parameters such as a list of abbreviations
+(without supervision) from portions of text. Using a ``PunktTrainer`` directly
+allows for incremental training and modification of the hyper-parameters used
+to decide what is considered an abbreviation, etc.
+
+:class:`.PunktWordTokenizer` uses a regular expression to divide a text into tokens,
+leaving all periods attached to words, but separating off other punctuation:
+
+ >>> from nltk.tokenize.punkt import PunktWordTokenizer
+ >>> s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\n\nThanks."
+ >>> PunktWordTokenizer().tokenize(s)
+ ['Good', 'muffins', 'cost', '$3.88', 'in', 'New', 'York.', 'Please',
+ 'buy', 'me', 'two', 'of', 'them.', 'Thanks.']
+ >>> PunktWordTokenizer().span_tokenize(s)
+ [(0, 4), (5, 12), (13, 17), (18, 23), (24, 26), (27, 30), (31, 36), (38, 44),
+ (45, 48), (49, 51), (52, 55), (56, 58), (59, 64), (66, 73)]
+
+The algorithm for this tokenizer is described in::
+
+ Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence
+ Boundary Detection. Computational Linguistics 32: 485-525.
+"""
+from __future__ import print_function, unicode_literals
+
+# TODO: Make orthographic heuristic less susceptible to overtraining
+# TODO: Frequent sentence starters optionally exclude always-capitalised words
+# FIXME: Problem with ending string with e.g. '!!!' -> '!! !'
+
+import re
+import math
+from collections import defaultdict
+
+from nltk.compat import unicode_repr, python_2_unicode_compatible, string_types
+from nltk.probability import FreqDist
+from nltk.tokenize.api import TokenizerI
+
+######################################################################
+#{ Orthographic Context Constants
+######################################################################
+# The following constants are used to describe the orthographic
+# contexts in which a word can occur. BEG=beginning, MID=middle,
+# UNK=unknown, UC=uppercase, LC=lowercase, NC=no case.
+
+_ORTHO_BEG_UC = 1 << 1
+"""Orthographic context: beginning of a sentence with upper case."""
+
+_ORTHO_MID_UC = 1 << 2
+"""Orthographic context: middle of a sentence with upper case."""
+
+_ORTHO_UNK_UC = 1 << 3
+"""Orthographic context: unknown position in a sentence with upper case."""
+
+_ORTHO_BEG_LC = 1 << 4
+"""Orthographic context: beginning of a sentence with lower case."""
+
+_ORTHO_MID_LC = 1 << 5
+"""Orthographic context: middle of a sentence with lower case."""
+
+_ORTHO_UNK_LC = 1 << 6
+"""Orthographic context: unknown position in a sentence with lower case."""
+
+_ORTHO_UC = _ORTHO_BEG_UC + _ORTHO_MID_UC + _ORTHO_UNK_UC
+"""Orthographic context: occurs with upper case."""
+
+_ORTHO_LC = _ORTHO_BEG_LC + _ORTHO_MID_LC + _ORTHO_UNK_LC
+"""Orthographic context: occurs with lower case."""
+
+_ORTHO_MAP = {
+ ('initial', 'upper'): _ORTHO_BEG_UC,
+ ('internal', 'upper'): _ORTHO_MID_UC,
+ ('unknown', 'upper'): _ORTHO_UNK_UC,
+ ('initial', 'lower'): _ORTHO_BEG_LC,
+ ('internal', 'lower'): _ORTHO_MID_LC,
+ ('unknown', 'lower'): _ORTHO_UNK_LC,
+}
+"""A map from context position and first-letter case to the
+appropriate orthographic context flag."""
+
+#} (end orthographic context constants)
+######################################################################
+
+######################################################################
+#{ Decision reasons for debugging
+######################################################################
+
+REASON_DEFAULT_DECISION = 'default decision'
+REASON_KNOWN_COLLOCATION = 'known collocation (both words)'
+REASON_ABBR_WITH_ORTHOGRAPHIC_HEURISTIC = 'abbreviation + orthographic heuristic'
+REASON_ABBR_WITH_SENTENCE_STARTER = 'abbreviation + frequent sentence starter'
+REASON_INITIAL_WITH_ORTHOGRAPHIC_HEURISTIC = 'initial + orthographic heuristic'
+REASON_NUMBER_WITH_ORTHOGRAPHIC_HEURISTIC = 'initial + orthographic heuristic'
+REASON_INITIAL_WITH_SPECIAL_ORTHOGRAPHIC_HEURISTIC = 'initial + special orthographic heuristic'
+
+#} (end decision reasons for debugging)
+######################################################################
+
+######################################################################
+#{ Language-dependent variables
+######################################################################
+
+class PunktLanguageVars(object):
+ """
+ Stores variables, mostly regular expressions, which may be
+ language-dependent for correct application of the algorithm.
+ An extension of this class may modify its properties to suit
+ a language other than English; an instance can then be passed
+ as an argument to PunktSentenceTokenizer and PunktTrainer
+ constructors.
+ """
+
+ __slots__ = ('_re_period_context', '_re_word_tokenizer')
+
+ def __getstate__(self):
+ # All modifications to the class are performed by inheritance.
+ # Non-default parameters to be pickled must be defined in the inherited
+ # class.
+ return 1
+
+ def __setstate__(self, state):
+ return 1
+
+ sent_end_chars = ('.', '?', '!')
+ """Characters which are candidates for sentence boundaries"""
+
+ @property
+ def _re_sent_end_chars(self):
+ return '[%s]' % re.escape(''.join(self.sent_end_chars))
+
+ internal_punctuation = ',:;' # might want to extend this..
+ """sentence internal punctuation, which indicates an abbreviation if
+ preceded by a period-final token."""
+
+ re_boundary_realignment = re.compile(r'["\')\]}]+?(?:\s+|(?=--)|$)',
+ re.MULTILINE)
+ """Used to realign punctuation that should be included in a sentence
+ although it follows the period (or ?, !)."""
+
+ _re_word_start = r"[^\(\"\`{\[:;&\#\*@\)}\]\-,]"
+ """Excludes some characters from starting word tokens"""
+
+ _re_non_word_chars = r"(?:[?!)\";}\]\*:@\'\({\[])"
+ """Characters that cannot appear within words"""
+
+ _re_multi_char_punct = r"(?:\-{2,}|\.{2,}|(?:\.\s){2,}\.)"
+ """Hyphen and ellipsis are multi-character punctuation"""
+
+ _word_tokenize_fmt = r'''(
+ %(MultiChar)s
+ |
+ (?=%(WordStart)s)\S+? # Accept word characters until end is found
+ (?= # Sequences marking a word's end
+ \s| # White-space
+ $| # End-of-string
+ %(NonWord)s|%(MultiChar)s| # Punctuation
+ ,(?=$|\s|%(NonWord)s|%(MultiChar)s) # Comma if at end of word
+ )
+ |
+ \S
+ )'''
+ """Format of a regular expression to split punctuation from words,
+ excluding period."""
+
+ def _word_tokenizer_re(self):
+ """Compiles and returns a regular expression for word tokenization"""
+ try:
+ return self._re_word_tokenizer
+ except AttributeError:
+ self._re_word_tokenizer = re.compile(
+ self._word_tokenize_fmt %
+ {
+ 'NonWord': self._re_non_word_chars,
+ 'MultiChar': self._re_multi_char_punct,
+ 'WordStart': self._re_word_start,
+ },
+ re.UNICODE | re.VERBOSE
+ )
+ return self._re_word_tokenizer
+
+ def word_tokenize(self, s):
+ """Tokenize a string to split off punctuation other than periods"""
+ return self._word_tokenizer_re().findall(s)
+
+ _period_context_fmt = r"""
+ \S* # some word material
+ %(SentEndChars)s # a potential sentence ending
+ (?=(?P<after_tok>
+ %(NonWord)s # either other punctuation
+ |
+ \s+(?P<next_tok>\S+) # or whitespace and some other token
+ ))"""
+ """Format of a regular expression to find contexts including possible
+ sentence boundaries. Matches token which the possible sentence boundary
+ ends, and matches the following token within a lookahead expression."""
+
+ def period_context_re(self):
+ """Compiles and returns a regular expression to find contexts
+ including possible sentence boundaries."""
+ try:
+ return self._re_period_context
+ except:
+ self._re_period_context = re.compile(
+ self._period_context_fmt %
+ {
+ 'NonWord': self._re_non_word_chars,
+ 'SentEndChars': self._re_sent_end_chars,
+ },
+ re.UNICODE | re.VERBOSE)
+ return self._re_period_context
+
+
+_re_non_punct = re.compile(r'[^\W\d]', re.UNICODE)
+"""Matches token types that are not merely punctuation. (Types for
+numeric tokens are changed to ##number## and hence contain alpha.)"""
+
+#}
+######################################################################
+
+
+######################################################################
+#{ Punkt Word Tokenizer
+######################################################################
+
+class PunktWordTokenizer(TokenizerI):
+ # Retained for backward compatibility
+ def __init__(self, lang_vars=PunktLanguageVars()):
+ self._lang_vars = lang_vars
+
+ def tokenize(self, text):
+ return self._lang_vars.word_tokenize(text)
+
+ def span_tokenize(self, text):
+ """
+ Given a text, returns a list of the (start, end) spans of words
+ in the text.
+ """
+ return [(sl.start, sl.stop) for sl in self._slices_from_text(text)]
+
+ def _slices_from_text(self, text):
+ last_break = 0
+ contains_no_words = True
+ for match in self._lang_vars._word_tokenizer_re().finditer(text):
+ contains_no_words = False
+ context = match.group()
+ yield slice(match.start(), match.end())
+ if contains_no_words:
+ yield slice(0, 0) # matches PunktSentenceTokenizer's functionality
+
+#}
+######################################################################
+
+
+#////////////////////////////////////////////////////////////
+#{ Helper Functions
+#////////////////////////////////////////////////////////////
+
+def _pair_iter(it):
+ """
+ Yields pairs of tokens from the given iterator such that each input
+ token will appear as the first element in a yielded tuple. The last
+ pair will have None as its second element.
+ """
+ it = iter(it)
+ prev = next(it)
+ for el in it:
+ yield (prev, el)
+ prev = el
+ yield (prev, None)
+
+######################################################################
+#{ Punkt Parameters
+######################################################################
+
+class PunktParameters(object):
+ """Stores data used to perform sentence boundary detection with Punkt."""
+
+ def __init__(self):
+ self.abbrev_types = set()
+ """A set of word types for known abbreviations."""
+
+ self.collocations = set()
+ """A set of word type tuples for known common collocations
+ where the first word ends in a period. E.g., ('S.', 'Bach')
+ is a common collocation in a text that discusses 'Johann
+ S. Bach'. These count as negative evidence for sentence
+ boundaries."""
+
+ self.sent_starters = set()
+ """A set of word types for words that often appear at the
+ beginning of sentences."""
+
+ self.ortho_context = defaultdict(int)
+ """A dictionary mapping word types to the set of orthographic
+ contexts that word type appears in. Contexts are represented
+ by adding orthographic context flags: ..."""
+
+ def clear_abbrevs(self):
+ self.abbrev_types = set()
+
+ def clear_collocations(self):
+ self.collocations = set()
+
+ def clear_sent_starters(self):
+ self.sent_starters = set()
+
+ def clear_ortho_context(self):
+ self.ortho_context = defaultdict(int)
+
+ def add_ortho_context(self, typ, flag):
+ self.ortho_context[typ] |= flag
+
+ def _debug_ortho_context(self, typ):
+ c = self.ortho_context[typ]
+ if c & _ORTHO_BEG_UC:
+ yield 'BEG-UC'
+ if c & _ORTHO_MID_UC:
+ yield 'MID-UC'
+ if c & _ORTHO_UNK_UC:
+ yield 'UNK-UC'
+ if c & _ORTHO_BEG_LC:
+ yield 'BEG-LC'
+ if c & _ORTHO_MID_LC:
+ yield 'MID-LC'
+ if c & _ORTHO_UNK_LC:
+ yield 'UNK-LC'
+
+######################################################################
+#{ PunktToken
+######################################################################
+
+ at python_2_unicode_compatible
+class PunktToken(object):
+ """Stores a token of text with annotations produced during
+ sentence boundary detection."""
+
+ _properties = [
+ 'parastart', 'linestart',
+ 'sentbreak', 'abbr', 'ellipsis'
+ ]
+ __slots__ = ['tok', 'type', 'period_final'] + _properties
+
+ def __init__(self, tok, **params):
+ self.tok = tok
+ self.type = self._get_type(tok)
+ self.period_final = tok.endswith('.')
+
+ for p in self._properties:
+ setattr(self, p, None)
+ for k in params:
+ setattr(self, k, params[k])
+
+ #////////////////////////////////////////////////////////////
+ #{ Regular expressions for properties
+ #////////////////////////////////////////////////////////////
+ # Note: [A-Za-z] is approximated by [^\W\d] in the general case.
+ _RE_ELLIPSIS = re.compile(r'\.\.+$')
+ _RE_NUMERIC = re.compile(r'^-?[\.,]?\d[\d,\.-]*\.?$')
+ _RE_INITIAL = re.compile(r'[^\W\d]\.$', re.UNICODE)
+ _RE_ALPHA = re.compile(r'[^\W\d]+$', re.UNICODE)
+
+ #////////////////////////////////////////////////////////////
+ #{ Derived properties
+ #////////////////////////////////////////////////////////////
+
+ def _get_type(self, tok):
+ """Returns a case-normalized representation of the token."""
+ return self._RE_NUMERIC.sub('##number##', tok.lower())
+
+ @property
+ def type_no_period(self):
+ """
+ The type with its final period removed if it has one.
+ """
+ if len(self.type) > 1 and self.type[-1] == '.':
+ return self.type[:-1]
+ return self.type
+
+ @property
+ def type_no_sentperiod(self):
+ """
+ The type with its final period removed if it is marked as a
+ sentence break.
+ """
+ if self.sentbreak:
+ return self.type_no_period
+ return self.type
+
+ @property
+ def first_upper(self):
+ """True if the token's first character is uppercase."""
+ return self.tok[0].isupper()
+
+ @property
+ def first_lower(self):
+ """True if the token's first character is lowercase."""
+ return self.tok[0].islower()
+
+ @property
+ def first_case(self):
+ if self.first_lower:
+ return 'lower'
+ elif self.first_upper:
+ return 'upper'
+ return 'none'
+
+ @property
+ def is_ellipsis(self):
+ """True if the token text is that of an ellipsis."""
+ return self._RE_ELLIPSIS.match(self.tok)
+
+ @property
+ def is_number(self):
+ """True if the token text is that of a number."""
+ return self.type.startswith('##number##')
+
+ @property
+ def is_initial(self):
+ """True if the token text is that of an initial."""
+ return self._RE_INITIAL.match(self.tok)
+
+ @property
+ def is_alpha(self):
+ """True if the token text is all alphabetic."""
+ return self._RE_ALPHA.match(self.tok)
+
+ @property
+ def is_non_punct(self):
+ """True if the token is either a number or is alphabetic."""
+ return _re_non_punct.search(self.type)
+
+ #////////////////////////////////////////////////////////////
+ #{ String representation
+ #////////////////////////////////////////////////////////////
+
+ def __repr__(self):
+ """
+ A string representation of the token that can reproduce it
+ with eval(), which lists all the token's non-default
+ annotations.
+ """
+ typestr = (' type=%s,' % unicode_repr(self.type)
+ if self.type != self.tok else '')
+
+ propvals = ', '.join(
+ '%s=%s' % (p, unicode_repr(getattr(self, p)))
+ for p in self._properties
+ if getattr(self, p)
+ )
+
+ return '%s(%s,%s %s)' % (self.__class__.__name__,
+ unicode_repr(self.tok), typestr, propvals)
+
+ def __str__(self):
+ """
+ A string representation akin to that used by Kiss and Strunk.
+ """
+ res = self.tok
+ if self.abbr:
+ res += '<A>'
+ if self.ellipsis:
+ res += '<E>'
+ if self.sentbreak:
+ res += '<S>'
+ return res
+
+######################################################################
+#{ Punkt base class
+######################################################################
+
+class PunktBaseClass(object):
+ """
+ Includes common components of PunktTrainer and PunktSentenceTokenizer.
+ """
+
+ def __init__(self, lang_vars=PunktLanguageVars(), token_cls=PunktToken,
+ params=PunktParameters()):
+ self._params = params
+ self._lang_vars = lang_vars
+ self._Token = token_cls
+ """The collection of parameters that determines the behavior
+ of the punkt tokenizer."""
+
+ #////////////////////////////////////////////////////////////
+ #{ Word tokenization
+ #////////////////////////////////////////////////////////////
+
+ def _tokenize_words(self, plaintext):
+ """
+ Divide the given text into tokens, using the punkt word
+ segmentation regular expression, and generate the resulting list
+ of tokens augmented as three-tuples with two boolean values for whether
+ the given token occurs at the start of a paragraph or a new line,
+ respectively.
+ """
+ parastart = False
+ for line in plaintext.split('\n'):
+ if line.strip():
+ line_toks = iter(self._lang_vars.word_tokenize(line))
+
+ yield self._Token(next(line_toks),
+ parastart=parastart, linestart=True)
+ parastart = False
+
+ for t in line_toks:
+ yield self._Token(t)
+ else:
+ parastart = True
+
+
+ #////////////////////////////////////////////////////////////
+ #{ Annotation Procedures
+ #////////////////////////////////////////////////////////////
+
+ def _annotate_first_pass(self, tokens):
+ """
+ Perform the first pass of annotation, which makes decisions
+ based purely based on the word type of each word:
+
+ - '?', '!', and '.' are marked as sentence breaks.
+ - sequences of two or more periods are marked as ellipsis.
+ - any word ending in '.' that's a known abbreviation is
+ marked as an abbreviation.
+ - any other word ending in '.' is marked as a sentence break.
+
+ Return these annotations as a tuple of three sets:
+
+ - sentbreak_toks: The indices of all sentence breaks.
+ - abbrev_toks: The indices of all abbreviations.
+ - ellipsis_toks: The indices of all ellipsis marks.
+ """
+ for aug_tok in tokens:
+ self._first_pass_annotation(aug_tok)
+ yield aug_tok
+
+ def _first_pass_annotation(self, aug_tok):
+ """
+ Performs type-based annotation on a single token.
+ """
+
+ tok = aug_tok.tok
+
+ if tok in self._lang_vars.sent_end_chars:
+ aug_tok.sentbreak = True
+ elif aug_tok.is_ellipsis:
+ aug_tok.ellipsis = True
+ elif aug_tok.period_final and not tok.endswith('..'):
+ if (tok[:-1].lower() in self._params.abbrev_types or
+ tok[:-1].lower().split('-')[-1] in self._params.abbrev_types):
+
+ aug_tok.abbr = True
+ else:
+ aug_tok.sentbreak = True
+
+ return
+
+######################################################################
+#{ Punkt Trainer
+######################################################################
+
+
+class PunktTrainer(PunktBaseClass):
+ """Learns parameters used in Punkt sentence boundary detection."""
+
+ def __init__(self, train_text=None, verbose=False,
+ lang_vars=PunktLanguageVars(), token_cls=PunktToken):
+
+ PunktBaseClass.__init__(self, lang_vars=lang_vars,
+ token_cls=token_cls)
+
+ self._type_fdist = FreqDist()
+ """A frequency distribution giving the frequency of each
+ case-normalized token type in the training data."""
+
+ self._num_period_toks = 0
+ """The number of words ending in period in the training data."""
+
+ self._collocation_fdist = FreqDist()
+ """A frequency distribution giving the frequency of all
+ bigrams in the training data where the first word ends in a
+ period. Bigrams are encoded as tuples of word types.
+ Especially common collocations are extracted from this
+ frequency distribution, and stored in
+ ``_params``.``collocations <PunktParameters.collocations>``."""
+
+ self._sent_starter_fdist = FreqDist()
+ """A frequency distribution giving the frequency of all words
+ that occur at the training data at the beginning of a sentence
+ (after the first pass of annotation). Especially common
+ sentence starters are extracted from this frequency
+ distribution, and stored in ``_params.sent_starters``.
+ """
+
+ self._sentbreak_count = 0
+ """The total number of sentence breaks identified in training, used for
+ calculating the frequent sentence starter heuristic."""
+
+ self._finalized = True
+ """A flag as to whether the training has been finalized by finding
+ collocations and sentence starters, or whether finalize_training()
+ still needs to be called."""
+
+ if train_text:
+ self.train(train_text, verbose, finalize=True)
+
+ def get_params(self):
+ """
+ Calculates and returns parameters for sentence boundary detection as
+ derived from training."""
+ if not self._finalized:
+ self.finalize_training()
+ return self._params
+
+ #////////////////////////////////////////////////////////////
+ #{ Customization Variables
+ #////////////////////////////////////////////////////////////
+
+ ABBREV = 0.3
+ """cut-off value whether a 'token' is an abbreviation"""
+
+ IGNORE_ABBREV_PENALTY = False
+ """allows the disabling of the abbreviation penalty heuristic, which
+ exponentially disadvantages words that are found at times without a
+ final period."""
+
+ ABBREV_BACKOFF = 5
+ """upper cut-off for Mikheev's(2002) abbreviation detection algorithm"""
+
+ COLLOCATION = 7.88
+ """minimal log-likelihood value that two tokens need to be considered
+ as a collocation"""
+
+ SENT_STARTER = 30
+ """minimal log-likelihood value that a token requires to be considered
+ as a frequent sentence starter"""
+
+ INCLUDE_ALL_COLLOCS = False
+ """this includes as potential collocations all word pairs where the first
+ word ends in a period. It may be useful in corpora where there is a lot
+ of variation that makes abbreviations like Mr difficult to identify."""
+
+ INCLUDE_ABBREV_COLLOCS = False
+ """this includes as potential collocations all word pairs where the first
+ word is an abbreviation. Such collocations override the orthographic
+ heuristic, but not the sentence starter heuristic. This is overridden by
+ INCLUDE_ALL_COLLOCS, and if both are false, only collocations with initials
+ and ordinals are considered."""
+ """"""
+
+ MIN_COLLOC_FREQ = 1
+ """this sets a minimum bound on the number of times a bigram needs to
+ appear before it can be considered a collocation, in addition to log
+ likelihood statistics. This is useful when INCLUDE_ALL_COLLOCS is True."""
+
+ #////////////////////////////////////////////////////////////
+ #{ Training..
+ #////////////////////////////////////////////////////////////
+
+ def train(self, text, verbose=False, finalize=True):
+ """
+ Collects training data from a given text. If finalize is True, it
+ will determine all the parameters for sentence boundary detection. If
+ not, this will be delayed until get_params() or finalize_training() is
+ called. If verbose is True, abbreviations found will be listed.
+ """
+ # Break the text into tokens; record which token indices correspond to
+ # line starts and paragraph starts; and determine their types.
+ self._train_tokens(self._tokenize_words(text), verbose)
+ if finalize:
+ self.finalize_training(verbose)
+
+ def train_tokens(self, tokens, verbose=False, finalize=True):
+ """
+ Collects training data from a given list of tokens.
+ """
+ self._train_tokens((self._Token(t) for t in tokens), verbose)
+ if finalize:
+ self.finalize_training(verbose)
+
+ def _train_tokens(self, tokens, verbose):
+ self._finalized = False
+
+ # Ensure tokens are a list
+ tokens = list(tokens)
+
+ # Find the frequency of each case-normalized type. (Don't
+ # strip off final periods.) Also keep track of the number of
+ # tokens that end in periods.
+ for aug_tok in tokens:
+ self._type_fdist[aug_tok.type] += 1
+ if aug_tok.period_final:
+ self._num_period_toks += 1
+
+ # Look for new abbreviations, and for types that no longer are
+ unique_types = self._unique_types(tokens)
+ for abbr, score, is_add in self._reclassify_abbrev_types(unique_types):
+ if score >= self.ABBREV:
+ if is_add:
+ self._params.abbrev_types.add(abbr)
+ if verbose:
+ print((' Abbreviation: [%6.4f] %s' %
+ (score, abbr)))
+ else:
+ if not is_add:
+ self._params.abbrev_types.remove(abbr)
+ if verbose:
+ print((' Removed abbreviation: [%6.4f] %s' %
+ (score, abbr)))
+
+ # Make a preliminary pass through the document, marking likely
+ # sentence breaks, abbreviations, and ellipsis tokens.
+ tokens = list(self._annotate_first_pass(tokens))
+
+ # Check what contexts each word type can appear in, given the
+ # case of its first letter.
+ self._get_orthography_data(tokens)
+
+ # We need total number of sentence breaks to find sentence starters
+ self._sentbreak_count += self._get_sentbreak_count(tokens)
+
+ # The remaining heuristics relate to pairs of tokens where the first
+ # ends in a period.
+ for aug_tok1, aug_tok2 in _pair_iter(tokens):
+ if not aug_tok1.period_final or not aug_tok2:
+ continue
+
+ # Is the first token a rare abbreviation?
+ if self._is_rare_abbrev_type(aug_tok1, aug_tok2):
+ self._params.abbrev_types.add(aug_tok1.type_no_period)
+ if verbose:
+ print((' Rare Abbrev: %s' % aug_tok1.type))
+
+ # Does second token have a high likelihood of starting a sentence?
+ if self._is_potential_sent_starter(aug_tok2, aug_tok1):
+ self._sent_starter_fdist[aug_tok2.type] += 1
+
+ # Is this bigram a potential collocation?
+ if self._is_potential_collocation(aug_tok1, aug_tok2):
+ self._collocation_fdist[
+ (aug_tok1.type_no_period, aug_tok2.type_no_sentperiod)] += 1
+
+ def _unique_types(self, tokens):
+ return set(aug_tok.type for aug_tok in tokens)
+
+ def finalize_training(self, verbose=False):
+ """
+ Uses data that has been gathered in training to determine likely
+ collocations and sentence starters.
+ """
+ self._params.clear_sent_starters()
+ for typ, ll in self._find_sent_starters():
+ self._params.sent_starters.add(typ)
+ if verbose:
+ print((' Sent Starter: [%6.4f] %r' % (ll, typ)))
+
+ self._params.clear_collocations()
+ for (typ1, typ2), ll in self._find_collocations():
+ self._params.collocations.add( (typ1,typ2) )
+ if verbose:
+ print((' Collocation: [%6.4f] %r+%r' %
+ (ll, typ1, typ2)))
+
+ self._finalized = True
+
+ #////////////////////////////////////////////////////////////
+ #{ Overhead reduction
+ #////////////////////////////////////////////////////////////
+
+ def freq_threshold(self, ortho_thresh=2, type_thresh=2, colloc_thres=2,
+ sentstart_thresh=2):
+ """
+ Allows memory use to be reduced after much training by removing data
+ about rare tokens that are unlikely to have a statistical effect with
+ further training. Entries occurring above the given thresholds will be
+ retained.
+ """
+ if ortho_thresh > 1:
+ old_oc = self._params.ortho_context
+ self._params.clear_ortho_context()
+ for tok in self._type_fdist:
+ count = self._type_fdist[tok]
+ if count >= ortho_thresh:
+ self._params.ortho_context[tok] = old_oc[tok]
+
+ self._type_fdist = self._freq_threshold(self._type_fdist, type_thresh)
+ self._collocation_fdist = self._freq_threshold(
+ self._collocation_fdist, colloc_thres)
+ self._sent_starter_fdist = self._freq_threshold(
+ self._sent_starter_fdist, sentstart_thresh)
+
+ def _freq_threshold(self, fdist, threshold):
+ """
+ Returns a FreqDist containing only data with counts below a given
+ threshold, as well as a mapping (None -> count_removed).
+ """
+ # We assume that there is more data below the threshold than above it
+ # and so create a new FreqDist rather than working in place.
+ res = FreqDist()
+ num_removed = 0
+ for tok in fdist:
+ count = fdist[tok]
+ if count < threshold:
+ num_removed += 1
+ else:
+ res[tok] += count
+ res[None] += num_removed
+ return res
+
+ #////////////////////////////////////////////////////////////
+ #{ Orthographic data
+ #////////////////////////////////////////////////////////////
+
+ def _get_orthography_data(self, tokens):
+ """
+ Collect information about whether each token type occurs
+ with different case patterns (i) overall, (ii) at
+ sentence-initial positions, and (iii) at sentence-internal
+ positions.
+ """
+ # 'initial' or 'internal' or 'unknown'
+ context = 'internal'
+ tokens = list(tokens)
+
+ for aug_tok in tokens:
+ # If we encounter a paragraph break, then it's a good sign
+ # that it's a sentence break. But err on the side of
+ # caution (by not positing a sentence break) if we just
+ # saw an abbreviation.
+ if aug_tok.parastart and context != 'unknown':
+ context = 'initial'
+
+ # If we're at the beginning of a line, then we can't decide
+ # between 'internal' and 'initial'.
+ if aug_tok.linestart and context == 'internal':
+ context = 'unknown'
+
+ # Find the case-normalized type of the token. If it's a
+ # sentence-final token, strip off the period.
+ typ = aug_tok.type_no_sentperiod
+
+ # Update the orthographic context table.
+ flag = _ORTHO_MAP.get((context, aug_tok.first_case), 0)
+ if flag:
+ self._params.add_ortho_context(typ, flag)
+
+ # Decide whether the next word is at a sentence boundary.
+ if aug_tok.sentbreak:
+ if not (aug_tok.is_number or aug_tok.is_initial):
+ context = 'initial'
+ else:
+ context = 'unknown'
+ elif aug_tok.ellipsis or aug_tok.abbr:
+ context = 'unknown'
+ else:
+ context = 'internal'
+
+ #////////////////////////////////////////////////////////////
+ #{ Abbreviations
+ #////////////////////////////////////////////////////////////
+
+ def _reclassify_abbrev_types(self, types):
+ """
+ (Re)classifies each given token if
+ - it is period-final and not a known abbreviation; or
+ - it is not period-final and is otherwise a known abbreviation
+ by checking whether its previous classification still holds according
+ to the heuristics of section 3.
+ Yields triples (abbr, score, is_add) where abbr is the type in question,
+ score is its log-likelihood with penalties applied, and is_add specifies
+ whether the present type is a candidate for inclusion or exclusion as an
+ abbreviation, such that:
+ - (is_add and score >= 0.3) suggests a new abbreviation; and
+ - (not is_add and score < 0.3) suggests excluding an abbreviation.
+ """
+ # (While one could recalculate abbreviations from all .-final tokens at
+ # every iteration, in cases requiring efficiency, the number of tokens
+ # in the present training document will be much less.)
+
+ for typ in types:
+ # Check some basic conditions, to rule out words that are
+ # clearly not abbrev_types.
+ if not _re_non_punct.search(typ) or typ == '##number##':
+ continue
+
+ if typ.endswith('.'):
+ if typ in self._params.abbrev_types:
+ continue
+ typ = typ[:-1]
+ is_add = True
+ else:
+ if typ not in self._params.abbrev_types:
+ continue
+ is_add = False
+
+ # Count how many periods & nonperiods are in the
+ # candidate.
+ num_periods = typ.count('.') + 1
+ num_nonperiods = len(typ) - num_periods + 1
+
+ # Let <a> be the candidate without the period, and <b>
+ # be the period. Find a log likelihood ratio that
+ # indicates whether <ab> occurs as a single unit (high
+ # value of ll), or as two independent units <a> and
+ # <b> (low value of ll).
+ count_with_period = self._type_fdist[typ + '.']
+ count_without_period = self._type_fdist[typ]
+ ll = self._dunning_log_likelihood(
+ count_with_period + count_without_period,
+ self._num_period_toks, count_with_period,
+ self._type_fdist.N())
+
+ # Apply three scaling factors to 'tweak' the basic log
+ # likelihood ratio:
+ # F_length: long word -> less likely to be an abbrev
+ # F_periods: more periods -> more likely to be an abbrev
+ # F_penalty: penalize occurrences w/o a period
+ f_length = math.exp(-num_nonperiods)
+ f_periods = num_periods
+ f_penalty = (int(self.IGNORE_ABBREV_PENALTY)
+ or math.pow(num_nonperiods, -count_without_period))
+ score = ll * f_length * f_periods * f_penalty
+
+ yield typ, score, is_add
+
+ def find_abbrev_types(self):
+ """
+ Recalculates abbreviations given type frequencies, despite no prior
+ determination of abbreviations.
+ This fails to include abbreviations otherwise found as "rare".
+ """
+ self._params.clear_abbrevs()
+ tokens = (typ for typ in self._type_fdist if typ and typ.endswith('.'))
+ for abbr, score, is_add in self._reclassify_abbrev_types(tokens):
+ if score >= self.ABBREV:
+ self._params.abbrev_types.add(abbr)
+
+ # This function combines the work done by the original code's
+ # functions `count_orthography_context`, `get_orthography_count`,
+ # and `get_rare_abbreviations`.
+ def _is_rare_abbrev_type(self, cur_tok, next_tok):
+ """
+ A word type is counted as a rare abbreviation if...
+ - it's not already marked as an abbreviation
+ - it occurs fewer than ABBREV_BACKOFF times
+ - either it is followed by a sentence-internal punctuation
+ mark, *or* it is followed by a lower-case word that
+ sometimes appears with upper case, but never occurs with
+ lower case at the beginning of sentences.
+ """
+ if cur_tok.abbr or not cur_tok.sentbreak:
+ return False
+
+ # Find the case-normalized type of the token. If it's
+ # a sentence-final token, strip off the period.
+ typ = cur_tok.type_no_sentperiod
+
+ # Proceed only if the type hasn't been categorized as an
+ # abbreviation already, and is sufficiently rare...
+ count = self._type_fdist[typ] + self._type_fdist[typ[:-1]]
+ if (typ in self._params.abbrev_types or count >= self.ABBREV_BACKOFF):
+ return False
+
+ # Record this token as an abbreviation if the next
+ # token is a sentence-internal punctuation mark.
+ # [XX] :1 or check the whole thing??
+ if next_tok.tok[:1] in self._lang_vars.internal_punctuation:
+ return True
+
+ # Record this type as an abbreviation if the next
+ # token... (i) starts with a lower case letter,
+ # (ii) sometimes occurs with an uppercase letter,
+ # and (iii) never occus with an uppercase letter
+ # sentence-internally.
+ # [xx] should the check for (ii) be modified??
+ elif next_tok.first_lower:
+ typ2 = next_tok.type_no_sentperiod
+ typ2ortho_context = self._params.ortho_context[typ2]
+ if ( (typ2ortho_context & _ORTHO_BEG_UC) and
+ not (typ2ortho_context & _ORTHO_MID_UC) ):
+ return True
+
+ #////////////////////////////////////////////////////////////
+ #{ Log Likelihoods
+ #////////////////////////////////////////////////////////////
+
+ # helper for _reclassify_abbrev_types:
+ @staticmethod
+ def _dunning_log_likelihood(count_a, count_b, count_ab, N):
+ """
+ A function that calculates the modified Dunning log-likelihood
+ ratio scores for abbreviation candidates. The details of how
+ this works is available in the paper.
+ """
+ p1 = float(count_b) / N
+ p2 = 0.99
+
+ null_hypo = (float(count_ab) * math.log(p1) +
+ (count_a - count_ab) * math.log(1.0 - p1))
+ alt_hypo = (float(count_ab) * math.log(p2) +
+ (count_a - count_ab) * math.log(1.0 - p2))
+
+ likelihood = null_hypo - alt_hypo
+
+ return (-2.0 * likelihood)
+
+ @staticmethod
+ def _col_log_likelihood(count_a, count_b, count_ab, N):
+ """
+ A function that will just compute log-likelihood estimate, in
+ the original paper it's described in algorithm 6 and 7.
+
+ This *should* be the original Dunning log-likelihood values,
+ unlike the previous log_l function where it used modified
+ Dunning log-likelihood values
+ """
+ import math
+
+ p = 1.0 * count_b / N
+ p1 = 1.0 * count_ab / count_a
+ p2 = 1.0 * (count_b - count_ab) / (N - count_a)
+
+ summand1 = (count_ab * math.log(p) +
+ (count_a - count_ab) * math.log(1.0 - p))
+
+ summand2 = ((count_b - count_ab) * math.log(p) +
+ (N - count_a - count_b + count_ab) * math.log(1.0 - p))
+
+ if count_a == count_ab:
+ summand3 = 0
+ else:
+ summand3 = (count_ab * math.log(p1) +
+ (count_a - count_ab) * math.log(1.0 - p1))
+
+ if count_b == count_ab:
+ summand4 = 0
+ else:
+ summand4 = ((count_b - count_ab) * math.log(p2) +
+ (N - count_a - count_b + count_ab) * math.log(1.0 - p2))
+
+ likelihood = summand1 + summand2 - summand3 - summand4
+
+ return (-2.0 * likelihood)
+
+ #////////////////////////////////////////////////////////////
+ #{ Collocation Finder
+ #////////////////////////////////////////////////////////////
+
+ def _is_potential_collocation(self, aug_tok1, aug_tok2):
+ """
+ Returns True if the pair of tokens may form a collocation given
+ log-likelihood statistics.
+ """
+ return ((self.INCLUDE_ALL_COLLOCS or
+ (self.INCLUDE_ABBREV_COLLOCS and aug_tok1.abbr) or
+ (aug_tok1.sentbreak and
+ (aug_tok1.is_number or aug_tok1.is_initial)))
+ and aug_tok1.is_non_punct
+ and aug_tok2.is_non_punct)
+
+ def _find_collocations(self):
+ """
+ Generates likely collocations and their log-likelihood.
+ """
+ for types in self._collocation_fdist:
+ try:
+ typ1, typ2 = types
+ except TypeError:
+ # types may be None after calling freq_threshold()
+ continue
+ if typ2 in self._params.sent_starters:
+ continue
+
+ col_count = self._collocation_fdist[types]
+ typ1_count = self._type_fdist[typ1]+self._type_fdist[typ1+'.']
+ typ2_count = self._type_fdist[typ2]+self._type_fdist[typ2+'.']
+ if (typ1_count > 1 and typ2_count > 1
+ and self.MIN_COLLOC_FREQ <
+ col_count <= min(typ1_count, typ2_count)):
+
+ ll = self._col_log_likelihood(typ1_count, typ2_count,
+ col_count, self._type_fdist.N())
+ # Filter out the not-so-collocative
+ if (ll >= self.COLLOCATION and
+ (float(self._type_fdist.N())/typ1_count >
+ float(typ2_count)/col_count)):
+ yield (typ1, typ2), ll
+
+ #////////////////////////////////////////////////////////////
+ #{ Sentence-Starter Finder
+ #////////////////////////////////////////////////////////////
+
+ def _is_potential_sent_starter(self, cur_tok, prev_tok):
+ """
+ Returns True given a token and the token that preceds it if it
+ seems clear that the token is beginning a sentence.
+ """
+ # If a token (i) is preceded by a sentece break that is
+ # not a potential ordinal number or initial, and (ii) is
+ # alphabetic, then it is a a sentence-starter.
+ return ( prev_tok.sentbreak and
+ not (prev_tok.is_number or prev_tok.is_initial) and
+ cur_tok.is_alpha )
+
+ def _find_sent_starters(self):
+ """
+ Uses collocation heuristics for each candidate token to
+ determine if it frequently starts sentences.
+ """
+ for typ in self._sent_starter_fdist:
+ if not typ:
+ continue
+
+ typ_at_break_count = self._sent_starter_fdist[typ]
+ typ_count = self._type_fdist[typ]+self._type_fdist[typ+'.']
+ if typ_count < typ_at_break_count:
+ # needed after freq_threshold
+ continue
+
+ ll = self._col_log_likelihood(self._sentbreak_count, typ_count,
+ typ_at_break_count,
+ self._type_fdist.N())
+
+ if (ll >= self.SENT_STARTER and
+ float(self._type_fdist.N())/self._sentbreak_count >
+ float(typ_count)/typ_at_break_count):
+
+ yield typ, ll
+
+ def _get_sentbreak_count(self, tokens):
+ """
+ Returns the number of sentence breaks marked in a given set of
+ augmented tokens.
+ """
+ return sum(1 for aug_tok in tokens if aug_tok.sentbreak)
+
+
+######################################################################
+#{ Punkt Sentence Tokenizer
+######################################################################
+
+
+class PunktSentenceTokenizer(PunktBaseClass,TokenizerI):
+ """
+ A sentence tokenizer which uses an unsupervised algorithm to build
+ a model for abbreviation words, collocations, and words that start
+ sentences; and then uses that model to find sentence boundaries.
+ This approach has been shown to work well for many European
+ languages.
+ """
+ def __init__(self, train_text=None, verbose=False,
+ lang_vars=PunktLanguageVars(), token_cls=PunktToken):
+ """
+ train_text can either be the sole training text for this sentence
+ boundary detector, or can be a PunktParameters object.
+ """
+ PunktBaseClass.__init__(self, lang_vars=lang_vars,
+ token_cls=token_cls)
+
+ if train_text:
+ self._params = self.train(train_text, verbose)
+
+ def train(self, train_text, verbose=False):
+ """
+ Derives parameters from a given training text, or uses the parameters
+ given. Repeated calls to this method destroy previous parameters. For
+ incremental training, instantiate a separate PunktTrainer instance.
+ """
+ if not isinstance(train_text, string_types):
+ return train_text
+ return PunktTrainer(train_text, lang_vars=self._lang_vars,
+ token_cls=self._Token).get_params()
+
+ #////////////////////////////////////////////////////////////
+ #{ Tokenization
+ #////////////////////////////////////////////////////////////
+
+ def tokenize(self, text, realign_boundaries=True):
+ """
+ Given a text, returns a list of the sentences in that text.
+ """
+ return list(self.sentences_from_text(text, realign_boundaries))
+
+ def debug_decisions(self, text):
+ """
+ Classifies candidate periods as sentence breaks, yielding a dict for
+ each that may be used to understand why the decision was made.
+
+ See format_debug_decision() to help make this output readable.
+ """
+
+ for match in self._lang_vars.period_context_re().finditer(text):
+ decision_text = match.group() + match.group('after_tok')
+ tokens = self._tokenize_words(decision_text)
+ tokens = list(self._annotate_first_pass(tokens))
+ while not tokens[0].period_final:
+ tokens.pop(0)
+ yield dict(period_index=match.end() - 1,
+ text=decision_text,
+ type1=tokens[0].type,
+ type2=tokens[1].type,
+ type1_in_abbrs=bool(tokens[0].abbr),
+ type1_is_initial=bool(tokens[0].is_initial),
+ type2_is_sent_starter=tokens[1].type_no_sentperiod in self._params.sent_starters,
+ type2_ortho_heuristic=self._ortho_heuristic(tokens[1]),
+ type2_ortho_contexts=set(self._params._debug_ortho_context(tokens[1].type_no_sentperiod)),
+ collocation=(tokens[0].type_no_sentperiod, tokens[1].type_no_sentperiod) in self._params.collocations,
+
+ reason=self._second_pass_annotation(tokens[0], tokens[1]) or REASON_DEFAULT_DECISION,
+ break_decision=tokens[0].sentbreak,
+ )
+
+ def span_tokenize(self, text, realign_boundaries=True):
+ """
+ Given a text, returns a list of the (start, end) spans of sentences
+ in the text.
+ """
+ slices = self._slices_from_text(text)
+ if realign_boundaries:
+ slices = self._realign_boundaries(text, slices)
+ return [(sl.start, sl.stop) for sl in slices]
+
+ def sentences_from_text(self, text, realign_boundaries=True):
+ """
+ Given a text, generates the sentences in that text by only
+ testing candidate sentence breaks. If realign_boundaries is
+ True, includes in the sentence closing punctuation that
+ follows the period.
+ """
+ return [text[s:e] for s, e in self.span_tokenize(text, realign_boundaries)]
+
+ def _slices_from_text(self, text):
+ last_break = 0
+ for match in self._lang_vars.period_context_re().finditer(text):
+ context = match.group() + match.group('after_tok')
+ if self.text_contains_sentbreak(context):
+ yield slice(last_break, match.end())
+ if match.group('next_tok'):
+ # next sentence starts after whitespace
+ last_break = match.start('next_tok')
+ else:
+ # next sentence starts at following punctuation
+ last_break = match.end()
+ yield slice(last_break, len(text))
+
+ def _realign_boundaries(self, text, slices):
+ """
+ Attempts to realign punctuation that falls after the period but
+ should otherwise be included in the same sentence.
+
+ For example: "(Sent1.) Sent2." will otherwise be split as::
+
+ ["(Sent1.", ") Sent1."].
+
+ This method will produce::
+
+ ["(Sent1.)", "Sent2."].
+ """
+ realign = 0
+ for sl1, sl2 in _pair_iter(slices):
+ sl1 = slice(sl1.start + realign, sl1.stop)
+ if not sl2:
+ if text[sl1]:
+ yield sl1
+ continue
+
+ m = self._lang_vars.re_boundary_realignment.match(text[sl2])
+ if m:
+ yield slice(sl1.start, sl2.start + len(m.group(0).rstrip()))
+ realign = m.end()
+ else:
+ realign = 0
+ if text[sl1]:
+ yield sl1
+
+ def text_contains_sentbreak(self, text):
+ """
+ Returns True if the given text includes a sentence break.
+ """
+ found = False # used to ignore last token
+ for t in self._annotate_tokens(self._tokenize_words(text)):
+ if found:
+ return True
+ if t.sentbreak:
+ found = True
+ return False
+
+ def sentences_from_text_legacy(self, text):
+ """
+ Given a text, generates the sentences in that text. Annotates all
+ tokens, rather than just those with possible sentence breaks. Should
+ produce the same results as ``sentences_from_text``.
+ """
+ tokens = self._annotate_tokens(self._tokenize_words(text))
+ return self._build_sentence_list(text, tokens)
+
+ def sentences_from_tokens(self, tokens):
+ """
+ Given a sequence of tokens, generates lists of tokens, each list
+ corresponding to a sentence.
+ """
+ tokens = iter(self._annotate_tokens(self._Token(t) for t in tokens))
+ sentence = []
+ for aug_tok in tokens:
+ sentence.append(aug_tok.tok)
+ if aug_tok.sentbreak:
+ yield sentence
+ sentence = []
+ if sentence:
+ yield sentence
+
+ def _annotate_tokens(self, tokens):
+ """
+ Given a set of tokens augmented with markers for line-start and
+ paragraph-start, returns an iterator through those tokens with full
+ annotation including predicted sentence breaks.
+ """
+ # Make a preliminary pass through the document, marking likely
+ # sentence breaks, abbreviations, and ellipsis tokens.
+ tokens = self._annotate_first_pass(tokens)
+
+ # Make a second pass through the document, using token context
+ # information to change our preliminary decisions about where
+ # sentence breaks, abbreviations, and ellipsis occurs.
+ tokens = self._annotate_second_pass(tokens)
+
+ ## [XX] TESTING
+ #tokens = list(tokens)
+ #self.dump(tokens)
+
+ return tokens
+
+ def _build_sentence_list(self, text, tokens):
+ """
+ Given the original text and the list of augmented word tokens,
+ construct and return a tokenized list of sentence strings.
+ """
+ # Most of the work here is making sure that we put the right
+ # pieces of whitespace back in all the right places.
+
+ # Our position in the source text, used to keep track of which
+ # whitespace to add:
+ pos = 0
+
+ # A regular expression that finds pieces of whitespace:
+ WS_REGEXP = re.compile(r'\s*')
+
+ sentence = ''
+ for aug_tok in tokens:
+ tok = aug_tok.tok
+
+ # Find the whitespace before this token, and update pos.
+ ws = WS_REGEXP.match(text, pos).group()
+ pos += len(ws)
+
+ # Some of the rules used by the punkt word tokenizer
+ # strip whitespace out of the text, resulting in tokens
+ # that contain whitespace in the source text. If our
+ # token doesn't match, see if adding whitespace helps.
+ # If so, then use the version with whitespace.
+ if text[pos:pos+len(tok)] != tok:
+ pat = '\s*'.join(re.escape(c) for c in tok)
+ m = re.compile(pat).match(text,pos)
+ if m: tok = m.group()
+
+ # Move our position pointer to the end of the token.
+ assert text[pos:pos+len(tok)] == tok
+ pos += len(tok)
+
+ # Add this token. If it's not at the beginning of the
+ # sentence, then include any whitespace that separated it
+ # from the previous token.
+ if sentence:
+ sentence += ws
+ sentence += tok
+
+ # If we're at a sentence break, then start a new sentence.
+ if aug_tok.sentbreak:
+ yield sentence
+ sentence = ''
+
+ # If the last sentence is emtpy, discard it.
+ if sentence:
+ yield sentence
+
+ # [XX] TESTING
+ def dump(self, tokens):
+ print('writing to /tmp/punkt.new...')
+ with open('/tmp/punkt.new', 'w') as outfile:
+ for aug_tok in tokens:
+ if aug_tok.parastart:
+ outfile.write('\n\n')
+ elif aug_tok.linestart:
+ outfile.write('\n')
+ else:
+ outfile.write(' ')
+
+ outfile.write(str(aug_tok))
+
+ #////////////////////////////////////////////////////////////
+ #{ Customization Variables
+ #////////////////////////////////////////////////////////////
+
+ PUNCTUATION = tuple(';:,.!?')
+
+ #////////////////////////////////////////////////////////////
+ #{ Annotation Procedures
+ #////////////////////////////////////////////////////////////
+
+ def _annotate_second_pass(self, tokens):
+ """
+ Performs a token-based classification (section 4) over the given
+ tokens, making use of the orthographic heuristic (4.1.1), collocation
+ heuristic (4.1.2) and frequent sentence starter heuristic (4.1.3).
+ """
+ for t1, t2 in _pair_iter(tokens):
+ self._second_pass_annotation(t1, t2)
+ yield t1
+
+ def _second_pass_annotation(self, aug_tok1, aug_tok2):
+ """
+ Performs token-based classification over a pair of contiguous tokens
+ updating the first.
+ """
+ # Is it the last token? We can't do anything then.
+ if not aug_tok2:
+ return
+
+ tok = aug_tok1.tok
+ if not aug_tok1.period_final:
+ # We only care about words ending in periods.
+ return
+
+ typ = aug_tok1.type_no_period
+ next_tok = aug_tok2.tok
+ next_typ = aug_tok2.type_no_sentperiod
+ tok_is_initial = aug_tok1.is_initial
+
+ # [4.1.2. Collocation Heuristic] If there's a
+ # collocation between the word before and after the
+ # period, then label tok as an abbreviation and NOT
+ # a sentence break. Note that collocations with
+ # frequent sentence starters as their second word are
+ # excluded in training.
+ if (typ, next_typ) in self._params.collocations:
+ aug_tok1.sentbreak = False
+ aug_tok1.abbr = True
+ return REASON_KNOWN_COLLOCATION
+
+ # [4.2. Token-Based Reclassification of Abbreviations] If
+ # the token is an abbreviation or an ellipsis, then decide
+ # whether we should *also* classify it as a sentbreak.
+ if ( (aug_tok1.abbr or aug_tok1.ellipsis) and
+ (not tok_is_initial) ):
+ # [4.1.1. Orthographic Heuristic] Check if there's
+ # orthogrpahic evidence about whether the next word
+ # starts a sentence or not.
+ is_sent_starter = self._ortho_heuristic(aug_tok2)
+ if is_sent_starter == True:
+ aug_tok1.sentbreak = True
+ return REASON_ABBR_WITH_ORTHOGRAPHIC_HEURISTIC
+
+ # [4.1.3. Frequent Sentence Starter Heruistic] If the
+ # next word is capitalized, and is a member of the
+ # frequent-sentence-starters list, then label tok as a
+ # sentence break.
+ if ( aug_tok2.first_upper and
+ next_typ in self._params.sent_starters):
+ aug_tok1.sentbreak = True
+ return REASON_ABBR_WITH_SENTENCE_STARTER
+
+ # [4.3. Token-Based Detection of Initials and Ordinals]
+ # Check if any initials or ordinals tokens that are marked
+ # as sentbreaks should be reclassified as abbreviations.
+ if tok_is_initial or typ == '##number##':
+
+ # [4.1.1. Orthographic Heuristic] Check if there's
+ # orthogrpahic evidence about whether the next word
+ # starts a sentence or not.
+ is_sent_starter = self._ortho_heuristic(aug_tok2)
+
+ if is_sent_starter == False:
+ aug_tok1.sentbreak = False
+ aug_tok1.abbr = True
+ if tok_is_initial:
+ return REASON_INITIAL_WITH_ORTHOGRAPHIC_HEURISTIC
+ else:
+ return REASON_NUMBER_WITH_ORTHOGRAPHIC_HEURISTIC
+
+ # Special heuristic for initials: if orthogrpahic
+ # heuristc is unknown, and next word is always
+ # capitalized, then mark as abbrev (eg: J. Bach).
+ if ( is_sent_starter == 'unknown' and tok_is_initial and
+ aug_tok2.first_upper and
+ not (self._params.ortho_context[next_typ] & _ORTHO_LC) ):
+ aug_tok1.sentbreak = False
+ aug_tok1.abbr = True
+ return REASON_INITIAL_WITH_SPECIAL_ORTHOGRAPHIC_HEURISTIC
+
+ return
+
+ def _ortho_heuristic(self, aug_tok):
+ """
+ Decide whether the given token is the first token in a sentence.
+ """
+ # Sentences don't start with punctuation marks:
+ if aug_tok.tok in self.PUNCTUATION:
+ return False
+
+ ortho_context = self._params.ortho_context[aug_tok.type_no_sentperiod]
+
+ # If the word is capitalized, occurs at least once with a
+ # lower case first letter, and never occurs with an upper case
+ # first letter sentence-internally, then it's a sentence starter.
+ if ( aug_tok.first_upper and
+ (ortho_context & _ORTHO_LC) and
+ not (ortho_context & _ORTHO_MID_UC) ):
+ return True
+
+ # If the word is lower case, and either (a) we've seen it used
+ # with upper case, or (b) we've never seen it used
+ # sentence-initially with lower case, then it's not a sentence
+ # starter.
+ if ( aug_tok.first_lower and
+ ((ortho_context & _ORTHO_UC) or
+ not (ortho_context & _ORTHO_BEG_LC)) ):
+ return False
+
+ # Otherwise, we're not sure.
+ return 'unknown'
+
+
+DEBUG_DECISION_FMT = '''Text: %(text)r (at offset %(period_index)d)
+Sentence break? %(break_decision)s (%(reason)s)
+Collocation? %(collocation)s
+%(type1)r:
+ known abbreviation: %(type1_in_abbrs)s
+ is initial: %(type1_is_initial)s
+%(type2)r:
+ known sentence starter: %(type2_is_sent_starter)s
+ orthographic heuristic suggests is a sentence starter? %(type2_ortho_heuristic)s
+ orthographic contexts in training: %(type2_ortho_contexts)s
+'''
+def format_debug_decision(d):
+ return DEBUG_DECISION_FMT % d
+
+def demo(text, tok_cls=PunktSentenceTokenizer, train_cls=PunktTrainer):
+ """Builds a punkt model and applies it to the same text"""
+ cleanup = lambda s: re.compile(r'(?:\r|^\s+)', re.MULTILINE).sub('', s).replace('\n', ' ')
+ trainer = train_cls()
+ trainer.INCLUDE_ALL_COLLOCS = True
+ trainer.train(text)
+ sbd = tok_cls(trainer.get_params())
+ for l in sbd.sentences_from_text(text):
+ print(cleanup(l))
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tokenize/regexp.py b/nltk/tokenize/regexp.py
new file mode 100644
index 0000000..476b9a2
--- /dev/null
+++ b/nltk/tokenize/regexp.py
@@ -0,0 +1,208 @@
+# Natural Language Toolkit: Tokenizers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# Trevor Cohn <tacohn at csse.unimelb.edu.au>
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+r"""
+Regular-Expression Tokenizers
+
+A ``RegexpTokenizer`` splits a string into substrings using a regular expression.
+For example, the following tokenizer forms tokens out of alphabetic sequences,
+money expressions, and any other non-whitespace sequences:
+
+ >>> from nltk.tokenize import RegexpTokenizer
+ >>> s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\n\nThanks."
+ >>> tokenizer = RegexpTokenizer('\w+|\$[\d\.]+|\S+')
+ >>> tokenizer.tokenize(s)
+ ['Good', 'muffins', 'cost', '$3.88', 'in', 'New', 'York', '.',
+ 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+
+A ``RegexpTokenizer`` can use its regexp to match delimiters instead:
+
+ >>> tokenizer = RegexpTokenizer('\s+', gaps=True)
+ >>> tokenizer.tokenize(s)
+ ['Good', 'muffins', 'cost', '$3.88', 'in', 'New', 'York.',
+ 'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks.']
+
+Note that empty tokens are not returned when the delimiter appears at
+the start or end of the string.
+
+The material between the tokens is discarded. For example,
+the following tokenizer selects just the capitalized words:
+
+ >>> capword_tokenizer = RegexpTokenizer('[A-Z]\w+')
+ >>> capword_tokenizer.tokenize(s)
+ ['Good', 'New', 'York', 'Please', 'Thanks']
+
+This module contains several subclasses of ``RegexpTokenizer``
+that use pre-defined regular expressions.
+
+ >>> from nltk.tokenize import BlanklineTokenizer
+ >>> # Uses '\s*\n\s*\n\s*':
+ >>> BlanklineTokenizer().tokenize(s)
+ ['Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.',
+ 'Thanks.']
+
+All of the regular expression tokenizers are also available as functions:
+
+ >>> from nltk.tokenize import regexp_tokenize, wordpunct_tokenize, blankline_tokenize
+ >>> regexp_tokenize(s, pattern='\w+|\$[\d\.]+|\S+')
+ ['Good', 'muffins', 'cost', '$3.88', 'in', 'New', 'York', '.',
+ 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+ >>> wordpunct_tokenize(s)
+ ['Good', 'muffins', 'cost', '$', '3', '.', '88', 'in', 'New', 'York',
+ '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+ >>> blankline_tokenize(s)
+ ['Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.', 'Thanks.']
+
+Caution: The function ``regexp_tokenize()`` takes the text as its
+first argument, and the regular expression pattern as its second
+argument. This differs from the conventions used by Python's
+``re`` functions, where the pattern is always the first argument.
+(This is for consistency with the other NLTK tokenizers.)
+"""
+from __future__ import unicode_literals
+
+import re
+import sre_constants
+
+from nltk.internals import compile_regexp_to_noncapturing
+from nltk.tokenize.api import TokenizerI
+from nltk.tokenize.util import regexp_span_tokenize
+from nltk.compat import python_2_unicode_compatible
+
+ at python_2_unicode_compatible
+class RegexpTokenizer(TokenizerI):
+ """
+ A tokenizer that splits a string using a regular expression, which
+ matches either the tokens or the separators between tokens.
+
+ >>> tokenizer = RegexpTokenizer('\w+|\$[\d\.]+|\S+')
+
+ :type pattern: str
+ :param pattern: The pattern used to build this tokenizer.
+ (This pattern may safely contain capturing parentheses.)
+ :type gaps: bool
+ :param gaps: True if this tokenizer's pattern should be used
+ to find separators between tokens; False if this
+ tokenizer's pattern should be used to find the tokens
+ themselves.
+ :type discard_empty: bool
+ :param discard_empty: True if any empty tokens `''`
+ generated by the tokenizer should be discarded. Empty
+ tokens can only be generated if `_gaps == True`.
+ :type flags: int
+ :param flags: The regexp flags used to compile this
+ tokenizer's pattern. By default, the following flags are
+ used: `re.UNICODE | re.MULTILINE | re.DOTALL`.
+
+ """
+ def __init__(self, pattern, gaps=False, discard_empty=True,
+ flags=re.UNICODE | re.MULTILINE | re.DOTALL):
+ # If they gave us a regexp object, extract the pattern.
+ pattern = getattr(pattern, 'pattern', pattern)
+
+ self._pattern = pattern
+ self._gaps = gaps
+ self._discard_empty = discard_empty
+ self._flags = flags
+ self._regexp = None
+
+ # Remove capturing parentheses -- if the regexp contains any
+ # capturing parentheses, then the behavior of re.findall and
+ # re.split will change.
+ try:
+ self._regexp = compile_regexp_to_noncapturing(pattern, flags)
+ except re.error as e:
+ raise ValueError('Error in regular expression %r: %s' %
+ (pattern, e))
+
+ def tokenize(self, text):
+ # If our regexp matches gaps, use re.split:
+ if self._gaps:
+ if self._discard_empty:
+ return [tok for tok in self._regexp.split(text) if tok]
+ else:
+ return self._regexp.split(text)
+
+ # If our regexp matches tokens, use re.findall:
+ else:
+ return self._regexp.findall(text)
+
+ def span_tokenize(self, text):
+ if self._gaps:
+ for left, right in regexp_span_tokenize(text, self._regexp):
+ if not (self._discard_empty and left == right):
+ yield left, right
+ else:
+ for m in re.finditer(self._regexp, text):
+ yield m.span()
+
+ def __repr__(self):
+ return ('%s(pattern=%r, gaps=%r, discard_empty=%r, flags=%r)' %
+ (self.__class__.__name__, self._pattern, self._gaps,
+ self._discard_empty, self._flags))
+
+class WhitespaceTokenizer(RegexpTokenizer):
+ r"""
+ Tokenize a string on whitespace (space, tab, newline).
+ In general, users should use the string ``split()`` method instead.
+
+ >>> from nltk.tokenize import WhitespaceTokenizer
+ >>> s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\n\nThanks."
+ >>> WhitespaceTokenizer().tokenize(s)
+ ['Good', 'muffins', 'cost', '$3.88', 'in', 'New', 'York.',
+ 'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks.']
+ """
+
+ def __init__(self):
+ RegexpTokenizer.__init__(self, r'\s+', gaps=True)
+
+class BlanklineTokenizer(RegexpTokenizer):
+ """
+ Tokenize a string, treating any sequence of blank lines as a delimiter.
+ Blank lines are defined as lines containing no characters, except for
+ space or tab characters.
+ """
+ def __init__(self):
+ RegexpTokenizer.__init__(self, r'\s*\n\s*\n\s*', gaps=True)
+
+class WordPunctTokenizer(RegexpTokenizer):
+ """
+ Tokenize a text into a sequence of alphabetic and
+ non-alphabetic characters, using the regexp ``\w+|[^\w\s]+``.
+
+ >>> from nltk.tokenize import WordPunctTokenizer
+ >>> s = "Good muffins cost $3.88\\nin New York. Please buy me\\ntwo of them.\\n\\nThanks."
+ >>> WordPunctTokenizer().tokenize(s)
+ ['Good', 'muffins', 'cost', '$', '3', '.', '88', 'in', 'New', 'York',
+ '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+ """
+ def __init__(self):
+ RegexpTokenizer.__init__(self, r'\w+|[^\w\s]+')
+
+######################################################################
+#{ Tokenization Functions
+######################################################################
+
+def regexp_tokenize(text, pattern, gaps=False, discard_empty=True,
+ flags=re.UNICODE | re.MULTILINE | re.DOTALL):
+ """
+ Return a tokenized copy of *text*. See :class:`.RegexpTokenizer`
+ for descriptions of the arguments.
+ """
+ tokenizer = RegexpTokenizer(pattern, gaps, discard_empty, flags)
+ return tokenizer.tokenize(text)
+
+blankline_tokenize = BlanklineTokenizer().tokenize
+wordpunct_tokenize = WordPunctTokenizer().tokenize
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
diff --git a/nltk/tokenize/sexpr.py b/nltk/tokenize/sexpr.py
new file mode 100644
index 0000000..78e733e
--- /dev/null
+++ b/nltk/tokenize/sexpr.py
@@ -0,0 +1,145 @@
+# Natural Language Toolkit: Tokenizers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Yoav Goldberg <yoavg at cs.bgu.ac.il>
+# Steven Bird <stevenbird1 at gmail.com> (minor edits)
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+"""
+S-Expression Tokenizer
+
+``SExprTokenizer`` is used to find parenthesized expressions in a
+string. In particular, it divides a string into a sequence of
+substrings that are either parenthesized expressions (including any
+nested parenthesized expressions), or other whitespace-separated
+tokens.
+
+ >>> from nltk.tokenize import SExprTokenizer
+ >>> SExprTokenizer().tokenize('(a b (c d)) e f (g)')
+ ['(a b (c d))', 'e', 'f', '(g)']
+
+By default, `SExprTokenizer` will raise a ``ValueError`` exception if
+used to tokenize an expression with non-matching parentheses:
+
+ >>> SExprTokenizer().tokenize('c) d) e (f (g')
+ Traceback (most recent call last):
+ ...
+ ValueError: Un-matched close paren at char 1
+
+The ``strict`` argument can be set to False to allow for
+non-matching parentheses. Any unmatched close parentheses will be
+listed as their own s-expression; and the last partial sexpr with
+unmatched open parentheses will be listed as its own sexpr:
+
+ >>> SExprTokenizer(strict=False).tokenize('c) d) e (f (g')
+ ['c', ')', 'd', ')', 'e', '(f (g']
+
+The characters used for open and close parentheses may be customized
+using the ``parens`` argument to the `SExprTokenizer` constructor:
+
+ >>> SExprTokenizer(parens='{}').tokenize('{a b {c d}} e f {g}')
+ ['{a b {c d}}', 'e', 'f', '{g}']
+
+The s-expression tokenizer is also available as a function:
+
+ >>> from nltk.tokenize import sexpr_tokenize
+ >>> sexpr_tokenize('(a b (c d)) e f (g)')
+ ['(a b (c d))', 'e', 'f', '(g)']
+
+"""
+
+import re
+
+from nltk.tokenize.api import TokenizerI
+
+class SExprTokenizer(TokenizerI):
+ """
+ A tokenizer that divides strings into s-expressions.
+ An s-expresion can be either:
+
+ - a parenthesized expression, including any nested parenthesized
+ expressions, or
+ - a sequence of non-whitespace non-parenthesis characters.
+
+ For example, the string ``(a (b c)) d e (f)`` consists of four
+ s-expressions: ``(a (b c))``, ``d``, ``e``, and ``(f)``.
+
+ By default, the characters ``(`` and ``)`` are treated as open and
+ close parentheses, but alternative strings may be specified.
+
+ :param parens: A two-element sequence specifying the open and close parentheses
+ that should be used to find sexprs. This will typically be either a
+ two-character string, or a list of two strings.
+ :type parens: str or list
+ :param strict: If true, then raise an exception when tokenizing an ill-formed sexpr.
+ """
+
+ def __init__(self, parens='()', strict=True):
+ if len(parens) != 2:
+ raise ValueError('parens must contain exactly two strings')
+ self._strict = strict
+ self._open_paren = parens[0]
+ self._close_paren = parens[1]
+ self._paren_regexp = re.compile('%s|%s' % (re.escape(parens[0]),
+ re.escape(parens[1])))
+
+ def tokenize(self, text):
+ """
+ Return a list of s-expressions extracted from *text*.
+ For example:
+
+ >>> SExprTokenizer().tokenize('(a b (c d)) e f (g)')
+ ['(a b (c d))', 'e', 'f', '(g)']
+
+ All parentheses are assumed to mark s-expressions.
+ (No special processing is done to exclude parentheses that occur
+ inside strings, or following backslash characters.)
+
+ If the given expression contains non-matching parentheses,
+ then the behavior of the tokenizer depends on the ``strict``
+ parameter to the constructor. If ``strict`` is ``True``, then
+ raise a ``ValueError``. If ``strict`` is ``False``, then any
+ unmatched close parentheses will be listed as their own
+ s-expression; and the last partial s-expression with unmatched open
+ parentheses will be listed as its own s-expression:
+
+ >>> SExprTokenizer(strict=False).tokenize('c) d) e (f (g')
+ ['c', ')', 'd', ')', 'e', '(f (g']
+
+ :param text: the string to be tokenized
+ :type text: str or iter(str)
+ :rtype: iter(str)
+ """
+ result = []
+ pos = 0
+ depth = 0
+ for m in self._paren_regexp.finditer(text):
+ paren = m.group()
+ if depth == 0:
+ result += text[pos:m.start()].split()
+ pos = m.start()
+ if paren == self._open_paren:
+ depth += 1
+ if paren == self._close_paren:
+ if self._strict and depth == 0:
+ raise ValueError('Un-matched close paren at char %d'
+ % m.start())
+ depth = max(0, depth-1)
+ if depth == 0:
+ result.append(text[pos:m.end()])
+ pos = m.end()
+ if self._strict and depth > 0:
+ raise ValueError('Un-matched open paren at char %d' % pos)
+ if pos < len(text):
+ result.append(text[pos:])
+ return result
+
+sexpr_tokenize = SExprTokenizer().tokenize
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
+
diff --git a/nltk/tokenize/simple.py b/nltk/tokenize/simple.py
new file mode 100644
index 0000000..b92a7e3
--- /dev/null
+++ b/nltk/tokenize/simple.py
@@ -0,0 +1,139 @@
+# Natural Language Toolkit: Simple Tokenizers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+r"""
+Simple Tokenizers
+
+These tokenizers divide strings into substrings using the string
+``split()`` method.
+When tokenizing using a particular delimiter string, use
+the string ``split()`` method directly, as this is more efficient.
+
+The simple tokenizers are *not* available as separate functions;
+instead, you should just use the string ``split()`` method directly:
+
+ >>> s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\n\nThanks."
+ >>> s.split()
+ ['Good', 'muffins', 'cost', '$3.88', 'in', 'New', 'York.',
+ 'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks.']
+ >>> s.split(' ')
+ ['Good', 'muffins', 'cost', '$3.88\nin', 'New', 'York.', '',
+ 'Please', 'buy', 'me\ntwo', 'of', 'them.\n\nThanks.']
+ >>> s.split('\n')
+ ['Good muffins cost $3.88', 'in New York. Please buy me',
+ 'two of them.', '', 'Thanks.']
+
+The simple tokenizers are mainly useful because they follow the
+standard ``TokenizerI`` interface, and so can be used with any code
+that expects a tokenizer. For example, these tokenizers can be used
+to specify the tokenization conventions when building a `CorpusReader`.
+
+"""
+from __future__ import unicode_literals
+from nltk.tokenize.api import TokenizerI, StringTokenizer
+from nltk.tokenize.util import string_span_tokenize, regexp_span_tokenize
+
+class SpaceTokenizer(StringTokenizer):
+ r"""Tokenize a string using the space character as a delimiter,
+ which is the same as ``s.split(' ')``.
+
+ >>> from nltk.tokenize import SpaceTokenizer
+ >>> s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\n\nThanks."
+ >>> SpaceTokenizer().tokenize(s)
+ ['Good', 'muffins', 'cost', '$3.88\nin', 'New', 'York.', '',
+ 'Please', 'buy', 'me\ntwo', 'of', 'them.\n\nThanks.']
+ """
+
+ _string = ' '
+
+class TabTokenizer(StringTokenizer):
+ r"""Tokenize a string use the tab character as a delimiter,
+ the same as ``s.split('\t')``.
+
+ >>> from nltk.tokenize import TabTokenizer
+ >>> TabTokenizer().tokenize('a\tb c\n\t d')
+ ['a', 'b c\n', ' d']
+ """
+
+ _string = '\t'
+
+class CharTokenizer(StringTokenizer):
+ """Tokenize a string into individual characters. If this functionality
+ is ever required directly, use ``for char in string``.
+ """
+
+ def tokenize(self, s):
+ return list(s)
+
+ def span_tokenize(self, s):
+ for i, j in enumerate(range(1, len(s) + 1)):
+ yield i, j
+
+class LineTokenizer(TokenizerI):
+ r"""Tokenize a string into its lines, optionally discarding blank lines.
+ This is similar to ``s.split('\n')``.
+
+ >>> from nltk.tokenize import LineTokenizer
+ >>> s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\n\nThanks."
+ >>> LineTokenizer(blanklines='keep').tokenize(s)
+ ['Good muffins cost $3.88', 'in New York. Please buy me',
+ 'two of them.', '', 'Thanks.']
+ >>> # same as [l for l in s.split('\n') if l.strip()]:
+ >>> LineTokenizer(blanklines='discard').tokenize(s)
+ ['Good muffins cost $3.88', 'in New York. Please buy me',
+ 'two of them.', 'Thanks.']
+
+ :param blanklines: Indicates how blank lines should be handled. Valid values are:
+
+ - ``discard``: strip blank lines out of the token list before returning it.
+ A line is considered blank if it contains only whitespace characters.
+ - ``keep``: leave all blank lines in the token list.
+ - ``discard-eof``: if the string ends with a newline, then do not generate
+ a corresponding token ``''`` after that newline.
+ """
+
+ def __init__(self, blanklines='discard'):
+ valid_blanklines = ('discard', 'keep', 'discard-eof')
+ if blanklines not in valid_blanklines:
+ raise ValueError('Blank lines must be one of: %s' %
+ ' '.join(valid_blanklines))
+
+ self._blanklines = blanklines
+
+ def tokenize(self, s):
+ lines = s.splitlines()
+ # If requested, strip off blank lines.
+ if self._blanklines == 'discard':
+ lines = [l for l in lines if l.rstrip()]
+ elif self._blanklines == 'discard-eof':
+ if lines and not lines[-1].strip():
+ lines.pop()
+ return lines
+
+ # discard-eof not implemented
+ def span_tokenize(self, s):
+ if self._blanklines == 'keep':
+ for span in string_span_tokenize(s, r'\n'):
+ yield span
+ else:
+ for span in regexp_span_tokenize(s, r'\n(\s+\n)*'):
+ yield span
+
+######################################################################
+#{ Tokenization Functions
+######################################################################
+# XXX: it is stated in module docs that there is no function versions
+
+def line_tokenize(text, blanklines='discard'):
+ return LineTokenizer(blanklines).tokenize(text)
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
+
diff --git a/nltk/tokenize/stanford.py b/nltk/tokenize/stanford.py
new file mode 100644
index 0000000..39f4ddb
--- /dev/null
+++ b/nltk/tokenize/stanford.py
@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Interface to the Stanford Tokenizer
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Xu <xxu at student.unimelb.edu.au>
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import unicode_literals, print_function
+
+import tempfile
+import os
+import json
+from subprocess import PIPE
+
+from nltk import compat
+from nltk.internals import find_jar, config_java, java, _java_options
+
+from nltk.tokenize.api import TokenizerI
+
+_stanford_url = 'http://nlp.stanford.edu/software/lex-parser.shtml'
+
+class StanfordTokenizer(TokenizerI):
+ r"""
+ Interface to the Stanford Tokenizer
+
+ >>> from nltk.tokenize.stanford import StanfordTokenizer
+ >>> s = "Good muffins cost $3.88\nin New York. Please buy me\ntwo of them.\nThanks."
+ >>> StanfordTokenizer().tokenize(s)
+ ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York', '.', 'Please', 'buy', 'me', 'two', 'of', 'them', '.', 'Thanks', '.']
+ >>> s = "The colour of the wall is blue."
+ >>> StanfordTokenizer(options={"americanize": True}).tokenize(s)
+ ['The', 'color', 'of', 'the', 'wall', 'is', 'blue', '.']
+ """
+
+ _JAR = 'stanford-postagger.jar'
+
+ def __init__(self, path_to_jar=None, encoding='UTF-8', options=None, verbose=False, java_options='-mx1000m'):
+ self._stanford_jar = find_jar(
+ self._JAR, path_to_jar,
+ env_vars=('STANFORD_POSTAGGER',),
+ searchpath=(), url=_stanford_url,
+ verbose=verbose
+ )
+
+ self._encoding = encoding
+ self.java_options = java_options
+ options = {} if options is None else options
+ self._options_cmd = ','.join('{0}={1}'.format(key, json.dumps(val)) for key, val in options.items())
+
+ @staticmethod
+ def _parse_tokenized_output(s):
+ return s.splitlines()
+
+ def tokenize(self, s):
+ """
+ Use stanford tokenizer's PTBTokenizer to tokenize multiple sentences.
+ """
+ cmd = [
+ 'edu.stanford.nlp.process.PTBTokenizer',
+ ]
+ return self._parse_tokenized_output(self._execute(cmd, s))
+
+ def _execute(self, cmd, input_, verbose=False):
+ encoding = self._encoding
+ cmd.extend(['-charset', encoding])
+ _options_cmd = self._options_cmd
+ if _options_cmd:
+ cmd.extend(['-options', self._options_cmd])
+
+ default_options = ' '.join(_java_options)
+
+ # Configure java.
+ config_java(options=self.java_options, verbose=verbose)
+
+ # Windows is incompatible with NamedTemporaryFile() without passing in delete=False.
+ with tempfile.NamedTemporaryFile(mode='wb', delete=False) as input_file:
+ # Write the actual sentences to the temporary input file
+ if isinstance(input_, compat.text_type) and encoding:
+ input_ = input_.encode(encoding)
+ input_file.write(input_)
+ input_file.flush()
+
+ cmd.append(input_file.name)
+
+ # Run the tagger and get the output.
+ stdout, stderr = java(cmd, classpath=self._stanford_jar,
+ stdout=PIPE, stderr=PIPE)
+ stdout = stdout.decode(encoding)
+
+ os.unlink(input_file.name)
+
+ # Return java configurations to their default values.
+ config_java(options=default_options, verbose=False)
+
+ return stdout
+
+
+def setup_module(module):
+ from nose import SkipTest
+
+ try:
+ StanfordTokenizer()
+ except LookupError:
+ raise SkipTest('doctests from nltk.tokenize.stanford are skipped because the stanford postagger jar doesn\'t exist')
diff --git a/nltk/tokenize/texttiling.py b/nltk/tokenize/texttiling.py
new file mode 100644
index 0000000..a5df71c
--- /dev/null
+++ b/nltk/tokenize/texttiling.py
@@ -0,0 +1,454 @@
+# Natural Language Toolkit: TextTiling
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: George Boutsioukis
+#
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+import re
+import math
+
+try:
+ import numpy
+except ImportError:
+ pass
+
+from nltk.tokenize.api import TokenizerI
+
+BLOCK_COMPARISON, VOCABULARY_INTRODUCTION = 0, 1
+LC, HC = 0, 1
+DEFAULT_SMOOTHING = [0]
+
+
+class TextTilingTokenizer(TokenizerI):
+ """Tokenize a document into topical sections using the TextTiling algorithm.
+ This algorithm detects subtopic shifts based on the analysis of lexical
+ co-occurrence patterns.
+
+ The process starts by tokenizing the text into pseudosentences of
+ a fixed size w. Then, depending on the method used, similarity
+ scores are assigned at sentence gaps. The algorithm proceeds by
+ detecting the peak differences between these scores and marking
+ them as boundaries. The boundaries are normalized to the closest
+ paragraph break and the segmented text is returned.
+
+ :param w: Pseudosentence size
+ :type w: int
+ :param k: Size (in sentences) of the block used in the block comparison method
+ :type k: int
+ :param similarity_method: The method used for determining similarity scores:
+ `BLOCK_COMPARISON` (default) or `VOCABULARY_INTRODUCTION`.
+ :type similarity_method: constant
+ :param stopwords: A list of stopwords that are filtered out (defaults to NLTK's stopwords corpus)
+ :type stopwords: list(str)
+ :param smoothing_method: The method used for smoothing the score plot:
+ `DEFAULT_SMOOTHING` (default)
+ :type smoothing_method: constant
+ :param smoothing_width: The width of the window used by the smoothing method
+ :type smoothing_width: int
+ :param smoothing_rounds: The number of smoothing passes
+ :type smoothing_rounds: int
+ :param cutoff_policy: The policy used to determine the number of boundaries:
+ `HC` (default) or `LC`
+ :type cutoff_policy: constant
+ """
+
+ def __init__(self,
+ w=20,
+ k=10,
+ similarity_method=BLOCK_COMPARISON,
+ stopwords=None,
+ smoothing_method=DEFAULT_SMOOTHING,
+ smoothing_width=2,
+ smoothing_rounds=1,
+ cutoff_policy=HC,
+ demo_mode=False):
+
+
+ if stopwords is None:
+ from nltk.corpus import stopwords
+ stopwords = stopwords.words('english')
+ self.__dict__.update(locals())
+ del self.__dict__['self']
+
+ def tokenize(self, text):
+ """Return a tokenized copy of *text*, where each "token" represents
+ a separate topic."""
+
+ lowercase_text = text.lower()
+ paragraph_breaks = self._mark_paragraph_breaks(text)
+ text_length = len(lowercase_text)
+
+ # Tokenization step starts here
+
+ # Remove punctuation
+ nopunct_text = ''.join(c for c in lowercase_text
+ if re.match("[a-z\-\' \n\t]", c))
+ nopunct_par_breaks = self._mark_paragraph_breaks(nopunct_text)
+
+ tokseqs = self._divide_to_tokensequences(nopunct_text)
+
+ # The morphological stemming step mentioned in the TextTile
+ # paper is not implemented. A comment in the original C
+ # implementation states that it offers no benefit to the
+ # process. It might be interesting to test the existing
+ # stemmers though.
+ #words = _stem_words(words)
+
+ # Filter stopwords
+ for ts in tokseqs:
+ ts.wrdindex_list = [wi for wi in ts.wrdindex_list
+ if wi[0] not in self.stopwords]
+
+ token_table = self._create_token_table(tokseqs, nopunct_par_breaks)
+ # End of the Tokenization step
+
+ # Lexical score determination
+ if self.similarity_method == BLOCK_COMPARISON:
+ gap_scores = self._block_comparison(tokseqs, token_table)
+ elif self.similarity_method == VOCABULARY_INTRODUCTION:
+ raise NotImplementedError("Vocabulary introduction not implemented")
+
+ if self.smoothing_method == DEFAULT_SMOOTHING:
+ smooth_scores = self._smooth_scores(gap_scores)
+ # End of Lexical score Determination
+
+ # Boundary identification
+ depth_scores = self._depth_scores(smooth_scores)
+ segment_boundaries = self._identify_boundaries(depth_scores)
+
+ normalized_boundaries = self._normalize_boundaries(text,
+ segment_boundaries,
+ paragraph_breaks)
+ # End of Boundary Identification
+ segmented_text = []
+ prevb = 0
+
+ for b in normalized_boundaries:
+ if b == 0:
+ continue
+ segmented_text.append(text[prevb:b])
+ prevb = b
+
+ if prevb < text_length: # append any text that may be remaining
+ segmented_text.append(text[prevb:])
+
+ if not segmented_text:
+ segmented_text = [text]
+
+ if self.demo_mode:
+ return gap_scores, smooth_scores, depth_scores, segment_boundaries
+ return segmented_text
+
+ def _block_comparison(self, tokseqs, token_table):
+ "Implements the block comparison method"
+ def blk_frq(tok, block):
+ ts_occs = filter(lambda o: o[0] in block,
+ token_table[tok].ts_occurences)
+ freq = sum([tsocc[1] for tsocc in ts_occs])
+ return freq
+
+ gap_scores = []
+ numgaps = len(tokseqs)-1
+
+ for curr_gap in range(numgaps):
+ score_dividend, score_divisor_b1, score_divisor_b2 = 0.0, 0.0, 0.0
+ score = 0.0
+ #adjust window size for boundary conditions
+ if curr_gap < self.k-1:
+ window_size = curr_gap + 1
+ elif curr_gap > numgaps-self.k:
+ window_size = numgaps - curr_gap
+ else:
+ window_size = self.k
+
+ b1 = [ts.index
+ for ts in tokseqs[curr_gap-window_size+1 : curr_gap+1]]
+ b2 = [ts.index
+ for ts in tokseqs[curr_gap+1 : curr_gap+window_size+1]]
+
+ for t in token_table:
+ score_dividend += blk_frq(t, b1)*blk_frq(t, b2)
+ score_divisor_b1 += blk_frq(t, b1)**2
+ score_divisor_b2 += blk_frq(t, b2)**2
+ try:
+ score = score_dividend/math.sqrt(score_divisor_b1*
+ score_divisor_b2)
+ except ZeroDivisionError:
+ pass # score += 0.0
+
+ gap_scores.append(score)
+
+ return gap_scores
+
+ def _smooth_scores(self, gap_scores):
+ "Wraps the smooth function from the SciPy Cookbook"
+ return list(smooth(numpy.array(gap_scores[:]),
+ window_len = self.smoothing_width+1))
+
+ def _mark_paragraph_breaks(self, text):
+ """Identifies indented text or line breaks as the beginning of
+ paragraphs"""
+
+ MIN_PARAGRAPH = 100
+ pattern = re.compile("[ \t\r\f\v]*\n[ \t\r\f\v]*\n[ \t\r\f\v]*")
+ matches = pattern.finditer(text)
+
+ last_break = 0
+ pbreaks = [0]
+ for pb in matches:
+ if pb.start()-last_break < MIN_PARAGRAPH:
+ continue
+ else:
+ pbreaks.append(pb.start())
+ last_break = pb.start()
+
+ return pbreaks
+
+ def _divide_to_tokensequences(self, text):
+ "Divides the text into pseudosentences of fixed size"
+ w = self.w
+ wrdindex_list = []
+ matches = re.finditer("\w+", text)
+ for match in matches:
+ wrdindex_list.append((match.group(), match.start()))
+ return [TokenSequence(i/w, wrdindex_list[i:i+w])
+ for i in range(0, len(wrdindex_list), w)]
+
+ def _create_token_table(self, token_sequences, par_breaks):
+ "Creates a table of TokenTableFields"
+ token_table = {}
+ current_par = 0
+ current_tok_seq = 0
+ pb_iter = par_breaks.__iter__()
+ current_par_break = next(pb_iter)
+ if current_par_break == 0:
+ try:
+ current_par_break = next(pb_iter) #skip break at 0
+ except StopIteration:
+ raise ValueError(
+ "No paragraph breaks were found(text too short perhaps?)"
+ )
+ for ts in token_sequences:
+ for word, index in ts.wrdindex_list:
+ try:
+ while index > current_par_break:
+ current_par_break = next(pb_iter)
+ current_par += 1
+ except StopIteration:
+ #hit bottom
+ pass
+
+ if word in token_table:
+ token_table[word].total_count += 1
+
+ if token_table[word].last_par != current_par:
+ token_table[word].last_par = current_par
+ token_table[word].par_count += 1
+
+ if token_table[word].last_tok_seq != current_tok_seq:
+ token_table[word].last_tok_seq = current_tok_seq
+ token_table[word]\
+ .ts_occurences.append([current_tok_seq,1])
+ else:
+ token_table[word].ts_occurences[-1][1] += 1
+ else: #new word
+ token_table[word] = TokenTableField(first_pos=index,
+ ts_occurences= \
+ [[current_tok_seq,1]],
+ total_count=1,
+ par_count=1,
+ last_par=current_par,
+ last_tok_seq= \
+ current_tok_seq)
+
+ current_tok_seq += 1
+
+ return token_table
+
+ def _identify_boundaries(self, depth_scores):
+ """Identifies boundaries at the peaks of similarity score
+ differences"""
+
+ boundaries = [0 for x in depth_scores]
+
+ avg = sum(depth_scores)/len(depth_scores)
+ stdev = numpy.std(depth_scores)
+
+ #SB: what is the purpose of this conditional?
+ if self.cutoff_policy == LC:
+ cutoff = avg-stdev/2.0
+ else:
+ cutoff = avg-stdev/2.0
+
+ depth_tuples = sorted(zip(depth_scores, range(len(depth_scores))))
+ depth_tuples.reverse()
+ hp = filter(lambda x:x[0]>cutoff, depth_tuples)
+
+ for dt in hp:
+ boundaries[dt[1]] = 1
+ for dt2 in hp: #undo if there is a boundary close already
+ if dt[1] != dt2[1] and abs(dt2[1]-dt[1]) < 4 \
+ and boundaries[dt2[1]] == 1:
+ boundaries[dt[1]] = 0
+ return boundaries
+
+ def _depth_scores(self, scores):
+ """Calculates the depth of each gap, i.e. the average difference
+ between the left and right peaks and the gap's score"""
+
+ depth_scores = [0 for x in scores]
+ #clip boundaries: this holds on the rule of thumb(my thumb)
+ #that a section shouldn't be smaller than at least 2
+ #pseudosentences for small texts and around 5 for larger ones.
+
+ clip = min(max(len(scores)/10, 2), 5)
+ index = clip
+
+ for gapscore in scores[clip:-clip]:
+ lpeak = gapscore
+ for score in scores[index::-1]:
+ if score >= lpeak:
+ lpeak = score
+ else:
+ break
+ rpeak = gapscore
+ for score in scores[index:]:
+ if score >= rpeak:
+ rpeak = score
+ else:
+ break
+ depth_scores[index] = lpeak + rpeak - 2 * gapscore
+ index += 1
+
+ return depth_scores
+
+ def _normalize_boundaries(self, text, boundaries, paragraph_breaks):
+ """Normalize the boundaries identified to the original text's
+ paragraph breaks"""
+
+ norm_boundaries = []
+ char_count, word_count, gaps_seen = 0, 0, 0
+ seen_word = False
+
+ for char in text:
+ char_count += 1
+ if char in " \t\n" and seen_word:
+ seen_word = False
+ word_count += 1
+ if char not in " \t\n" and not seen_word:
+ seen_word=True
+ if gaps_seen < len(boundaries) and word_count > \
+ (max(gaps_seen*self.w, self.w)):
+ if boundaries[gaps_seen] == 1:
+ #find closest paragraph break
+ best_fit = len(text)
+ for br in paragraph_breaks:
+ if best_fit > abs(br-char_count):
+ best_fit = abs(br-char_count)
+ bestbr = br
+ else:
+ break
+ if bestbr not in norm_boundaries: #avoid duplicates
+ norm_boundaries.append(bestbr)
+ gaps_seen += 1
+
+ return norm_boundaries
+
+
+class TokenTableField(object):
+ """A field in the token table holding parameters for each token,
+ used later in the process"""
+ def __init__(self,
+ first_pos,
+ ts_occurences,
+ total_count=1,
+ par_count=1,
+ last_par=0,
+ last_tok_seq=None):
+ self.__dict__.update(locals())
+ del self.__dict__['self']
+
+class TokenSequence(object):
+ "A token list with its original length and its index"
+ def __init__(self,
+ index,
+ wrdindex_list,
+ original_length=None):
+ original_length=original_length or len(wrdindex_list)
+ self.__dict__.update(locals())
+ del self.__dict__['self']
+
+#Pasted from the SciPy cookbook: http://www.scipy.org/Cookbook/SignalSmooth
+def smooth(x,window_len=11,window='flat'):
+ """smooth the data using a window with requested size.
+
+ This method is based on the convolution of a scaled window with the signal.
+ The signal is prepared by introducing reflected copies of the signal
+ (with the window size) in both ends so that transient parts are minimized
+ in the beginning and end part of the output signal.
+
+ :param x: the input signal
+ :param window_len: the dimension of the smoothing window; should be an odd integer
+ :param window: the type of window from 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'
+ flat window will produce a moving average smoothing.
+
+ :return: the smoothed signal
+
+ example::
+
+ t=linspace(-2,2,0.1)
+ x=sin(t)+randn(len(t))*0.1
+ y=smooth(x)
+
+ :see also: numpy.hanning, numpy.hamming, numpy.bartlett, numpy.blackman, numpy.convolve,
+ scipy.signal.lfilter
+
+ TODO: the window parameter could be the window itself if an array instead of a string
+ """
+
+ if x.ndim != 1:
+ raise ValueError("smooth only accepts 1 dimension arrays.")
+
+ if x.size < window_len:
+ raise ValueError("Input vector needs to be bigger than window size.")
+
+ if window_len<3:
+ return x
+
+ if not window in ['flat', 'hanning', 'hamming', 'bartlett', 'blackman']:
+ raise ValueError("Window is on of 'flat', 'hanning', 'hamming', 'bartlett', 'blackman'")
+
+ s=numpy.r_[2*x[0]-x[window_len:1:-1],x,2*x[-1]-x[-1:-window_len:-1]]
+
+ #print(len(s))
+ if window == 'flat': #moving average
+ w=numpy.ones(window_len,'d')
+ else:
+ w=eval('numpy.'+window+'(window_len)')
+
+ y=numpy.convolve(w/w.sum(),s,mode='same')
+
+ return y[window_len-1:-window_len+1]
+
+
+def demo(text=None):
+ from nltk.corpus import brown
+ import pylab
+ tt=TextTilingTokenizer(demo_mode=True)
+ if text is None: text=brown.raw()[:10000]
+ s,ss,d,b=tt.tokenize(text)
+ pylab.xlabel("Sentence Gap index")
+ pylab.ylabel("Gap Scores")
+ pylab.plot(range(len(s)), s, label="Gap Scores")
+ pylab.plot(range(len(ss)), ss, label="Smoothed Gap scores")
+ pylab.plot(range(len(d)), d, label="Depth scores")
+ pylab.stem(range(len(b)),b)
+ pylab.legend()
+ pylab.show()
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tokenize/treebank.py b/nltk/tokenize/treebank.py
new file mode 100644
index 0000000..aabde29
--- /dev/null
+++ b/nltk/tokenize/treebank.py
@@ -0,0 +1,104 @@
+# Natural Language Toolkit: Tokenizers
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Michael Heilman <mheilman at cmu.edu> (re-port from http://www.cis.upenn.edu/~treebank/tokenizer.sed)
+#
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+r"""
+
+Penn Treebank Tokenizer
+
+The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
+This implementation is a port of the tokenizer sed script written by Robert McIntyre
+and available at http://www.cis.upenn.edu/~treebank/tokenizer.sed.
+"""
+
+import re
+from nltk.tokenize.api import TokenizerI
+
+
+class TreebankWordTokenizer(TokenizerI):
+ """
+ The Treebank tokenizer uses regular expressions to tokenize text as in Penn Treebank.
+ This is the method that is invoked by ``word_tokenize()``. It assumes that the
+ text has already been segmented into sentences, e.g. using ``sent_tokenize()``.
+
+ This tokenizer performs the following steps:
+
+ - split standard contractions, e.g. ``don't`` -> ``do n't`` and ``they'll`` -> ``they 'll``
+ - treat most punctuation characters as separate tokens
+ - split off commas and single quotes, when followed by whitespace
+ - separate periods that appear at the end of line
+
+ >>> from nltk.tokenize import TreebankWordTokenizer
+ >>> s = '''Good muffins cost $3.88\\nin New York. Please buy me\\ntwo of them.\\nThanks.'''
+ >>> TreebankWordTokenizer().tokenize(s)
+ ['Good', 'muffins', 'cost', '$', '3.88', 'in', 'New', 'York.', 'Please', 'buy', 'me', 'two', 'of', 'them.', 'Thanks', '.']
+ >>> s = "They'll save and invest more."
+ >>> TreebankWordTokenizer().tokenize(s)
+ ['They', "'ll", 'save', 'and', 'invest', 'more', '.']
+ """
+
+ # List of contractions adapted from Robert MacIntyre's tokenizer.
+ CONTRACTIONS2 = [re.compile(r"(?i)\b(can)(not)\b"),
+ re.compile(r"(?i)\b(d)('ye)\b"),
+ re.compile(r"(?i)\b(gim)(me)\b"),
+ re.compile(r"(?i)\b(gon)(na)\b"),
+ re.compile(r"(?i)\b(got)(ta)\b"),
+ re.compile(r"(?i)\b(lem)(me)\b"),
+ re.compile(r"(?i)\b(mor)('n)\b"),
+ re.compile(r"(?i)\b(wan)(na) ")]
+ CONTRACTIONS3 = [re.compile(r"(?i) ('t)(is)\b"),
+ re.compile(r"(?i) ('t)(was)\b")]
+ CONTRACTIONS4 = [re.compile(r"(?i)\b(whad)(dd)(ya)\b"),
+ re.compile(r"(?i)\b(wha)(t)(cha)\b")]
+
+ def tokenize(self, text):
+ #starting quotes
+ text = re.sub(r'^\"', r'``', text)
+ text = re.sub(r'(``)', r' \1 ', text)
+ text = re.sub(r'([ (\[{<])"', r'\1 `` ', text)
+
+ #punctuation
+ text = re.sub(r'([:,])([^\d])', r' \1 \2', text)
+ text = re.sub(r'\.\.\.', r' ... ', text)
+ text = re.sub(r'[;@#$%&]', r' \g<0> ', text)
+ text = re.sub(r'([^\.])(\.)([\]\)}>"\']*)\s*$', r'\1 \2\3 ', text)
+ text = re.sub(r'[?!]', r' \g<0> ', text)
+
+ text = re.sub(r"([^'])' ", r"\1 ' ", text)
+
+ #parens, brackets, etc.
+ text = re.sub(r'[\]\[\(\)\{\}\<\>]', r' \g<0> ', text)
+ text = re.sub(r'--', r' -- ', text)
+
+ #add extra space to make things easier
+ text = " " + text + " "
+
+ #ending quotes
+ text = re.sub(r'"', " '' ", text)
+ text = re.sub(r'(\S)(\'\')', r'\1 \2 ', text)
+
+ text = re.sub(r"([^' ])('[sS]|'[mM]|'[dD]|') ", r"\1 \2 ", text)
+ text = re.sub(r"([^' ])('ll|'LL|'re|'RE|'ve|'VE|n't|N'T) ", r"\1 \2 ",
+ text)
+
+ for regexp in self.CONTRACTIONS2:
+ text = regexp.sub(r' \1 \2 ', text)
+ for regexp in self.CONTRACTIONS3:
+ text = regexp.sub(r' \1 \2 ', text)
+
+ # We are not using CONTRACTIONS4 since
+ # they are also commented out in the SED scripts
+ # for regexp in self.CONTRACTIONS4:
+ # text = regexp.sub(r' \1 \2 \3 ', text)
+
+ return text.split()
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/tokenize/util.py b/nltk/tokenize/util.py
new file mode 100644
index 0000000..d9d1342
--- /dev/null
+++ b/nltk/tokenize/util.py
@@ -0,0 +1,93 @@
+# Natural Language Toolkit: Tokenizer Utilities
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.sourceforge.net>
+# For license information, see LICENSE.TXT
+
+from re import finditer
+
+def string_span_tokenize(s, sep):
+ r"""
+ Return the offsets of the tokens in *s*, as a sequence of ``(start, end)``
+ tuples, by splitting the string at each occurrence of *sep*.
+
+ >>> from nltk.tokenize.util import string_span_tokenize
+ >>> s = '''Good muffins cost $3.88\nin New York. Please buy me
+ ... two of them.\n\nThanks.'''
+ >>> list(string_span_tokenize(s, " "))
+ [(0, 4), (5, 12), (13, 17), (18, 26), (27, 30), (31, 36), (37, 37),
+ (38, 44), (45, 48), (49, 55), (56, 58), (59, 73)]
+
+ :param s: the string to be tokenized
+ :type s: str
+ :param sep: the token separator
+ :type sep: str
+ :rtype: iter(tuple(int, int))
+ """
+ if len(sep) == 0:
+ raise ValueError("Token delimiter must not be empty")
+ left = 0
+ while True:
+ try:
+ right = s.index(sep, left)
+ if right != 0:
+ yield left, right
+ except ValueError:
+ if left != len(s):
+ yield left, len(s)
+ break
+
+ left = right + len(sep)
+
+def regexp_span_tokenize(s, regexp):
+ r"""
+ Return the offsets of the tokens in *s*, as a sequence of ``(start, end)``
+ tuples, by splitting the string at each successive match of *regexp*.
+
+ >>> from nltk.tokenize import WhitespaceTokenizer
+ >>> s = '''Good muffins cost $3.88\nin New York. Please buy me
+ ... two of them.\n\nThanks.'''
+ >>> list(WhitespaceTokenizer().span_tokenize(s))
+ [(0, 4), (5, 12), (13, 17), (18, 23), (24, 26), (27, 30), (31, 36),
+ (38, 44), (45, 48), (49, 51), (52, 55), (56, 58), (59, 64), (66, 73)]
+
+ :param s: the string to be tokenized
+ :type s: str
+ :param regexp: regular expression that matches token separators
+ :type regexp: str
+ :rtype: iter(tuple(int, int))
+ """
+ left = 0
+ for m in finditer(regexp, s):
+ right, next = m.span()
+ if right != 0:
+ yield left, right
+ left = next
+ yield left, len(s)
+
+def spans_to_relative(spans):
+ r"""
+ Return a sequence of relative spans, given a sequence of spans.
+
+ >>> from nltk.tokenize import WhitespaceTokenizer
+ >>> from nltk.tokenize.util import spans_to_relative
+ >>> s = '''Good muffins cost $3.88\nin New York. Please buy me
+ ... two of them.\n\nThanks.'''
+ >>> list(spans_to_relative(WhitespaceTokenizer().span_tokenize(s)))
+ [(0, 4), (1, 7), (1, 4), (1, 5), (1, 2), (1, 3), (1, 5), (2, 6),
+ (1, 3), (1, 2), (1, 3), (1, 2), (1, 5), (2, 7)]
+
+ :param spans: a sequence of (start, end) offsets of the tokens
+ :type spans: iter(tuple(int, int))
+ :rtype: iter(tuple(int, int))
+ """
+ prev = 0
+ for left, right in spans:
+ yield left - prev, right - left
+ prev = right
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/toolbox.py b/nltk/toolbox.py
new file mode 100644
index 0000000..46bf42b
--- /dev/null
+++ b/nltk/toolbox.py
@@ -0,0 +1,497 @@
+# coding: utf-8
+# Natural Language Toolkit: Toolbox Reader
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Greg Aumann <greg_aumann at sil.org>
+# URL: <http://nltk.org>
+# For license information, see LICENSE.TXT
+
+"""
+Module for reading, writing and manipulating
+Toolbox databases and settings files.
+"""
+from __future__ import print_function
+
+import os, re, codecs
+from xml.etree.ElementTree import ElementTree, TreeBuilder, Element, SubElement
+
+from nltk.compat import StringIO, u, PY3
+from nltk.data import PathPointer, ZipFilePathPointer, find
+
+
+class StandardFormat(object):
+ """
+ Class for reading and processing standard format marker files and strings.
+ """
+ def __init__(self, filename=None, encoding=None):
+ self._encoding = encoding
+ if filename is not None:
+ self.open(filename)
+
+ def open(self, sfm_file):
+ """
+ Open a standard format marker file for sequential reading.
+
+ :param sfm_file: name of the standard format marker input file
+ :type sfm_file: str
+ """
+ if isinstance(sfm_file, PathPointer):
+ # [xx] We don't use 'rU' mode here -- do we need to?
+ # (PathPointer.open doesn't take a mode option)
+ self._file = sfm_file.open(self._encoding)
+ else:
+ self._file = codecs.open(sfm_file, 'rU', self._encoding)
+
+ def open_string(self, s):
+ """
+ Open a standard format marker string for sequential reading.
+
+ :param s: string to parse as a standard format marker input file
+ :type s: str
+ """
+ self._file = StringIO(s)
+
+ def raw_fields(self):
+ """
+ Return an iterator that returns the next field in a (marker, value)
+ tuple. Linebreaks and trailing white space are preserved except
+ for the final newline in each field.
+
+ :rtype: iter(tuple(str, str))
+ """
+ join_string = '\n'
+ line_regexp = r'^%s(?:\\(\S+)\s*)?(.*)$'
+ # discard a BOM in the first line
+ first_line_pat = re.compile(line_regexp % '(?:\xef\xbb\xbf)?')
+ line_pat = re.compile(line_regexp % '')
+ # need to get first line outside the loop for correct handling
+ # of the first marker if it spans multiple lines
+ file_iter = iter(self._file)
+ line = next(file_iter)
+ mobj = re.match(first_line_pat, line)
+ mkr, line_value = mobj.groups()
+ value_lines = [line_value,]
+ self.line_num = 0
+ for line in file_iter:
+ self.line_num += 1
+ mobj = re.match(line_pat, line)
+ line_mkr, line_value = mobj.groups()
+ if line_mkr:
+ yield (mkr, join_string.join(value_lines))
+ mkr = line_mkr
+ value_lines = [line_value,]
+ else:
+ value_lines.append(line_value)
+ self.line_num += 1
+ yield (mkr, join_string.join(value_lines))
+
+ def fields(self, strip=True, unwrap=True, encoding=None, errors='strict', unicode_fields=None):
+ """
+ Return an iterator that returns the next field in a ``(marker, value)``
+ tuple, where ``marker`` and ``value`` are unicode strings if an ``encoding``
+ was specified in the ``fields()`` method. Otherwise they are non-unicode strings.
+
+ :param strip: strip trailing whitespace from the last line of each field
+ :type strip: bool
+ :param unwrap: Convert newlines in a field to spaces.
+ :type unwrap: bool
+ :param encoding: Name of an encoding to use. If it is specified then
+ the ``fields()`` method returns unicode strings rather than non
+ unicode strings.
+ :type encoding: str or None
+ :param errors: Error handling scheme for codec. Same as the ``decode()``
+ builtin string method.
+ :type errors: str
+ :param unicode_fields: Set of marker names whose values are UTF-8 encoded.
+ Ignored if encoding is None. If the whole file is UTF-8 encoded set
+ ``encoding='utf8'`` and leave ``unicode_fields`` with its default
+ value of None.
+ :type unicode_fields: sequence
+ :rtype: iter(tuple(str, str))
+ """
+ if encoding is None and unicode_fields is not None:
+ raise ValueError('unicode_fields is set but not encoding.')
+ unwrap_pat = re.compile(r'\n+')
+ for mkr, val in self.raw_fields():
+ if encoding and not PY3: # kludge - already decoded in PY3?
+ if unicode_fields is not None and mkr in unicode_fields:
+ val = val.decode('utf8', errors)
+ else:
+ val = val.decode(encoding, errors)
+ mkr = mkr.decode(encoding, errors)
+ if unwrap:
+ val = unwrap_pat.sub(' ', val)
+ if strip:
+ val = val.rstrip()
+ yield (mkr, val)
+
+ def close(self):
+ """Close a previously opened standard format marker file or string."""
+ self._file.close()
+ try:
+ del self.line_num
+ except AttributeError:
+ pass
+
+class ToolboxData(StandardFormat):
+ def parse(self, grammar=None, **kwargs):
+ if grammar:
+ return self._chunk_parse(grammar=grammar, **kwargs)
+ else:
+ return self._record_parse(**kwargs)
+
+ def _record_parse(self, key=None, **kwargs):
+ """
+ Returns an element tree structure corresponding to a toolbox data file with
+ all markers at the same level.
+
+ Thus the following Toolbox database::
+ \_sh v3.0 400 Rotokas Dictionary
+ \_DateStampHasFourDigitYear
+
+ \lx kaa
+ \ps V.A
+ \ge gag
+ \gp nek i pas
+
+ \lx kaa
+ \ps V.B
+ \ge strangle
+ \gp pasim nek
+
+ after parsing will end up with the same structure (ignoring the extra
+ whitespace) as the following XML fragment after being parsed by
+ ElementTree::
+ <toolbox_data>
+ <header>
+ <_sh>v3.0 400 Rotokas Dictionary</_sh>
+ <_DateStampHasFourDigitYear/>
+ </header>
+
+ <record>
+ <lx>kaa</lx>
+ <ps>V.A</ps>
+ <ge>gag</ge>
+ <gp>nek i pas</gp>
+ </record>
+
+ <record>
+ <lx>kaa</lx>
+ <ps>V.B</ps>
+ <ge>strangle</ge>
+ <gp>pasim nek</gp>
+ </record>
+ </toolbox_data>
+
+ :param key: Name of key marker at the start of each record. If set to
+ None (the default value) the first marker that doesn't begin with
+ an underscore is assumed to be the key.
+ :type key: str
+ :param kwargs: Keyword arguments passed to ``StandardFormat.fields()``
+ :type kwargs: dict
+ :rtype: ElementTree._ElementInterface
+ :return: contents of toolbox data divided into header and records
+ """
+ builder = TreeBuilder()
+ builder.start('toolbox_data', {})
+ builder.start('header', {})
+ in_records = False
+ for mkr, value in self.fields(**kwargs):
+ if key is None and not in_records and mkr[0] != '_':
+ key = mkr
+ if mkr == key:
+ if in_records:
+ builder.end('record')
+ else:
+ builder.end('header')
+ in_records = True
+ builder.start('record', {})
+ builder.start(mkr, {})
+ builder.data(value)
+ builder.end(mkr)
+ if in_records:
+ builder.end('record')
+ else:
+ builder.end('header')
+ builder.end('toolbox_data')
+ return builder.close()
+
+ def _tree2etree(self, parent):
+ from nltk.tree import Tree
+
+ root = Element(parent.label())
+ for child in parent:
+ if isinstance(child, Tree):
+ root.append(self._tree2etree(child))
+ else:
+ text, tag = child
+ e = SubElement(root, tag)
+ e.text = text
+ return root
+
+ def _chunk_parse(self, grammar=None, root_label='record', trace=0, **kwargs):
+ """
+ Returns an element tree structure corresponding to a toolbox data file
+ parsed according to the chunk grammar.
+
+ :type grammar: str
+ :param grammar: Contains the chunking rules used to parse the
+ database. See ``chunk.RegExp`` for documentation.
+ :type root_label: str
+ :param root_label: The node value that should be used for the
+ top node of the chunk structure.
+ :type trace: int
+ :param trace: The level of tracing that should be used when
+ parsing a text. ``0`` will generate no tracing output;
+ ``1`` will generate normal tracing output; and ``2`` or
+ higher will generate verbose tracing output.
+ :type kwargs: dict
+ :param kwargs: Keyword arguments passed to ``toolbox.StandardFormat.fields()``
+ :rtype: ElementTree._ElementInterface
+ """
+ from nltk import chunk
+ from nltk.tree import Tree
+
+ cp = chunk.RegexpParser(grammar, root_label=root_label, trace=trace)
+ db = self.parse(**kwargs)
+ tb_etree = Element('toolbox_data')
+ header = db.find('header')
+ tb_etree.append(header)
+ for record in db.findall('record'):
+ parsed = cp.parse([(elem.text, elem.tag) for elem in record])
+ tb_etree.append(self._tree2etree(parsed))
+ return tb_etree
+
+_is_value = re.compile(r"\S")
+
+def to_sfm_string(tree, encoding=None, errors='strict', unicode_fields=None):
+ """
+ Return a string with a standard format representation of the toolbox
+ data in tree (tree can be a toolbox database or a single record).
+
+ :param tree: flat representation of toolbox data (whole database or single record)
+ :type tree: ElementTree._ElementInterface
+ :param encoding: Name of an encoding to use.
+ :type encoding: str
+ :param errors: Error handling scheme for codec. Same as the ``encode()``
+ builtin string method.
+ :type errors: str
+ :param unicode_fields:
+ :type unicode_fields: dict(str) or set(str)
+ :rtype: str
+ """
+ if tree.tag == 'record':
+ root = Element('toolbox_data')
+ root.append(tree)
+ tree = root
+
+ if tree.tag != 'toolbox_data':
+ raise ValueError("not a toolbox_data element structure")
+ if encoding is None and unicode_fields is not None:
+ raise ValueError("if encoding is not specified then neither should unicode_fields")
+ l = []
+ for rec in tree:
+ l.append('\n')
+ for field in rec:
+ mkr = field.tag
+ value = field.text
+ if encoding is not None:
+ if unicode_fields is not None and mkr in unicode_fields:
+ cur_encoding = 'utf8'
+ else:
+ cur_encoding = encoding
+ if re.search(_is_value, value):
+ l.append((u("\\%s %s\n") % (mkr, value)).encode(cur_encoding, errors))
+ else:
+ l.append((u("\\%s%s\n") % (mkr, value)).encode(cur_encoding, errors))
+ else:
+ if re.search(_is_value, value):
+ l.append("\\%s %s\n" % (mkr, value))
+ else:
+ l.append("\\%s%s\n" % (mkr, value))
+ return ''.join(l[1:])
+
+class ToolboxSettings(StandardFormat):
+ """This class is the base class for settings files."""
+
+ def __init__(self):
+ super(ToolboxSettings, self).__init__()
+
+ def parse(self, encoding=None, errors='strict', **kwargs):
+ """
+ Return the contents of toolbox settings file with a nested structure.
+
+ :param encoding: encoding used by settings file
+ :type encoding: str
+ :param errors: Error handling scheme for codec. Same as ``decode()`` builtin method.
+ :type errors: str
+ :param kwargs: Keyword arguments passed to ``StandardFormat.fields()``
+ :type kwargs: dict
+ :rtype: ElementTree._ElementInterface
+ """
+ builder = TreeBuilder()
+ for mkr, value in self.fields(encoding=encoding, errors=errors, **kwargs):
+ # Check whether the first char of the field marker
+ # indicates a block start (+) or end (-)
+ block=mkr[0]
+ if block in ("+", "-"):
+ mkr=mkr[1:]
+ else:
+ block=None
+ # Build tree on the basis of block char
+ if block == "+":
+ builder.start(mkr, {})
+ builder.data(value)
+ elif block == '-':
+ builder.end(mkr)
+ else:
+ builder.start(mkr, {})
+ builder.data(value)
+ builder.end(mkr)
+ return builder.close()
+
+def to_settings_string(tree, encoding=None, errors='strict', unicode_fields=None):
+ # write XML to file
+ l = list()
+ _to_settings_string(tree.getroot(), l, encoding=encoding, errors=errors, unicode_fields=unicode_fields)
+ return ''.join(l)
+
+def _to_settings_string(node, l, **kwargs):
+ # write XML to file
+ tag = node.tag
+ text = node.text
+ if len(node) == 0:
+ if text:
+ l.append('\\%s %s\n' % (tag, text))
+ else:
+ l.append('\\%s\n' % tag)
+ else:
+ if text:
+ l.append('\\+%s %s\n' % (tag, text))
+ else:
+ l.append('\\+%s\n' % tag)
+ for n in node:
+ _to_settings_string(n, l, **kwargs)
+ l.append('\\-%s\n' % tag)
+ return
+
+def remove_blanks(elem):
+ """
+ Remove all elements and subelements with no text and no child elements.
+
+ :param elem: toolbox data in an elementtree structure
+ :type elem: ElementTree._ElementInterface
+ """
+ out = list()
+ for child in elem:
+ remove_blanks(child)
+ if child.text or len(child) > 0:
+ out.append(child)
+ elem[:] = out
+
+def add_default_fields(elem, default_fields):
+ """
+ Add blank elements and subelements specified in default_fields.
+
+ :param elem: toolbox data in an elementtree structure
+ :type elem: ElementTree._ElementInterface
+ :param default_fields: fields to add to each type of element and subelement
+ :type default_fields: dict(tuple)
+ """
+ for field in default_fields.get(elem.tag, []):
+ if elem.find(field) is None:
+ SubElement(elem, field)
+ for child in elem:
+ add_default_fields(child, default_fields)
+
+def sort_fields(elem, field_orders):
+ """
+ Sort the elements and subelements in order specified in field_orders.
+
+ :param elem: toolbox data in an elementtree structure
+ :type elem: ElementTree._ElementInterface
+ :param field_orders: order of fields for each type of element and subelement
+ :type field_orders: dict(tuple)
+ """
+ order_dicts = dict()
+ for field, order in field_orders.items():
+ order_dicts[field] = order_key = dict()
+ for i, subfield in enumerate(order):
+ order_key[subfield] = i
+ _sort_fields(elem, order_dicts)
+
+def _sort_fields(elem, orders_dicts):
+ """sort the children of elem"""
+ try:
+ order = orders_dicts[elem.tag]
+ except KeyError:
+ pass
+ else:
+ tmp = sorted([((order.get(child.tag, 1e9), i), child) for i, child in enumerate(elem)])
+ elem[:] = [child for key, child in tmp]
+ for child in elem:
+ if len(child):
+ _sort_fields(child, orders_dicts)
+
+def add_blank_lines(tree, blanks_before, blanks_between):
+ """
+ Add blank lines before all elements and subelements specified in blank_before.
+
+ :param elem: toolbox data in an elementtree structure
+ :type elem: ElementTree._ElementInterface
+ :param blank_before: elements and subelements to add blank lines before
+ :type blank_before: dict(tuple)
+ """
+ try:
+ before = blanks_before[tree.tag]
+ between = blanks_between[tree.tag]
+ except KeyError:
+ for elem in tree:
+ if len(elem):
+ add_blank_lines(elem, blanks_before, blanks_between)
+ else:
+ last_elem = None
+ for elem in tree:
+ tag = elem.tag
+ if last_elem is not None and last_elem.tag != tag:
+ if tag in before and last_elem is not None:
+ e = last_elem.getiterator()[-1]
+ e.text = (e.text or "") + "\n"
+ else:
+ if tag in between:
+ e = last_elem.getiterator()[-1]
+ e.text = (e.text or "") + "\n"
+ if len(elem):
+ add_blank_lines(elem, blanks_before, blanks_between)
+ last_elem = elem
+
+def demo():
+ from itertools import islice
+
+# zip_path = find('corpora/toolbox.zip')
+# lexicon = ToolboxData(ZipFilePathPointer(zip_path, 'toolbox/rotokas.dic')).parse()
+ file_path = find('corpora/toolbox/rotokas.dic')
+ lexicon = ToolboxData(file_path).parse()
+ print('first field in fourth record:')
+ print(lexicon[3][0].tag)
+ print(lexicon[3][0].text)
+
+ print('\nfields in sequential order:')
+ for field in islice(lexicon.find('record'), 10):
+ print(field.tag, field.text)
+
+ print('\nlx fields:')
+ for field in islice(lexicon.findall('record/lx'), 10):
+ print(field.text)
+
+ settings = ToolboxSettings()
+ file_path = find('corpora/toolbox/MDF/MDF_AltH.typ')
+ settings.open(file_path)
+# settings.open(ZipFilePathPointer(zip_path, entry='toolbox/MDF/MDF_AltH.typ'))
+ tree = settings.parse(unwrap=False, encoding='cp1252')
+ print(tree.find('expset/expMDF/rtfPageSetup/paperSize').text)
+ settings_tree = ElementTree(tree)
+ print(to_settings_string(settings_tree).encode('utf8'))
+
+if __name__ == '__main__':
+ demo()
diff --git a/nltk/tree.py b/nltk/tree.py
new file mode 100644
index 0000000..6bc0472
--- /dev/null
+++ b/nltk/tree.py
@@ -0,0 +1,1585 @@
+# -*- coding: utf-8 -*-
+# Natural Language Toolkit: Text Trees
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# Peter Ljunglöf <peter.ljunglof at gu.se>
+# Nathan Bodenstab <bodenstab at cslu.ogi.edu> (tree transforms)
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+Class for representing hierarchical language structures, such as
+syntax trees and morphological trees.
+"""
+from __future__ import print_function, unicode_literals
+
+# TODO: add LabelledTree (can be used for dependency trees)
+
+import re
+
+from nltk.grammar import Production, Nonterminal
+from nltk.probability import ProbabilisticMixIn
+from nltk.util import slice_bounds
+from nltk.compat import string_types, python_2_unicode_compatible, unicode_repr
+from nltk.internals import raise_unorderable_types
+
+######################################################################
+## Trees
+######################################################################
+
+ at python_2_unicode_compatible
+class Tree(list):
+ """
+ A Tree represents a hierarchical grouping of leaves and subtrees.
+ For example, each constituent in a syntax tree is represented by a single Tree.
+
+ A tree's children are encoded as a list of leaves and subtrees,
+ where a leaf is a basic (non-tree) value; and a subtree is a
+ nested Tree.
+
+ >>> from nltk.tree import Tree
+ >>> print(Tree(1, [2, Tree(3, [4]), 5]))
+ (1 2 (3 4) 5)
+ >>> vp = Tree('VP', [Tree('V', ['saw']),
+ ... Tree('NP', ['him'])])
+ >>> s = Tree('S', [Tree('NP', ['I']), vp])
+ >>> print(s)
+ (S (NP I) (VP (V saw) (NP him)))
+ >>> print(s[1])
+ (VP (V saw) (NP him))
+ >>> print(s[1,1])
+ (NP him)
+ >>> t = Tree.fromstring("(S (NP I) (VP (V saw) (NP him)))")
+ >>> s == t
+ True
+ >>> t[1][1].set_label('X')
+ >>> t[1][1].label()
+ 'X'
+ >>> print(t)
+ (S (NP I) (VP (V saw) (X him)))
+ >>> t[0], t[1,1] = t[1,1], t[0]
+ >>> print(t)
+ (S (X him) (VP (V saw) (NP I)))
+
+ The length of a tree is the number of children it has.
+
+ >>> len(t)
+ 2
+
+ The set_label() and label() methods allow individual constituents
+ to be labeled. For example, syntax trees use this label to specify
+ phrase tags, such as "NP" and "VP".
+
+ Several Tree methods use "tree positions" to specify
+ children or descendants of a tree. Tree positions are defined as
+ follows:
+
+ - The tree position *i* specifies a Tree's *i*\ th child.
+ - The tree position ``()`` specifies the Tree itself.
+ - If *p* is the tree position of descendant *d*, then
+ *p+i* specifies the *i*\ th child of *d*.
+
+ I.e., every tree position is either a single index *i*,
+ specifying ``tree[i]``; or a sequence *i1, i2, ..., iN*,
+ specifying ``tree[i1][i2]...[iN]``.
+
+ Construct a new tree. This constructor can be called in one
+ of two ways:
+
+ - ``Tree(label, children)`` constructs a new tree with the
+ specified label and list of children.
+
+ - ``Tree.fromstring(s)`` constructs a new tree by parsing the string ``s``.
+ """
+ def __init__(self, node, children=None):
+ if children is None:
+ raise TypeError("%s: Expected a node value and child list "
+ % type(self).__name__)
+ elif isinstance(children, string_types):
+ raise TypeError("%s() argument 2 should be a list, not a "
+ "string" % type(self).__name__)
+ else:
+ list.__init__(self, children)
+ self._label = node
+
+ #////////////////////////////////////////////////////////////
+ # Comparison operators
+ #////////////////////////////////////////////////////////////
+
+ def __eq__(self, other):
+ return (self.__class__ is other.__class__ and
+ (self._label, list(self)) == (other._label, list(other)))
+
+ def __lt__(self, other):
+ if not isinstance(other, Tree):
+ # raise_unorderable_types("<", self, other)
+ # Sometimes children can be pure strings,
+ # so we need to be able to compare with non-trees:
+ return self.__class__.__name__ < other.__class__.__name__
+ elif self.__class__ is other.__class__:
+ return (self._label, list(self)) < (other._label, list(other))
+ else:
+ return self.__class__.__name__ < other.__class__.__name__
+
+ # @total_ordering doesn't work here, since the class inherits from a builtin class
+ __ne__ = lambda self, other: not self == other
+ __gt__ = lambda self, other: not (self < other or self == other)
+ __le__ = lambda self, other: self < other or self == other
+ __ge__ = lambda self, other: not self < other
+
+ #////////////////////////////////////////////////////////////
+ # Disabled list operations
+ #////////////////////////////////////////////////////////////
+
+ def __mul__(self, v):
+ raise TypeError('Tree does not support multiplication')
+ def __rmul__(self, v):
+ raise TypeError('Tree does not support multiplication')
+ def __add__(self, v):
+ raise TypeError('Tree does not support addition')
+ def __radd__(self, v):
+ raise TypeError('Tree does not support addition')
+
+ #////////////////////////////////////////////////////////////
+ # Indexing (with support for tree positions)
+ #////////////////////////////////////////////////////////////
+
+ def __getitem__(self, index):
+ if isinstance(index, (int, slice)):
+ return list.__getitem__(self, index)
+ elif isinstance(index, (list, tuple)):
+ if len(index) == 0:
+ return self
+ elif len(index) == 1:
+ return self[index[0]]
+ else:
+ return self[index[0]][index[1:]]
+ else:
+ raise TypeError("%s indices must be integers, not %s" %
+ (type(self).__name__, type(index).__name__))
+
+ def __setitem__(self, index, value):
+ if isinstance(index, (int, slice)):
+ return list.__setitem__(self, index, value)
+ elif isinstance(index, (list, tuple)):
+ if len(index) == 0:
+ raise IndexError('The tree position () may not be '
+ 'assigned to.')
+ elif len(index) == 1:
+ self[index[0]] = value
+ else:
+ self[index[0]][index[1:]] = value
+ else:
+ raise TypeError("%s indices must be integers, not %s" %
+ (type(self).__name__, type(index).__name__))
+
+ def __delitem__(self, index):
+ if isinstance(index, (int, slice)):
+ return list.__delitem__(self, index)
+ elif isinstance(index, (list, tuple)):
+ if len(index) == 0:
+ raise IndexError('The tree position () may not be deleted.')
+ elif len(index) == 1:
+ del self[index[0]]
+ else:
+ del self[index[0]][index[1:]]
+ else:
+ raise TypeError("%s indices must be integers, not %s" %
+ (type(self).__name__, type(index).__name__))
+
+ #////////////////////////////////////////////////////////////
+ # Basic tree operations
+ #////////////////////////////////////////////////////////////
+
+ def _get_node(self):
+ """Outdated method to access the node value; use the label() method instead."""
+ raise NotImplementedError("Use label() to access a node label.")
+ def _set_node(self, value):
+ """Outdated method to set the node value; use the set_label() method instead."""
+ raise NotImplementedError("Use set_label() method to set a node label.")
+ node = property(_get_node, _set_node)
+
+ def label(self):
+ """
+ Return the node label of the tree.
+
+ >>> t = Tree.fromstring('(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))')
+ >>> t.label()
+ 'S'
+
+ :return: the node label (typically a string)
+ :rtype: any
+ """
+ return self._label
+
+ def set_label(self, label):
+ """
+ Set the node label of the tree.
+
+ >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+ >>> t.set_label("T")
+ >>> print(t)
+ (T (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))
+
+ :param label: the node label (typically a string)
+ :type label: any
+ """
+ self._label = label
+
+ def leaves(self):
+ """
+ Return the leaves of the tree.
+
+ >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+ >>> t.leaves()
+ ['the', 'dog', 'chased', 'the', 'cat']
+
+ :return: a list containing this tree's leaves.
+ The order reflects the order of the
+ leaves in the tree's hierarchical structure.
+ :rtype: list
+ """
+ leaves = []
+ for child in self:
+ if isinstance(child, Tree):
+ leaves.extend(child.leaves())
+ else:
+ leaves.append(child)
+ return leaves
+
+ def flatten(self):
+ """
+ Return a flat version of the tree, with all non-root non-terminals removed.
+
+ >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+ >>> print(t.flatten())
+ (S the dog chased the cat)
+
+ :return: a tree consisting of this tree's root connected directly to
+ its leaves, omitting all intervening non-terminal nodes.
+ :rtype: Tree
+ """
+ return Tree(self.label(), self.leaves())
+
+ def height(self):
+ """
+ Return the height of the tree.
+
+ >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+ >>> t.height()
+ 5
+ >>> print(t[0,0])
+ (D the)
+ >>> t[0,0].height()
+ 2
+
+ :return: The height of this tree. The height of a tree
+ containing no children is 1; the height of a tree
+ containing only leaves is 2; and the height of any other
+ tree is one plus the maximum of its children's
+ heights.
+ :rtype: int
+ """
+ max_child_height = 0
+ for child in self:
+ if isinstance(child, Tree):
+ max_child_height = max(max_child_height, child.height())
+ else:
+ max_child_height = max(max_child_height, 1)
+ return 1 + max_child_height
+
+ def treepositions(self, order='preorder'):
+ """
+ >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+ >>> t.treepositions() # doctest: +ELLIPSIS
+ [(), (0,), (0, 0), (0, 0, 0), (0, 1), (0, 1, 0), (1,), (1, 0), (1, 0, 0), ...]
+ >>> for pos in t.treepositions('leaves'):
+ ... t[pos] = t[pos][::-1].upper()
+ >>> print(t)
+ (S (NP (D EHT) (N GOD)) (VP (V DESAHC) (NP (D EHT) (N TAC))))
+
+ :param order: One of: ``preorder``, ``postorder``, ``bothorder``,
+ ``leaves``.
+ """
+ positions = []
+ if order in ('preorder', 'bothorder'): positions.append( () )
+ for i, child in enumerate(self):
+ if isinstance(child, Tree):
+ childpos = child.treepositions(order)
+ positions.extend((i,)+p for p in childpos)
+ else:
+ positions.append( (i,) )
+ if order in ('postorder', 'bothorder'): positions.append( () )
+ return positions
+
+ def subtrees(self, filter=None):
+ """
+ Generate all the subtrees of this tree, optionally restricted
+ to trees matching the filter function.
+
+ >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+ >>> for s in t.subtrees(lambda t: t.height() == 2):
+ ... print(s)
+ (D the)
+ (N dog)
+ (V chased)
+ (D the)
+ (N cat)
+
+ :type filter: function
+ :param filter: the function to filter all local trees
+ """
+ if not filter or filter(self):
+ yield self
+ for child in self:
+ if isinstance(child, Tree):
+ for subtree in child.subtrees(filter):
+ yield subtree
+
+ def productions(self):
+ """
+ Generate the productions that correspond to the non-terminal nodes of the tree.
+ For each subtree of the form (P: C1 C2 ... Cn) this produces a production of the
+ form P -> C1 C2 ... Cn.
+
+ >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+ >>> t.productions()
+ [S -> NP VP, NP -> D N, D -> 'the', N -> 'dog', VP -> V NP, V -> 'chased',
+ NP -> D N, D -> 'the', N -> 'cat']
+
+ :rtype: list(Production)
+ """
+
+ if not isinstance(self._label, string_types):
+ raise TypeError('Productions can only be generated from trees having node labels that are strings')
+
+ prods = [Production(Nonterminal(self._label), _child_names(self))]
+ for child in self:
+ if isinstance(child, Tree):
+ prods += child.productions()
+ return prods
+
+ def pos(self):
+ """
+ Return a sequence of pos-tagged words extracted from the tree.
+
+ >>> t = Tree.fromstring("(S (NP (D the) (N dog)) (VP (V chased) (NP (D the) (N cat))))")
+ >>> t.pos()
+ [('the', 'D'), ('dog', 'N'), ('chased', 'V'), ('the', 'D'), ('cat', 'N')]
+
+ :return: a list of tuples containing leaves and pre-terminals (part-of-speech tags).
+ The order reflects the order of the leaves in the tree's hierarchical structure.
+ :rtype: list(tuple)
+ """
+ pos = []
+ for child in self:
+ if isinstance(child, Tree):
+ pos.extend(child.pos())
+ else:
+ pos.append((child, self._label))
+ return pos
+
+ def leaf_treeposition(self, index):
+ """
+ :return: The tree position of the ``index``-th leaf in this
+ tree. I.e., if ``tp=self.leaf_treeposition(i)``, then
+ ``self[tp]==self.leaves()[i]``.
+
+ :raise IndexError: If this tree contains fewer than ``index+1``
+ leaves, or if ``index<0``.
+ """
+ if index < 0: raise IndexError('index must be non-negative')
+
+ stack = [(self, ())]
+ while stack:
+ value, treepos = stack.pop()
+ if not isinstance(value, Tree):
+ if index == 0: return treepos
+ else: index -= 1
+ else:
+ for i in range(len(value)-1, -1, -1):
+ stack.append( (value[i], treepos+(i,)) )
+
+ raise IndexError('index must be less than or equal to len(self)')
+
+ def treeposition_spanning_leaves(self, start, end):
+ """
+ :return: The tree position of the lowest descendant of this
+ tree that dominates ``self.leaves()[start:end]``.
+ :raise ValueError: if ``end <= start``
+ """
+ if end <= start:
+ raise ValueError('end must be greater than start')
+ # Find the tree positions of the start & end leaves, and
+ # take the longest common subsequence.
+ start_treepos = self.leaf_treeposition(start)
+ end_treepos = self.leaf_treeposition(end-1)
+ # Find the first index where they mismatch:
+ for i in range(len(start_treepos)):
+ if i == len(end_treepos) or start_treepos[i] != end_treepos[i]:
+ return start_treepos[:i]
+ return start_treepos
+
+ #////////////////////////////////////////////////////////////
+ # Transforms
+ #////////////////////////////////////////////////////////////
+
+ def chomsky_normal_form(self, factor="right", horzMarkov=None, vertMarkov=0, childChar="|", parentChar="^"):
+ """
+ This method can modify a tree in three ways:
+
+ 1. Convert a tree into its Chomsky Normal Form (CNF)
+ equivalent -- Every subtree has either two non-terminals
+ or one terminal as its children. This process requires
+ the creation of more"artificial" non-terminal nodes.
+ 2. Markov (vertical) smoothing of children in new artificial
+ nodes
+ 3. Horizontal (parent) annotation of nodes
+
+ :param factor: Right or left factoring method (default = "right")
+ :type factor: str = [left|right]
+ :param horzMarkov: Markov order for sibling smoothing in artificial nodes (None (default) = include all siblings)
+ :type horzMarkov: int | None
+ :param vertMarkov: Markov order for parent smoothing (0 (default) = no vertical annotation)
+ :type vertMarkov: int | None
+ :param childChar: A string used in construction of the artificial nodes, separating the head of the
+ original subtree from the child nodes that have yet to be expanded (default = "|")
+ :type childChar: str
+ :param parentChar: A string used to separate the node representation from its vertical annotation
+ :type parentChar: str
+ """
+ from nltk.treetransforms import chomsky_normal_form
+ chomsky_normal_form(self, factor, horzMarkov, vertMarkov, childChar, parentChar)
+
+ def un_chomsky_normal_form(self, expandUnary = True, childChar = "|", parentChar = "^", unaryChar = "+"):
+ """
+ This method modifies the tree in three ways:
+
+ 1. Transforms a tree in Chomsky Normal Form back to its
+ original structure (branching greater than two)
+ 2. Removes any parent annotation (if it exists)
+ 3. (optional) expands unary subtrees (if previously
+ collapsed with collapseUnary(...) )
+
+ :param expandUnary: Flag to expand unary or not (default = True)
+ :type expandUnary: bool
+ :param childChar: A string separating the head node from its children in an artificial node (default = "|")
+ :type childChar: str
+ :param parentChar: A sting separating the node label from its parent annotation (default = "^")
+ :type parentChar: str
+ :param unaryChar: A string joining two non-terminals in a unary production (default = "+")
+ :type unaryChar: str
+ """
+ from nltk.treetransforms import un_chomsky_normal_form
+ un_chomsky_normal_form(self, expandUnary, childChar, parentChar, unaryChar)
+
+ def collapse_unary(self, collapsePOS = False, collapseRoot = False, joinChar = "+"):
+ """
+ Collapse subtrees with a single child (ie. unary productions)
+ into a new non-terminal (Tree node) joined by 'joinChar'.
+ This is useful when working with algorithms that do not allow
+ unary productions, and completely removing the unary productions
+ would require loss of useful information. The Tree is modified
+ directly (since it is passed by reference) and no value is returned.
+
+ :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie.
+ Part-of-Speech tags) since they are always unary productions
+ :type collapsePOS: bool
+ :param collapseRoot: 'False' (default) will not modify the root production
+ if it is unary. For the Penn WSJ treebank corpus, this corresponds
+ to the TOP -> productions.
+ :type collapseRoot: bool
+ :param joinChar: A string used to connect collapsed node values (default = "+")
+ :type joinChar: str
+ """
+ from nltk.treetransforms import collapse_unary
+ collapse_unary(self, collapsePOS, collapseRoot, joinChar)
+
+ #////////////////////////////////////////////////////////////
+ # Convert, copy
+ #////////////////////////////////////////////////////////////
+
+ @classmethod
+ def convert(cls, tree):
+ """
+ Convert a tree between different subtypes of Tree. ``cls`` determines
+ which class will be used to encode the new tree.
+
+ :type tree: Tree
+ :param tree: The tree that should be converted.
+ :return: The new Tree.
+ """
+ if isinstance(tree, Tree):
+ children = [cls.convert(child) for child in tree]
+ return cls(tree._label, children)
+ else:
+ return tree
+
+ def copy(self, deep=False):
+ if not deep: return type(self)(self._label, self)
+ else: return type(self).convert(self)
+
+ def _frozen_class(self): return ImmutableTree
+ def freeze(self, leaf_freezer=None):
+ frozen_class = self._frozen_class()
+ if leaf_freezer is None:
+ newcopy = frozen_class.convert(self)
+ else:
+ newcopy = self.copy(deep=True)
+ for pos in newcopy.treepositions('leaves'):
+ newcopy[pos] = leaf_freezer(newcopy[pos])
+ newcopy = frozen_class.convert(newcopy)
+ hash(newcopy) # Make sure the leaves are hashable.
+ return newcopy
+
+ #////////////////////////////////////////////////////////////
+ # Parsing
+ #////////////////////////////////////////////////////////////
+
+ @classmethod
+ def fromstring(cls, s, brackets='()', read_node=None, read_leaf=None,
+ node_pattern=None, leaf_pattern=None,
+ remove_empty_top_bracketing=False):
+ """
+ Read a bracketed tree string and return the resulting tree.
+ Trees are represented as nested brackettings, such as::
+
+ (S (NP (NNP John)) (VP (V runs)))
+
+ :type s: str
+ :param s: The string to read
+
+ :type brackets: str (length=2)
+ :param brackets: The bracket characters used to mark the
+ beginning and end of trees and subtrees.
+
+ :type read_node: function
+ :type read_leaf: function
+ :param read_node, read_leaf: If specified, these functions
+ are applied to the substrings of ``s`` corresponding to
+ nodes and leaves (respectively) to obtain the values for
+ those nodes and leaves. They should have the following
+ signature:
+
+ read_node(str) -> value
+
+ For example, these functions could be used to process nodes
+ and leaves whose values should be some type other than
+ string (such as ``FeatStruct``).
+ Note that by default, node strings and leaf strings are
+ delimited by whitespace and brackets; to override this
+ default, use the ``node_pattern`` and ``leaf_pattern``
+ arguments.
+
+ :type node_pattern: str
+ :type leaf_pattern: str
+ :param node_pattern, leaf_pattern: Regular expression patterns
+ used to find node and leaf substrings in ``s``. By
+ default, both nodes patterns are defined to match any
+ sequence of non-whitespace non-bracket characters.
+
+ :type remove_empty_top_bracketing: bool
+ :param remove_empty_top_bracketing: If the resulting tree has
+ an empty node label, and is length one, then return its
+ single child instead. This is useful for treebank trees,
+ which sometimes contain an extra level of bracketing.
+
+ :return: A tree corresponding to the string representation ``s``.
+ If this class method is called using a subclass of Tree,
+ then it will return a tree of that type.
+ :rtype: Tree
+ """
+ if not isinstance(brackets, string_types) or len(brackets) != 2:
+ raise TypeError('brackets must be a length-2 string')
+ if re.search('\s', brackets):
+ raise TypeError('whitespace brackets not allowed')
+ # Construct a regexp that will tokenize the string.
+ open_b, close_b = brackets
+ open_pattern, close_pattern = (re.escape(open_b), re.escape(close_b))
+ if node_pattern is None:
+ node_pattern = '[^\s%s%s]+' % (open_pattern, close_pattern)
+ if leaf_pattern is None:
+ leaf_pattern = '[^\s%s%s]+' % (open_pattern, close_pattern)
+ token_re = re.compile('%s\s*(%s)?|%s|(%s)' % (
+ open_pattern, node_pattern, close_pattern, leaf_pattern))
+ # Walk through each token, updating a stack of trees.
+ stack = [(None, [])] # list of (node, children) tuples
+ for match in token_re.finditer(s):
+ token = match.group()
+ # Beginning of a tree/subtree
+ if token[0] == open_b:
+ if len(stack) == 1 and len(stack[0][1]) > 0:
+ cls._parse_error(s, match, 'end-of-string')
+ label = token[1:].lstrip()
+ if read_node is not None: label = read_node(label)
+ stack.append((label, []))
+ # End of a tree/subtree
+ elif token == close_b:
+ if len(stack) == 1:
+ if len(stack[0][1]) == 0:
+ cls._parse_error(s, match, open_b)
+ else:
+ cls._parse_error(s, match, 'end-of-string')
+ label, children = stack.pop()
+ stack[-1][1].append(cls(label, children))
+ # Leaf node
+ else:
+ if len(stack) == 1:
+ cls._parse_error(s, match, open_b)
+ if read_leaf is not None: token = read_leaf(token)
+ stack[-1][1].append(token)
+
+ # check that we got exactly one complete tree.
+ if len(stack) > 1:
+ cls._parse_error(s, 'end-of-string', close_b)
+ elif len(stack[0][1]) == 0:
+ cls._parse_error(s, 'end-of-string', open_b)
+ else:
+ assert stack[0][0] is None
+ assert len(stack[0][1]) == 1
+ tree = stack[0][1][0]
+
+ # If the tree has an extra level with node='', then get rid of
+ # it. E.g.: "((S (NP ...) (VP ...)))"
+ if remove_empty_top_bracketing and tree._label == '' and len(tree) == 1:
+ tree = tree[0]
+ # return the tree.
+ return tree
+
+ @classmethod
+ def _parse_error(cls, s, match, expecting):
+ """
+ Display a friendly error message when parsing a tree string fails.
+ :param s: The string we're parsing.
+ :param match: regexp match of the problem token.
+ :param expecting: what we expected to see instead.
+ """
+ # Construct a basic error message
+ if match == 'end-of-string':
+ pos, token = len(s), 'end-of-string'
+ else:
+ pos, token = match.start(), match.group()
+ msg = '%s.read(): expected %r but got %r\n%sat index %d.' % (
+ cls.__name__, expecting, token, ' '*12, pos)
+ # Add a display showing the error token itsels:
+ s = s.replace('\n', ' ').replace('\t', ' ')
+ offset = pos
+ if len(s) > pos+10:
+ s = s[:pos+10]+'...'
+ if pos > 10:
+ s = '...'+s[pos-10:]
+ offset = 13
+ msg += '\n%s"%s"\n%s^' % (' '*16, s, ' '*(17+offset))
+ raise ValueError(msg)
+
+ #////////////////////////////////////////////////////////////
+ # Visualization & String Representation
+ #////////////////////////////////////////////////////////////
+
+ def draw(self):
+ """
+ Open a new window containing a graphical diagram of this tree.
+ """
+ from nltk.draw.tree import draw_trees
+ draw_trees(self)
+
+ def __repr__(self):
+ childstr = ", ".join(unicode_repr(c) for c in self)
+ return '%s(%s, [%s])' % (type(self).__name__, unicode_repr(self._label), childstr)
+
+ def _repr_png_(self):
+ """
+ Draws and outputs in PNG for ipython.
+ PNG is used instead of PDF, since it can be displayed in the qt console and
+ has wider browser support.
+ """
+ import os
+ import base64
+ import subprocess
+ import tempfile
+ from nltk.draw.tree import tree_to_treesegment
+ from nltk.draw.util import CanvasFrame
+ from nltk.internals import find_binary
+ _canvas_frame = CanvasFrame()
+ widget = tree_to_treesegment(_canvas_frame.canvas(), self)
+ _canvas_frame.add_widget(widget)
+ x, y, w, h = widget.bbox()
+ # print_to_file uses scrollregion to set the width and height of the pdf.
+ _canvas_frame.canvas()['scrollregion'] = (0, 0, w, h)
+ with tempfile.NamedTemporaryFile() as file:
+ in_path = '{0:}.ps'.format(file.name)
+ out_path = '{0:}.png'.format(file.name)
+ _canvas_frame.print_to_file(in_path)
+ _canvas_frame.destroy_widget(widget)
+ subprocess.call([find_binary('gs', binary_names=['gswin32c.exe', 'gswin64c.exe'], env_vars=['PATH'], verbose=False)] +
+ '-q -dEPSCrop -sDEVICE=png16m -r90 -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -dSAFER -dBATCH -dNOPAUSE -sOutputFile={0:} {1:}'
+ .format(out_path, in_path).split(),
+ shell=True)
+ with open(out_path, 'rb') as sr:
+ res = sr.read()
+ os.remove(in_path)
+ os.remove(out_path)
+ return base64.b64encode(res).decode()
+
+ def __str__(self):
+ return self.pprint()
+
+ def pprint(self, margin=70, indent=0, nodesep='', parens='()', quotes=False):
+ """
+ :return: A pretty-printed string representation of this tree.
+ :rtype: str
+ :param margin: The right margin at which to do line-wrapping.
+ :type margin: int
+ :param indent: The indentation level at which printing
+ begins. This number is used to decide how far to indent
+ subsequent lines.
+ :type indent: int
+ :param nodesep: A string that is used to separate the node
+ from the children. E.g., the default value ``':'`` gives
+ trees like ``(S: (NP: I) (VP: (V: saw) (NP: it)))``.
+ """
+
+ # Try writing it on one line.
+ s = self._pprint_flat(nodesep, parens, quotes)
+ if len(s)+indent < margin:
+ return s
+
+ # If it doesn't fit on one line, then write it on multi-lines.
+ if isinstance(self._label, string_types):
+ s = '%s%s%s' % (parens[0], self._label, nodesep)
+ else:
+ s = '%s%s%s' % (parens[0], unicode_repr(self._label), nodesep)
+ for child in self:
+ if isinstance(child, Tree):
+ s += '\n'+' '*(indent+2)+child.pprint(margin, indent+2,
+ nodesep, parens, quotes)
+ elif isinstance(child, tuple):
+ s += '\n'+' '*(indent+2)+ "/".join(child)
+ elif isinstance(child, string_types) and not quotes:
+ s += '\n'+' '*(indent+2)+ '%s' % child
+ else:
+ s += '\n'+' '*(indent+2)+ unicode_repr(child)
+ return s+parens[1]
+
+ def pprint_latex_qtree(self):
+ r"""
+ Returns a representation of the tree compatible with the
+ LaTeX qtree package. This consists of the string ``\Tree``
+ followed by the tree represented in bracketed notation.
+
+ For example, the following result was generated from a parse tree of
+ the sentence ``The announcement astounded us``::
+
+ \Tree [.I'' [.N'' [.D The ] [.N' [.N announcement ] ] ]
+ [.I' [.V'' [.V' [.V astounded ] [.N'' [.N' [.N us ] ] ] ] ] ] ]
+
+ See http://www.ling.upenn.edu/advice/latex.html for the LaTeX
+ style file for the qtree package.
+
+ :return: A latex qtree representation of this tree.
+ :rtype: str
+ """
+ reserved_chars = re.compile('([#\$%&~_\{\}])')
+
+ pprint = self.pprint(indent=6, nodesep='', parens=('[.', ' ]'))
+ return r'\Tree ' + re.sub(reserved_chars, r'\\\1', pprint)
+
+ def _pprint_flat(self, nodesep, parens, quotes):
+ childstrs = []
+ for child in self:
+ if isinstance(child, Tree):
+ childstrs.append(child._pprint_flat(nodesep, parens, quotes))
+ elif isinstance(child, tuple):
+ childstrs.append("/".join(child))
+ elif isinstance(child, string_types) and not quotes:
+ childstrs.append('%s' % child)
+ else:
+ childstrs.append(unicode_repr(child))
+ if isinstance(self._label, string_types):
+ return '%s%s%s %s%s' % (parens[0], self._label, nodesep,
+ " ".join(childstrs), parens[1])
+ else:
+ return '%s%s%s %s%s' % (parens[0], unicode_repr(self._label), nodesep,
+ " ".join(childstrs), parens[1])
+
+
+class ImmutableTree(Tree):
+ def __init__(self, node, children=None):
+ super(ImmutableTree, self).__init__(node, children)
+ # Precompute our hash value. This ensures that we're really
+ # immutable. It also means we only have to calculate it once.
+ try:
+ self._hash = hash((self._label, tuple(self)))
+ except (TypeError, ValueError):
+ raise ValueError("%s: node value and children "
+ "must be immutable" % type(self).__name__)
+
+ def __setitem__(self, index, value):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def __setslice__(self, i, j, value):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def __delitem__(self, index):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def __delslice__(self, i, j):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def __iadd__(self, other):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def __imul__(self, other):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def append(self, v):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def extend(self, v):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def pop(self, v=None):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def remove(self, v):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def reverse(self):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def sort(self):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ def __hash__(self):
+ return self._hash
+
+ def set_label(self, value):
+ """
+ Set the node label. This will only succeed the first time the
+ node label is set, which should occur in ImmutableTree.__init__().
+ """
+ if hasattr(self, '_label'):
+ raise ValueError('%s may not be modified' % type(self).__name__)
+ self._label = value
+
+
+######################################################################
+## Parented trees
+######################################################################
+
+class AbstractParentedTree(Tree):
+ """
+ An abstract base class for a ``Tree`` that automatically maintains
+ pointers to parent nodes. These parent pointers are updated
+ whenever any change is made to a tree's structure. Two subclasses
+ are currently defined:
+
+ - ``ParentedTree`` is used for tree structures where each subtree
+ has at most one parent. This class should be used in cases
+ where there is no"sharing" of subtrees.
+
+ - ``MultiParentedTree`` is used for tree structures where a
+ subtree may have zero or more parents. This class should be
+ used in cases where subtrees may be shared.
+
+ Subclassing
+ ===========
+ The ``AbstractParentedTree`` class redefines all operations that
+ modify a tree's structure to call two methods, which are used by
+ subclasses to update parent information:
+
+ - ``_setparent()`` is called whenever a new child is added.
+ - ``_delparent()`` is called whenever a child is removed.
+ """
+
+ def __init__(self, node, children=None):
+ super(AbstractParentedTree, self).__init__(node, children)
+ # If children is None, the tree is read from node, and
+ # all parents will be set during parsing.
+ if children is not None:
+ # Otherwise we have to set the parent of the children.
+ # Iterate over self, and *not* children, because children
+ # might be an iterator.
+ for i, child in enumerate(self):
+ if isinstance(child, Tree):
+ self._setparent(child, i, dry_run=True)
+ for i, child in enumerate(self):
+ if isinstance(child, Tree):
+ self._setparent(child, i)
+
+ #////////////////////////////////////////////////////////////
+ # Parent management
+ #////////////////////////////////////////////////////////////
+
+ def _setparent(self, child, index, dry_run=False):
+ """
+ Update the parent pointer of ``child`` to point to ``self``. This
+ method is only called if the type of ``child`` is ``Tree``;
+ i.e., it is not called when adding a leaf to a tree. This method
+ is always called before the child is actually added to the
+ child list of ``self``.
+
+ :type child: Tree
+ :type index: int
+ :param index: The index of ``child`` in ``self``.
+ :raise TypeError: If ``child`` is a tree with an impropriate
+ type. Typically, if ``child`` is a tree, then its type needs
+ to match the type of ``self``. This prevents mixing of
+ different tree types (single-parented, multi-parented, and
+ non-parented).
+ :param dry_run: If true, the don't actually set the child's
+ parent pointer; just check for any error conditions, and
+ raise an exception if one is found.
+ """
+ raise NotImplementedError()
+
+ def _delparent(self, child, index):
+ """
+ Update the parent pointer of ``child`` to not point to self. This
+ method is only called if the type of ``child`` is ``Tree``; i.e., it
+ is not called when removing a leaf from a tree. This method
+ is always called before the child is actually removed from the
+ child list of ``self``.
+
+ :type child: Tree
+ :type index: int
+ :param index: The index of ``child`` in ``self``.
+ """
+ raise NotImplementedError()
+
+ #////////////////////////////////////////////////////////////
+ # Methods that add/remove children
+ #////////////////////////////////////////////////////////////
+ # Every method that adds or removes a child must make
+ # appropriate calls to _setparent() and _delparent().
+
+ def __delitem__(self, index):
+ # del ptree[start:stop]
+ if isinstance(index, slice):
+ start, stop, step = slice_bounds(self, index, allow_step=True)
+ # Clear all the children pointers.
+ for i in range(start, stop, step):
+ if isinstance(self[i], Tree):
+ self._delparent(self[i], i)
+ # Delete the children from our child list.
+ super(AbstractParentedTree, self).__delitem__(index)
+
+ # del ptree[i]
+ elif isinstance(index, int):
+ if index < 0: index += len(self)
+ if index < 0: raise IndexError('index out of range')
+ # Clear the child's parent pointer.
+ if isinstance(self[index], Tree):
+ self._delparent(self[index], index)
+ # Remove the child from our child list.
+ super(AbstractParentedTree, self).__delitem__(index)
+
+ elif isinstance(index, (list, tuple)):
+ # del ptree[()]
+ if len(index) == 0:
+ raise IndexError('The tree position () may not be deleted.')
+ # del ptree[(i,)]
+ elif len(index) == 1:
+ del self[index[0]]
+ # del ptree[i1, i2, i3]
+ else:
+ del self[index[0]][index[1:]]
+
+ else:
+ raise TypeError("%s indices must be integers, not %s" %
+ (type(self).__name__, type(index).__name__))
+
+ def __setitem__(self, index, value):
+ # ptree[start:stop] = value
+ if isinstance(index, slice):
+ start, stop, step = slice_bounds(self, index, allow_step=True)
+ # make a copy of value, in case it's an iterator
+ if not isinstance(value, (list, tuple)):
+ value = list(value)
+ # Check for any error conditions, so we can avoid ending
+ # up in an inconsistent state if an error does occur.
+ for i, child in enumerate(value):
+ if isinstance(child, Tree):
+ self._setparent(child, start + i*step, dry_run=True)
+ # clear the child pointers of all parents we're removing
+ for i in range(start, stop, step):
+ if isinstance(self[i], Tree):
+ self._delparent(self[i], i)
+ # set the child pointers of the new children. We do this
+ # after clearing *all* child pointers, in case we're e.g.
+ # reversing the elements in a tree.
+ for i, child in enumerate(value):
+ if isinstance(child, Tree):
+ self._setparent(child, start + i*step)
+ # finally, update the content of the child list itself.
+ super(AbstractParentedTree, self).__setitem__(index, value)
+
+ # ptree[i] = value
+ elif isinstance(index, int):
+ if index < 0: index += len(self)
+ if index < 0: raise IndexError('index out of range')
+ # if the value is not changing, do nothing.
+ if value is self[index]:
+ return
+ # Set the new child's parent pointer.
+ if isinstance(value, Tree):
+ self._setparent(value, index)
+ # Remove the old child's parent pointer
+ if isinstance(self[index], Tree):
+ self._delparent(self[index], index)
+ # Update our child list.
+ super(AbstractParentedTree, self).__setitem__(index, value)
+
+ elif isinstance(index, (list, tuple)):
+ # ptree[()] = value
+ if len(index) == 0:
+ raise IndexError('The tree position () may not be assigned to.')
+ # ptree[(i,)] = value
+ elif len(index) == 1:
+ self[index[0]] = value
+ # ptree[i1, i2, i3] = value
+ else:
+ self[index[0]][index[1:]] = value
+
+ else:
+ raise TypeError("%s indices must be integers, not %s" %
+ (type(self).__name__, type(index).__name__))
+
+ def append(self, child):
+ if isinstance(child, Tree):
+ self._setparent(child, len(self))
+ super(AbstractParentedTree, self).append(child)
+
+ def extend(self, children):
+ for child in children:
+ if isinstance(child, Tree):
+ self._setparent(child, len(self))
+ super(AbstractParentedTree, self).append(child)
+
+ def insert(self, index, child):
+ # Handle negative indexes. Note that if index < -len(self),
+ # we do *not* raise an IndexError, unlike __getitem__. This
+ # is done for consistency with list.__getitem__ and list.index.
+ if index < 0: index += len(self)
+ if index < 0: index = 0
+ # Set the child's parent, and update our child list.
+ if isinstance(child, Tree):
+ self._setparent(child, index)
+ super(AbstractParentedTree, self).insert(index, child)
+
+ def pop(self, index=-1):
+ if index < 0: index += len(self)
+ if index < 0: raise IndexError('index out of range')
+ if isinstance(self[index], Tree):
+ self._delparent(self[index], index)
+ return super(AbstractParentedTree, self).pop(index)
+
+ # n.b.: like `list`, this is done by equality, not identity!
+ # To remove a specific child, use del ptree[i].
+ def remove(self, child):
+ index = self.index(child)
+ if isinstance(self[index], Tree):
+ self._delparent(self[index], index)
+ super(AbstractParentedTree, self).remove(child)
+
+ # We need to implement __getslice__ and friends, even though
+ # they're deprecated, because otherwise list.__getslice__ will get
+ # called (since we're subclassing from list). Just delegate to
+ # __getitem__ etc., but use max(0, start) and max(0, stop) because
+ # because negative indices are already handled *before*
+ # __getslice__ is called; and we don't want to double-count them.
+ if hasattr(list, '__getslice__'):
+ def __getslice__(self, start, stop):
+ return self.__getitem__(slice(max(0, start), max(0, stop)))
+ def __delslice__(self, start, stop):
+ return self.__delitem__(slice(max(0, start), max(0, stop)))
+ def __setslice__(self, start, stop, value):
+ return self.__setitem__(slice(max(0, start), max(0, stop)), value)
+
+class ParentedTree(AbstractParentedTree):
+ """
+ A ``Tree`` that automatically maintains parent pointers for
+ single-parented trees. The following are methods for querying
+ the structure of a parented tree: ``parent``, ``parent_index``,
+ ``left_sibling``, ``right_sibling``, ``root``, ``treeposition``.
+
+ Each ``ParentedTree`` may have at most one parent. In
+ particular, subtrees may not be shared. Any attempt to reuse a
+ single ``ParentedTree`` as a child of more than one parent (or
+ as multiple children of the same parent) will cause a
+ ``ValueError`` exception to be raised.
+
+ ``ParentedTrees`` should never be used in the same tree as ``Trees``
+ or ``MultiParentedTrees``. Mixing tree implementations may result
+ in incorrect parent pointers and in ``TypeError`` exceptions.
+ """
+ def __init__(self, node, children=None):
+ self._parent = None
+ """The parent of this Tree, or None if it has no parent."""
+ super(ParentedTree, self).__init__(node, children)
+ if children is None:
+ # If children is None, the tree is read from node.
+ # After parsing, the parent of the immediate children
+ # will point to an intermediate tree, not self.
+ # We fix this by brute force:
+ for i, child in enumerate(self):
+ if isinstance(child, Tree):
+ child._parent = None
+ self._setparent(child, i)
+
+ def _frozen_class(self): return ImmutableParentedTree
+
+ #/////////////////////////////////////////////////////////////////
+ # Methods
+ #/////////////////////////////////////////////////////////////////
+
+ def parent(self):
+ """The parent of this tree, or None if it has no parent."""
+ return self._parent
+
+ def parent_index(self):
+ """
+ The index of this tree in its parent. I.e.,
+ ``ptree.parent()[ptree.parent_index()] is ptree``. Note that
+ ``ptree.parent_index()`` is not necessarily equal to
+ ``ptree.parent.index(ptree)``, since the ``index()`` method
+ returns the first child that is equal to its argument.
+ """
+ if self._parent is None: return None
+ for i, child in enumerate(self._parent):
+ if child is self: return i
+ assert False, 'expected to find self in self._parent!'
+
+ def left_sibling(self):
+ """The left sibling of this tree, or None if it has none."""
+ parent_index = self.parent_index()
+ if self._parent and parent_index > 0:
+ return self._parent[parent_index-1]
+ return None # no left sibling
+
+ def right_sibling(self):
+ """The right sibling of this tree, or None if it has none."""
+ parent_index = self.parent_index()
+ if self._parent and parent_index < (len(self._parent)-1):
+ return self._parent[parent_index+1]
+ return None # no right sibling
+
+ def root(self):
+ """
+ The root of this tree. I.e., the unique ancestor of this tree
+ whose parent is None. If ``ptree.parent()`` is None, then
+ ``ptree`` is its own root.
+ """
+ root = self
+ while root.parent() is not None:
+ root = root.parent()
+ return root
+
+ def treeposition(self):
+ """
+ The tree position of this tree, relative to the root of the
+ tree. I.e., ``ptree.root[ptree.treeposition] is ptree``.
+ """
+ if self.parent() is None:
+ return ()
+ else:
+ return self.parent().treeposition() + (self.parent_index(),)
+
+
+ #/////////////////////////////////////////////////////////////////
+ # Parent Management
+ #/////////////////////////////////////////////////////////////////
+
+ def _delparent(self, child, index):
+ # Sanity checks
+ assert isinstance(child, ParentedTree)
+ assert self[index] is child
+ assert child._parent is self
+
+ # Delete child's parent pointer.
+ child._parent = None
+
+ def _setparent(self, child, index, dry_run=False):
+ # If the child's type is incorrect, then complain.
+ if not isinstance(child, ParentedTree):
+ raise TypeError('Can not insert a non-ParentedTree '+
+ 'into a ParentedTree')
+
+ # If child already has a parent, then complain.
+ if child._parent is not None:
+ raise ValueError('Can not insert a subtree that already '
+ 'has a parent.')
+
+ # Set child's parent pointer & index.
+ if not dry_run:
+ child._parent = self
+
+
+class MultiParentedTree(AbstractParentedTree):
+ """
+ A ``Tree`` that automatically maintains parent pointers for
+ multi-parented trees. The following are methods for querying the
+ structure of a multi-parented tree: ``parents()``, ``parent_indices()``,
+ ``left_siblings()``, ``right_siblings()``, ``roots``, ``treepositions``.
+
+ Each ``MultiParentedTree`` may have zero or more parents. In
+ particular, subtrees may be shared. If a single
+ ``MultiParentedTree`` is used as multiple children of the same
+ parent, then that parent will appear multiple times in its
+ ``parents()`` method.
+
+ ``MultiParentedTrees`` should never be used in the same tree as
+ ``Trees`` or ``ParentedTrees``. Mixing tree implementations may
+ result in incorrect parent pointers and in ``TypeError`` exceptions.
+ """
+ def __init__(self, node, children=None):
+ self._parents = []
+ """A list of this tree's parents. This list should not
+ contain duplicates, even if a parent contains this tree
+ multiple times."""
+ super(MultiParentedTree, self).__init__(node, children)
+ if children is None:
+ # If children is None, the tree is read from node.
+ # After parsing, the parent(s) of the immediate children
+ # will point to an intermediate tree, not self.
+ # We fix this by brute force:
+ for i, child in enumerate(self):
+ if isinstance(child, Tree):
+ child._parents = []
+ self._setparent(child, i)
+
+ def _frozen_class(self): return ImmutableMultiParentedTree
+
+ #/////////////////////////////////////////////////////////////////
+ # Methods
+ #/////////////////////////////////////////////////////////////////
+
+ def parents(self):
+ """
+ The set of parents of this tree. If this tree has no parents,
+ then ``parents`` is the empty set. To check if a tree is used
+ as multiple children of the same parent, use the
+ ``parent_indices()`` method.
+
+ :type: list(MultiParentedTree)
+ """
+ return list(self._parents)
+
+ def left_siblings(self):
+ """
+ A list of all left siblings of this tree, in any of its parent
+ trees. A tree may be its own left sibling if it is used as
+ multiple contiguous children of the same parent. A tree may
+ appear multiple times in this list if it is the left sibling
+ of this tree with respect to multiple parents.
+
+ :type: list(MultiParentedTree)
+ """
+ return [parent[index-1]
+ for (parent, index) in self._get_parent_indices()
+ if index > 0]
+
+ def right_siblings(self):
+ """
+ A list of all right siblings of this tree, in any of its parent
+ trees. A tree may be its own right sibling if it is used as
+ multiple contiguous children of the same parent. A tree may
+ appear multiple times in this list if it is the right sibling
+ of this tree with respect to multiple parents.
+
+ :type: list(MultiParentedTree)
+ """
+ return [parent[index+1]
+ for (parent, index) in self._get_parent_indices()
+ if index < (len(parent)-1)]
+
+ def _get_parent_indices(self):
+ return [(parent, index)
+ for parent in self._parents
+ for index, child in enumerate(parent)
+ if child is self]
+
+ def roots(self):
+ """
+ The set of all roots of this tree. This set is formed by
+ tracing all possible parent paths until trees with no parents
+ are found.
+
+ :type: list(MultiParentedTree)
+ """
+ return list(self._get_roots_helper({}).values())
+
+ def _get_roots_helper(self, result):
+ if self._parents:
+ for parent in self._parents:
+ parent._get_roots_helper(result)
+ else:
+ result[id(self)] = self
+ return result
+
+ def parent_indices(self, parent):
+ """
+ Return a list of the indices where this tree occurs as a child
+ of ``parent``. If this child does not occur as a child of
+ ``parent``, then the empty list is returned. The following is
+ always true::
+
+ for parent_index in ptree.parent_indices(parent):
+ parent[parent_index] is ptree
+ """
+ if parent not in self._parents: return []
+ else: return [index for (index, child) in enumerate(parent)
+ if child is self]
+
+ def treepositions(self, root):
+ """
+ Return a list of all tree positions that can be used to reach
+ this multi-parented tree starting from ``root``. I.e., the
+ following is always true::
+
+ for treepos in ptree.treepositions(root):
+ root[treepos] is ptree
+ """
+ if self is root:
+ return [()]
+ else:
+ return [treepos+(index,)
+ for parent in self._parents
+ for treepos in parent.treepositions(root)
+ for (index, child) in enumerate(parent) if child is self]
+
+
+ #/////////////////////////////////////////////////////////////////
+ # Parent Management
+ #/////////////////////////////////////////////////////////////////
+
+ def _delparent(self, child, index):
+ # Sanity checks
+ assert isinstance(child, MultiParentedTree)
+ assert self[index] is child
+ assert len([p for p in child._parents if p is self]) == 1
+
+ # If the only copy of child in self is at index, then delete
+ # self from child's parent list.
+ for i, c in enumerate(self):
+ if c is child and i != index: break
+ else:
+ child._parents.remove(self)
+
+ def _setparent(self, child, index, dry_run=False):
+ # If the child's type is incorrect, then complain.
+ if not isinstance(child, MultiParentedTree):
+ raise TypeError('Can not insert a non-MultiParentedTree '+
+ 'into a MultiParentedTree')
+
+ # Add self as a parent pointer if it's not already listed.
+ if not dry_run:
+ for parent in child._parents:
+ if parent is self: break
+ else:
+ child._parents.append(self)
+
+class ImmutableParentedTree(ImmutableTree, ParentedTree):
+ pass
+
+class ImmutableMultiParentedTree(ImmutableTree, MultiParentedTree):
+ pass
+
+
+######################################################################
+## Probabilistic trees
+######################################################################
+
+ at python_2_unicode_compatible
+class ProbabilisticTree(Tree, ProbabilisticMixIn):
+ def __init__(self, node, children=None, **prob_kwargs):
+ Tree.__init__(self, node, children)
+ ProbabilisticMixIn.__init__(self, **prob_kwargs)
+
+ # We have to patch up these methods to make them work right:
+ def _frozen_class(self): return ImmutableProbabilisticTree
+ def __repr__(self):
+ return '%s (p=%r)' % (Tree.unicode_repr(self), self.prob())
+ def __str__(self):
+ return '%s (p=%.6g)' % (self.pprint(margin=60), self.prob())
+ def copy(self, deep=False):
+ if not deep: return type(self)(self._label, self, prob=self.prob())
+ else: return type(self).convert(self)
+ @classmethod
+ def convert(cls, val):
+ if isinstance(val, Tree):
+ children = [cls.convert(child) for child in val]
+ if isinstance(val, ProbabilisticMixIn):
+ return cls(val._label, children, prob=val.prob())
+ else:
+ return cls(val._label, children, prob=1.0)
+ else:
+ return val
+
+ def __eq__(self, other):
+ return (self.__class__ is other.__class__ and
+ (self._label, list(self), self.prob()) ==
+ (other._label, list(other), other.prob()))
+
+ def __lt__(self, other):
+ if not isinstance(other, Tree):
+ raise_unorderable_types("<", self, other)
+ if self.__class__ is other.__class__:
+ return ((self._label, list(self), self.prob()) <
+ (other._label, list(other), other.prob()))
+ else:
+ return self.__class__.__name__ < other.__class__.__name__
+
+
+ at python_2_unicode_compatible
+class ImmutableProbabilisticTree(ImmutableTree, ProbabilisticMixIn):
+ def __init__(self, node, children=None, **prob_kwargs):
+ ImmutableTree.__init__(self, node, children)
+ ProbabilisticMixIn.__init__(self, **prob_kwargs)
+ self._hash = hash((self._label, tuple(self), self.prob()))
+
+ # We have to patch up these methods to make them work right:
+ def _frozen_class(self): return ImmutableProbabilisticTree
+ def __repr__(self):
+ return '%s [%s]' % (Tree.unicode_repr(self), self.prob())
+ def __str__(self):
+ return '%s [%s]' % (self.pprint(margin=60), self.prob())
+ def copy(self, deep=False):
+ if not deep: return type(self)(self._label, self, prob=self.prob())
+ else: return type(self).convert(self)
+ @classmethod
+ def convert(cls, val):
+ if isinstance(val, Tree):
+ children = [cls.convert(child) for child in val]
+ if isinstance(val, ProbabilisticMixIn):
+ return cls(val._label, children, prob=val.prob())
+ else:
+ return cls(val._label, children, prob=1.0)
+ else:
+ return val
+
+
+def _child_names(tree):
+ names = []
+ for child in tree:
+ if isinstance(child, Tree):
+ names.append(Nonterminal(child._label))
+ else:
+ names.append(child)
+ return names
+
+######################################################################
+## Parsing
+######################################################################
+
+def bracket_parse(s):
+ """
+ Use Tree.read(s, remove_empty_top_bracketing=True) instead.
+ """
+ raise NameError("Use Tree.read(s, remove_empty_top_bracketing=True) instead.")
+
+def sinica_parse(s):
+ """
+ Parse a Sinica Treebank string and return a tree. Trees are represented as nested brackettings,
+ as shown in the following example (X represents a Chinese character):
+ S(goal:NP(Head:Nep:XX)|theme:NP(Head:Nhaa:X)|quantity:Dab:X|Head:VL2:X)#0(PERIODCATEGORY)
+
+ :return: A tree corresponding to the string representation.
+ :rtype: Tree
+ :param s: The string to be converted
+ :type s: str
+ """
+ tokens = re.split(r'([()| ])', s)
+ for i in range(len(tokens)):
+ if tokens[i] == '(':
+ tokens[i-1], tokens[i] = tokens[i], tokens[i-1] # pull nonterminal inside parens
+ elif ':' in tokens[i]:
+ fields = tokens[i].split(':')
+ if len(fields) == 2: # non-terminal
+ tokens[i] = fields[1]
+ else:
+ tokens[i] = "(" + fields[-2] + " " + fields[-1] + ")"
+ elif tokens[i] == '|':
+ tokens[i] = ''
+
+ treebank_string = " ".join(tokens)
+ return Tree.fromstring(treebank_string, remove_empty_top_bracketing=True)
+
+# s = re.sub(r'^#[^\s]*\s', '', s) # remove leading identifier
+# s = re.sub(r'\w+:', '', s) # remove role tags
+
+# return s
+
+######################################################################
+## Demonstration
+######################################################################
+
+def demo():
+ """
+ A demonstration showing how Trees and Trees can be
+ used. This demonstration creates a Tree, and loads a
+ Tree from the Treebank corpus,
+ and shows the results of calling several of their methods.
+ """
+
+ from nltk import Tree
+
+ # Demonstrate tree parsing.
+ s = '(S (NP (DT the) (NN cat)) (VP (VBD ate) (NP (DT a) (NN cookie))))'
+ t = Tree.fromstring(s)
+ print("Convert bracketed string into tree:")
+ print(t)
+ print(t.__repr__())
+
+ print("Display tree properties:")
+ print(t.label()) # tree's constituent type
+ print(t[0]) # tree's first child
+ print(t[1]) # tree's second child
+ print(t.height())
+ print(t.leaves())
+ print(t[1])
+ print(t[1,1])
+ print(t[1,1,0])
+
+ # Demonstrate tree modification.
+ the_cat = t[0]
+ the_cat.insert(1, Tree.fromstring('(JJ big)'))
+ print("Tree modification:")
+ print(t)
+ t[1,1,1] = Tree.fromstring('(NN cake)')
+ print(t)
+ print()
+
+ # Tree transforms
+ print("Collapse unary:")
+ t.collapse_unary()
+ print(t)
+ print("Chomsky normal form:")
+ t.chomsky_normal_form()
+ print(t)
+ print()
+
+ # Demonstrate probabilistic trees.
+ pt = tree.ProbabilisticTree('x', ['y', 'z'], prob=0.5)
+ print("Probabilistic Tree:")
+ print(pt)
+ print()
+
+ # Demonstrate parsing of treebank output format.
+ t = Tree.fromstring(t.pprint())
+ print("Convert tree to bracketed string and back again:")
+ print(t)
+ print()
+
+ # Demonstrate LaTeX output
+ print("LaTeX output:")
+ print(t.pprint_latex_qtree())
+ print()
+
+ # Demonstrate Productions
+ print("Production output:")
+ print(t.productions())
+ print()
+
+ # Demonstrate tree nodes containing objects other than strings
+ t.set_label(('test', 3))
+ print(t)
+
+__all__ = ['ImmutableProbabilisticTree', 'ImmutableTree', 'ProbabilisticMixIn',
+ 'ProbabilisticTree', 'Tree', 'bracket_parse',
+ 'sinica_parse', 'ParentedTree', 'MultiParentedTree',
+ 'ImmutableParentedTree', 'ImmutableMultiParentedTree']
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/nltk/treetransforms.py b/nltk/treetransforms.py
new file mode 100644
index 0000000..cf21528
--- /dev/null
+++ b/nltk/treetransforms.py
@@ -0,0 +1,309 @@
+# Natural Language Toolkit: Tree Transformations
+#
+# Copyright (C) 2005-2007 Oregon Graduate Institute
+# Author: Nathan Bodenstab <bodenstab at cslu.ogi.edu>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+"""
+A collection of methods for tree (grammar) transformations used
+in parsing natural language.
+
+Although many of these methods are technically grammar transformations
+(ie. Chomsky Norm Form), when working with treebanks it is much more
+natural to visualize these modifications in a tree structure. Hence,
+we will do all transformation directly to the tree itself.
+Transforming the tree directly also allows us to do parent annotation.
+A grammar can then be simply induced from the modified tree.
+
+The following is a short tutorial on the available transformations.
+
+ 1. Chomsky Normal Form (binarization)
+
+ It is well known that any grammar has a Chomsky Normal Form (CNF)
+ equivalent grammar where CNF is defined by every production having
+ either two non-terminals or one terminal on its right hand side.
+ When we have hierarchically structured data (ie. a treebank), it is
+ natural to view this in terms of productions where the root of every
+ subtree is the head (left hand side) of the production and all of
+ its children are the right hand side constituents. In order to
+ convert a tree into CNF, we simply need to ensure that every subtree
+ has either two subtrees as children (binarization), or one leaf node
+ (non-terminal). In order to binarize a subtree with more than two
+ children, we must introduce artificial nodes.
+
+ There are two popular methods to convert a tree into CNF: left
+ factoring and right factoring. The following example demonstrates
+ the difference between them. Example::
+
+ Original Right-Factored Left-Factored
+
+ A A A
+ / | \ / \ / \
+ B C D ==> B A|<C-D> OR A|<B-C> D
+ / \ / \
+ C D B C
+
+ 2. Parent Annotation
+
+ In addition to binarizing the tree, there are two standard
+ modifications to node labels we can do in the same traversal: parent
+ annotation and Markov order-N smoothing (or sibling smoothing).
+
+ The purpose of parent annotation is to refine the probabilities of
+ productions by adding a small amount of context. With this simple
+ addition, a CYK (inside-outside, dynamic programming chart parse)
+ can improve from 74% to 79% accuracy. A natural generalization from
+ parent annotation is to grandparent annotation and beyond. The
+ tradeoff becomes accuracy gain vs. computational complexity. We
+ must also keep in mind data sparcity issues. Example::
+
+ Original Parent Annotation
+
+ A A^<?>
+ / | \ / \
+ B C D ==> B^<A> A|<C-D>^<?> where ? is the
+ / \ parent of A
+ C^<A> D^<A>
+
+
+ 3. Markov order-N smoothing
+
+ Markov smoothing combats data sparcity issues as well as decreasing
+ computational requirements by limiting the number of children
+ included in artificial nodes. In practice, most people use an order
+ 2 grammar. Example::
+
+ Original No Smoothing Markov order 1 Markov order 2 etc.
+
+ __A__ A A A
+ / /|\ \ / \ / \ / \
+ B C D E F ==> B A|<C-D-E-F> ==> B A|<C> ==> B A|<C-D>
+ / \ / \ / \
+ C ... C ... C ...
+
+
+
+ Annotation decisions can be thought about in the vertical direction
+ (parent, grandparent, etc) and the horizontal direction (number of
+ siblings to keep). Parameters to the following functions specify
+ these values. For more information see:
+
+ Dan Klein and Chris Manning (2003) "Accurate Unlexicalized
+ Parsing", ACL-03. http://www.aclweb.org/anthology/P03-1054
+
+ 4. Unary Collapsing
+
+ Collapse unary productions (ie. subtrees with a single child) into a
+ new non-terminal (Tree node). This is useful when working with
+ algorithms that do not allow unary productions, yet you do not wish
+ to lose the parent information. Example::
+
+ A
+ |
+ B ==> A+B
+ / \ / \
+ C D C D
+
+"""
+from __future__ import print_function
+
+from nltk.tree import Tree
+
+def chomsky_normal_form(tree, factor="right", horzMarkov=None, vertMarkov=0, childChar="|", parentChar="^"):
+ # assume all subtrees have homogeneous children
+ # assume all terminals have no siblings
+
+ # A semi-hack to have elegant looking code below. As a result,
+ # any subtree with a branching factor greater than 999 will be incorrectly truncated.
+ if horzMarkov is None: horzMarkov = 999
+
+ # Traverse the tree depth-first keeping a list of ancestor nodes to the root.
+ # I chose not to use the tree.treepositions() method since it requires
+ # two traversals of the tree (one to get the positions, one to iterate
+ # over them) and node access time is proportional to the height of the node.
+ # This method is 7x faster which helps when parsing 40,000 sentences.
+
+ nodeList = [(tree, [tree.label()])]
+ while nodeList != []:
+ node, parent = nodeList.pop()
+ if isinstance(node,Tree):
+
+ # parent annotation
+ parentString = ""
+ originalNode = node.label()
+ if vertMarkov != 0 and node != tree and isinstance(node[0],Tree):
+ parentString = "%s<%s>" % (parentChar, "-".join(parent))
+ node.set_label(node.label() + parentString)
+ parent = [originalNode] + parent[:vertMarkov - 1]
+
+ # add children to the agenda before we mess with them
+ for child in node:
+ nodeList.append((child, parent))
+
+ # chomsky normal form factorization
+ if len(node) > 2:
+ childNodes = [child.label() for child in node]
+ nodeCopy = node.copy()
+ node[0:] = [] # delete the children
+
+ curNode = node
+ numChildren = len(nodeCopy)
+ for i in range(1,numChildren - 1):
+ if factor == "right":
+ newHead = "%s%s<%s>%s" % (originalNode, childChar, "-".join(childNodes[i:min([i+horzMarkov,numChildren])]),parentString) # create new head
+ newNode = Tree(newHead, [])
+ curNode[0:] = [nodeCopy.pop(0), newNode]
+ else:
+ newHead = "%s%s<%s>%s" % (originalNode, childChar, "-".join(childNodes[max([numChildren-i-horzMarkov,0]):-i]),parentString)
+ newNode = Tree(newHead, [])
+ curNode[0:] = [newNode, nodeCopy.pop()]
+
+ curNode = newNode
+
+ curNode[0:] = [child for child in nodeCopy]
+
+
+def un_chomsky_normal_form(tree, expandUnary = True, childChar = "|", parentChar = "^", unaryChar = "+"):
+ # Traverse the tree-depth first keeping a pointer to the parent for modification purposes.
+ nodeList = [(tree,[])]
+ while nodeList != []:
+ node,parent = nodeList.pop()
+ if isinstance(node,Tree):
+ # if the node contains the 'childChar' character it means that
+ # it is an artificial node and can be removed, although we still need
+ # to move its children to its parent
+ childIndex = node.label().find(childChar)
+ if childIndex != -1:
+ nodeIndex = parent.index(node)
+ parent.remove(parent[nodeIndex])
+ # Generated node was on the left if the nodeIndex is 0 which
+ # means the grammar was left factored. We must insert the children
+ # at the beginning of the parent's children
+ if nodeIndex == 0:
+ parent.insert(0,node[0])
+ parent.insert(1,node[1])
+ else:
+ parent.extend([node[0],node[1]])
+
+ # parent is now the current node so the children of parent will be added to the agenda
+ node = parent
+ else:
+ parentIndex = node.label().find(parentChar)
+ if parentIndex != -1:
+ # strip the node name of the parent annotation
+ node.set_label(node.label()[:parentIndex])
+
+ # expand collapsed unary productions
+ if expandUnary == True:
+ unaryIndex = node.label().find(unaryChar)
+ if unaryIndex != -1:
+ newNode = Tree(node.label()[unaryIndex + 1:], [i for i in node])
+ node.set_label(node.label()[:unaryIndex])
+ node[0:] = [newNode]
+
+ for child in node:
+ nodeList.append((child,node))
+
+
+def collapse_unary(tree, collapsePOS = False, collapseRoot = False, joinChar = "+"):
+ """
+ Collapse subtrees with a single child (ie. unary productions)
+ into a new non-terminal (Tree node) joined by 'joinChar'.
+ This is useful when working with algorithms that do not allow
+ unary productions, and completely removing the unary productions
+ would require loss of useful information. The Tree is modified
+ directly (since it is passed by reference) and no value is returned.
+
+ :param tree: The Tree to be collapsed
+ :type tree: Tree
+ :param collapsePOS: 'False' (default) will not collapse the parent of leaf nodes (ie.
+ Part-of-Speech tags) since they are always unary productions
+ :type collapsePOS: bool
+ :param collapseRoot: 'False' (default) will not modify the root production
+ if it is unary. For the Penn WSJ treebank corpus, this corresponds
+ to the TOP -> productions.
+ :type collapseRoot: bool
+ :param joinChar: A string used to connect collapsed node values (default = "+")
+ :type joinChar: str
+ """
+
+ if collapseRoot == False and isinstance(tree, Tree) and len(tree) == 1:
+ nodeList = [tree[0]]
+ else:
+ nodeList = [tree]
+
+ # depth-first traversal of tree
+ while nodeList != []:
+ node = nodeList.pop()
+ if isinstance(node,Tree):
+ if len(node) == 1 and isinstance(node[0], Tree) and (collapsePOS == True or isinstance(node[0,0], Tree)):
+ node.set_label(node.label() + joinChar + node[0].label())
+ node[0:] = [child for child in node[0]]
+ # since we assigned the child's children to the current node,
+ # evaluate the current node again
+ nodeList.append(node)
+ else:
+ for child in node:
+ nodeList.append(child)
+
+#################################################################
+# Demonstration
+#################################################################
+
+def demo():
+ """
+ A demonstration showing how each tree transform can be used.
+ """
+
+ from nltk.draw.tree import draw_trees
+ from nltk import tree, treetransforms
+ from copy import deepcopy
+
+ # original tree from WSJ bracketed text
+ sentence = """(TOP
+ (S
+ (S
+ (VP
+ (VBN Turned)
+ (ADVP (RB loose))
+ (PP
+ (IN in)
+ (NP
+ (NP (NNP Shane) (NNP Longman) (POS 's))
+ (NN trading)
+ (NN room)))))
+ (, ,)
+ (NP (DT the) (NN yuppie) (NNS dealers))
+ (VP (AUX do) (NP (NP (RB little)) (ADJP (RB right))))
+ (. .)))"""
+ t = tree.Tree.fromstring(sentence, remove_empty_top_bracketing=True)
+
+ # collapse subtrees with only one child
+ collapsedTree = deepcopy(t)
+ treetransforms.collapse_unary(collapsedTree)
+
+ # convert the tree to CNF
+ cnfTree = deepcopy(collapsedTree)
+ treetransforms.chomsky_normal_form(cnfTree)
+
+ # convert the tree to CNF with parent annotation (one level) and horizontal smoothing of order two
+ parentTree = deepcopy(collapsedTree)
+ treetransforms.chomsky_normal_form(parentTree, horzMarkov=2, vertMarkov=1)
+
+ # convert the tree back to its original form (used to make CYK results comparable)
+ original = deepcopy(parentTree)
+ treetransforms.un_chomsky_normal_form(original)
+
+ # convert tree back to bracketed text
+ sentence2 = original.pprint()
+ print(sentence)
+ print(sentence2)
+ print("Sentences the same? ", sentence == sentence2)
+
+ draw_trees(t, collapsedTree, cnfTree, parentTree, original)
+
+if __name__ == '__main__':
+ demo()
+
+__all__ = ["chomsky_normal_form", "un_chomsky_normal_form", "collapse_unary"]
diff --git a/nltk/util.py b/nltk/util.py
new file mode 100644
index 0000000..85fbe69
--- /dev/null
+++ b/nltk/util.py
@@ -0,0 +1,1100 @@
+# Natural Language Toolkit: Utility functions
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+from __future__ import print_function
+
+import locale
+import re
+import types
+import textwrap
+import pydoc
+import bisect
+import os
+
+from itertools import islice, chain
+from pprint import pprint
+from collections import defaultdict, deque
+from sys import version_info
+
+from nltk.internals import slice_bounds, raise_unorderable_types
+from nltk.compat import (class_types, text_type, string_types, total_ordering,
+ python_2_unicode_compatible, getproxies,
+ ProxyHandler, build_opener, install_opener,
+ HTTPPasswordMgrWithDefaultRealm,
+ ProxyBasicAuthHandler, ProxyDigestAuthHandler)
+
+######################################################################
+# Short usage message
+######################################################################
+
+def usage(obj, selfname='self'):
+ import inspect
+ str(obj) # In case it's lazy, this will load it.
+
+ if not isinstance(obj, class_types):
+ obj = obj.__class__
+
+ print('%s supports the following operations:' % obj.__name__)
+ for (name, method) in sorted(pydoc.allmethods(obj).items()):
+ if name.startswith('_'): continue
+ if getattr(method, '__deprecated__', False): continue
+
+ args, varargs, varkw, defaults = inspect.getargspec(method)
+ if (args and args[0]=='self' and
+ (defaults is None or len(args)>len(defaults))):
+ args = args[1:]
+ name = '%s.%s' % (selfname, name)
+ argspec = inspect.formatargspec(
+ args, varargs, varkw, defaults)
+ print(textwrap.fill('%s%s' % (name, argspec),
+ initial_indent=' - ',
+ subsequent_indent=' '*(len(name)+5)))
+
+##########################################################################
+# IDLE
+##########################################################################
+
+def in_idle():
+ """
+ Return True if this function is run within idle. Tkinter
+ programs that are run in idle should never call ``Tk.mainloop``; so
+ this function should be used to gate all calls to ``Tk.mainloop``.
+
+ :warning: This function works by checking ``sys.stdin``. If the
+ user has modified ``sys.stdin``, then it may return incorrect
+ results.
+ :rtype: bool
+ """
+ import sys
+ return sys.stdin.__class__.__name__ in ('PyShell', 'RPCProxy')
+
+##########################################################################
+# PRETTY PRINTING
+##########################################################################
+
+def pr(data, start=0, end=None):
+ """
+ Pretty print a sequence of data items
+
+ :param data: the data stream to print
+ :type data: sequence or iter
+ :param start: the start position
+ :type start: int
+ :param end: the end position
+ :type end: int
+ """
+ pprint(list(islice(data, start, end)))
+
+def print_string(s, width=70):
+ """
+ Pretty print a string, breaking lines on whitespace
+
+ :param s: the string to print, consisting of words and spaces
+ :type s: str
+ :param width: the display width
+ :type width: int
+ """
+ print('\n'.join(textwrap.wrap(s, width=width)))
+
+def tokenwrap(tokens, separator=" ", width=70):
+ """
+ Pretty print a list of text tokens, breaking lines on whitespace
+
+ :param tokens: the tokens to print
+ :type tokens: list
+ :param separator: the string to use to separate tokens
+ :type separator: str
+ :param width: the display width (default=70)
+ :type width: int
+ """
+ return '\n'.join(textwrap.wrap(separator.join(tokens), width=width))
+
+
+##########################################################################
+# Python version
+##########################################################################
+
+def py25():
+ return version_info[0] == 2 and version_info[1] == 5
+def py26():
+ return version_info[0] == 2 and version_info[1] == 6
+def py27():
+ return version_info[0] == 2 and version_info[1] == 7
+
+
+##########################################################################
+# Indexing
+##########################################################################
+
+class Index(defaultdict):
+
+ def __init__(self, pairs):
+ defaultdict.__init__(self, list)
+ for key, value in pairs:
+ self[key].append(value)
+
+
+######################################################################
+## Regexp display (thanks to David Mertz)
+######################################################################
+
+def re_show(regexp, string, left="{", right="}"):
+ """
+ Return a string with markers surrounding the matched substrings.
+ Search str for substrings matching ``regexp`` and wrap the matches
+ with braces. This is convenient for learning about regular expressions.
+
+ :param regexp: The regular expression.
+ :type regexp: str
+ :param string: The string being matched.
+ :type string: str
+ :param left: The left delimiter (printed before the matched substring)
+ :type left: str
+ :param right: The right delimiter (printed after the matched substring)
+ :type right: str
+ :rtype: str
+ """
+ print(re.compile(regexp, re.M).sub(left + r"\g<0>" + right, string.rstrip()))
+
+
+##########################################################################
+# READ FROM FILE OR STRING
+##########################################################################
+
+# recipe from David Mertz
+def filestring(f):
+ if hasattr(f, 'read'):
+ return f.read()
+ elif isinstance(f, string_types):
+ with open(f, 'r') as infile:
+ return infile.read()
+ else:
+ raise ValueError("Must be called with a filename or file-like object")
+
+##########################################################################
+# Breadth-First Search
+##########################################################################
+
+def breadth_first(tree, children=iter, maxdepth=-1):
+ """Traverse the nodes of a tree in breadth-first order.
+ (No need to check for cycles.)
+ The first argument should be the tree root;
+ children should be a function taking as argument a tree node
+ and returning an iterator of the node's children.
+ """
+ queue = deque([(tree, 0)])
+
+ while queue:
+ node, depth = queue.popleft()
+ yield node
+
+ if depth != maxdepth:
+ try:
+ queue.extend((c, depth + 1) for c in children(node))
+ except TypeError:
+ pass
+
+##########################################################################
+# Guess Character Encoding
+##########################################################################
+
+# adapted from io.py in the docutils extension module (http://docutils.sourceforge.net)
+# http://www.pyzine.com/Issue008/Section_Articles/article_Encodings.html
+
+def guess_encoding(data):
+ """
+ Given a byte string, attempt to decode it.
+ Tries the standard 'UTF8' and 'latin-1' encodings,
+ Plus several gathered from locale information.
+
+ The calling program *must* first call::
+
+ locale.setlocale(locale.LC_ALL, '')
+
+ If successful it returns ``(decoded_unicode, successful_encoding)``.
+ If unsuccessful it raises a ``UnicodeError``.
+ """
+ successful_encoding = None
+ # we make 'utf-8' the first encoding
+ encodings = ['utf-8']
+ #
+ # next we add anything we can learn from the locale
+ try:
+ encodings.append(locale.nl_langinfo(locale.CODESET))
+ except AttributeError:
+ pass
+ try:
+ encodings.append(locale.getlocale()[1])
+ except (AttributeError, IndexError):
+ pass
+ try:
+ encodings.append(locale.getdefaultlocale()[1])
+ except (AttributeError, IndexError):
+ pass
+ #
+ # we try 'latin-1' last
+ encodings.append('latin-1')
+ for enc in encodings:
+ # some of the locale calls
+ # may have returned None
+ if not enc:
+ continue
+ try:
+ decoded = text_type(data, enc)
+ successful_encoding = enc
+
+ except (UnicodeError, LookupError):
+ pass
+ else:
+ break
+ if not successful_encoding:
+ raise UnicodeError(
+ 'Unable to decode input data. Tried the following encodings: %s.'
+ % ', '.join([repr(enc) for enc in encodings if enc]))
+ else:
+ return (decoded, successful_encoding)
+
+
+##########################################################################
+# Remove repeated elements from a list deterministcally
+##########################################################################
+
+def unique_list(xs):
+ seen = set()
+ # not seen.add(x) here acts to make the code shorter without using if statements, seen.add(x) always returns None.
+ return [x for x in xs if x not in seen and not seen.add(x)]
+
+##########################################################################
+# Invert a dictionary
+##########################################################################
+
+def invert_dict(d):
+ inverted_dict = defaultdict(list)
+ for key in d:
+ if hasattr(d[key], '__iter__'):
+ for term in d[key]:
+ inverted_dict[term].append(key)
+ else:
+ inverted_dict[d[key]] = key
+ return inverted_dict
+
+
+##########################################################################
+# Utilities for directed graphs: transitive closure, and inversion
+# The graph is represented as a dictionary of sets
+##########################################################################
+
+def transitive_closure(graph, reflexive=False):
+ """
+ Calculate the transitive closure of a directed graph,
+ optionally the reflexive transitive closure.
+
+ The algorithm is a slight modification of the "Marking Algorithm" of
+ Ioannidis & Ramakrishnan (1998) "Efficient Transitive Closure Algorithms".
+
+ :param graph: the initial graph, represented as a dictionary of sets
+ :type graph: dict(set)
+ :param reflexive: if set, also make the closure reflexive
+ :type reflexive: bool
+ :rtype: dict(set)
+ """
+ if reflexive:
+ base_set = lambda k: set([k])
+ else:
+ base_set = lambda k: set()
+ # The graph U_i in the article:
+ agenda_graph = dict((k, graph[k].copy()) for k in graph)
+ # The graph M_i in the article:
+ closure_graph = dict((k, base_set(k)) for k in graph)
+ for i in graph:
+ agenda = agenda_graph[i]
+ closure = closure_graph[i]
+ while agenda:
+ j = agenda.pop()
+ closure.add(j)
+ closure |= closure_graph.setdefault(j, base_set(j))
+ agenda |= agenda_graph.get(j, base_set(j))
+ agenda -= closure
+ return closure_graph
+
+
+def invert_graph(graph):
+ """
+ Inverts a directed graph.
+
+ :param graph: the graph, represented as a dictionary of sets
+ :type graph: dict(set)
+ :return: the inverted graph
+ :rtype: dict(set)
+ """
+ inverted = {}
+ for key in graph:
+ for value in graph[key]:
+ inverted.setdefault(value, set()).add(key)
+ return inverted
+
+
+
+##########################################################################
+# HTML Cleaning
+##########################################################################
+
+def clean_html(html):
+ raise NotImplementedError ("To remove HTML markup, use BeautifulSoup's get_text() function")
+
+def clean_url(url):
+ raise NotImplementedError ("To remove HTML markup, use BeautifulSoup's get_text() function")
+
+##########################################################################
+# FLATTEN LISTS
+##########################################################################
+
+def flatten(*args):
+ """
+ Flatten a list.
+
+ >>> from nltk.util import flatten
+ >>> flatten(1, 2, ['b', 'a' , ['c', 'd']], 3)
+ [1, 2, 'b', 'a', 'c', 'd', 3]
+
+ :param args: items and lists to be combined into a single list
+ :rtype: list
+ """
+
+ x = []
+ for l in args:
+ if not isinstance(l, (list, tuple)): l = [l]
+ for item in l:
+ if isinstance(item, (list, tuple)):
+ x.extend(flatten(item))
+ else:
+ x.append(item)
+ return x
+
+##########################################################################
+# Ngram iteration
+##########################################################################
+
+# add a flag to pad the sequence so we get peripheral ngrams?
+
+def ngrams(sequence, n, pad_left=False, pad_right=False, pad_symbol=None):
+ """
+ Return the ngrams generated from a sequence of items, as an iterator.
+ For example:
+
+ >>> from nltk.util import ngrams
+ >>> list(ngrams([1,2,3,4,5], 3))
+ [(1, 2, 3), (2, 3, 4), (3, 4, 5)]
+
+ Use ngrams for a list version of this function. Set pad_left
+ or pad_right to true in order to get additional ngrams:
+
+ >>> list(ngrams([1,2,3,4,5], 2, pad_right=True))
+ [(1, 2), (2, 3), (3, 4), (4, 5), (5, None)]
+
+ :param sequence: the source data to be converted into ngrams
+ :type sequence: sequence or iter
+ :param n: the degree of the ngrams
+ :type n: int
+ :param pad_left: whether the ngrams should be left-padded
+ :type pad_left: bool
+ :param pad_right: whether the ngrams should be right-padded
+ :type pad_right: bool
+ :param pad_symbol: the symbol to use for padding (default is None)
+ :type pad_symbol: any
+ :rtype: iter(tuple)
+ """
+
+ sequence = iter(sequence)
+ if pad_left:
+ sequence = chain((pad_symbol,) * (n-1), sequence)
+ if pad_right:
+ sequence = chain(sequence, (pad_symbol,) * (n-1))
+
+ history = []
+ while n > 1:
+ history.append(next(sequence))
+ n -= 1
+ for item in sequence:
+ history.append(item)
+ yield tuple(history)
+ del history[0]
+
+def bigrams(sequence, **kwargs):
+ """
+ Return the bigrams generated from a sequence of items, as an iterator.
+ For example:
+
+ >>> from nltk.util import bigrams
+ >>> list(bigrams([1,2,3,4,5]))
+ [(1, 2), (2, 3), (3, 4), (4, 5)]
+
+ Use bigrams for a list version of this function.
+
+ :param sequence: the source data to be converted into bigrams
+ :type sequence: sequence or iter
+ :rtype: iter(tuple)
+ """
+
+ for item in ngrams(sequence, 2, **kwargs):
+ yield item
+
+def trigrams(sequence, **kwargs):
+ """
+ Return the trigrams generated from a sequence of items, as an iterator.
+ For example:
+
+ >>> from nltk.util import trigrams
+ >>> list(trigrams([1,2,3,4,5]))
+ [(1, 2, 3), (2, 3, 4), (3, 4, 5)]
+
+ Use trigrams for a list version of this function.
+
+ :param sequence: the source data to be converted into trigrams
+ :type sequence: sequence or iter
+ :rtype: iter(tuple)
+ """
+
+ for item in ngrams(sequence, 3, **kwargs):
+ yield item
+
+##########################################################################
+# Ordered Dictionary
+##########################################################################
+
+class OrderedDict(dict):
+ def __init__(self, data=None, **kwargs):
+ self._keys = self.keys(data, kwargs.get('keys'))
+ self._default_factory = kwargs.get('default_factory')
+ if data is None:
+ dict.__init__(self)
+ else:
+ dict.__init__(self, data)
+
+ def __delitem__(self, key):
+ dict.__delitem__(self, key)
+ self._keys.remove(key)
+
+ def __getitem__(self, key):
+ try:
+ return dict.__getitem__(self, key)
+ except KeyError:
+ return self.__missing__(key)
+
+ def __iter__(self):
+ return (key for key in self.keys())
+
+ def __missing__(self, key):
+ if not self._default_factory and key not in self._keys:
+ raise KeyError()
+ return self._default_factory()
+
+ def __setitem__(self, key, item):
+ dict.__setitem__(self, key, item)
+ if key not in self._keys:
+ self._keys.append(key)
+
+ def clear(self):
+ dict.clear(self)
+ self._keys.clear()
+
+ def copy(self):
+ d = dict.copy(self)
+ d._keys = self._keys
+ return d
+
+ def items(self):
+ # returns iterator under python 3 and list under python 2
+ return zip(self.keys(), self.values())
+
+ def keys(self, data=None, keys=None):
+ if data:
+ if keys:
+ assert isinstance(keys, list)
+ assert len(data) == len(keys)
+ return keys
+ else:
+ assert isinstance(data, dict) or \
+ isinstance(data, OrderedDict) or \
+ isinstance(data, list)
+ if isinstance(data, dict) or isinstance(data, OrderedDict):
+ return data.keys()
+ elif isinstance(data, list):
+ return [key for (key, value) in data]
+ elif '_keys' in self.__dict__:
+ return self._keys
+ else:
+ return []
+
+ def popitem(self):
+ if not self._keys:
+ raise KeyError()
+
+ key = self._keys.pop()
+ value = self[key]
+ del self[key]
+ return (key, value)
+
+ def setdefault(self, key, failobj=None):
+ dict.setdefault(self, key, failobj)
+ if key not in self._keys:
+ self._keys.append(key)
+
+ def update(self, data):
+ dict.update(self, data)
+ for key in self.keys(data):
+ if key not in self._keys:
+ self._keys.append(key)
+
+ def values(self):
+ # returns iterator under python 3
+ return map(self.get, self._keys)
+
+######################################################################
+# Lazy Sequences
+######################################################################
+
+ at total_ordering
+ at python_2_unicode_compatible
+class AbstractLazySequence(object):
+ """
+ An abstract base class for read-only sequences whose values are
+ computed as needed. Lazy sequences act like tuples -- they can be
+ indexed, sliced, and iterated over; but they may not be modified.
+
+ The most common application of lazy sequences in NLTK is for
+ corpus view objects, which provide access to the contents of a
+ corpus without loading the entire corpus into memory, by loading
+ pieces of the corpus from disk as needed.
+
+ The result of modifying a mutable element of a lazy sequence is
+ undefined. In particular, the modifications made to the element
+ may or may not persist, depending on whether and when the lazy
+ sequence caches that element's value or reconstructs it from
+ scratch.
+
+ Subclasses are required to define two methods: ``__len__()``
+ and ``iterate_from()``.
+ """
+ def __len__(self):
+ """
+ Return the number of tokens in the corpus file underlying this
+ corpus view.
+ """
+ raise NotImplementedError('should be implemented by subclass')
+
+ def iterate_from(self, start):
+ """
+ Return an iterator that generates the tokens in the corpus
+ file underlying this corpus view, starting at the token number
+ ``start``. If ``start>=len(self)``, then this iterator will
+ generate no tokens.
+ """
+ raise NotImplementedError('should be implemented by subclass')
+
+ def __getitem__(self, i):
+ """
+ Return the *i* th token in the corpus file underlying this
+ corpus view. Negative indices and spans are both supported.
+ """
+ if isinstance(i, slice):
+ start, stop = slice_bounds(self, i)
+ return LazySubsequence(self, start, stop)
+ else:
+ # Handle negative indices
+ if i < 0: i += len(self)
+ if i < 0: raise IndexError('index out of range')
+ # Use iterate_from to extract it.
+ try:
+ return next(self.iterate_from(i))
+ except StopIteration:
+ raise IndexError('index out of range')
+
+ def __iter__(self):
+ """Return an iterator that generates the tokens in the corpus
+ file underlying this corpus view."""
+ return self.iterate_from(0)
+
+ def count(self, value):
+ """Return the number of times this list contains ``value``."""
+ return sum(1 for elt in self if elt==value)
+
+ def index(self, value, start=None, stop=None):
+ """Return the index of the first occurrence of ``value`` in this
+ list that is greater than or equal to ``start`` and less than
+ ``stop``. Negative start and stop values are treated like negative
+ slice bounds -- i.e., they count from the end of the list."""
+ start, stop = slice_bounds(self, slice(start, stop))
+ for i, elt in enumerate(islice(self, start, stop)):
+ if elt == value: return i+start
+ raise ValueError('index(x): x not in list')
+
+ def __contains__(self, value):
+ """Return true if this list contains ``value``."""
+ return bool(self.count(value))
+
+ def __add__(self, other):
+ """Return a list concatenating self with other."""
+ return LazyConcatenation([self, other])
+
+ def __radd__(self, other):
+ """Return a list concatenating other with self."""
+ return LazyConcatenation([other, self])
+
+ def __mul__(self, count):
+ """Return a list concatenating self with itself ``count`` times."""
+ return LazyConcatenation([self] * count)
+
+ def __rmul__(self, count):
+ """Return a list concatenating self with itself ``count`` times."""
+ return LazyConcatenation([self] * count)
+
+ _MAX_REPR_SIZE = 60
+ def __repr__(self):
+ """
+ Return a string representation for this corpus view that is
+ similar to a list's representation; but if it would be more
+ than 60 characters long, it is truncated.
+ """
+ pieces = []
+ length = 5
+ for elt in self:
+ pieces.append(repr(elt))
+ length += len(pieces[-1]) + 2
+ if length > self._MAX_REPR_SIZE and len(pieces) > 2:
+ return '[%s, ...]' % text_type(', ').join(pieces[:-1])
+ else:
+ return '[%s]' % text_type(', ').join(pieces)
+
+ def __eq__(self, other):
+ return (type(self) == type(other) and list(self) == list(other))
+
+ def __ne__(self, other):
+ return not self == other
+
+ def __lt__(self, other):
+ if type(other) != type(self):
+ raise_unorderable_types("<", self, other)
+ return list(self) < list(other)
+
+ def __hash__(self):
+ """
+ :raise ValueError: Corpus view objects are unhashable.
+ """
+ raise ValueError('%s objects are unhashable' %
+ self.__class__.__name__)
+
+
+class LazySubsequence(AbstractLazySequence):
+ """
+ A subsequence produced by slicing a lazy sequence. This slice
+ keeps a reference to its source sequence, and generates its values
+ by looking them up in the source sequence.
+ """
+
+ MIN_SIZE = 100
+ """
+ The minimum size for which lazy slices should be created. If
+ ``LazySubsequence()`` is called with a subsequence that is
+ shorter than ``MIN_SIZE``, then a tuple will be returned instead.
+ """
+
+ def __new__(cls, source, start, stop):
+ """
+ Construct a new slice from a given underlying sequence. The
+ ``start`` and ``stop`` indices should be absolute indices --
+ i.e., they should not be negative (for indexing from the back
+ of a list) or greater than the length of ``source``.
+ """
+ # If the slice is small enough, just use a tuple.
+ if stop-start < cls.MIN_SIZE:
+ return list(islice(source.iterate_from(start), stop-start))
+ else:
+ return object.__new__(cls)
+
+ def __init__(self, source, start, stop):
+ self._source = source
+ self._start = start
+ self._stop = stop
+
+ def __len__(self):
+ return self._stop - self._start
+
+ def iterate_from(self, start):
+ return islice(self._source.iterate_from(start+self._start),
+ max(0, len(self)-start))
+
+
+class LazyConcatenation(AbstractLazySequence):
+ """
+ A lazy sequence formed by concatenating a list of lists. This
+ underlying list of lists may itself be lazy. ``LazyConcatenation``
+ maintains an index that it uses to keep track of the relationship
+ between offsets in the concatenated lists and offsets in the
+ sublists.
+ """
+ def __init__(self, list_of_lists):
+ self._list = list_of_lists
+ self._offsets = [0]
+
+ def __len__(self):
+ if len(self._offsets) <= len(self._list):
+ for tok in self.iterate_from(self._offsets[-1]): pass
+ return self._offsets[-1]
+
+ def iterate_from(self, start_index):
+ if start_index < self._offsets[-1]:
+ sublist_index = bisect.bisect_right(self._offsets, start_index)-1
+ else:
+ sublist_index = len(self._offsets)-1
+
+ index = self._offsets[sublist_index]
+
+ # Construct an iterator over the sublists.
+ if isinstance(self._list, AbstractLazySequence):
+ sublist_iter = self._list.iterate_from(sublist_index)
+ else:
+ sublist_iter = islice(self._list, sublist_index, None)
+
+ for sublist in sublist_iter:
+ if sublist_index == (len(self._offsets)-1):
+ assert index+len(sublist) >= self._offsets[-1], (
+ 'offests not monotonic increasing!')
+ self._offsets.append(index+len(sublist))
+ else:
+ assert self._offsets[sublist_index+1] == index+len(sublist), (
+ 'inconsistent list value (num elts)')
+
+ for value in sublist[max(0, start_index-index):]:
+ yield value
+
+ index += len(sublist)
+ sublist_index += 1
+
+
+class LazyMap(AbstractLazySequence):
+ """
+ A lazy sequence whose elements are formed by applying a given
+ function to each element in one or more underlying lists. The
+ function is applied lazily -- i.e., when you read a value from the
+ list, ``LazyMap`` will calculate that value by applying its
+ function to the underlying lists' value(s). ``LazyMap`` is
+ essentially a lazy version of the Python primitive function
+ ``map``. In particular, the following two expressions are
+ equivalent:
+
+ >>> from nltk.util import LazyMap
+ >>> function = str
+ >>> sequence = [1,2,3]
+ >>> map(function, sequence) # doctest: +SKIP
+ ['1', '2', '3']
+ >>> list(LazyMap(function, sequence))
+ ['1', '2', '3']
+
+ Like the Python ``map`` primitive, if the source lists do not have
+ equal size, then the value None will be supplied for the
+ 'missing' elements.
+
+ Lazy maps can be useful for conserving memory, in cases where
+ individual values take up a lot of space. This is especially true
+ if the underlying list's values are constructed lazily, as is the
+ case with many corpus readers.
+
+ A typical example of a use case for this class is performing
+ feature detection on the tokens in a corpus. Since featuresets
+ are encoded as dictionaries, which can take up a lot of memory,
+ using a ``LazyMap`` can significantly reduce memory usage when
+ training and running classifiers.
+ """
+ def __init__(self, function, *lists, **config):
+ """
+ :param function: The function that should be applied to
+ elements of ``lists``. It should take as many arguments
+ as there are ``lists``.
+ :param lists: The underlying lists.
+ :param cache_size: Determines the size of the cache used
+ by this lazy map. (default=5)
+ """
+ if not lists:
+ raise TypeError('LazyMap requires at least two args')
+
+ self._lists = lists
+ self._func = function
+ self._cache_size = config.get('cache_size', 5)
+ self._cache = ({} if self._cache_size > 0 else None)
+
+ # If you just take bool() of sum() here _all_lazy will be true just
+ # in case n >= 1 list is an AbstractLazySequence. Presumably this
+ # isn't what's intended.
+ self._all_lazy = sum(isinstance(lst, AbstractLazySequence)
+ for lst in lists) == len(lists)
+
+ def iterate_from(self, index):
+ # Special case: one lazy sublist
+ if len(self._lists) == 1 and self._all_lazy:
+ for value in self._lists[0].iterate_from(index):
+ yield self._func(value)
+ return
+
+ # Special case: one non-lazy sublist
+ elif len(self._lists) == 1:
+ while True:
+ try: yield self._func(self._lists[0][index])
+ except IndexError: return
+ index += 1
+
+ # Special case: n lazy sublists
+ elif self._all_lazy:
+ iterators = [lst.iterate_from(index) for lst in self._lists]
+ while True:
+ elements = []
+ for iterator in iterators:
+ try: elements.append(next(iterator))
+ except: elements.append(None)
+ if elements == [None] * len(self._lists):
+ return
+ yield self._func(*elements)
+ index += 1
+
+ # general case
+ else:
+ while True:
+ try: elements = [lst[index] for lst in self._lists]
+ except IndexError:
+ elements = [None] * len(self._lists)
+ for i, lst in enumerate(self._lists):
+ try: elements[i] = lst[index]
+ except IndexError: pass
+ if elements == [None] * len(self._lists):
+ return
+ yield self._func(*elements)
+ index += 1
+
+ def __getitem__(self, index):
+ if isinstance(index, slice):
+ sliced_lists = [lst[index] for lst in self._lists]
+ return LazyMap(self._func, *sliced_lists)
+ else:
+ # Handle negative indices
+ if index < 0: index += len(self)
+ if index < 0: raise IndexError('index out of range')
+ # Check the cache
+ if self._cache is not None and index in self._cache:
+ return self._cache[index]
+ # Calculate the value
+ try: val = next(self.iterate_from(index))
+ except StopIteration:
+ raise IndexError('index out of range')
+ # Update the cache
+ if self._cache is not None:
+ if len(self._cache) > self._cache_size:
+ self._cache.popitem() # discard random entry
+ self._cache[index] = val
+ # Return the value
+ return val
+
+ def __len__(self):
+ return max(len(lst) for lst in self._lists)
+
+
+class LazyZip(LazyMap):
+ """
+ A lazy sequence whose elements are tuples, each containing the i-th
+ element from each of the argument sequences. The returned list is
+ truncated in length to the length of the shortest argument sequence. The
+ tuples are constructed lazily -- i.e., when you read a value from the
+ list, ``LazyZip`` will calculate that value by forming a tuple from
+ the i-th element of each of the argument sequences.
+
+ ``LazyZip`` is essentially a lazy version of the Python primitive function
+ ``zip``. In particular, an evaluated LazyZip is equivalent to a zip:
+
+ >>> from nltk.util import LazyZip
+ >>> sequence1, sequence2 = [1, 2, 3], ['a', 'b', 'c']
+ >>> zip(sequence1, sequence2) # doctest: +SKIP
+ [(1, 'a'), (2, 'b'), (3, 'c')]
+ >>> list(LazyZip(sequence1, sequence2))
+ [(1, 'a'), (2, 'b'), (3, 'c')]
+ >>> sequences = [sequence1, sequence2, [6,7,8,9]]
+ >>> list(zip(*sequences)) == list(LazyZip(*sequences))
+ True
+
+ Lazy zips can be useful for conserving memory in cases where the argument
+ sequences are particularly long.
+
+ A typical example of a use case for this class is combining long sequences
+ of gold standard and predicted values in a classification or tagging task
+ in order to calculate accuracy. By constructing tuples lazily and
+ avoiding the creation of an additional long sequence, memory usage can be
+ significantly reduced.
+ """
+ def __init__(self, *lists):
+ """
+ :param lists: the underlying lists
+ :type lists: list(list)
+ """
+ LazyMap.__init__(self, lambda *elts: elts, *lists)
+
+ def iterate_from(self, index):
+ iterator = LazyMap.iterate_from(self, index)
+ while index < len(self):
+ yield next(iterator)
+ index += 1
+ return
+
+ def __len__(self):
+ return min(len(lst) for lst in self._lists)
+
+
+class LazyEnumerate(LazyZip):
+ """
+ A lazy sequence whose elements are tuples, each ontaining a count (from
+ zero) and a value yielded by underlying sequence. ``LazyEnumerate`` is
+ useful for obtaining an indexed list. The tuples are constructed lazily
+ -- i.e., when you read a value from the list, ``LazyEnumerate`` will
+ calculate that value by forming a tuple from the count of the i-th
+ element and the i-th element of the underlying sequence.
+
+ ``LazyEnumerate`` is essentially a lazy version of the Python primitive
+ function ``enumerate``. In particular, the following two expressions are
+ equivalent:
+
+ >>> from nltk.util import LazyEnumerate
+ >>> sequence = ['first', 'second', 'third']
+ >>> list(enumerate(sequence))
+ [(0, 'first'), (1, 'second'), (2, 'third')]
+ >>> list(LazyEnumerate(sequence))
+ [(0, 'first'), (1, 'second'), (2, 'third')]
+
+ Lazy enumerations can be useful for conserving memory in cases where the
+ argument sequences are particularly long.
+
+ A typical example of a use case for this class is obtaining an indexed
+ list for a long sequence of values. By constructing tuples lazily and
+ avoiding the creation of an additional long sequence, memory usage can be
+ significantly reduced.
+ """
+
+ def __init__(self, lst):
+ """
+ :param lst: the underlying list
+ :type lst: list
+ """
+ LazyZip.__init__(self, range(len(lst)), lst)
+
+
+######################################################################
+# Binary Search in a File
+######################################################################
+
+# inherited from pywordnet, by Oliver Steele
+def binary_search_file(file, key, cache={}, cacheDepth=-1):
+ """
+ Return the line from the file with first word key.
+ Searches through a sorted file using the binary search algorithm.
+
+ :type file: file
+ :param file: the file to be searched through.
+ :type key: str
+ :param key: the identifier we are searching for.
+ """
+
+ key = key + ' '
+ keylen = len(key)
+ start = 0
+ currentDepth = 0
+
+ if hasattr(file, 'name'):
+ end = os.stat(file.name).st_size - 1
+ else:
+ file.seek(0, 2)
+ end = file.tell() - 1
+ file.seek(0)
+
+ while start < end:
+ lastState = start, end
+ middle = (start + end) // 2
+
+ if cache.get(middle):
+ offset, line = cache[middle]
+
+ else:
+ line = ""
+ while True:
+ file.seek(max(0, middle - 1))
+ if middle > 0:
+ file.readline()
+ offset = file.tell()
+ line = file.readline()
+ if line != "": break
+ # at EOF; try to find start of the last line
+ middle = (start + middle)//2
+ if middle == end -1:
+ return None
+ if currentDepth < cacheDepth:
+ cache[middle] = (offset, line)
+
+ if offset > end:
+ assert end != middle - 1, "infinite loop"
+ end = middle - 1
+ elif line[:keylen] == key:
+ return line
+ elif line > key:
+ assert end != middle - 1, "infinite loop"
+ end = middle - 1
+ elif line < key:
+ start = offset + len(line) - 1
+
+ currentDepth += 1
+ thisState = start, end
+
+ if lastState == thisState:
+ # Detects the condition where we're searching past the end
+ # of the file, which is otherwise difficult to detect
+ return None
+
+ return None
+
+######################################################################
+# Proxy configuration
+######################################################################
+
+def set_proxy(proxy, user=None, password=''):
+ """
+ Set the HTTP proxy for Python to download through.
+
+ If ``proxy`` is None then tries to set proxy from environment or system
+ settings.
+
+ :param proxy: The HTTP proxy server to use. For example:
+ 'http://proxy.example.com:3128/'
+ :param user: The username to authenticate with. Use None to disable
+ authentication.
+ :param password: The password to authenticate with.
+ """
+ from nltk import compat
+
+ if proxy is None:
+ # Try and find the system proxy settings
+ try:
+ proxy = getproxies()['http']
+ except KeyError:
+ raise ValueError('Could not detect default proxy settings')
+
+ # Set up the proxy handler
+ proxy_handler = ProxyHandler({'http': proxy})
+ opener = build_opener(proxy_handler)
+
+ if user is not None:
+ # Set up basic proxy authentication if provided
+ password_manager = HTTPPasswordMgrWithDefaultRealm()
+ password_manager.add_password(realm=None, uri=proxy, user=user,
+ passwd=password)
+ opener.add_handler(ProxyBasicAuthHandler(password_manager))
+ opener.add_handler(ProxyDigestAuthHandler(password_manager))
+
+ # Overide the existing url opener
+ install_opener(opener)
diff --git a/nltk/wsd.py b/nltk/wsd.py
new file mode 100644
index 0000000..f4e5282
--- /dev/null
+++ b/nltk/wsd.py
@@ -0,0 +1,66 @@
+# Natural Language Toolkit: Word Sense Disambiguation Algorithms
+#
+# Author: Liling Tan <alvations at gmail.com>
+#
+# Copyright (C) 2001-2014 NLTK Project
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from nltk.corpus import wordnet as wn
+
+############################################################
+# Lesk Algorithm
+############################################################
+
+def _compare_overlaps_greedy(context, synsets_signatures, pos=None):
+ """
+ Calculate overlaps between the context sentence and the synset_signature
+ and returns the synset with the highest overlap.
+
+ :param context: ``context_sentence`` The context sentence where the ambiguous word occurs.
+ :param synsets_signatures: ``dictionary`` A list of words that 'signifies' the ambiguous word.
+ :param pos: ``pos`` A specified Part-of-Speech (POS).
+ :return: ``lesk_sense`` The Synset() object with the highest signature overlaps.
+ """
+ max_overlaps = 0
+ lesk_sense = None
+ for ss in synsets_signatures:
+ if pos and str(ss.pos()) != pos: # Skips different POS.
+ continue
+ overlaps = set(synsets_signatures[ss]).intersection(context)
+ if len(overlaps) > max_overlaps:
+ lesk_sense = ss
+ max_overlaps = len(overlaps)
+ return lesk_sense
+
+def lesk(context_sentence, ambiguous_word, pos=None, dictionary=None):
+ """
+ This function is the implementation of the original Lesk algorithm (1986).
+ It requires a dictionary which contains the definition of the different
+ sense of each word. See http://goo.gl/8TB15w
+
+ >>> from nltk import word_tokenize
+ >>> sent = word_tokenize("I went to the bank to deposit money.")
+ >>> word = "bank"
+ >>> pos = "n"
+ >>> lesk(sent, word, pos)
+ Synset('bank.n.07')
+
+ :param context_sentence: The context sentence where the ambiguous word occurs.
+ :param ambiguous_word: The ambiguous word that requires WSD.
+ :param pos: A specified Part-of-Speech (POS).
+ :param dictionary: A list of words that 'signifies' the ambiguous word.
+ :return: ``lesk_sense`` The Synset() object with the highest signature overlaps.
+ """
+ if not dictionary:
+ dictionary = {}
+ for ss in wn.synsets(ambiguous_word):
+ dictionary[ss] = ss.definition().split()
+ best_sense = _compare_overlaps_greedy(context_sentence,
+ dictionary, pos)
+ return best_sense
+
+
+if __name__ == "__main__":
+ import doctest
+ doctest.testmod(optionflags=doctest.NORMALIZE_WHITESPACE)
diff --git a/papers/acl-02/.cvsignore b/papers/acl-02/.cvsignore
new file mode 100644
index 0000000..a027086
--- /dev/null
+++ b/papers/acl-02/.cvsignore
@@ -0,0 +1,7 @@
+*.aux
+*.dvi
+*.log
+*.ps
+*.bbl
+*.blg
+*.pdf
diff --git a/papers/acl-02/Makefile b/papers/acl-02/Makefile
new file mode 100644
index 0000000..27db9fc
--- /dev/null
+++ b/papers/acl-02/Makefile
@@ -0,0 +1,62 @@
+# Natural Language Toolkit: Technical report Makefile
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+##############################################
+## The name of the report
+REPORT = acl02
+
+help: usage
+usage:
+ @echo
+ @echo make '[dvi | ps | pdf | clean]'
+ @echo
+
+# We're using bibtex:
+$(REPORT).dvi: $(REPORT).bbl
+BIBFILE = nltk.bib
+
+##############################################
+## Figure dependancies
+
+
+##############################################
+## You shouldn't have to change anything below here.
+
+# Find the name of the dvi and ps files.
+DVI := $(REPORT).dvi
+PS := $(REPORT).ps
+PDF := $(REPORT).pdf
+
+# Top-level rules.
+dvi: $(DVI)
+ps: $(PS)
+pdf: $(PDF)
+clean:
+ rm -f *.log *.aux *.dvi *.ps *.toc *.pdf *.bbl *.blg
+
+%.bbl: %.tex $(BIBFILE)
+ latex $*.tex || (rm -f $*.dvi && false)
+ bibtex $* || (rm -f $*.dvi $@ && false)
+
+%.dvi: %.tex
+ latex $*.tex || (rm -f $@ && false)
+ latex $*.tex || (rm -f $@ && false)
+
+%.ps: %.dvi
+ dvips -t letter -o $@ $< -G0 -Ppdf
+
+%.eps: %.dot
+ dot -Tps -o $@ $<
+
+%.eps: %.obj
+ tgif -print -eps $<
+
+%.pdf: %.ps
+ ps2pdf -sPAPERSIZE=letter -dMaxSubsetPct=100 \
+ -dCompatibilityLevel=1.2 -dSubsetFonts=true \
+ -dEmbedAllFonts=true $< $@
+
diff --git a/papers/acl-02/acl-02.tex b/papers/acl-02/acl-02.tex
new file mode 100644
index 0000000..5e527ee
--- /dev/null
+++ b/papers/acl-02/acl-02.tex
@@ -0,0 +1,705 @@
+%
+% Submission for ``Effective Tools And Methodologies For Teaching
+% Natural Language Processing And Computational Linguistics'' workshop
+% at ACL 2002
+%
+% Authors: Edward Loper and Steven Bird
+%
+\documentclass[11pt]{article}
+\usepackage{acl2002,url,alltt,epsfig}
+
+% hyphenation control
+\pretolerance 250
+\tolerance 500
+\hyphenpenalty 200
+\exhyphenpenalty 100
+\doublehyphendemerits 7500
+\finalhyphendemerits 7500
+\brokenpenalty 10000
+\lefthyphenmin 3
+\righthyphenmin 3
+\widowpenalty 10000
+\clubpenalty 10000
+\displaywidowpenalty 10000
+\looseness 1
+
+\title{NLTK: The Natural Language Toolkit}
+\author{
+Edward Loper and Steven Bird\\
+Department of Computer and Information Science \\
+University of Pennsylvania, Philadelphia, PA 19104-6389, USA
+}
+\date{}
+
+% Outline:
+% - abstract (summary, above.)
+% - introduction
+% - choice of programming language
+% - design criteria
+% - modules
+% - uses
+% - assignments
+% - class demonstrations
+% - projects
+% - evaluation
+% - other approaches
+% - conclusion
+
+
+\newenvironment{sv}{\small\begin{alltt}}{\end{alltt}\normalsize}
+
+\begin{document}
+\maketitle
+
+\begin{abstract}
+NLTK, the Natural Language Toolkit, is a suite of open source program modules,
+tutorials and problem sets, providing ready-to-use computational
+linguistics courseware. NLTK covers symbolic and statistical natural
+language processing, and is interfaced to annotated corpora. Students
+augment and replace existing components, learn structured
+programming by example, and manipulate sophisticated models from the
+outset.
+\end{abstract}
+
+% ===================== Introduction =====================
+\section{Introduction}
+
+Teachers of introductory courses on computational linguistics are
+often faced with the challenge of setting up a practical programming
+component for student assignments and projects. This is a difficult
+task because different computational linguistics domains require a
+variety of different data structures and functions, and because a
+diverse range of topics may need to be included in the syllabus.
+
+A widespread practice is to employ multiple programming languages,
+where each language provides native data structures and functions that
+are a good fit for the task at hand. For example, a course might use
+Prolog for parsing, Perl for corpus processing, and a finite-state
+toolkit for morphological analysis. By relying on the built-in
+features of various languages, the teacher avoids having to develop a
+lot of software infrastructure.
+
+An unfortunate consequence is that a significant part of such courses
+must be devoted to teaching programming languages. Further, many
+interesting projects span a variety of domains, and would require that
+multiple languages be bridged. For example, a student project that
+involved syntactic parsing of corpus data from a morphologically rich
+language might involve all three of the languages mentioned above:
+Perl for string processing; a finite state toolkit for morphological
+analysis; and Prolog for parsing.
+It is clear that these considerable overheads and shortcomings warrant
+a fresh approach.
+
+Apart from the practical component, computational linguistics courses
+may also depend on software for in-class demonstrations. This context
+calls for highly interactive graphical user interfaces, making it
+possible to view program state (e.g. the chart of a chart parser),
+observe program execution step-by-step (e.g. execution of a
+finite-state machine), and even make minor modifications to programs
+in response to ``what if'' questions from the class. Because of these
+difficulties it is common to avoid live demonstrations, and keep
+classes for theoretical presentations only. Apart from being dull,
+this approach leaves students to solve important practical problems on
+their own, or to deal with them less efficiently in office hours.
+
+In this paper we introduce a new approach to the above challenges, a
+streamlined and flexible way of organizing the practical component of
+an introductory computational linguistics course. We describe NLTK,
+the Natural Language Toolkit, which we have developed in conjunction
+with a course we have taught at the University of Pennsylvania.
+
+The Natural Language Toolkit is available under an open source license
+from \mbox{\url{http://nltk.sf.net/}}. NLTK runs on all platforms
+supported by Python, including Windows, OS X, Linux, and Unix.
+
+% ===================== Python =====================
+\section{Choice of Programming Language}
+\label{sec:python}
+
+The most basic step in setting up a practical component is choosing a
+suitable programming language. A number of considerations influenced
+our choice. First, the language must have a shallow learning curve,
+so that novice programmers get immediate rewards for their efforts.
+Second, the language must support rapid prototyping and a short
+develop/test cycle; an obligatory compilation step is a serious
+detraction. Third, the code should be self-documenting, with a
+transparent syntax and semantics. Fourth, it should be easy to write
+structured programs, ideally object-oriented but without the burden
+associated with languages like C++. Finally, the language must have
+an easy-to-use graphics library to support the development of
+graphical user interfaces.
+
+% [EL] ``punitive'' seems like a strange word.
+% [EL] I couldn't find a REF for ``executable pseudocode'' -- closest
+% I got was http://www.thinkware.se/cgi-bin/thinki.cgi/PythonQuotes
+% [SB] changed Python citation to URL
+In surveying the available languages, we believe that Python offers an
+especially good fit to the above requirements. Python is an
+object-oriented scripting language developed by Guido van Rossum
+and available on all platforms (\url{www.python.org}). Python offers a
+shallow learning curve; it was designed to be easily learnt by
+children \cite{rossum99}. As an interpreted language, Python is
+suitable for rapid prototyping. Python code is exceptionally
+readable, and it has been praised as ``executable pseudocode.''
+Python is an object-oriented language, but not punitively so, and it
+is easy to encapsulate data and methods inside Python classes.
+Finally, Python has an interface to the Tk graphics toolkit
+\cite{tkinter}, and writing graphical interfaces is straightforward.
+
+% ===================== Design Criteria =====================
+\section{Design Criteria}
+\label{sec:criteria}
+
+% [EL] Is this ambiguous between descending/ascending?
+Several criteria were considered in the design and implementation of
+the toolkit. These design criteria are listed in the order of their
+importance. It was also important to decide what goals the toolkit
+would \emph{not} attempt to accomplish; we therefore include an
+explicit set of non-requirements, which the toolkit is not expected to
+satisfy.
+
+\subsection{Requirements}
+
+\paragraph{\textit{Ease of Use.}} The primary purpose of the toolkit is
+to allow students to concentrate on building natural language
+processing (NLP) systems. The more time students must spend learning
+to use the toolkit, the less useful it is.
+
+\paragraph{\textit{Consistency.}} The toolkit should use consistent data
+structures and interfaces.
+
+\paragraph{\textit{Extensibility.}} The toolkit should easily
+accommodate new components, whether those components replicate or
+extend the toolkit's existing functionality. The toolkit should be
+structured in such a way that it is obvious where new extensions would
+fit into the toolkit's infrastructure.
+
+% [EL] I commented out the list of documentation types, since we have
+% a section that talks about the documentation.
+\paragraph{\textit{Documentation.}} The toolkit, its data structures,
+and its implementation all need to be carefully and thoroughly
+documented. All nomenclature must be carefully chosen and
+consistently used.
+
+\paragraph{\textit{Simplicity.}} The toolkit should structure the
+complexities of building NLP systems, not hide them. Therefore, each
+class defined by the toolkit should be simple enough that a student
+could implement it by the time they finish an introductory course in
+computational linguistics.
+
+\paragraph{\textit{Modularity.}} The interaction between different
+components of the toolkit should be kept to a minimum, using simple,
+well-defined interfaces. In particular, it should be possible to
+complete individual projects using small parts of the toolkit, without
+worrying about how they interact with the rest of the toolkit. This
+allows students to learn how to use the toolkit incrementally
+throughout a course. Modularity also makes it easier to change and
+extend the toolkit.
+
+\subsection{Non-Requirements}
+
+\paragraph{\textit{Comprehensiveness.}} The toolkit is not intended to
+provide a comprehensive set of tools. Indeed, there should be a wide
+variety of ways in which students can extend the toolkit.
+
+\paragraph{\textit{Efficiency.}} The toolkit does not need to be highly
+optimized for runtime performance. However, it should be efficient
+enough that students can use their NLP systems to perform real tasks.
+
+\paragraph{\textit{Cleverness.}} Clear designs and implementations are
+far preferable to ingenious yet indecipherable ones.
+
+% ===================== Modules =====================
+\section{Modules}
+\label{sec:modules}
+% What order should the subsections be in? Put more impressive stuff
+% at the beginning and end?
+
+% [EL] is ``defines'' the right word?
+The toolkit is implemented as a collection of independent
+\emph{modules}, each of which defines a specific data structure or
+task.
+
+A set of core modules defines basic data types and processing systems
+that are used throughout the toolkit. The \texttt{token} module
+provides basic classes for processing individual elements of text,
+such as words or sentences. The \texttt{tree} module defines data
+structures for representing tree structures over text, such as syntax
+trees and morphological trees. The \texttt{probability} module
+implements classes that encode frequency distributions and probability
+distributions, including a variety of statistical smoothing
+techniques.
+
+The remaining modules define data structures and interfaces for
+performing specific NLP tasks. This list of modules will grow over
+time, as we add new tasks and algorithms to the toolkit.
+
+\subsection*{Parsing Modules}
+
+The \texttt{parser} module defines a high-level interface for
+producing trees that represent the structures of texts. The
+\texttt{chunkparser} module defines a sub-interface for parsers that
+identify non-overlapping linguistic groups (such as base noun phrases)
+in unrestricted text.
+
+Four modules provide implementations for these abstract interfaces.
+The \texttt{srparser} module implements a simple shift-reduce parser.
+The \texttt{chartparser} module defines a flexible parser that uses a
+\emph{chart} to record hypotheses about syntactic constituents. The
+\texttt{pcfgparser} module provides a variety of different parsers for
+probabilistic grammars. And the \texttt{rechunkparser} module defines
+a transformational regular-expression based implementation of the
+chunk parser interface.
+
+\subsection*{Tagging Modules}
+
+The \texttt{tagger} module defines a standard interface for augmenting
+each token of a text with supplementary information, such as its part
+of speech or its WordNet synset tag; and provides several different
+implementations for this interface.
+
+\subsection*{Finite State Automata}
+
+% [EL] Steven, do you want to add anything here? Say anything about HMMS?
+% Will you have implemented more interfaces by the time of ACL?
+The \texttt{fsa} module defines a data type for encoding finite state
+automata; and an interface for creating automata from regular
+expressions.
+
+\subsection*{Type Checking}
+
+% [EL] This could use another pass or two. :)
+Debugging time is an important factor in the toolkit's ease of use.
+To reduce the amount of time students must spend debugging their code,
+we provide a type checking module, which can be used to ensure that
+functions are given valid arguments. The type checking module is
+used by all of the basic data types and processing classes.
+
+Since type checking is done explicitly, it can slow the toolkit down.
+However, when efficiency is an issue, type checking can be easily
+turned off; and with type checking is disabled, there is no
+performance penalty.
+
+\subsection*{Visualization}
+
+Visualization modules define graphical interfaces for viewing and
+manipulating data structures, and graphical tools for experimenting
+with NLP tasks. The \texttt{draw.tree} module provides a simple
+graphical interface for displaying tree structures. The
+\texttt{draw.tree\_edit} module provides an interface for building and
+modifying tree structures. The \texttt{draw.plot\_graph} module can be
+used to graph mathematical functions. The \texttt{draw.fsa} module
+provides a graphical tool for displaying and simulating finite state
+automata. The \texttt{draw.chart} module provides an interactive
+graphical tool for experimenting with chart parsers.
+
+% [EL] This still needs work...
+The visualization modules provide interfaces for interaction and
+experimentation; they do not directly implement NLP data structures or
+tasks. Simplicity of implementation is therefore less of an issue for
+the visualization modules than it is for the rest of the toolkit.
+
+\subsection*{Text Classification}
+
+The \texttt{classifier} module defines a standard interface for
+classifying texts into categories. This interface is currently
+implemented by two modules. The \texttt{classifier.naivebayes} module
+defines a text classifier based on the Naive Bayes assumption. The
+\texttt{classifier.maxent} module defines the maximum entropy model
+for text classification, and implements two algorithms for training
+the model: Generalized Iterative Scaling and Improved Iterative
+Scaling.
+
+The \texttt{classifier.feature} module provides a standard encoding
+for the information that is used to make decisions for a particular
+classification task. This standard encoding allows students to
+experiment with the differences between different text classification
+algorithms, using identical feature sets.
+
+The \texttt{classifier.featureselection} module defines a standard
+interface for choosing which features are relevant for a particular
+classification task. Good feature selection can significantly improve
+classification performance.
+
+% ===================== Documentation =====================
+\section{Documentation}
+\label{sec:documentation}
+
+The toolkit is accompanied by extensive documentation that explains
+the toolkit, and describes how to use and extend it. This
+documentation is divided into three primary categories:
+
+% [EL] Provide some indication of how much tutorial material is
+% available? (Currently, 9 tutorials, mean=16pp, range=6pp-52pp.)
+\paragraph{\textit{Tutorials}} teach students how to use the toolkit,
+in the context of performing specific tasks. Each tutorial focuses on
+a single domain, such as tagging, probabilistic systems, or text
+classification. The tutorials include a high-level discussion that
+explains and motivates the domain, followed by a detailed
+walk-through that uses examples to show how NLTK can be used to
+perform specific tasks.
+
+\paragraph{\textit{Reference Documentation}} provides precise
+definitions for every module, interface, class, method, function, and
+variable in the toolkit. It is automatically extracted from docstring
+comments in the Python source code, using Epydoc \cite{epydoc}.
+
+\paragraph{\textit{Technical Reports}} explain and justify the
+toolkit's design and implementation. They are used by the developers
+of the toolkit to guide and document the toolkit's construction.
+Students can also consult these reports if they would like further
+information about how the toolkit is designed, and why it is designed
+that way.
+
+% ===================== Uses =====================
+\section{Uses of NLTK}
+\label{sec:uses}
+
+\subsection{Assignments}
+
+NLTK can be used to create student assignments of varying difficulty
+and scope.
+% Use a module
+In the simplest assignments, students experiment with an existing
+module. The wide variety of existing modules provide many opportunities
+for creating these simple assignments.
+% Edit/extend a module.
+Once students become more familiar with the toolkit, they can be asked
+to make minor changes or extensions to an existing module.
+% Develop a new module.
+A more challenging task is to develop a new module. Here, NLTK
+provides some useful starting points: predefined interfaces and data
+structures, and existing modules that implement the same interface.
+
+\subsubsection*{Example: Chunk Parsing}
+
+As an example of a moderately difficult assignment, we asked students
+to construct a chunk parser that correctly identifies base noun phrase
+chunks in a given text, by defining a cascade of transformational
+chunking rules. The NLTK \texttt{rechunkparser} module provides a
+variety of regular-expression based rule types, which the students can
+instantiate to construct complete rules. For example,
+\texttt{ChunkRule('<NN.*>')} builds chunks from sequences of
+consecutive nouns; \texttt{ChinkRule('<VB.>')} excises verbs from
+existing chunks; \texttt{SplitRule('<NN>', '<DT>')} splits any
+existing chunk that contains a singular noun followed by determiner
+into two pieces; and \texttt{MergeRule('<JJ>', '<JJ>')} combines two
+adjacent chunks where the first chunk ends and the second chunk starts
+with adjectives.
+
+The chunking tutorial motivates chunk parsing, describes each rule
+type, and provides all the necessary code for the assignment. The
+provided code is responsible for loading the chunked, part-of-speech
+tagged text using an existing tokenizer, creating an unchunked version
+of the text, applying the chunk rules to the unchunked text, and
+scoring the result. Students focus on the NLP task only -- providing
+a rule set with the best coverage.
+
+In the remainder of this section we reproduce some of the cascades
+created by the students. The first example illustrates a combination
+of several rule types:
+
+\begin{sv}
+cascade = [
+ ChunkRule('<DT><NN.*><VB.><NN.*>'),
+ ChunkRule('<DT><VB.><NN.*>'),
+ ChunkRule('<.*>'),
+ UnChunkRule('<IN|VB.*|CC|MD|RB.*>'),
+ UnChunkRule("<,|{\textbackslash}{\textbackslash}.|``|''>"),
+ MergeRule('<NN.*|DT|JJ.*|CD>',
+ '<NN.*|DT|JJ.*|CD>'),
+ SplitRule('<NN.*>', '<DT|JJ>')
+]
+\end{sv}
+
+The next example illustrates a brute-force statistical approach. The
+student calculated how often each part-of-speech tag was included in a
+noun phrase. They then constructed chunks from any sequence of tags
+that occurred in a noun phrase more than 50\% of the time.
+
+\begin{sv}
+cascade = [
+ ChunkRule('<{\textbackslash}{\textbackslash}\$|CD|DT|EX|PDT
+ |PRP.*|WP.*|{\textbackslash}{\textbackslash}\#|FW
+ |JJ.*|NN.*|POS|RBS|WDT>*')
+]
+\end{sv}
+
+In the third example, the student constructed a single chunk
+containing the entire text, and then excised all elements that did not
+belong.
+
+\begin{sv}
+cascade = [
+ ChunkRule('<.*>+')
+ ChinkRule('<VB.*|IN|CC|R.*|MD|WRB|TO|.|,>+')
+]
+\end{sv}
+
+%% [EL] This makes it sound like we GRADED them using their precision,
+%% recall, and f-measure. Either say something about the contest, or
+%% leave this out:
+%Each student's project was scored using its precision, recall, and
+%F-measure.
+
+% [EL] Leave out the picture.
+%Figure~\ref{fig:contest} shows the results; students with the best two
+%scores were presented with prizes.
+%
+%\begin{figure}
+%\centerline{\epsfig{figure=contest.ps,width=\linewidth}}
+%\caption{Precision/Recall Graph for Chunking Competition}\label{fig:contest}
+%\vspace*{2ex}\hrule
+%\end{figure}
+
+\subsection{Class demonstrations}
+
+NLTK provides graphical tools that can be used in class demonstrations
+to help explain basic NLP concepts and algorithms. These interactive
+tools can be used to display relevant data structures and to show the
+step-by-step execution of algorithms. Both data structures and
+control flow can be easily modified during the demonstration, in
+response to questions from the class.
+
+Since these graphical tools are included with the toolkit, they can
+also be used by students. This allows students to experiment at home
+with the algorithms that they have seen presented in class.
+
+\subsubsection*{Example: The Chart Parsing Tool}
+
+The chart parsing tool is an example of a graphical tool provided by
+NLTK. This tool can be used to explain the basic concepts behind
+chart parsing, and to show how the algorithm works. Chart parsing is
+a flexible parsing algorithm that uses a data structure called a
+\emph{chart} to record hypotheses about syntactic constituents. Each
+hypothesis is represented by a single \emph{edge} on the chart. A set
+of \emph{rules} determine when new edges can be added to the chart.
+This set of rules controls the overall behavior of the parser (e.g.,
+whether it parses top-down or bottom-up).
+
+The chart parsing tool demonstrates the process of parsing a single
+sentence, with a given grammar and lexicon. Its display is divided
+into three sections: the bottom section displays the chart; the middle
+section displays the sentence; and the top section displays the
+partial syntax tree corresponding to the selected edge. Buttons along
+the bottom of the window are used to control the execution of the
+algorithm. The main display window for the chart parsing tool is
+shown in Figure~\ref{fig:chartparse}.
+
+\begin{figure}
+\centerline{\epsfig{figure=chartparse.eps,width=\linewidth}}
+\caption{Chart Parsing Tool}\label{fig:chartparse}
+\vspace*{2ex}\hrule
+\end{figure}
+
+This tool can be used to explain several different aspects of chart
+parsing. First, it can be used to explain the basic chart data
+structure, and to show how edges can represent hypotheses about
+syntactic constituents. It can then be used to demonstrate and
+explain the individual rules that the chart parser uses to create new
+edges. Finally, it can be used to show how these individual rules
+combine to find a complete parse for a given sentence.
+
+% Is ``user'' a good word here? ``lecturer''?
+To reduce the overhead of setting up demonstrations during lecture,
+the user can define a list of preset charts. The tool can then be
+reset to any one of these charts at any time.
+
+The chart parsing tool allows for flexible control of the parsing
+algorithm. At each step of the algorithm, the user can select which
+rule or strategy they wish to apply. This allows the user to
+experiment with mixing different strategies (e.g., top-down and
+bottom-up). The user can exercise fine-grained control over the
+algorithm by selecting which edge they wish to apply a rule to. This
+flexibility allows lecturers to use the tool to respond to a wide
+variety of questions; and allows students to experiment with different
+variations on the chart parsing algorithm.
+
+\subsection{Advanced Projects}
+
+NLTK provides students with a flexible framework for advanced
+projects. Typical projects involve the development of entirely new
+functionality for a previously unsupported NLP task, or the
+development of a complete system out of existing and new modules.
+
+% Broad coverage
+The toolkit's broad coverage allows students to explore a wide variety
+of topics. In our introductory computational linguistics course,
+topics for student projects included text generation, word sense
+disambiguation, collocation analysis, and morphological analysis.
+
+% Less grunt-work
+NLTK eliminates the tedious infrastructure-building that is typically
+associated with advanced student projects by providing students with
+the basic data structures, tools, and interfaces that they need. This
+allows the students to concentrate on the problems that interest them.
+
+% Meaningful contribution
+The collaborative, open-source nature of the toolkit can provide
+students with a sense that their projects are meaningful
+contributions, and not just exercises. Several of the students in our
+course have expressed interest in incorporating their projects into
+the toolkit.
+
+% Good examples
+Finally, many of the modules included in the toolkit provide students
+with good examples of what projects should look like, with well
+thought-out interfaces, clean code structure, and thorough
+documentation.
+
+\subsubsection*{Example: Probabilistic Parsing}
+% [EL] I'm a bit uncomfortable including this description without
+% mentioning that I'm the author -- seems a little misleading. But at
+% the same time, it seems like it's much less interesting if I say
+% that I wrote it, given that I wrote the majority of the toolkit
+% anyway. :-/ Perhaps we should just leave out this section. Any
+% ideas?
+
+The probabilistic parsing module was created as a class project for a
+statistical NLP course. The toolkit provided the basic data types and
+interfaces for parsing. The project extended these, adding a new
+probabilistic parsing interface, and using subclasses to create a
+probabilistic version of the context free grammar data structure.
+These new components were used in conjunction with several existing
+components, such as the chart data structure, to define two
+implementations of the probabilistic parsing interface. Finally, a
+tutorial was written that explained the basic motivations and concepts
+behind probabilistic parsing, and described the new interfaces, data
+structures, and parsers.
+
+% ===================== Evaluation =====================
+\section{Evaluation}
+\label{sec:evaluation}
+
+% Do we want to mention that there were 40h of lecture time?
+% Should we be more explicit about who took the class?
+We used NLTK as a basis for the assignments and student projects in
+CIS-530, an introductory computational linguistics class taught at the
+University of Pennsylvania. CIS-530 is a graduate level class,
+although some advanced undergraduates were also enrolled. Most
+students had a background in either computer science or linguistics
+(and occasionally both). Students were required to complete five assignments,
+two exams, and a final project. All class materials are available
+from the course website \mbox{\url{http://www.cis.upenn.edu/~cis530/}}.
+
+The experience of using NLTK was very positive, both for us and for
+the students. The students liked the fact that they could do
+interesting projects from the outset. They also liked being able to
+run everything on their computer at home. The students found the
+extensive documentation very helpful for learning to use the toolkit.
+They found the interfaces defined by NLTK intuitive, and appreciated
+the ease with which they could combine different components to create
+complete NLP systems.
+
+We did encounter a few difficulties during the semester. One problem
+was finding large clean corpora that the students could use for their
+assignments. Several of the students needed assistance finding
+suitable corpora for their final projects. Another issue was the fact
+that we were actively developing NLTK during the semester; some
+modules were only completed one or two weeks before the students used
+them. As a result, students who worked at home needed to download new
+versions of the toolkit several times throughout the semester.
+Luckily, Python has extensive support for installation scripts, which
+made these upgrades simple. The students encountered a couple
+of bugs in the toolkit, but none were serious, and all were quickly
+corrected.
+
+% ===================== Other Approaches =====================
+\section{Other Approaches}
+\label{sec:approaches}
+
+The computational component of computational linguistics courses takes
+many forms. In this section we briefly review a selection of approaches,
+classified according to the (original) target audience.
+
+{\bf Linguistics Students.}
+Various books introduce programming or computing to linguists.
+These are elementary on the computational side, providing a
+gentle introduction to students having no prior experience
+in computer science. Examples of such books are:
+\emph{Using Computers in Linguistics}
+\cite{Lawler98}, and
+\emph{Programming for Linguistics: Java Technology for Language
+Researchers} \cite{Hammond02}.
+
+{\bf Grammar Developers.}
+Infrastructure for grammar development has a long history in
+unification-based (or constraint-based) grammar frameworks, from DCG
+\cite{PereiraWarren80} to HPSG \cite{PollardSag94}. Recent work includes
+\cite{Copestake00,Baldridge02}. A concurrent development has been the
+finite state toolkits, such as the Xerox toolkit \cite{Beesley02}. This
+work has found widespread pedagogical application.
+
+{\bf Other Researchers and Developers.}
+A variety of toolkits have been created for research or R\&D
+purposes. Examples include
+the \emph{CMU-Cambridge Statistical Language Modeling Toolkit}
+\cite{Clarkson97},
+the \emph{EMU Speech Database System} \cite{Harrington99},
+the \emph{General Architecture for Text Engineering} \cite{Bontcheva02},
+the \emph{Maxent Package for Maximum Entropy Models} \cite{maxent},
+and the \emph{Annotation Graph Toolkit} \cite{MaedaBird02}.
+Although not originally motivated by pedagogical needs, all of these
+toolkits have pedagogical applications and many have already been
+used in teaching.
+
+% ===================== Conclusion =====================
+\section{Conclusions and Future Work}
+\label{sec:conclusion}
+
+%Overview of the paper:
+% [EL] Pick better 3 adjectives :)
+NLTK provides a simple, extensible, uniform framework for assignments,
+projects, and class demonstrations. It is well documented, easy to
+learn, and simple to use. We hope that NLTK will allow computational
+linguistics classes to include more hands-on experience with using and
+building NLP components and systems.
+
+NLTK is unique in its combination of three factors.
+First, it was deliberately designed as courseware and gives pedagogical
+goals primary status. Second, its target audience consists of both
+linguists and computer scientists, and it is accessible and challenging
+at many levels of prior computational skill. Finally, it is based on
+an object-oriented scripting language supporting rapid prototyping and
+literate programming.
+
+% [EL] No room for this for now...
+%NOTES: Many CL textbooks provide good exercises at the end of each
+%chapter. This isn't enough for many situations. E.g. students spend
+%lots of time writing interface code (e.g. to process a corpus).
+%E.g. teachers spend lots of time creating do-able assignments...
+
+% [EL] Add something saying explicitly that there's still a lot to
+% do? (In response to reviewer comment)
+%Future work:
+We plan to continue extending the breadth of materials covered by
+the toolkit. We are currently working on NLTK modules for Hidden
+Markov Models, language modeling, and tree adjoining grammars. We
+also plan to increase the number of algorithms implemented by some
+existing modules, such as the text classification module.
+
+Finding suitable corpora is a prerequisite for many student assignments and
+projects. We are therefore putting together a collection of corpora
+containing data appropriate for every module defined by the toolkit.
+
+NLTK is an open source project, and we welcome any contributions.
+Readers who are interested in contributing to NLTK, or who have
+suggestions for improvements, are encouraged to contact the authors.
+
+% include URL (a second time; 1st was in the intro) in conclusion??
+
+% ===================== Acknowledgements =====================
+\section{Acknowledgments}
+\label{sec:acknowledgments}
+
+We are indebted to our students for feedback on the toolkit, and to
+anonymous reviewers, Jee Bang, and the workshop organizers for
+comments on an earlier version of this paper. We are grateful to
+Mitch Marcus and the Department of Computer and Information Science at
+the University of Pennsylvania for sponsoring the work reported here.
+
+\bibliographystyle{acl}
+\bibliography{nltk}
+
+\end{document}
+% LocalWords: stevenbird Exp pcfgparser subclasses CFG DCG HPSG toolkits CMU
diff --git a/papers/acl-02/acl.bst b/papers/acl-02/acl.bst
new file mode 100644
index 0000000..b95ec04
--- /dev/null
+++ b/papers/acl-02/acl.bst
@@ -0,0 +1,1322 @@
+
+% BibTeX `acl' style file for BibTeX version 0.99c, LaTeX version 2.09
+% This version was made by modifying `aaai-named' format based on the master
+% file by Oren Patashnik (PATASHNIK at SCORE.STANFORD.EDU)
+
+% Copyright (C) 1985, all rights reserved.
+% Modifications Copyright 1988, Peter F. Patel-Schneider
+% Further modifictions by Stuart Shieber, 1991, and Fernando Pereira, 1992.
+% Copying of this file is authorized only if either
+% (1) you make absolutely no changes to your copy, including name, or
+% (2) if you do make changes, you name it something other than
+% btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst.
+% This restriction helps ensure that all standard styles are identical.
+
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at spar.slb.com
+
+% Citation format: [author-last-name, year]
+% [author-last-name and author-last-name, year]
+% [author-last-name {\em et al.}, year]
+%
+% Reference list ordering: alphabetical by author or whatever passes
+% for author in the absence of one.
+%
+% This BibTeX style has support for short (year only) citations. This
+% is done by having the citations actually look like
+% \citename{name-info, }year
+% The LaTeX style has to have the following
+% \let\@internalcite\cite
+% \def\cite{\def\citename##1{##1}\@internalcite}
+% \def\shortcite{\def\citename##1{}\@internalcite}
+% \def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+% which makes \shortcite the macro for short citations.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Changes made by SMS for thesis style
+% no emphasis on "et al."
+% "Ph.D." includes periods (not "PhD")
+% moved year to immediately after author's name
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ENTRY
+ { address
+ author
+ booktitle
+ chapter
+ edition
+ editor
+ howpublished
+ institution
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ publisher
+ school
+ series
+ title
+ type
+ volume
+ year
+ }
+ {}
+ { label extra.label sort.label }
+
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+
+STRINGS { s t }
+
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { add.period$ " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+
+ "\bibitem[" write$
+ label write$
+ "]{" write$
+
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+
+FUNCTION {new.block.checka}
+{ empty$
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.sentence.checka}
+{ empty$
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {new.sentence.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+
+FUNCTION {emphasize}
+{ duplicate$ empty$
+ { pop$ "" }
+ { "{\em " swap$ * "}" * }
+ if$
+}
+
+INTEGERS { nameptr namesleft numnames }
+
+FUNCTION {format.names}
+{ 's :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+
+ { s nameptr "{ff~}{vv~}{ll}{, jj}" format.name$ 't :=
+
+ nameptr #1 >
+ { namesleft #1 >
+ { ", " * t * }
+ { numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ t "others" =
+ { " et~al." * }
+ { " and " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {format.authors}
+{ author empty$
+ { "" }
+ { author format.names }
+ if$
+}
+
+FUNCTION {format.editors}
+{ editor empty$
+ { "" }
+ { editor format.names
+ editor num.names$ #1 >
+ { ", editors" * }
+ { ", editor" * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title empty$
+ { "" }
+
+ { title "t" change.case$ }
+
+ if$
+}
+
+FUNCTION {n.dashify}
+{ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {format.date}
+{ year empty$
+ { month empty$
+ { "" }
+ { "there's a month but no year in " cite$ * warning$
+ month
+ }
+ if$
+ }
+ { month empty$
+ { "" }
+ { month }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.btitle}
+{ title emphasize
+}
+
+FUNCTION {tie.or.space.connect}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$ * *
+}
+
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { "volume" volume tie.or.space.connect
+ series empty$
+ 'skip$
+ { " of " * series emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { output.state mid.sentence =
+ { "number" }
+ { "Number" }
+ if$
+ number tie.or.space.connect
+ series empty$
+ { "there's a number but no series in " cite$ * warning$ }
+ { " in " * series * }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition empty$
+ { "" }
+ { output.state mid.sentence =
+ { edition "l" change.case$ " edition" * }
+ { edition "t" change.case$ " edition" * }
+ if$
+ }
+ if$
+}
+
+INTEGERS { multiresult }
+
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+
+FUNCTION {format.pages}
+{ pages empty$
+ { "" }
+ { pages multi.page.check
+ { "pages" pages n.dashify tie.or.space.connect }
+ { "page" pages tie.or.space.connect }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.year.label}
+{ year extra.label *
+}
+
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ number empty$
+ 'skip$
+ { "(" number * ")" * *
+ volume empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ }
+ if$
+ pages empty$
+ 'skip$
+ { duplicate$ empty$
+ { pop$ format.pages }
+ { ":" * pages n.dashify * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { "chapter" }
+ { type "l" change.case$ }
+ if$
+ chapter tie.or.space.connect
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.in.ed.booktitle}
+{ booktitle empty$
+ { "" }
+ { editor empty$
+ { "In " booktitle emphasize * }
+ { "In " format.editors * ", " * booktitle emphasize * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {empty.misc.check}
+{ author empty$ title empty$ howpublished empty$
+ month empty$ year empty$ note empty$
+ and and and and and
+
+ key empty$ not and
+
+ { "all relevant fields are empty in " cite$ * warning$ }
+ 'skip$
+ if$
+}
+
+FUNCTION {format.thesis.type}
+{ type empty$
+ 'skip$
+ { pop$
+ type "t" change.case$
+ }
+ if$
+}
+
+FUNCTION {format.tr.number}
+{ type empty$
+ { "Technical Report" }
+ 'type
+ if$
+ number empty$
+ { "t" change.case$ }
+ { number tie.or.space.connect }
+ if$
+}
+
+FUNCTION {format.article.crossref}
+{ key empty$
+ { journal empty$
+ { "need key or journal for " cite$ * " to crossref " * crossref *
+ warning$
+ ""
+ }
+ { "In {\em " journal * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.crossref.editor}
+{ editor #1 "{vv~}{ll}" format.name$
+ editor num.names$ duplicate$
+ #2 >
+ { pop$ " et~al." * }
+ { #2 <
+ 'skip$
+ { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" =
+ { " et~al." * }
+ { " and " * editor #2 "{vv~}{ll}" format.name$ * }
+ if$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.book.crossref}
+{ volume empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ "In "
+ }
+ { "Volume" volume tie.or.space.connect
+ " of " *
+ }
+ if$
+ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { series empty$
+ { "need editor, key, or series for " cite$ * " to crossref " *
+ crossref * warning$
+ "" *
+ }
+ { "{\em " * series * "\/}" * }
+ if$
+ }
+ { key * }
+ if$
+ }
+ { format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.incoll.inproc.crossref}
+{ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { booktitle empty$
+ { "need editor, key, or booktitle for " cite$ * " to crossref " *
+ crossref * warning$
+ ""
+ }
+ { "In {\em " booktitle * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ }
+ { "In " format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { journal emphasize "journal" output.check
+ format.vol.num.pages output
+ format.date output
+ }
+ { format.article.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {booklet}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ howpublished address new.block.checkb
+ howpublished output
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.chapter.pages output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ format.edition output
+ format.date output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.chapter.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.pages output
+ address empty$
+ { organization publisher new.sentence.checkb
+ organization output
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ organization output
+ publisher output
+ }
+ if$
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {conference} { inproceedings }
+
+FUNCTION {manual}
+{ output.bibitem
+ author empty$
+ { organization empty$
+ 'skip$
+ { organization output.nonnull
+ address output
+ }
+ if$
+ }
+ { format.authors output.nonnull }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ author empty$
+ { organization empty$
+ { address new.block.checka
+ address output
+ }
+ 'skip$
+ if$
+ }
+ { organization address new.block.checkb
+ organization output
+ address output
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ "Master's thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label output
+ new.block
+ title howpublished new.block.checkb
+ format.title output
+ howpublished new.block.checka
+ howpublished output
+ format.date output
+ new.block
+ note output
+ fin.entry
+ empty.misc.check
+}
+
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ new.block
+ "{Ph.D.} thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ editor empty$
+ { organization output }
+ { format.editors output.nonnull }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ format.bvolume output
+ format.number.series output
+ address empty$
+ { editor empty$
+ { publisher new.sentence.checka }
+ { organization publisher new.sentence.checkb
+ organization output
+ }
+ if$
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ editor empty$
+ 'skip$
+ { organization output }
+ if$
+ publisher output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ format.tr.number output.nonnull
+ institution "institution" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ note "note" output.check
+ format.date output
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+
+MACRO {jan} {"January"}
+
+MACRO {feb} {"February"}
+
+MACRO {mar} {"March"}
+
+MACRO {apr} {"April"}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"August"}
+
+MACRO {sep} {"September"}
+
+MACRO {oct} {"October"}
+
+MACRO {nov} {"November"}
+
+MACRO {dec} {"December"}
+
+MACRO {acmcs} {"ACM Computing Surveys"}
+
+MACRO {acta} {"Acta Informatica"}
+
+MACRO {cacm} {"Communications of the ACM"}
+
+MACRO {ibmjrd} {"IBM Journal of Research and Development"}
+
+MACRO {ibmsj} {"IBM Systems Journal"}
+
+MACRO {ieeese} {"IEEE Transactions on Software Engineering"}
+
+MACRO {ieeetc} {"IEEE Transactions on Computers"}
+
+MACRO {ieeetcad}
+ {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"}
+
+MACRO {ipl} {"Information Processing Letters"}
+
+MACRO {jacm} {"Journal of the ACM"}
+
+MACRO {jcss} {"Journal of Computer and System Sciences"}
+
+MACRO {scp} {"Science of Computer Programming"}
+
+MACRO {sicomp} {"SIAM Journal on Computing"}
+
+MACRO {tocs} {"ACM Transactions on Computer Systems"}
+
+MACRO {tods} {"ACM Transactions on Database Systems"}
+
+MACRO {tog} {"ACM Transactions on Graphics"}
+
+MACRO {toms} {"ACM Transactions on Mathematical Software"}
+
+MACRO {toois} {"ACM Transactions on Office Information Systems"}
+
+MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"}
+
+MACRO {tcs} {"Theoretical Computer Science"}
+
+READ
+
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+
+INTEGERS { len }
+
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+
+INTEGERS { et.al.char.used }
+
+FUNCTION {initialize.et.al.char.used}
+{ #0 'et.al.char.used :=
+}
+
+EXECUTE {initialize.et.al.char.used}
+
+FUNCTION {format.lab.names}
+{ 's :=
+ s num.names$ 'numnames :=
+
+ numnames #1 =
+ { s #1 "{vv }{ll}" format.name$ }
+ { numnames #2 =
+ { s #1 "{vv }{ll }and " format.name$ s #2 "{vv }{ll}" format.name$ *
+ }
+ { s #1 "{vv }{ll }\bgroup et al.\egroup " format.name$ }
+ if$
+ }
+ if$
+
+}
+
+FUNCTION {author.key.label}
+{ author empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.editor.key.label}
+{ author empty$
+ { editor empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.key.organization.label}
+{ author empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {editor.key.organization.label}
+{ editor empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+}
+
+FUNCTION {calc.label}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.label
+ { type$ "proceedings" =
+ 'editor.key.organization.label
+ { type$ "manual" =
+ 'author.key.organization.label
+ 'author.key.label
+ if$
+ }
+ if$
+ }
+ if$
+ duplicate$
+
+ "\protect\citename{" swap$ * "}" *
+ year field.or.null purify$ *
+ 'label :=
+ year field.or.null purify$ *
+
+ sortify 'sort.label :=
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { nameptr #1 >
+ { " " * }
+ 'skip$
+ if$
+
+ s nameptr "{vv{ } }{ll{ }}{ ff{ }}{ jj{ }}" format.name$ 't :=
+
+ nameptr numnames = t "others" = and
+ { "et al" * }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.organization.sort}
+{ author empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need author, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {editor.organization.sort}
+{ editor empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need editor, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+
+FUNCTION {presort}
+
+{ calc.label
+ sort.label
+ " "
+ *
+ type$ "book" =
+
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.organization.sort
+ { type$ "manual" =
+ 'author.organization.sort
+ 'author.sort
+ if$
+ }
+ if$
+ }
+ if$
+
+ *
+
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+
+SORT
+
+STRINGS { longest.label last.sort.label next.extra }
+
+INTEGERS { longest.label.width last.extra.num }
+
+FUNCTION {initialize.longest.label}
+{ "" 'longest.label :=
+ #0 int.to.chr$ 'last.sort.label :=
+ "" 'next.extra :=
+ #0 'longest.label.width :=
+ #0 'last.extra.num :=
+}
+
+FUNCTION {forward.pass}
+{ last.sort.label sort.label =
+ { last.extra.num #1 + 'last.extra.num :=
+ last.extra.num int.to.chr$ 'extra.label :=
+ }
+ { "a" chr.to.int$ 'last.extra.num :=
+ "" 'extra.label :=
+ sort.label 'last.sort.label :=
+ }
+ if$
+}
+
+FUNCTION {reverse.pass}
+{ next.extra "b" =
+ { "a" 'extra.label := }
+ 'skip$
+ if$
+ label extra.label * 'label :=
+ label width$ longest.label.width >
+ { label 'longest.label :=
+ label width$ 'longest.label.width :=
+ }
+ 'skip$
+ if$
+ extra.label 'next.extra :=
+}
+
+EXECUTE {initialize.longest.label}
+
+ITERATE {forward.pass}
+
+REVERSE {reverse.pass}
+
+FUNCTION {begin.bib}
+
+{ et.al.char.used
+ { "\newcommand{\etalchar}[1]{$^{#1}$}" write$ newline$ }
+ 'skip$
+ if$
+ preamble$ empty$
+
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+
+ "\begin{thebibliography}{" "}" * write$ newline$
+
+}
+
+EXECUTE {begin.bib}
+
+EXECUTE {init.state.consts}
+
+ITERATE {call.type$}
+
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+
+EXECUTE {end.bib}
+
diff --git a/papers/acl-02/acl2002.sty b/papers/acl-02/acl2002.sty
new file mode 100644
index 0000000..161734e
--- /dev/null
+++ b/papers/acl-02/acl2002.sty
@@ -0,0 +1,340 @@
+% File acl2002.sty
+% October 1, 2002
+% Contact: lindek at cs.ualberta.ca
+
+% This is the LaTeX style file for ACL 2002. It is nearly identical to the
+% style files for ACL 2001, ACL 2000, EACL 95 and EACL 99. Minor changes concern
+% changes in the dimentions for margins.
+%
+% -- Roberto Zamparelli, March 26, 2001
+% -- Dekang Lin, October 1, 2001
+
+% This is the LaTeX style file for ACL 2000. It is nearly identical to the
+% style files for EACL 95 and EACL 99. Minor changes include editing the
+% instructions to reflect use of \documentclass rather than \documentstyle
+% and removing the white space before the title on the first page
+% -- John Chen, June 29, 2000
+
+% To convert from submissions prepared using the style file aclsub.sty
+% prepared for the ACL 2000 conference, proceed as follows:
+% 1) Remove submission-specific information: \whichsession, \id,
+% \wordcount, \otherconferences, \area, \keywords
+% 2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% 3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% 4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+% 5) Change the style reference from aclsub to acl2000, and be sure
+% this style file is in your TeX search path
+
+
+% This is the LaTeX style file for EACL-95. It is identical to the
+% style file for ANLP '94 except that the margins are adjusted for A4
+% paper. -- abney 13 Dec 94
+
+% The ANLP '94 style file is a slightly modified
+% version of the style used for AAAI and IJCAI, using some changes
+% prepared by Fernando Pereira and others and some minor changes
+% by Paul Jacobs.
+
+% Papers prepared using the aclsub.sty file and acl.bst bibtex style
+% should be easily converted to final format using this style.
+% (1) Submission information (\wordcount, \subject, and \makeidpage)
+% should be removed.
+% (2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% (between \begin{abstract} and \end{abstract}).
+% (3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% (4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+
+% Place this in a file called aclap.sty in the TeX search path.
+% (Placing it in the same directory as the paper should also work.)
+
+% Prepared by Peter F. Patel-Schneider, liberally using the ideas of
+% other style hackers, including Barbara Beeton.
+% This style is NOT guaranteed to work. It is provided in the hope
+% that it will make the preparation of papers easier.
+%
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at research.att.com
+
+% Papers are to be prepared using the ``acl'' bibliography style,
+% as follows:
+% \documentclass[11pt]{article}
+% \usepackage{acl2000}
+% \title{Title}
+% \author{Author 1 \and Author 2 \\ Address line \\ Address line \And
+% Author 3 \\ Address line \\ Address line}
+% \begin{document}
+% ...
+% \bibliography{bibliography-file}
+% \bibliographystyle{acl}
+% \end{document}
+
+% Author information can be set in various styles:
+% For several authors from the same institution:
+% \author{Author 1 \and ... \and Author n \\
+% Address line \\ ... \\ Address line}
+% if the names do not fit well on one line use
+% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
+% For authors from different institutions:
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \And ... \And
+% Author n \\ Address line \\ ... \\ Address line}
+% To start a seperate ``row'' of authors use \AND, as in
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \AND
+% Author 2 \\ Address line \\ ... \\ Address line \And
+% Author 3 \\ Address line \\ ... \\ Address line}
+
+% If the title and author information does not fit in the area allocated,
+% place \setlength\titlebox{<new height>} right after
+% \usepackage{acl2000}
+% where <new height> can be something larger than 2.25in
+
+% \typeout{Conference Style for ACL 2000 -- released June 20, 2000}
+\typeout{Conference Style for ACL 2002 -- released April 8, 2002}
+
+% NOTE: Some laser printers have a serious problem printing TeX output.
+% These printing devices, commonly known as ``write-white'' laser
+% printers, tend to make characters too light. To get around this
+% problem, a darker set of fonts must be created for these devices.
+%
+
+% Physical page layout - slightly modified from IJCAI by pj
+\setlength\topmargin{0.0in} \setlength\oddsidemargin{-0.0in}
+\setlength\textheight{9.0in} \setlength\textwidth{6.5in}
+\setlength\columnsep{0.2in}
+\newlength\titlebox
+\setlength\titlebox{2.25in}
+\setlength\headheight{0pt} \setlength\headsep{0pt}
+%\setlength\footheight{0pt}
+\setlength\footskip{0pt}
+\thispagestyle{empty} \pagestyle{empty}
+\flushbottom \twocolumn \sloppy
+
+%% A4 version of page layout
+%\setlength\topmargin{-0.45cm} % changed by Rz -1.4
+%\setlength\oddsidemargin{.8mm} % was -0cm, changed by Rz
+%\setlength\textheight{23.5cm}
+%\setlength\textwidth{15.8cm}
+%\setlength\columnsep{0.6cm}
+%\newlength\titlebox
+%\setlength\titlebox{2.00in}
+%\setlength\headheight{5pt}
+%\setlength\headsep{0pt}
+%%\setlength\footheight{0pt}
+%\setlength\footskip{0pt}
+%\thispagestyle{empty}
+%\pagestyle{empty}
+
+\flushbottom \twocolumn \sloppy
+
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip at sandia-2
+\def\addcontentsline#1#2#3{}
+
+% Title stuff, taken from deproc.
+\def\maketitle{\par
+ \begingroup
+ \def\thefootnote{\fnsymbol{footnote}}
+ \def\@makefnmark{\hbox to 0pt{$^{\@thefnmark}$\hss}}
+ \twocolumn[\@maketitle] \@thanks
+ \endgroup
+ \setcounter{footnote}{0}
+ \let\maketitle\relax \let\@maketitle\relax
+ \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax}
+\def\@maketitle{\vbox to \titlebox{\hsize\textwidth
+ \linewidth\hsize \vskip 0.125in minus 0.125in \centering
+ {\Large\bf \@title \par} \vskip 0.2in plus 1fil minus 0.1in
+ {\def\and{\unskip\enspace{\rm and}\enspace}%
+ \def\And{\end{tabular}\hss \egroup \hskip 1in plus 2fil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}%
+ \def\AND{\end{tabular}\hss\egroup \hfil\hfil\egroup
+ \vskip 0.25in plus 1fil minus 0.125in
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf\@author
+ \end{tabular}\hss\egroup
+ \hfil\hfil\egroup}
+ \vskip 0.3in plus 2fil minus 0.1in
+}}
+\renewenvironment{abstract}{\centerline{\large\bf
+ Abstract}\vspace{0.5ex}\begin{quote}}{\par\end{quote}\vskip 1ex}
+
+
+% bibliography
+
+\def\thebibliography#1{\section*{References}
+ \global\def\@listi{\leftmargin\leftmargini
+ \labelwidth\leftmargini \advance\labelwidth-\labelsep
+ \topsep 1pt plus 2pt minus 1pt
+ \parsep 0.25ex plus 1pt \itemsep 0.25ex plus 1pt}
+ \list {[\arabic{enumi}]}{\settowidth\labelwidth{[#1]}\leftmargin\labelwidth
+ \advance\leftmargin\labelsep\usecounter{enumi}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy
+ \sfcode`\.=1000\relax}
+
+\def\@up#1{\raise.2ex\hbox{#1}}
+
+% most of cite format is from aclsub.sty by SMS
+
+% don't box citations, separate with ; and a space
+% also, make the penalty between citations negative: a good place to break
+% changed comma back to semicolon pj 2/1/90
+% \def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+% \def\@citea{}\@cite{\@for\@citeb:=#2\do
+% {\@citea\def\@citea{;\penalty\@citeseppen\ }\@ifundefined
+% {b@\@citeb}{{\bf ?}\@warning
+% {Citation `\@citeb' on page \thepage \space undefined}}%
+% {\csname b@\@citeb\endcsname}}}{#1}}
+
+% don't box citations, separate with ; and a space
+% Replaced for multiple citations (pj)
+% don't box citations and also add space, semicolon between multiple citations
+\def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@cite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{; }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+ {\csname b@\@citeb\endcsname}}}{#1}}
+
+% Allow short (name-less) citations, when used in
+% conjunction with a bibliography style that creates labels like
+% \citename{<names>, }<year>
+%
+\let\@internalcite\cite
+\def\cite{\def\citename##1{##1, }\@internalcite}
+\def\shortcite{\def\citename##1{}\@internalcite}
+\def\newcite{\leavevmode\def\citename##1{{##1} (}\@internalciteb}
+
+% Macros for \newcite, which leaves name in running text, and is
+% otherwise like \shortcite.
+\def\@citexb[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@newcite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{;\penalty\@m\ }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+\hbox{\csname b@\@citeb\endcsname}}}{#1}}
+\def\@internalciteb{\@ifnextchar [{\@tempswatrue\@citexb}{\@tempswafalse\@citexb[]}}
+
+\def\@newcite#1#2{{#1\if at tempswa, #2\fi)}}
+
+\def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+
+%%% More changes made by SMS (originals in latex.tex)
+% Use parentheses instead of square brackets in the text.
+\def\@cite#1#2{({#1\if at tempswa , #2\fi})}
+
+% Don't put a label in the bibliography at all. Just use the unlabeled format
+% instead.
+\def\thebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{References\@mkboth
+ {References}{References}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthebibliography=\endlist
+
+% Allow for a bibliography of sources of attested examples
+\def\thesourcebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{Sources of Attested Examples\@mkboth
+ {Sources of Attested Examples}{Sources of Attested Examples}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthesourcebibliography=\endlist
+
+\def\@lbibitem[#1]#2{\item[]\if at filesw
+ { \def\protect##1{\string ##1\space}\immediate
+ \write\@auxout{\string\bibcite{#2}{#1}}\fi\ignorespaces}}
+
+\def\@bibitem#1{\item\if at filesw \immediate\write\@auxout
+ {\string\bibcite{#1}{\the\c at enumi}}\fi\ignorespaces}
+
+% sections with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+ -0.5ex minus -.2ex}{1.5ex plus 0.3ex minus .2ex}{\large\bf\raggedright}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-1.8ex plus
+ -0.5ex minus -.2ex}{0.8ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{0.5ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\paragraph{\@startsection{paragraph}{4}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+\def\subparagraph{\@startsection{subparagraph}{5}{\parindent}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+
+% Footnotes
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+
+% Lists and paragraphs
+\parindent 1em
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 2pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+
+\leftmargin 2em \leftmargini\leftmargin \leftmarginii 2em
+\leftmarginiii 1.5em \leftmarginiv 1.0em \leftmarginv .5em \leftmarginvi .5em
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+ \labelwidth\leftmarginii\advance\labelwidth-\labelsep
+ \topsep 2pt plus 1pt minus 0.5pt
+ \parsep 1pt plus 0.5pt minus 0.5pt
+ \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+ \labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+ \topsep 1pt plus 0.5pt minus 0.5pt
+ \parsep \z@ \partopsep 0.5pt plus 0pt minus 0.5pt
+ \itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+ \labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+ \labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+ \labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\@normalsize{\@setsize\normalsize{11pt}\xpt\@xpt}
+\def\small{\@setsize\small{10pt}\ixpt\@ixpt}
+\def\footnotesize{\@setsize\footnotesize{10pt}\ixpt\@ixpt}
+\def\scriptsize{\@setsize\scriptsize{8pt}\viipt\@viipt}
+\def\tiny{\@setsize\tiny{7pt}\vipt\@vipt}
+\def\large{\@setsize\large{14pt}\xiipt\@xiipt}
+\def\Large{\@setsize\Large{16pt}\xivpt\@xivpt}
+\def\LARGE{\@setsize\LARGE{20pt}\xviipt\@xviipt}
+\def\huge{\@setsize\huge{23pt}\xxpt\@xxpt}
+\def\Huge{\@setsize\Huge{28pt}\xxvpt\@xxvpt}
diff --git a/papers/acl-02/chartparse.eps.gz b/papers/acl-02/chartparse.eps.gz
new file mode 100644
index 0000000..d1969aa
Binary files /dev/null and b/papers/acl-02/chartparse.eps.gz differ
diff --git a/papers/acl-02/contest.ps.gz b/papers/acl-02/contest.ps.gz
new file mode 100644
index 0000000..1f78840
Binary files /dev/null and b/papers/acl-02/contest.ps.gz differ
diff --git a/papers/acl-02/nltk.bib b/papers/acl-02/nltk.bib
new file mode 100644
index 0000000..bdee454
--- /dev/null
+++ b/papers/acl-02/nltk.bib
@@ -0,0 +1,154 @@
+
+ at InProceedings{Baldridge02,
+ author = {Jason Baldridge and John Dowding and Susana Early},
+ title = {Leo: an architecture for sharing resources for unification-based grammars},
+ booktitle = {Proceedings of the Third Language Resources and Evaluation Conference},
+ year = 2002,
+ publisher = {Paris: European Language Resources Association},
+ note = {\\\url{http://www.iccs.informatics.ed.ac.uk/~jmb/leo-lrec.ps.gz}}
+}
+
+ at book{Beesley02,
+ author = {Kenneth R.~Beesley and Lauri Karttunen},
+ title = {Finite-State Morphology: Xerox Tools and Techniques},
+ publisher = {Cambridge University Press},
+ series = {Studies in Natural Language Processing},
+ year = 2002
+}
+
+ at InProceedings{Bontcheva02,
+ author = {Kalina Bontcheva and Hamish Cunningham and Valentin Tablan and Diana Maynard and Oana Hamza},
+ title = {Using {GATE} as an Environment for Teaching {NLP}},
+ booktitle = {Proceedings of the ACL Workshop on Effective Tools
+ and Methodologies for Teaching NLP and CL},
+ year = 2002,
+ publisher = {Somerset, NJ: Association for Computational Linguistics}
+}
+
+ at InProceedings{Clarkson97,
+ author = {Philip R.~Clarkson and Ronald Rosenfeld},
+ title = {Statistical language modeling using the {CMU-Cambridge Toolkit}},
+ booktitle = {Proceedings of the 5th European Conference on Speech
+ Communication and Technology (EUROSPEECH '97)},
+ year = 1997,
+ note = {\url{http://svr-www.eng.cam.ac.uk/~prc14/eurospeech97.ps}}
+}
+
+ at Misc{Copestake00,
+ author = {Ann Copestake},
+ title = {The (new) {LKB} system},
+ year = 2000,
+ note = {\\\url{http://www-csli.stanford.edu/~aac/doc5-2.pdf}}
+}
+
+ at Book{Hammond02,
+ author = {Michael Hammond},
+ title = {Programming for Linguistics: Java
+ Technology for Language Researchers},
+ publisher = {Oxford: Blackwell},
+ year = 2002,
+ note = {In press.}
+}
+
+ at Book{Lawler98,
+ editor = {John M.~Lawler and Helen Aristar Dry},
+ title = {Using Computers in Linguistics},
+ publisher = {London: Routledge},
+ year = 1998
+}
+
+ at Book{Harrington99,
+ author = {Jonathan Harrington and Steve Cassidy},
+ title = {Techniques in Speech Acoustics},
+ publisher = {Kluwer},
+ year = 1999
+}
+
+ at InProceedings{MaedaBird02,
+ author = {Kazuaki Maeda and Steven Bird and Xiaoyi Ma and Haejoong Lee},
+ title = {Creating Annotation Tools with the Annotation Graph Toolkit},
+ booktitle = {Proceedings of the Third International Conference
+ on Language Resources and Evaluation},
+ year = 2002,
+ note = {\url{http://arXiv.org/abs/cs/0204005}}
+}
+
+ at article{PereiraWarren80,
+ author={Fernando C.~N.~Pereira and David H.~D.~Warren},
+ title={Definite Clause Grammars for language analysis -- a
+ survey of the formalism and a comparison with
+ Augmented Transition Grammars},
+ journal={Artificial Intelligence},
+ year={1980},
+ volume={13},
+ pages={231--78}
+}
+
+ at book{PollardSag94,
+ author={Carl Pollard and Ivan A.~Sag},
+ title={Head-Driven Phrase Structure Grammar},
+ publisher={Chicago University Press},
+ year={1994}
+}
+
+ at Misc{epydoc,
+ author = {Edward Loper},
+ title = {Epydoc},
+ year = 2002,
+ note = {\\\url{http://epydoc.sourceforge.net/}}
+}
+
+ at Misc{nltk,
+ title = {Natural Language Processing Toolkit},
+ author = {Edward Loper and Steven Bird},
+ year = 2002,
+ note = {\\\url{http://nltk.sourceforge.net/}}
+}
+
+ at Misc{opennlp,
+ title = {The OpenNLP Project},
+ author = {Jason Baldridge},
+ year = 2002,
+ note = {\\\url{http://opennlp.sourceforge.net/}}
+}
+
+ at Misc{maxent,
+ title = {The {MaxEnt} Project},
+ author = {Jason Baldridge and Thomas Morton and Gann Bierner},
+ year = 2002,
+ note = {\\\url{http://maxent.sourceforge.net/}}
+}
+
+% I couldn't find any "original paper" or anything like that.
+% Guido has some essays at http://www.python.org/doc/essays/
+% We could cite one of those instead?
+% Also, what year? It was created in the winter of 89-90, but it's
+% evolved a lot over time.
+% [SB - I don't think we need a formal citation for this]
+ at Misc{python,
+ author = {Guido van Rossum},
+ title = {The Python Programming Language},
+ note = {\url{http://www.python.org}},
+ year = 1990
+}
+
+% Is this right? It's a proposal. (Note: change "url=" to "note=" if
+% we want to display the url in the bibliography)
+% [SB - yes the only thing on this I've seen is the proposal]
+% [SB - done, we want URLs in the bibliography]
+% CNRI Proposal # 90120-1a
+ at TechReport{rossum99,
+ author = {Guido van Rossum},
+ title = {Computer Programming for Everybody},
+ institution = {Corporation for National Research Initiatives},
+ year = 1999,
+ note = {\url{http://www.python.org/doc/essays/cp4e.html}}
+}
+
+ at Misc{tkinter,
+ author = {Fredrik Lundh},
+ title = {An Introduction to Tkinter},
+ note = {\\\url{http://www.pythonware.com/library/tkinter/introduction/index.htm}},
+ year = 1999
+}
+
diff --git a/papers/acl-04/.cvsignore b/papers/acl-04/.cvsignore
new file mode 100644
index 0000000..ede195c
--- /dev/null
+++ b/papers/acl-04/.cvsignore
@@ -0,0 +1,12 @@
+!
+#*
+*.aux
+*.dvi
+*.eps
+*.log
+*.pdf
+*.ps
+*.toc
+*~
+acl04.bbl
+acl04.blg
diff --git a/papers/acl-04/Makefile b/papers/acl-04/Makefile
new file mode 100644
index 0000000..1a49016
--- /dev/null
+++ b/papers/acl-04/Makefile
@@ -0,0 +1,45 @@
+# Natural Language Toolkit: Technical report Makefile
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+##############################################
+## The name of the report
+REPORT = acl04
+
+##############################################
+## Figure dependancies
+
+##############################################
+## You shouldn't have to change anything below here.
+
+# Find the name of the dvi and ps files.
+DVI := $(REPORT).dvi
+PS := $(REPORT).ps
+PDF := $(REPORT).pdf
+
+# Top-level rules.
+dvi: $(DVI)
+ps: $(PS)
+pdf: $(PDF)
+clean:
+ rm -f *.eps *.log *.aux *.dvi *.ps *.toc *.pdf
+
+# General rules
+%.dvi: %.tex
+ latex $<
+ latex $<
+
+%.ps: %.dvi
+ dvips -t letter -o $@ $<
+
+%.eps: %.dot
+ dot -Tps -o $@ $<
+
+%.eps: %.obj
+ tgif -print -eps $<
+
+%.pdf: %.ps
+ ps2pdf $< $@
diff --git a/papers/acl-04/acl-04.tex b/papers/acl-04/acl-04.tex
new file mode 100644
index 0000000..9bfd16e
--- /dev/null
+++ b/papers/acl-04/acl-04.tex
@@ -0,0 +1,508 @@
+\documentclass[11pt]{article}
+\usepackage{colacl}
+\usepackage{times}
+\usepackage{latexsym}
+\usepackage{url,alltt,epsfig,boxedminipage}
+
+% hyphenation control
+\pretolerance 250
+\tolerance 500
+\hyphenpenalty 200
+\exhyphenpenalty 100
+\doublehyphendemerits 7500
+\finalhyphendemerits 7500
+\brokenpenalty 10000
+\lefthyphenmin 3
+\righthyphenmin 3
+\widowpenalty 10000
+\clubpenalty 10000
+\displaywidowpenalty 10000
+\looseness 1
+
+\def\UrlFont{\tt\small}
+\def\object#1{\texttt{\small #1}}
+
+\title{NLTK: The Natural Language Toolkit}
+
+\author{
+ Steven Bird \\
+ Department of Computer Science \\
+ \indent and Software Engineering \\
+ University of Melbourne \\
+ Victoria 3010, Australia \\
+ {\tt\small stevenbird1 at gmail.com}
+\And
+ Edward Loper\\
+ Department of Computer \\
+ \indent and Information Science \\
+ University of Pennsylvania\\
+ Philadelphia PA 19104-6389, USA\\
+ {\tt\small edloper at gmail.com}
+}
+
+\newenvironment{sv}{\scriptsize\begin{alltt}}{\end{alltt}\normalsize}
+
+\begin{document}
+
+\maketitle
+
+\begin{abstract}\small
+ The Natural Language Toolkit is a suite of program modules, data
+ sets, tutorials and exercises, covering symbolic and statistical
+ natural language processing. NLTK is written in Python and
+ distributed under the GPL open source license. Over the past three
+ years, NLTK has become popular in teaching and research. We
+ describe the toolkit and report on its current state of development.
+\end{abstract}
+
+%========================= Introduction =========================
+\section{Introduction}
+
+The Natural Language Toolkit (NLTK) was developed in conjunction with
+a computational linguistics course at the University of Pennsylvania
+in 2001 \cite{LoperBird02}. It was designed with three pedagogical
+applications in mind: assignments, demonstrations, and projects.
+
+\textbf{Assignments.}
+NLTK supports assignments of varying difficulty
+and scope. In the simplest assignments, students experiment with
+existing components to perform a wide variety of NLP tasks. As students
+become more familiar with the toolkit, they can be asked to modify
+existing components, or to create complete systems out of existing
+components.
+
+\textbf{Demonstrations.}
+NLTK's interactive graphical demonstrations have proven to be very
+useful for students learning NLP concepts.
+The demonstrations give a step-by-step execution of important
+algorithms, displaying the current state of key data structures.
+A screenshot of the chart parsing demonstration is shown in Figure~\ref{fig:chart}.
+
+\textbf{Projects.} NLTK provides students with a flexible framework
+for advanced projects. Typical projects might involve implementing a
+new algorithm, developing a new component, or implementing a new
+task.
+
+We chose Python because it has a shallow learning curve, its syntax
+and semantics are transparent, and it has good string-handling
+functionality. As an interpreted language, Python facilitates
+interactive exploration. As an object-oriented language, Python
+permits data and methods to be encapsulated and re-used easily. Python comes with an extensive
+standard library, including tools for graphical programming and
+numerical processing. The recently added generator
+syntax makes it easy to create interactive implementations of
+algorithms \cite{Loper04,Rossum03intro,Rossum03ref}.
+
+\begin{figure}[bth]
+\epsfig{file=chart.eps, width=\linewidth}
+\caption{Interactive Chart Parsing Demonstration}
+\label{fig:chart}
+\end{figure}
+
+\section{Design}
+
+NLTK is implemented as a large collection of minimally interdependent
+modules, organized into a shallow hierarchy. A set of core
+modules defines basic data types that are used throughout the toolkit.
+The remaining modules are \emph{task modules}, each devoted to an
+individual natural language processing task. For example, the
+\object{nltk.parser} module encompasses to the task of
+\emph{parsing}, or deriving the syntactic structure of a sentence; and
+the \object{nltk.tokenizer} module is devoted to the task of
+\emph{tokenizing}, or dividing a text into its constituent parts.
+
+\subsection{Tokens and other core data types}
+
+To maximize interoperability between modules, we use a
+single class to encode information about natural language texts -- the
+\object{Token} class. Each \object{Token} instance represents a
+unit of text such as a word, sentence, or document, and is
+defined by a (partial) mapping from property names to values. For
+example, the \object{TEXT} property is used to encode a token's text
+content:\footnote{Some code samples are specific to NLTK
+ version 1.4.}
+
+\begin{alltt}\small
+\textbf{>>> from nltk.token import *}
+\textbf{>>> Token(TEXT="Hello World!")}
+<Hello World!>
+\end{alltt}
+%
+The \object{TAG} property is used to encode a token's part-of-speech
+tag:
+
+\begin{alltt}\small
+\textbf{>>> Token(TEXT="python", TAG="NN")}
+<python/NN>
+\end{alltt}
+%
+The \object{SUBTOKENS} property is used to store a tokenized text:
+
+\begin{alltt}\small
+\textbf{>>> from nltk.tokenizer import *}
+\textbf{>>> tok = Token(TEXT="Hello World!")}
+\textbf{>>> WhitespaceTokenizer().tokenize(tok)}
+\textbf{>>> print tok['SUBTOKENS'])}
+[<Hello>, <World!>]
+\end{alltt}
+%
+In a similar fashion, other language processing tasks such as
+word-sense disambiguation, chunking and parsing all add properties to
+the \object{Token} data structure.
+
+In general, language processing tasks are formulated as
+annotations and transformations involving \object{Tokens}. In
+particular, each processing task takes a token and extends it to
+include new information. These modifications are typically
+\emph{monotonic}; new information is added but
+existing information is not deleted or modified. Thus, tokens serve
+as a \emph{blackboard}, where information about a piece of text is
+collated. This architecture contrasts with the more typical
+\emph{pipeline} architecture where each processing task's output
+discards its input information. We chose the blackboard approach
+over the pipeline approach because it allows more flexibility when
+combining tasks into a single system.
+
+In addition to the \object{Token} class and its derivatives, NLTK
+defines a variety of other data types. For instance, the
+\object{probability} module defines classes for
+probability distributions and statistical smoothing techniques; and
+the \object{cfg} module defines classes for encoding context free
+grammars and probabilistic context free grammars.
+
+\subsection{The corpus module}
+
+\begin{table*}
+\small\noindent
+\begin{boxedminipage}{\linewidth}
+\begin{tabular}{llll}
+\setlength{\tabcolsep}{0.5\tabcolsep}
+\textbf{Corpus} &
+\textbf{Contents and Wordcount} &
+\textbf{Example Application} \\
+
+20 Newsgroups (selection) &
+3 newsgroups, 4000 posts, 780kw &
+text classification \\
+
+Brown Corpus &
+15 genres, 1.15Mw, tagged &
+training \& testing taggers, text classification \\
+
+CoNLL 2000 Chunking Data &
+270kw, tagged and chunked &
+training \& testing chunk parsers \\
+
+Project Gutenberg (selection) &
+14 texts, 1.7Mw &
+text classification, language modelling \\
+
+NIST 1999 IEER (selection) &
+63kw, named-entity markup &
+training \& testing named-entity recognizers \\
+
+Levin Verb Index &
+3k verbs with Levin classes &
+parser development \\
+
+Names Corpus &
+8k male \& female names &
+text classification \\
+
+PP Attachment Corpus &
+28k prepositional phrases, tagged &
+parser development \\
+
+Roget's Thesaurus &
+200kw, formatted text &
+word-sense disambiguation \\
+
+SEMCOR &
+880kw, POS \& sense tagged &
+word-sense disambiguation \\
+
+SENSEVAL 2 Corpus &
+600kw, POS \& sense tagged &
+word-sense disambiguation \\
+
+Stopwords Corpus &
+2,400 stopwords for 11 lgs &
+text retrieval \\
+
+Penn Treebank (sample) &
+40kw, tagged \& parsed &
+parser development \\
+
+Wordnet 1.7 &
+180kw in a semantic network &
+WSD, NL understanding \\
+
+Wordlist Corpus &
+960kw and 20k affixes for 8 lgs &
+spell checking
+ \\
+\end{tabular}
+\caption{Corpora and Corpus Samples Distributed with NLTK}\label{tab:data}
+\end{boxedminipage}
+\end{table*}
+
+Many language processing tasks must be developed and tested using
+annotated data sets or corpora. Several such corpora are distributed
+with NLTK, as listed in Table~\ref{tab:data}. The
+\object{corpus} module defines classes for reading and processing
+many of these corpora. The following code fragment illustrates
+how the Brown Corpus is accessed.
+
+\begin{alltt}\small
+\textbf{>>> from nltk.corpus import brown}
+\textbf{>>> brown.groups()}
+['skill and hobbies', 'popular lore',
+'humor', 'fiction: mystery', ...]
+\textbf{>>> brown.items('humor')}
+('cr01', 'cr02', 'cr03', 'cr04', 'cr05',
+'cr06', 'cr07', 'cr08', 'cr09')
+\textbf{>>> brown.tokenize('cr01')}
+<[<It/pps>, <was/bedz>, <among/in>,
+<these/dts>, <that/cs>, <Hinkle/np>,
+<identified/vbd>, <a/at>, ...]>
+\end{alltt}
+%
+A selection of 5\% of the Penn Treebank corpus is included with
+NLTK, and it is accessed as follows:
+
+\begin{alltt}\small
+\textbf{>>> from nltk.corpus import treebank}
+\textbf{>>> treebank.groups()}
+('raw', 'tagged', 'parsed', 'merged')
+\textbf{>>> treebank.items('parsed')}
+['wsj_0001.prd', 'wsj_0002.prd', ...]
+\textbf{>>> item = 'parsed/wsj_0001.prd'}
+\textbf{>>> sentences = treebank.tokenize(item)}
+\textbf{>>> for sent in sentences['SUBTOKENS']:}
+\textbf{... print sent.pp()} \emph{# pretty-print}
+(S:
+ (NP-SBJ:
+ (NP: <Pierre> <Vinken>)
+ (ADJP:
+ (NP: <61> <years>)
+ <old>
+ )
+ ...
+\end{alltt}
+%
+
+\subsection{Processing modules}
+
+Each language processing algorithm is implemented as a class. For
+example, the \object{ChartParser} and
+\object{Recursive\-Descent\-Parser} classes each define a single
+algorithm for parsing a text. We implement language processing
+algorithms using classes instead of functions for three reasons.
+First, all algorithm-specific options can be passed to the
+constructor, allowing a consistent interface for applying the
+algorithms. Second, a number of algorithms need to have their state
+initialized before they can be used. For example, the
+\object{NthOrderTagger} class must be initialized by training on a
+tagged corpus before it can be used. Third, subclassing can be used
+to create specialized versions of a given algorithm.
+
+Each processing module defines an \emph{interface} for its task.
+Interface classes are distinguished by naming them with a trailing
+capital ``\object{I},'' such as \object{ParserI}.
+Each interface defines a single \emph{action method} which
+performs the task defined by the interface. For example, the
+\object{ParserI} interface defines the \object{parse} method and the
+\object{Tokenizer} interface defines the \object{tokenize} method.
+When appropriate, an interface defines \emph{extended action
+ methods}, which provide variations on the basic action method. For
+example, the \object{ParserI} interface defines the \object{parse\_n}
+method which finds at most $n$ parses for a given sentence; and
+the \object{TokenizerI} interface defines the \object{xtokenize}
+method, which outputs an iterator over subtokens instead of a list of
+subtokens.
+
+NLTK includes the following modules:
+\object{cfg},
+\object{corpus},
+\object{draw}
+(\object{cfg},
+\object{chart},
+\object{corpus},
+\object{featurestruct},
+\object{fsa},
+\object{graph},
+\object{plot},
+\object{rdparser},
+\object{srparser},
+\object{tree}),
+\object{eval},
+\object{featurestruct},
+\object{parser}
+(\object{chart},
+\object{chunk},
+\object{probabilistic}),
+\object{probability},
+\object{sense},
+\object{set},
+\object{stemmer}
+(\object{porter}),
+\object{tagger},
+\object{test},
+\object{token},
+\object{tokenizer},
+\object{tree}, and
+\object{util}.
+Please see the online documentation for details.
+
+\subsection{Documentation}
+
+Three different types of documentation are available. Tutorials
+explain how to use the toolkit, with detailed worked examples. The
+API documentation describes every module, interface, class, method,
+function, and variable in the toolkit. Technical reports explain and
+justify the toolkit's design and implementation. All are available
+from \url{http://nltk.sf.net/docs.html}.
+
+\section{Installing NLTK}
+
+NLTK is available from \url{nltk.sf.net}, and is packaged for
+easy installation under Unix, Mac OS X and Windows. The full
+distribution consists of four packages: the Python source code
+(\object{nltk}); the corpora (\object{nltk-data}); the documentation
+(\object{nltk-docs}); and third-party contributions
+(\object{nltk-contrib}). Before installing NLTK, it is necessary to
+install Python version 2.3 or later, available from
+\url{www.python.org}. Full installation instructions and a quick
+start guide are available from the NLTK homepage.
+
+As soon as NLTK is installed, users can run the demonstrations. On
+Windows, the demonstrations can be run by double-clicking on their
+Python source files. Alternatively, from the Python interpreter, this
+can be done as follows:
+
+\begin{alltt} \small
+\textbf{>>> import nltk.draw.rdparser}
+\textbf{>>> nltk.draw.rdparser.demo()}
+\textbf{>>> nltk.draw.srparser.demo()}
+\textbf{>>> nltk.draw.chart.demo()}
+\end{alltt}
+
+\section{Using and contributing to NLTK}
+
+NLTK has been used at the University of Pennsylvania since 2001, and has
+subsequently been adopted by several NLP courses at other
+universities, including those listed in Table~\ref{tab:courses}.
+
+Third party contributions to NLTK include: Brill tagger (Chris Maloof),
+hidden Markov model tagger (Trevor Cohn, Phil Blunsom), GPSG-style feature-based grammar
+and parser (Rob Speer, Bob Berwick), finite-state morphological analyzer
+(Carl de Marcken, Beracah Yankama, Bob Berwick), decision list and decision tree
+classifiers (Trevor Cohn), and Discourse Representation Theory
+implementation (Edward Ivanovic).
+
+NLTK is an open source project, and we welcome any contributions.
+There are several ways to contribute: users can report bugs, suggest
+features, or contribute patches on Sourceforge; users can participate
+in discussions on the \textit{NLTK-Devel} mailing
+list\footnote{\url{http://lists.sourceforge.net/lists/listinfo/nltk-devel}}
+or in the NLTK public
+forums; and users can submit their own NLTK-based projects for
+inclusion in the nltk\_contrib directory. New code modules that are
+relevant, substantial, original and well-documented will be considered
+for inclusion in NLTK proper.
+All source code is distributed under the GNU General Public License,
+and all documentation is distributed under a Creative Commons
+non-commercial license. Thus, potential contributors can be confident that their
+work will remain freely available to all. Further information about
+contributing to NLTK is available at \url{http://nltk.sf.net/contrib.html}.
+
+\begin{table}[bt]
+\small\noindent
+\begin{boxedminipage}{\linewidth}
+\begin{tabular}{l}
+
+Graz University of Technology, Austria \\
+\hspace{2ex}
+\textit{Information Search and Retrieval} \\[.5ex]
+
+Macquarie University, Australia \\
+\hspace{2ex}
+\textit{Intelligent Text Processing} \\[.5ex]
+
+Massachusetts Institute of Technology, USA \\
+\hspace{2ex}
+\textit{Natural Language Processing} \\[.5ex]
+
+National Autonomous University of Mexico, Mexico \\
+\hspace{2ex}
+\textit{Introduction to Natural Language Processing}\\
+\hspace{2ex}
+\textit{in Python} \\[.5ex]
+
+Ohio State University, USA \\
+\hspace{2ex}
+\textit{Statistical Natural Language Processing}\\[.5ex]
+
+University of Amsterdam, Netherlands \\
+\hspace{2ex}
+\textit{Language Processing and Information Access} \\[.5ex]
+
+University of Colorado, USA \\
+\hspace{2ex}
+\textit{Natural Language Processing} \\[.5ex]
+
+University of Edinburgh, UK \\
+\hspace{2ex}
+\textit{Introduction to Computational Linguistics} \\[.5ex]
+
+University of Magdeburg, Germany \\
+\hspace{2ex}
+\textit{Natural Language Systems} \\[.5ex]
+
+University of Malta, Malta \\
+\hspace{2ex}
+\textit{Natural Language Algorithms}\\[.5ex]
+
+University of Melbourne, Australia \\
+\hspace{2ex}
+\textit{Human Language Technology} \\[.5ex]
+
+University of Pennsylvania, USA \\
+\hspace{2ex}
+\textit{Introduction to Computational Linguistics} \\[.5ex]
+
+University of Pittsburgh, USA \\
+\hspace{2ex}
+\textit{Artificial Intelligence Application Development} \\[.5ex]
+
+Simon Fraser University, Canada \\
+\hspace{2ex}
+\textit{Computational Linguistics} \\[.5ex]
+
+\end{tabular}
+\caption{University Courses using NLTK}\label{tab:courses}
+\end{boxedminipage}
+\end{table}
+
+\section{Conclusion}
+
+NLTK is a broad-coverage natural language toolkit that provides a simple, extensible,
+uniform framework for assignments, demonstrations and projects. It is
+thoroughly documented, easy to learn, and simple to use. NLTK is now widely
+used in research and teaching. Readers who
+would like to receive occasional announcements about NLTK are
+encouraged to sign up for the low-volume, moderated mailing list
+\textit{NLTK-Announce}.\footnote{\url{http://lists.sourceforge.net/lists/listinfo/nltk-announce}}
+
+\section{Acknowledgements}
+
+We are indebted to our students and colleagues for feedback on the
+toolkit, and to many contributors listed on the NLTK website.
+
+\pagebreak
+
+\bibliographystyle{acl}
+\bibliography{nltk}
+
+\end{document}
+
diff --git a/papers/acl-04/acl.bst b/papers/acl-04/acl.bst
new file mode 100644
index 0000000..8c7d392
--- /dev/null
+++ b/papers/acl-04/acl.bst
@@ -0,0 +1,1323 @@
+% BibTeX `acl' style file for BibTeX version 0.99c, LaTeX version 2.09
+% This version was made by modifying `aaai-named' format based on the master
+% file by Oren Patashnik (PATASHNIK at SCORE.STANFORD.EDU)
+
+% Copyright (C) 1985, all rights reserved.
+% Modifications Copyright 1988, Peter F. Patel-Schneider
+% Further modifictions by Stuart Shieber, 1991, and Fernando Pereira, 1992.
+% Copying of this file is authorized only if either
+% (1) you make absolutely no changes to your copy, including name, or
+% (2) if you do make changes, you name it something other than
+% btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst.
+% This restriction helps ensure that all standard styles are identical.
+
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at spar.slb.com
+
+% Citation format: [author-last-name, year]
+% [author-last-name and author-last-name, year]
+% [author-last-name {\em et al.}, year]
+%
+% Reference list ordering: alphabetical by author or whatever passes
+% for author in the absence of one.
+%
+% This BibTeX style has support for short (year only) citations. This
+% is done by having the citations actually look like
+% \citename{name-info, }year
+% The LaTeX style has to have the following
+% \let\@internalcite\cite
+% \def\cite{\def\citename##1{##1}\@internalcite}
+% \def\shortcite{\def\citename##1{}\@internalcite}
+% \def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+% which makes \shortcite the macro for short citations.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Changes made by SMS for thesis style
+% no emphasis on "et al."
+% "Ph.D." includes periods (not "PhD")
+% moved year to immediately after author's name
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ENTRY
+ { address
+ author
+ booktitle
+ chapter
+ edition
+ editor
+ howpublished
+ institution
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ publisher
+ school
+ series
+ title
+ type
+ volume
+ year
+ }
+ {}
+ { label extra.label sort.label }
+
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+
+STRINGS { s t }
+
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { add.period$ " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+
+ "\bibitem[" write$
+ label write$
+ "]{" write$
+
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+
+FUNCTION {new.block.checka}
+{ empty$
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.sentence.checka}
+{ empty$
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {new.sentence.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+
+FUNCTION {emphasize}
+{ duplicate$ empty$
+ { pop$ "" }
+ { "{\em " swap$ * "}" * }
+ if$
+}
+
+INTEGERS { nameptr namesleft numnames }
+
+FUNCTION {format.names}
+{ 's :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+
+ { s nameptr "{ff~}{vv~}{ll}{, jj}" format.name$ 't :=
+
+ nameptr #1 >
+ { namesleft #1 >
+ { ", " * t * }
+ { numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ t "others" =
+ { " et~al." * }
+ { " and " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {format.authors}
+{ author empty$
+ { "" }
+ { author format.names }
+ if$
+}
+
+FUNCTION {format.editors}
+{ editor empty$
+ { "" }
+ { editor format.names
+ editor num.names$ #1 >
+ { ", editors" * }
+ { ", editor" * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title empty$
+ { "" }
+
+ { title "t" change.case$ }
+
+ if$
+}
+
+FUNCTION {n.dashify}
+{ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {format.date}
+{ year empty$
+ { month empty$
+ { "" }
+ { "there's a month but no year in " cite$ * warning$
+ month
+ }
+ if$
+ }
+ { month empty$
+ { "" }
+ { month }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.btitle}
+{ title emphasize
+}
+
+FUNCTION {tie.or.space.connect}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$ * *
+}
+
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { "volume" volume tie.or.space.connect
+ series empty$
+ 'skip$
+ { " of " * series emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { output.state mid.sentence =
+ { "number" }
+ { "Number" }
+ if$
+ number tie.or.space.connect
+ series empty$
+ { "there's a number but no series in " cite$ * warning$ }
+ { " in " * series * }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition empty$
+ { "" }
+ { output.state mid.sentence =
+ { edition "l" change.case$ " edition" * }
+ { edition "t" change.case$ " edition" * }
+ if$
+ }
+ if$
+}
+
+INTEGERS { multiresult }
+
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+
+FUNCTION {format.pages}
+{ pages empty$
+ { "" }
+ { pages multi.page.check
+ { "pages" pages n.dashify tie.or.space.connect }
+ { "page" pages tie.or.space.connect }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.year.label}
+{ year extra.label *
+}
+
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ number empty$
+ 'skip$
+ { "(" number * ")" * *
+ volume empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ }
+ if$
+ pages empty$
+ 'skip$
+ { duplicate$ empty$
+ { pop$ format.pages }
+ { ":" * pages n.dashify * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { "chapter" }
+ { type "l" change.case$ }
+ if$
+ chapter tie.or.space.connect
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.in.ed.booktitle}
+{ booktitle empty$
+ { "" }
+ { editor empty$
+ { "In " booktitle emphasize * }
+ { "In " format.editors * ", " * booktitle emphasize * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {empty.misc.check}
+{ author empty$ title empty$ howpublished empty$
+ month empty$ year empty$ note empty$
+ and and and and and
+
+ key empty$ not and
+
+ { "all relevant fields are empty in " cite$ * warning$ }
+ 'skip$
+ if$
+}
+
+FUNCTION {format.thesis.type}
+{ type empty$
+ 'skip$
+ { pop$
+ type "t" change.case$
+ }
+ if$
+}
+
+FUNCTION {format.tr.number}
+{ type empty$
+ { "Technical Report" }
+ 'type
+ if$
+ number empty$
+ { "t" change.case$ }
+ { number tie.or.space.connect }
+ if$
+}
+
+FUNCTION {format.article.crossref}
+{ key empty$
+ { journal empty$
+ { "need key or journal for " cite$ * " to crossref " * crossref *
+ warning$
+ ""
+ }
+ { "In {\em " journal * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.crossref.editor}
+{ editor #1 "{vv~}{ll}" format.name$
+ editor num.names$ duplicate$
+ #2 >
+ { pop$ " et~al." * }
+ { #2 <
+ 'skip$
+ { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" =
+ { " et~al." * }
+ { " and " * editor #2 "{vv~}{ll}" format.name$ * }
+ if$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.book.crossref}
+{ volume empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ "In "
+ }
+ { "Volume" volume tie.or.space.connect
+ " of " *
+ }
+ if$
+ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { series empty$
+ { "need editor, key, or series for " cite$ * " to crossref " *
+ crossref * warning$
+ "" *
+ }
+ { "{\em " * series * "\/}" * }
+ if$
+ }
+ { key * }
+ if$
+ }
+ { format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.incoll.inproc.crossref}
+{ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { booktitle empty$
+ { "need editor, key, or booktitle for " cite$ * " to crossref " *
+ crossref * warning$
+ ""
+ }
+ { "In {\em " booktitle * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ }
+ { "In " format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { journal emphasize "journal" output.check
+ format.vol.num.pages output
+ format.date output
+ }
+ { format.article.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {booklet}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ howpublished address new.block.checkb
+ howpublished output
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.chapter.pages output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ format.edition output
+ format.date output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.chapter.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.pages output
+ address empty$
+ { organization publisher new.sentence.checkb
+ organization output
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ organization output
+ publisher output
+ }
+ if$
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {conference} { inproceedings }
+
+FUNCTION {manual}
+{ output.bibitem
+ author empty$
+ { organization empty$
+ 'skip$
+ { organization output.nonnull
+ address output
+ }
+ if$
+ }
+ { format.authors output.nonnull }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ author empty$
+ { organization empty$
+ { address new.block.checka
+ address output
+ }
+ 'skip$
+ if$
+ }
+ { organization address new.block.checkb
+ organization output
+ address output
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ "Master's thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label output
+ new.block
+ title howpublished new.block.checkb
+ format.title output
+ howpublished new.block.checka
+ howpublished output
+ format.date output
+ new.block
+ note output
+ fin.entry
+ empty.misc.check
+}
+
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ new.block
+ "{Ph.D.} thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ editor empty$
+ { organization output }
+ { format.editors output.nonnull }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ format.bvolume output
+ format.number.series output
+ address empty$
+ { editor empty$
+ { publisher new.sentence.checka }
+ { organization publisher new.sentence.checkb
+ organization output
+ }
+ if$
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ editor empty$
+ 'skip$
+ { organization output }
+ if$
+ publisher output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ format.tr.number output.nonnull
+ institution "institution" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ note "note" output.check
+ format.date output
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+
+MACRO {jan} {"January"}
+
+MACRO {feb} {"February"}
+
+MACRO {mar} {"March"}
+
+MACRO {apr} {"April"}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"August"}
+
+MACRO {sep} {"September"}
+
+MACRO {oct} {"October"}
+
+MACRO {nov} {"November"}
+
+MACRO {dec} {"December"}
+
+MACRO {acmcs} {"ACM Computing Surveys"}
+
+MACRO {acta} {"Acta Informatica"}
+
+MACRO {cacm} {"Communications of the ACM"}
+
+MACRO {ibmjrd} {"IBM Journal of Research and Development"}
+
+MACRO {ibmsj} {"IBM Systems Journal"}
+
+MACRO {ieeese} {"IEEE Transactions on Software Engineering"}
+
+MACRO {ieeetc} {"IEEE Transactions on Computers"}
+
+MACRO {ieeetcad}
+ {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"}
+
+MACRO {ipl} {"Information Processing Letters"}
+
+MACRO {jacm} {"Journal of the ACM"}
+
+MACRO {jcss} {"Journal of Computer and System Sciences"}
+
+MACRO {scp} {"Science of Computer Programming"}
+
+MACRO {sicomp} {"SIAM Journal on Computing"}
+
+MACRO {tocs} {"ACM Transactions on Computer Systems"}
+
+MACRO {tods} {"ACM Transactions on Database Systems"}
+
+MACRO {tog} {"ACM Transactions on Graphics"}
+
+MACRO {toms} {"ACM Transactions on Mathematical Software"}
+
+MACRO {toois} {"ACM Transactions on Office Information Systems"}
+
+MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"}
+
+MACRO {tcs} {"Theoretical Computer Science"}
+
+READ
+
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+
+INTEGERS { len }
+
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+
+INTEGERS { et.al.char.used }
+
+FUNCTION {initialize.et.al.char.used}
+{ #0 'et.al.char.used :=
+}
+
+EXECUTE {initialize.et.al.char.used}
+
+FUNCTION {format.lab.names}
+{ 's :=
+ s num.names$ 'numnames :=
+
+ numnames #1 =
+ { s #1 "{vv }{ll}" format.name$ }
+ { numnames #2 =
+ { s #1 "{vv }{ll }and " format.name$ s #2 "{vv }{ll}" format.name$ *
+ }
+ { s #1 "{vv }{ll }\bgroup et al.\egroup " format.name$ }
+ if$
+ }
+ if$
+
+}
+
+FUNCTION {author.key.label}
+{ author empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.editor.key.label}
+{ author empty$
+ { editor empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.key.organization.label}
+{ author empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {editor.key.organization.label}
+{ editor empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+}
+
+FUNCTION {calc.label}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.label
+ { type$ "proceedings" =
+ 'editor.key.organization.label
+ { type$ "manual" =
+ 'author.key.organization.label
+ 'author.key.label
+ if$
+ }
+ if$
+ }
+ if$
+ duplicate$
+
+ "\protect\citename{" swap$ * "}" *
+ year field.or.null purify$ *
+ 'label :=
+ year field.or.null purify$ *
+
+ sortify 'sort.label :=
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { nameptr #1 >
+ { " " * }
+ 'skip$
+ if$
+
+ s nameptr "{vv{ } }{ll{ }}{ ff{ }}{ jj{ }}" format.name$ 't :=
+
+ nameptr numnames = t "others" = and
+ { "et al" * }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.organization.sort}
+{ author empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need author, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {editor.organization.sort}
+{ editor empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need editor, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+
+FUNCTION {presort}
+
+{ calc.label
+ sort.label
+ " "
+ *
+ type$ "book" =
+
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.organization.sort
+ { type$ "manual" =
+ 'author.organization.sort
+ 'author.sort
+ if$
+ }
+ if$
+ }
+ if$
+
+ *
+
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+
+SORT
+
+STRINGS { longest.label last.sort.label next.extra }
+
+INTEGERS { longest.label.width last.extra.num }
+
+FUNCTION {initialize.longest.label}
+{ "" 'longest.label :=
+ #0 int.to.chr$ 'last.sort.label :=
+ "" 'next.extra :=
+ #0 'longest.label.width :=
+ #0 'last.extra.num :=
+}
+
+FUNCTION {forward.pass}
+{ last.sort.label sort.label =
+ { last.extra.num #1 + 'last.extra.num :=
+ last.extra.num int.to.chr$ 'extra.label :=
+ }
+ { "a" chr.to.int$ 'last.extra.num :=
+ "" 'extra.label :=
+ sort.label 'last.sort.label :=
+ }
+ if$
+}
+
+FUNCTION {reverse.pass}
+{ next.extra "b" =
+ { "a" 'extra.label := }
+ 'skip$
+ if$
+ label extra.label * 'label :=
+ label width$ longest.label.width >
+ { label 'longest.label :=
+ label width$ 'longest.label.width :=
+ }
+ 'skip$
+ if$
+ extra.label 'next.extra :=
+}
+
+EXECUTE {initialize.longest.label}
+
+ITERATE {forward.pass}
+
+REVERSE {reverse.pass}
+
+FUNCTION {begin.bib}
+
+{ et.al.char.used
+ { "\newcommand{\etalchar}[1]{$^{#1}$}" write$ newline$ }
+ 'skip$
+ if$
+ preamble$ empty$
+
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+
+ "\begin{thebibliography}{" "}" * write$ newline$
+
+}
+
+EXECUTE {begin.bib}
+
+EXECUTE {init.state.consts}
+
+ITERATE {call.type$}
+
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+
+EXECUTE {end.bib}
+
+
+
diff --git a/papers/acl-04/acl04.sty b/papers/acl-04/acl04.sty
new file mode 100644
index 0000000..558bf82
--- /dev/null
+++ b/papers/acl-04/acl04.sty
@@ -0,0 +1,361 @@
+% File acl04.sty
+% January 07, 2004
+% Contact: rambow at cs.columbia.edu
+
+% This is the LaTeX style file for ACL 2004. It is identical to the
+% style files for ACL 2003, ACL 2002, ACL 2001, ACL 2000, EACL 95 and EACL
+% 99.
+%
+% -- Roberto Zamparelli, March 26, 2001
+% -- Dekang Lin, October 1, 2001
+
+% This is the LaTeX style file for ACL 2000. It is nearly identical to the
+% style files for EACL 95 and EACL 99. Minor changes include editing the
+% instructions to reflect use of \documentclass rather than \documentstyle
+% and removing the white space before the title on the first page
+% -- John Chen, June 29, 2000
+
+% To convert from submissions prepared using the style file aclsub.sty
+% prepared for the ACL 2000 conference, proceed as follows:
+% 1) Remove submission-specific information: \whichsession, \id,
+% \wordcount, \otherconferences, \area, \keywords
+% 2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% 3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% 4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+% 5) Change the style reference from aclsub to acl2000, and be sure
+% this style file is in your TeX search path
+
+
+% This is the LaTeX style file for EACL-95. It is identical to the
+% style file for ANLP '94 except that the margins are adjusted for A4
+% paper. -- abney 13 Dec 94
+
+% The ANLP '94 style file is a slightly modified
+% version of the style used for AAAI and IJCAI, using some changes
+% prepared by Fernando Pereira and others and some minor changes
+% by Paul Jacobs.
+
+% Papers prepared using the aclsub.sty file and acl.bst bibtex style
+% should be easily converted to final format using this style.
+% (1) Submission information (\wordcount, \subject, and \makeidpage)
+% should be removed.
+% (2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% (between \begin{abstract} and \end{abstract}).
+% (3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% (4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+
+% Place this in a file called aclap.sty in the TeX search path.
+% (Placing it in the same directory as the paper should also work.)
+
+% Prepared by Peter F. Patel-Schneider, liberally using the ideas of
+% other style hackers, including Barbara Beeton.
+% This style is NOT guaranteed to work. It is provided in the hope
+% that it will make the preparation of papers easier.
+%
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at research.att.com
+
+% Papers are to be prepared using the ``acl'' bibliography style,
+% as follows:
+% \documentclass[11pt]{article}
+% \usepackage{acl2000}
+% \title{Title}
+% \author{Author 1 \and Author 2 \\ Address line \\ Address line \And
+% Author 3 \\ Address line \\ Address line}
+% \begin{document}
+% ...
+% \bibliography{bibliography-file}
+% \bibliographystyle{acl}
+% \end{document}
+
+% Author information can be set in various styles:
+% For several authors from the same institution:
+% \author{Author 1 \and ... \and Author n \\
+% Address line \\ ... \\ Address line}
+% if the names do not fit well on one line use
+% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
+% For authors from different institutions:
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \And ... \And
+% Author n \\ Address line \\ ... \\ Address line}
+% To start a seperate ``row'' of authors use \AND, as in
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \AND
+% Author 2 \\ Address line \\ ... \\ Address line \And
+% Author 3 \\ Address line \\ ... \\ Address line}
+
+% If the title and author information does not fit in the area allocated,
+% place \setlength\titlebox{<new height>} right after
+% \usepackage{acl2000}
+% where <new height> can be something larger than 2.25in
+
+% \typeout{Conference Style for ACL 2000 -- released June 20, 2000}
+\typeout{Conference Style for ACL 2002 -- released April 8, 2002}
+
+% NOTE: Some laser printers have a serious problem printing TeX output.
+% These printing devices, commonly known as ``write-white'' laser
+% printers, tend to make characters too light. To get around this
+% problem, a darker set of fonts must be created for these devices.
+%
+
+% % Physical page layout - slightly modified from IJCAI by pj
+% \setlength\topmargin{0.0in} \setlength\oddsidemargin{-0.0in}
+% \setlength\textheight{9.0in} \setlength\textwidth{6.5in}
+% \setlength\columnsep{0.2in}
+% \newlength\titlebox
+% \setlength\titlebox{2.25in}
+% \setlength\headheight{0pt} \setlength\headsep{0pt}
+% %\setlength\footheight{0pt}
+% \setlength\footskip{0pt}
+% \thispagestyle{empty} \pagestyle{empty}
+% \flushbottom \twocolumn \sloppy
+
+% %% A4 version of page layout
+% \setlength\topmargin{-0.45cm} % changed by Rz -1.4
+% \setlength\oddsidemargin{.8mm} % was -0cm, changed by Rz
+% \setlength\textheight{23.5cm}
+% \setlength\textwidth{15.8cm}
+% \setlength\columnsep{0.6cm}
+% \newlength\titlebox
+% \setlength\titlebox{2.00in}
+% \setlength\headheight{5pt}
+% \setlength\headsep{0pt}
+% %\setlength\footheight{0pt}
+% \setlength\footskip{0pt}
+% \thispagestyle{empty}
+% \pagestyle{empty}
+
+
+% EACL 2003 A4 version of page layout
+\setlength\topmargin{3mm}
+\setlength\oddsidemargin{11mm}
+\setlength\evensidemargin{-7mm}
+\setlength\textheight{230mm}
+\setlength\textwidth{160mm}
+\setlength\columnsep{6mm}
+\newlength\titlebox
+\setlength\titlebox{50mm}
+\setlength\headheight{5pt}
+\setlength\headsep{0pt}
+%\setlength\footheight{0pt}
+\setlength\footskip{0pt}
+\thispagestyle{empty}
+\pagestyle{empty}
+
+
+\flushbottom \twocolumn \sloppy
+
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip at sandia-2
+\def\addcontentsline#1#2#3{}
+
+% Title stuff, taken from deproc.
+\def\maketitle{\par
+ \begingroup
+ \def\thefootnote{\fnsymbol{footnote}}
+ \def\@makefnmark{\hbox to 0pt{$^{\@thefnmark}$\hss}}
+ \twocolumn[\@maketitle] \@thanks
+ \endgroup
+ \setcounter{footnote}{0}
+ \let\maketitle\relax \let\@maketitle\relax
+ \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax}
+\def\@maketitle{\vbox to \titlebox{\hsize\textwidth
+ \linewidth\hsize \vskip 0.125in minus 0.125in \centering
+ {\Large\bf \@title \par} \vskip 0.2in plus 1fil minus 0.1in
+ {\def\and{\unskip\enspace{\rm and}\enspace}%
+ \def\And{\end{tabular}\hss \egroup \hskip 1in plus 2fil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}%
+ \def\AND{\end{tabular}\hss\egroup \hfil\hfil\egroup
+ \vskip 0.25in plus 1fil minus 0.125in
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf\@author
+ \end{tabular}\hss\egroup
+ \hfil\hfil\egroup}
+ \vskip 0.3in plus 2fil minus 0.1in
+}}
+\renewenvironment{abstract}{\centerline{\large\bf
+ Abstract}\vspace{0.5ex}\begin{quote}}{\par\end{quote}\vskip 1ex}
+
+
+% bibliography
+
+\def\thebibliography#1{\section*{References}
+ \global\def\@listi{\leftmargin\leftmargini
+ \labelwidth\leftmargini \advance\labelwidth-\labelsep
+ \topsep 1pt plus 2pt minus 1pt
+ \parsep 0.25ex plus 1pt \itemsep 0.25ex plus 1pt}
+ \list {[\arabic{enumi}]}{\settowidth\labelwidth{[#1]}\leftmargin\labelwidth
+ \advance\leftmargin\labelsep\usecounter{enumi}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy
+ \sfcode`\.=1000\relax}
+
+\def\@up#1{\raise.2ex\hbox{#1}}
+
+% most of cite format is from aclsub.sty by SMS
+
+% don't box citations, separate with ; and a space
+% also, make the penalty between citations negative: a good place to break
+% changed comma back to semicolon pj 2/1/90
+% \def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+% \def\@citea{}\@cite{\@for\@citeb:=#2\do
+% {\@citea\def\@citea{;\penalty\@citeseppen\ }\@ifundefined
+% {b@\@citeb}{{\bf ?}\@warning
+% {Citation `\@citeb' on page \thepage \space undefined}}%
+% {\csname b@\@citeb\endcsname}}}{#1}}
+
+% don't box citations, separate with ; and a space
+% Replaced for multiple citations (pj)
+% don't box citations and also add space, semicolon between multiple citations
+\def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@cite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{; }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+ {\csname b@\@citeb\endcsname}}}{#1}}
+
+% Allow short (name-less) citations, when used in
+% conjunction with a bibliography style that creates labels like
+% \citename{<names>, }<year>
+%
+\let\@internalcite\cite
+\def\cite{\def\citename##1{##1, }\@internalcite}
+\def\shortcite{\def\citename##1{}\@internalcite}
+\def\newcite{\leavevmode\def\citename##1{{##1} (}\@internalciteb}
+
+% Macros for \newcite, which leaves name in running text, and is
+% otherwise like \shortcite.
+\def\@citexb[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@newcite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{;\penalty\@m\ }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+\hbox{\csname b@\@citeb\endcsname}}}{#1}}
+\def\@internalciteb{\@ifnextchar [{\@tempswatrue\@citexb}{\@tempswafalse\@citexb[]}}
+
+\def\@newcite#1#2{{#1\if at tempswa, #2\fi)}}
+
+\def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+
+%%% More changes made by SMS (originals in latex.tex)
+% Use parentheses instead of square brackets in the text.
+\def\@cite#1#2{({#1\if at tempswa , #2\fi})}
+
+% Don't put a label in the bibliography at all. Just use the unlabeled format
+% instead.
+\def\thebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{References\@mkboth
+ {References}{References}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthebibliography=\endlist
+
+% Allow for a bibliography of sources of attested examples
+\def\thesourcebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{Sources of Attested Examples\@mkboth
+ {Sources of Attested Examples}{Sources of Attested Examples}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthesourcebibliography=\endlist
+
+\def\@lbibitem[#1]#2{\item[]\if at filesw
+ { \def\protect##1{\string ##1\space}\immediate
+ \write\@auxout{\string\bibcite{#2}{#1}}\fi\ignorespaces}}
+
+\def\@bibitem#1{\item\if at filesw \immediate\write\@auxout
+ {\string\bibcite{#1}{\the\c at enumi}}\fi\ignorespaces}
+
+% sections with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+ -0.5ex minus -.2ex}{1.5ex plus 0.3ex minus .2ex}{\large\bf\raggedright}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-1.8ex plus
+ -0.5ex minus -.2ex}{0.8ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{0.5ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\paragraph{\@startsection{paragraph}{4}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+\def\subparagraph{\@startsection{subparagraph}{5}{\parindent}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+
+% Footnotes
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+
+% Lists and paragraphs
+\parindent 1em
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 2pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+
+\leftmargin 2em \leftmargini\leftmargin \leftmarginii 2em
+\leftmarginiii 1.5em \leftmarginiv 1.0em \leftmarginv .5em \leftmarginvi .5em
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+ \labelwidth\leftmarginii\advance\labelwidth-\labelsep
+ \topsep 2pt plus 1pt minus 0.5pt
+ \parsep 1pt plus 0.5pt minus 0.5pt
+ \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+ \labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+ \topsep 1pt plus 0.5pt minus 0.5pt
+ \parsep \z@ \partopsep 0.5pt plus 0pt minus 0.5pt
+ \itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+ \labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+ \labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+ \labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\@normalsize{\@setsize\normalsize{11pt}\xpt\@xpt}
+\def\small{\@setsize\small{10pt}\ixpt\@ixpt}
+\def\footnotesize{\@setsize\footnotesize{10pt}\ixpt\@ixpt}
+\def\scriptsize{\@setsize\scriptsize{8pt}\viipt\@viipt}
+\def\tiny{\@setsize\tiny{7pt}\vipt\@vipt}
+\def\large{\@setsize\large{14pt}\xiipt\@xiipt}
+\def\Large{\@setsize\Large{16pt}\xivpt\@xivpt}
+\def\LARGE{\@setsize\LARGE{20pt}\xviipt\@xviipt}
+\def\huge{\@setsize\huge{23pt}\xxpt\@xxpt}
+\def\Huge{\@setsize\Huge{28pt}\xxvpt\@xxvpt}
+
+% Expanding the titlebox
+\setlength\titlebox{6.5cm}
diff --git a/papers/acl-04/chart-matrix.gif b/papers/acl-04/chart-matrix.gif
new file mode 100644
index 0000000..699211e
Binary files /dev/null and b/papers/acl-04/chart-matrix.gif differ
diff --git a/papers/acl-04/chart.eps.gz b/papers/acl-04/chart.eps.gz
new file mode 100644
index 0000000..5d333ae
Binary files /dev/null and b/papers/acl-04/chart.eps.gz differ
diff --git a/papers/acl-04/nltk.bib b/papers/acl-04/nltk.bib
new file mode 100644
index 0000000..1f6c599
--- /dev/null
+++ b/papers/acl-04/nltk.bib
@@ -0,0 +1,50 @@
+
+ at Book{Rossum03intro,
+ author = {Guido Van Rossum},
+ title = {An Introduction to Python},
+ publisher = {Network Theory Ltd},
+ year = 2003
+}
+
+ at Book{Rossum03ref,
+ author = {Guido Van Rossum},
+ title = {The Python Language Reference},
+ publisher = {Network Theory Ltd},
+ year = 2003
+}
+
+ at InProceedings{LoperBird02,
+ author = {Edward Loper and Steven Bird},
+ title = {{NLTK: The Natural Language Toolkit}},
+ booktitle = {Proceedings of the ACL Workshop on Effective Tools and
+ Methodologies for Teaching Natural Language Processing and Computational
+ Linguistics},
+ year = 2002,
+ publisher={Somerset, NJ: Association for Computational Linguistics},
+ pages={62--69},
+ note = {\url{http://arXiv.org/abs/cs/0205028}},
+}
+
+ at InProceedings{Loper04,
+ author = {Edward Loper},
+ title = {{NLTK}: Building a Pedagogical Toolkit in {Python}},
+ booktitle = {PyCon DC 2004},
+ year = 2004,
+ publisher = {Python Software Foundation},
+ note = {\url{http://www.python.org/pycon/dc2004/papers/}}
+}
+
+ at Misc{tkinter,
+ author = {Fredrik Lundh},
+ title = {An Introduction to Tkinter},
+ note = {\url{http://www.pythonware.com/library/tkinter/introduction/index.htm}},
+ year = 1999
+}
+
+ at Misc{epydoc,
+ author = {Edward Loper},
+ title = {Epydoc},
+ year = 2002,
+ note = {\url{http://epydoc.sourceforge.net/}}
+}
+
diff --git a/papers/acl-06/acl-06.tex b/papers/acl-06/acl-06.tex
new file mode 100644
index 0000000..7c20d0a
--- /dev/null
+++ b/papers/acl-06/acl-06.tex
@@ -0,0 +1,405 @@
+\documentclass[11pt]{article}
+\usepackage{colacl06}
+\usepackage{times,url}
+\usepackage{latexsym}
+\usepackage{hyphen}
+\usepackage{epsfig}
+\setlength\titlebox{6.5cm} % Expanding the titlebox
+
+\title{NLTK: The Natural Language Toolkit}
+
+\author{Steven Bird\\[.5ex]
+ Department of Computer Science and Software Engineering\\
+ University of Melbourne, Victoria 3010, AUSTRALIA\\[.5ex]
+ Linguistic Data Consortium, University of Pennsylvania,\\
+ Philadelphia PA 19104-2653, USA}
+\date{}
+
+\begin{document}
+\maketitle
+\begin{abstract}
+ The Natural Language Toolkit is a suite of program modules, data
+ sets and tutorials supporting research and teaching in computational
+ linguistics and natural language processing. NLTK is written in
+ Python and distributed under the GPL open source license. Over the
+ past year the toolkit has been rewritten, simplifying many
+ linguistic data structures and taking advantage of recent
+ enhancements in the Python language. This paper reports on the
+ simplified toolkit and explains how it is used in teaching NLP.
+\end{abstract}
+
+\section{Introduction}
+
+NLTK, the Natural Language Toolkit, is a suite of Python modules
+providing many NLP data types, processing tasks, corpus samples and
+readers, together with animated algorithms, tutorials, and problem
+sets \cite{LoperBird02}. Data types include tokens, tags, chunks,
+trees, and feature structures. Interface definitions and reference
+implementations are provided for tokenizers, stemmers, taggers
+(regexp, ngram, Brill), chunkers, parsers (recursive-descent,
+shift-reduce, chart, probabilistic), clusterers, and classifiers.
+Corpus samples and readers include: Brown Corpus, CoNLL-2000 Chunking
+Corpus, CMU Pronunciation Dictionary, NIST IEER Corpus, PP Attachment
+Corpus, Penn Treebank, and the SIL Shoebox corpus format.
+
+NLTK is ideally suited to students who are learning NLP or conducting
+research in NLP or closely related areas.
+NLTK has been used successfully as a
+teaching tool, as an individual study tool, and as a platform for
+prototyping and building research systems \cite{Liddy05,Satre05}.
+
+We chose Python for its shallow learning curve, transparent syntax,
+and good string-handling. Python permits exploration via its
+interactive interpreter. As an object-oriented language, Python
+permits data and code to be encapsulated and re-used easily. Python
+comes with an extensive library, including tools for graphical
+programming and numerical processing \cite{Beasley06}.
+
+Over the past four years the toolkit grew rapidly and the data
+structures became significantly more complex. Each new processing
+task added new requirements on input and output
+representations. It was not clear how to generalize tasks so they
+could be applied independently of each other.
+
+As a simple example, consider the independent tasks of tagging and
+stemming, which both operate on sequences of tokens. If stemming is
+done first, we lose information required for tagging. If tagging is
+done first, the stemming must be able to skip over the tags. If both
+are done independently, we need to be able to align the results. As
+task combinations multiply, managing the data becomes extremely
+difficult.
+
+To address this problem, NLTK 1.4 introduced a blackboard architecture
+for tokens, unifying many data types, and permitting distinct tasks to
+be run independently. Unfortunately this architecture also came with
+a significant overhead for programmers, who were often forced to use
+``rather awkward code structures'' \cite{Hearst05}. It was clear that
+the re-engineering done in NLTK 1.4 unduly complicated the
+programmer's task.
+
+This paper presents a brief overview and tutorial on a new, simplified
+toolkit, and describes how it is used in teaching.
+
+\section{Simple Processing Tasks}
+
+\subsection{Tokenization and Stemming}
+
+The following three-line program imports the \texttt{tokenize}
+package, defines a text string, and tokenizes the string on
+whitespace to create a list of tokens. (NB.\ `\url{>>>}' is
+Python's interactive prompt; `\url{...}' is the continuation prompt.)
+
+{\small\begin{verbatim}
+>>> text = 'This is a test.'
+>>> list(tokenize.whitespace(text))
+['This', 'is', 'a', 'test.']
+\end{verbatim}}
+
+\noindent
+Several other tokenizers are provided. We can stem the output of tokenization
+using the Porter Stemmer as follows:
+
+{\small\begin{verbatim}
+>>> text = 'stemming is exciting'
+>>> tokens = tokenize.whitespace(text)
+>>> porter = stem.Porter()
+>>> for token in tokens:
+... print porter.stem(token),
+stem is excit
+\end{verbatim}}
+
+The corpora included with NLTK come with corpus readers that
+understand the file structure of the corpus, and load the data into
+Python data structures. For example, the following code reads part \textit{a}
+of the Brown Corpus. It prints a list of tuples, where each tuple
+consists of a word and its tag.
+
+{\small\begin{verbatim}
+>>> for sent in brown.tagged('a'):
+... print sent
+[('The', 'at'), ('Fulton', 'np-tl'),
+('County', 'nn-tl'), ('Grand', 'jj-tl'),
+('Jury', 'nn-tl'), ('said', 'vbd'), ...]
+\end{verbatim}}
+
+NLTK provides support for conditional frequency distributions, making
+it easy to count up items of interest in specified contexts. Such
+information may be useful for studies in stylistics or in text
+categorization.
+
+\subsection{Tagging}
+
+The simplest possible tagger assigns the same tag to each token:
+
+{\small\begin{verbatim}
+>>> my_tagger = tag.Default('nn')
+>>> list(my_tagger.tag(tokens))
+[('John', 'nn'), ('saw', 'nn'),
+ ('3', 'nn'), ('polar', 'nn'),
+ ('bears', 'nn'), ('.', 'nn')]
+\end{verbatim}}
+
+On its own, this will tag only 10--20\% of the tokens correctly.
+However, it is a reasonable tagger to use as a default if a more
+advanced tagger fails to determine a token's tag.
+
+The regular expression tagger assigns a tag to a token according to a
+series of string patterns. For instance, the following tagger assigns
+\texttt{cd} to cardinal numbers, \texttt{nns} to words ending in the
+letter \textit{s}, and \texttt{nn} to everything else:
+
+{\small\begin{verbatim}
+>>> patterns = [
+... (r'\d+(.\d+)?$', 'cd'),
+... (r'\.*s$', 'nns'),
+... (r'.*', 'nn')]
+>>> simple_tagger = tag.Regexp(patterns)
+>>> list(simple_tagger.tag(tokens))
+[('John', 'nn'), ('saw', 'nn'),
+ ('3', 'cd'), ('polar', 'nn'),
+ ('bears', 'nns'), ('.', 'nn')]
+\end{verbatim}}
+
+The \texttt{tag.Unigram} class implements a simple statistical tagging
+algorithm: for each token, it assigns the tag that is most likely
+for that token. For example, it will assign the tag \texttt{jj} to
+any occurrence of the word \textit{frequent}, since \textit{frequent} is used as an
+adjective (e.g.\ \textit{a frequent word}) more often than it is used as a
+verb (e.g.\ \textit{I frequent this cafe}).
+Before a unigram tagger can be used, it must be trained on
+a corpus, as shown below for the first section of the Brown Corpus.
+
+{\small\begin{verbatim}
+>>> unigram_tagger = tag.Unigram()
+>>> unigram_tagger.train(brown('a'))
+\end{verbatim}}
+
+\noindent
+Once a unigram tagger has been trained, it can be used to tag new text.
+Note that it assigns the default tag \texttt{None} to any token that was not
+encountered during training.
+
+{\small\begin{verbatim}
+>>> text = "John saw the books on the table"
+>>> tokens = list(tokenize.whitespace(text))
+>>> list(unigram_tagger.tag(tokens))
+[('John', 'np'), ('saw', 'vbd'),
+ ('the', 'at'), ('books', None),
+ ('on', 'in'), ('the', 'at'),
+ ('table', None)]
+\end{verbatim}}
+
+\noindent
+We can instruct the unigram tagger to back off to our default
+\url{simple_tagger} when it cannot assign a tag itself. Now all
+the words are guaranteed to be tagged:
+
+{\small\begin{verbatim}
+>>> unigram_tagger =
+... tag.Unigram(backoff=simple_tagger)
+>>> unigram_tagger.train(train_sents)
+>>> list(unigram_tagger.tag(tokens))
+[('John', 'np'), ('saw', 'vbd'),
+ ('the', 'at'), ('books', 'nns'),
+ ('on', 'in'), ('the', 'at'),
+ ('table', 'nn')]
+\end{verbatim}}
+
+\noindent
+We can go on to define and train a bigram tagger, as shown below:
+
+{\small\begin{verbatim}
+>>> bigram_tagger =\
+... tag.Bigram(backoff=unigram_tagger)
+>>> bigram_tagger.train(brown.tagged('a'))
+\end{verbatim}}
+
+\noindent
+We can easily evaluate this tagger against some gold-standard tagged
+text, using the \url{tag.accuracy()} function.
+
+NLTK also includes a Brill tagger (contributed by Christopher
+Maloof) and an HMM tagger (contributed by Trevor Cohn).
+
+\section{Chunking and Parsing}
+
+Chunking is a technique for shallow syntactic analysis of (tagged)
+text. Chunk data can be loaded from files that use the common bracket or
+IOB notations. We can define a regular-expression based chunk parser
+for use in chunking tagged text. NLTK also supports simple cascading
+of chunk parsers. Corpus readers for chunked data in Penn Treebank
+and CoNLL-2000 are provided, along with comprehensive support for
+evaluation and error analysis.
+
+NLTK provides several parsers for context-free phrase-structure
+grammars. Grammars can be defined using a series of productions as follows:
+
+{\small\begin{verbatim}
+>>> grammar = cfg.parse_grammar('''
+... S -> NP VP
+... VP -> V NP | V NP PP
+... V -> "saw" | "ate"
+... NP -> "John" | Det N | Det N PP
+... Det -> "a" | "an" | "the" | "my"
+... N -> "dog" | "cat" | "ball"
+... PP -> P NP
+... P -> "on" | "by" | "with"
+... ''')
+\end{verbatim}}
+
+\noindent
+Now we can tokenize and parse a sentence with a recursive descent
+parser. Note that we avoided left-recursive productions in the above
+grammar, so that this parser does not get into an infinite loop.
+
+{\small\begin{verbatim}
+>>> text = "John saw a cat with my ball"
+>>> sent = list(tokenize.whitespace(text))
+>>> rd = parse.RecursiveDescent(grammar)
+\end{verbatim}}
+
+Now we apply it to our sentence, and iterate over all the parses that
+it generates. Observe that two parses are possible, due to
+prepositional phrase attachment ambiguity.
+
+{\small\begin{verbatim}
+>>> for p in rd.get_parse_list(sent):
+... print p
+(S:
+ (NP: 'John')
+ (VP:
+ (V: 'saw')
+ (NP:
+ (Det: 'a')
+ (N: 'cat')
+ (PP: (P: 'with')
+ (NP: (Det: 'my') (N: 'ball'))))))
+(S:
+ (NP: 'John')
+ (VP:
+ (V: 'saw')
+ (NP: (Det: 'a') (N: 'cat'))
+ (PP: (P: 'with')
+ (NP: (Det: 'my') (N: 'ball')))))
+\end{verbatim}}
+
+\noindent
+The same sentence can be parsed using a grammar with left-recursive
+productions, so long as we use a chart parser. We can invoke NLTK's
+chart parser with a bottom-up rule-invocation strategy with
+\texttt{chart.ChartParse(grammar, chart.BU$\_$STRATEGY)}. Tracing can
+be turned on in order to display each step of the process. NLTK also
+supports probabilistic context free grammars, and provides a
+Viterbi-style PCFG parser, together with a suite of bottom-up
+probabilistic chart parsers.
+
+\begin{figure*}[tb]
+\centerline{\epsfig{file=srparser.eps, scale=.5}}
+\vspace{4ex}
+\centerline{\epsfig{file=rdparser.eps, scale=.5}}
+\caption{Two Parser Demonstrations: Shift-Reduce and Recursive Descent Parsers}
+\label{fig:parser}
+\end{figure*}
+\section{Teaching with NLTK}
+
+Natural language processing is often taught within the confines of a
+single-semester course, either at advanced undergraduate level or at
+postgraduate level. Unfortunately, it turns out to be rather
+difficult to cover both the theoretical and practical sides of the
+subject in such a short span of time. Some courses focus on theory to
+the exclusion of practical exercises, and deprive students of the
+challenge and excitement of writing programs to automatically process
+natural language. Other courses are simply designed to teach
+programming for linguists, and do not manage to cover any significant
+NLP content. NLTK was developed to address this problem, making
+it feasible to cover a substantial amount of theory and practice
+within a single-semester course.
+
+A significant fraction of any NLP course is made up of fundamental
+data structures and algorithms. These are usually taught with the
+help of formal notations and complex diagrams. Large trees and charts
+are copied onto the board and edited in tedious slow motion, or
+laboriously prepared for presentation slides. A more effective method
+is to use live demonstrations in which those diagrams are generated
+and updated automatically. NLTK provides interactive graphical user
+interfaces, making it possible to view program state and to study
+program execution step-by-step (e.g.\ see Figure~\ref{fig:parser}).
+Most NLTK components have a demonstration mode, and will perform an
+interesting task without requiring any special input from the user.
+It is even possible to make minor modifications to programs in
+response to ``what if'' questions. In this way, students learn the
+mechanics of NLP quickly, gain deeper insights into the data
+structures and algorithms, and acquire new problem-solving skills.
+Since these demonstrations are distributed with the toolkit, students
+can experiment on their own with the algorithms that they have seen
+presented in class.
+
+NLTK can be used to create student assignments of varying difficulty
+and scope. In the simplest assignments, students experiment with one
+of the existing modules. Once students become more familiar with the
+toolkit, they can be asked to make minor changes or extensions to an
+existing module (e.g.\ build a left-corner parser by modifying the
+recursive descent parser). A bigger challenge is to develop one or
+more new modules and integrate them with existing modules to perform
+a sophisticated NLP task. Here, NLTK provides a useful starting
+point with its existing components and its extensive tutorials and
+API documentation.
+
+NLTK is a unique framework for teaching natural language processing.
+NLTK provides comprehensive support for a first course in NLP which
+tightly couples theory and practice. Its extensive documentation
+maximizes the potential for independent learning. For more
+information, including documentation, download pointers, and links to
+dozens of courses that have adopted NLTK, please see:
+\url{http://nltk.sourceforge.net/} .
+
+\section*{Acknowledgements}
+
+I am grateful to Edward Loper, co-developer of NLTK, and to dozens of
+people who have contributed code and provided helpful feedback.
+\vspace{-2ex}
+
+\bibliographystyle{acl}
+
+\begin{thebibliography}{}
+\setlength{\parskip}{0pt}
+\setlength{\itemsep}{0pt}
+
+\bibitem[\protect\citename{Hearst}2005]{Hearst05}
+Marti Hearst.
+\newblock 2005.
+\newblock Teaching applied natural language processing: Triumphs and
+ tribulations.
+\newblock In {\em Proc 2nd ACL Workshop on Effective Tools and
+ Methodologies for Teaching NLP and CL}, pages 1--8, ACL
+
+\bibitem[\protect\citename{Liddy and McCracken}2005]{Liddy05}
+Elizabeth Liddy and Nancy McCracken.
+\newblock 2005.
+\newblock Hands-on {NLP} for an interdisciplinary audience.
+\newblock In {\em Proc 2nd ACL Workshop on Effective Tools and
+ Methodologies for Teaching NLP and CL}, pages 62--68, ACL
+
+\bibitem[\protect\citename{Loper and Bird}2002]{LoperBird02}
+Edward Loper and Steven Bird.
+\newblock 2002.
+\newblock {NLTK}: The Natural Language Toolkit.
+\newblock In {\em Proc ACL Workshop on Effective Tools and
+ Methodologies for Teaching Natural Language Processing and Computational
+ Linguistics}, pages 62--69. ACL.
+
+\bibitem[\protect\citename{Beasley}2006]{Beasley06}
+David Beasley.
+\newblock 2006.
+\newblock {\em Python Essential Reference, 3rd Edition}.
+\newblock Sams.
+
+\bibitem[\protect\citename{S{\ae}tre \bgroup et al.\egroup }2005]{Satre05}
+Rune S{\ae}tre, Amund Tveit, Tonje~S. Steigedal, and Astrid L{\ae}greid.
+\newblock 2005.
+\newblock Semantic annotation of biomedical literature using Google.
+\newblock In {\em Data Mining and Bioinformatics Workshop}, volume 3482 of {\em
+ Lecture Notes in Computer Science}. Springer.
+
+\end{thebibliography}
+\end{document}
diff --git a/papers/acl-06/acl.bst b/papers/acl-06/acl.bst
new file mode 100644
index 0000000..b95ec04
--- /dev/null
+++ b/papers/acl-06/acl.bst
@@ -0,0 +1,1322 @@
+
+% BibTeX `acl' style file for BibTeX version 0.99c, LaTeX version 2.09
+% This version was made by modifying `aaai-named' format based on the master
+% file by Oren Patashnik (PATASHNIK at SCORE.STANFORD.EDU)
+
+% Copyright (C) 1985, all rights reserved.
+% Modifications Copyright 1988, Peter F. Patel-Schneider
+% Further modifictions by Stuart Shieber, 1991, and Fernando Pereira, 1992.
+% Copying of this file is authorized only if either
+% (1) you make absolutely no changes to your copy, including name, or
+% (2) if you do make changes, you name it something other than
+% btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst.
+% This restriction helps ensure that all standard styles are identical.
+
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at spar.slb.com
+
+% Citation format: [author-last-name, year]
+% [author-last-name and author-last-name, year]
+% [author-last-name {\em et al.}, year]
+%
+% Reference list ordering: alphabetical by author or whatever passes
+% for author in the absence of one.
+%
+% This BibTeX style has support for short (year only) citations. This
+% is done by having the citations actually look like
+% \citename{name-info, }year
+% The LaTeX style has to have the following
+% \let\@internalcite\cite
+% \def\cite{\def\citename##1{##1}\@internalcite}
+% \def\shortcite{\def\citename##1{}\@internalcite}
+% \def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+% which makes \shortcite the macro for short citations.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Changes made by SMS for thesis style
+% no emphasis on "et al."
+% "Ph.D." includes periods (not "PhD")
+% moved year to immediately after author's name
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ENTRY
+ { address
+ author
+ booktitle
+ chapter
+ edition
+ editor
+ howpublished
+ institution
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ publisher
+ school
+ series
+ title
+ type
+ volume
+ year
+ }
+ {}
+ { label extra.label sort.label }
+
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+
+STRINGS { s t }
+
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { add.period$ " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+
+ "\bibitem[" write$
+ label write$
+ "]{" write$
+
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+
+FUNCTION {new.block.checka}
+{ empty$
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.sentence.checka}
+{ empty$
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {new.sentence.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+
+FUNCTION {emphasize}
+{ duplicate$ empty$
+ { pop$ "" }
+ { "{\em " swap$ * "}" * }
+ if$
+}
+
+INTEGERS { nameptr namesleft numnames }
+
+FUNCTION {format.names}
+{ 's :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+
+ { s nameptr "{ff~}{vv~}{ll}{, jj}" format.name$ 't :=
+
+ nameptr #1 >
+ { namesleft #1 >
+ { ", " * t * }
+ { numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ t "others" =
+ { " et~al." * }
+ { " and " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {format.authors}
+{ author empty$
+ { "" }
+ { author format.names }
+ if$
+}
+
+FUNCTION {format.editors}
+{ editor empty$
+ { "" }
+ { editor format.names
+ editor num.names$ #1 >
+ { ", editors" * }
+ { ", editor" * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title empty$
+ { "" }
+
+ { title "t" change.case$ }
+
+ if$
+}
+
+FUNCTION {n.dashify}
+{ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {format.date}
+{ year empty$
+ { month empty$
+ { "" }
+ { "there's a month but no year in " cite$ * warning$
+ month
+ }
+ if$
+ }
+ { month empty$
+ { "" }
+ { month }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.btitle}
+{ title emphasize
+}
+
+FUNCTION {tie.or.space.connect}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$ * *
+}
+
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { "volume" volume tie.or.space.connect
+ series empty$
+ 'skip$
+ { " of " * series emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { output.state mid.sentence =
+ { "number" }
+ { "Number" }
+ if$
+ number tie.or.space.connect
+ series empty$
+ { "there's a number but no series in " cite$ * warning$ }
+ { " in " * series * }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition empty$
+ { "" }
+ { output.state mid.sentence =
+ { edition "l" change.case$ " edition" * }
+ { edition "t" change.case$ " edition" * }
+ if$
+ }
+ if$
+}
+
+INTEGERS { multiresult }
+
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+
+FUNCTION {format.pages}
+{ pages empty$
+ { "" }
+ { pages multi.page.check
+ { "pages" pages n.dashify tie.or.space.connect }
+ { "page" pages tie.or.space.connect }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.year.label}
+{ year extra.label *
+}
+
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ number empty$
+ 'skip$
+ { "(" number * ")" * *
+ volume empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ }
+ if$
+ pages empty$
+ 'skip$
+ { duplicate$ empty$
+ { pop$ format.pages }
+ { ":" * pages n.dashify * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { "chapter" }
+ { type "l" change.case$ }
+ if$
+ chapter tie.or.space.connect
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.in.ed.booktitle}
+{ booktitle empty$
+ { "" }
+ { editor empty$
+ { "In " booktitle emphasize * }
+ { "In " format.editors * ", " * booktitle emphasize * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {empty.misc.check}
+{ author empty$ title empty$ howpublished empty$
+ month empty$ year empty$ note empty$
+ and and and and and
+
+ key empty$ not and
+
+ { "all relevant fields are empty in " cite$ * warning$ }
+ 'skip$
+ if$
+}
+
+FUNCTION {format.thesis.type}
+{ type empty$
+ 'skip$
+ { pop$
+ type "t" change.case$
+ }
+ if$
+}
+
+FUNCTION {format.tr.number}
+{ type empty$
+ { "Technical Report" }
+ 'type
+ if$
+ number empty$
+ { "t" change.case$ }
+ { number tie.or.space.connect }
+ if$
+}
+
+FUNCTION {format.article.crossref}
+{ key empty$
+ { journal empty$
+ { "need key or journal for " cite$ * " to crossref " * crossref *
+ warning$
+ ""
+ }
+ { "In {\em " journal * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.crossref.editor}
+{ editor #1 "{vv~}{ll}" format.name$
+ editor num.names$ duplicate$
+ #2 >
+ { pop$ " et~al." * }
+ { #2 <
+ 'skip$
+ { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" =
+ { " et~al." * }
+ { " and " * editor #2 "{vv~}{ll}" format.name$ * }
+ if$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.book.crossref}
+{ volume empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ "In "
+ }
+ { "Volume" volume tie.or.space.connect
+ " of " *
+ }
+ if$
+ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { series empty$
+ { "need editor, key, or series for " cite$ * " to crossref " *
+ crossref * warning$
+ "" *
+ }
+ { "{\em " * series * "\/}" * }
+ if$
+ }
+ { key * }
+ if$
+ }
+ { format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.incoll.inproc.crossref}
+{ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { booktitle empty$
+ { "need editor, key, or booktitle for " cite$ * " to crossref " *
+ crossref * warning$
+ ""
+ }
+ { "In {\em " booktitle * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ }
+ { "In " format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { journal emphasize "journal" output.check
+ format.vol.num.pages output
+ format.date output
+ }
+ { format.article.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {booklet}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ howpublished address new.block.checkb
+ howpublished output
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.chapter.pages output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ format.edition output
+ format.date output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.chapter.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.pages output
+ address empty$
+ { organization publisher new.sentence.checkb
+ organization output
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ organization output
+ publisher output
+ }
+ if$
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {conference} { inproceedings }
+
+FUNCTION {manual}
+{ output.bibitem
+ author empty$
+ { organization empty$
+ 'skip$
+ { organization output.nonnull
+ address output
+ }
+ if$
+ }
+ { format.authors output.nonnull }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ author empty$
+ { organization empty$
+ { address new.block.checka
+ address output
+ }
+ 'skip$
+ if$
+ }
+ { organization address new.block.checkb
+ organization output
+ address output
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ "Master's thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label output
+ new.block
+ title howpublished new.block.checkb
+ format.title output
+ howpublished new.block.checka
+ howpublished output
+ format.date output
+ new.block
+ note output
+ fin.entry
+ empty.misc.check
+}
+
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ new.block
+ "{Ph.D.} thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ editor empty$
+ { organization output }
+ { format.editors output.nonnull }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ format.bvolume output
+ format.number.series output
+ address empty$
+ { editor empty$
+ { publisher new.sentence.checka }
+ { organization publisher new.sentence.checkb
+ organization output
+ }
+ if$
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ editor empty$
+ 'skip$
+ { organization output }
+ if$
+ publisher output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ format.tr.number output.nonnull
+ institution "institution" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ note "note" output.check
+ format.date output
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+
+MACRO {jan} {"January"}
+
+MACRO {feb} {"February"}
+
+MACRO {mar} {"March"}
+
+MACRO {apr} {"April"}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"August"}
+
+MACRO {sep} {"September"}
+
+MACRO {oct} {"October"}
+
+MACRO {nov} {"November"}
+
+MACRO {dec} {"December"}
+
+MACRO {acmcs} {"ACM Computing Surveys"}
+
+MACRO {acta} {"Acta Informatica"}
+
+MACRO {cacm} {"Communications of the ACM"}
+
+MACRO {ibmjrd} {"IBM Journal of Research and Development"}
+
+MACRO {ibmsj} {"IBM Systems Journal"}
+
+MACRO {ieeese} {"IEEE Transactions on Software Engineering"}
+
+MACRO {ieeetc} {"IEEE Transactions on Computers"}
+
+MACRO {ieeetcad}
+ {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"}
+
+MACRO {ipl} {"Information Processing Letters"}
+
+MACRO {jacm} {"Journal of the ACM"}
+
+MACRO {jcss} {"Journal of Computer and System Sciences"}
+
+MACRO {scp} {"Science of Computer Programming"}
+
+MACRO {sicomp} {"SIAM Journal on Computing"}
+
+MACRO {tocs} {"ACM Transactions on Computer Systems"}
+
+MACRO {tods} {"ACM Transactions on Database Systems"}
+
+MACRO {tog} {"ACM Transactions on Graphics"}
+
+MACRO {toms} {"ACM Transactions on Mathematical Software"}
+
+MACRO {toois} {"ACM Transactions on Office Information Systems"}
+
+MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"}
+
+MACRO {tcs} {"Theoretical Computer Science"}
+
+READ
+
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+
+INTEGERS { len }
+
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+
+INTEGERS { et.al.char.used }
+
+FUNCTION {initialize.et.al.char.used}
+{ #0 'et.al.char.used :=
+}
+
+EXECUTE {initialize.et.al.char.used}
+
+FUNCTION {format.lab.names}
+{ 's :=
+ s num.names$ 'numnames :=
+
+ numnames #1 =
+ { s #1 "{vv }{ll}" format.name$ }
+ { numnames #2 =
+ { s #1 "{vv }{ll }and " format.name$ s #2 "{vv }{ll}" format.name$ *
+ }
+ { s #1 "{vv }{ll }\bgroup et al.\egroup " format.name$ }
+ if$
+ }
+ if$
+
+}
+
+FUNCTION {author.key.label}
+{ author empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.editor.key.label}
+{ author empty$
+ { editor empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.key.organization.label}
+{ author empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {editor.key.organization.label}
+{ editor empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+}
+
+FUNCTION {calc.label}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.label
+ { type$ "proceedings" =
+ 'editor.key.organization.label
+ { type$ "manual" =
+ 'author.key.organization.label
+ 'author.key.label
+ if$
+ }
+ if$
+ }
+ if$
+ duplicate$
+
+ "\protect\citename{" swap$ * "}" *
+ year field.or.null purify$ *
+ 'label :=
+ year field.or.null purify$ *
+
+ sortify 'sort.label :=
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { nameptr #1 >
+ { " " * }
+ 'skip$
+ if$
+
+ s nameptr "{vv{ } }{ll{ }}{ ff{ }}{ jj{ }}" format.name$ 't :=
+
+ nameptr numnames = t "others" = and
+ { "et al" * }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.organization.sort}
+{ author empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need author, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {editor.organization.sort}
+{ editor empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need editor, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+
+FUNCTION {presort}
+
+{ calc.label
+ sort.label
+ " "
+ *
+ type$ "book" =
+
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.organization.sort
+ { type$ "manual" =
+ 'author.organization.sort
+ 'author.sort
+ if$
+ }
+ if$
+ }
+ if$
+
+ *
+
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+
+SORT
+
+STRINGS { longest.label last.sort.label next.extra }
+
+INTEGERS { longest.label.width last.extra.num }
+
+FUNCTION {initialize.longest.label}
+{ "" 'longest.label :=
+ #0 int.to.chr$ 'last.sort.label :=
+ "" 'next.extra :=
+ #0 'longest.label.width :=
+ #0 'last.extra.num :=
+}
+
+FUNCTION {forward.pass}
+{ last.sort.label sort.label =
+ { last.extra.num #1 + 'last.extra.num :=
+ last.extra.num int.to.chr$ 'extra.label :=
+ }
+ { "a" chr.to.int$ 'last.extra.num :=
+ "" 'extra.label :=
+ sort.label 'last.sort.label :=
+ }
+ if$
+}
+
+FUNCTION {reverse.pass}
+{ next.extra "b" =
+ { "a" 'extra.label := }
+ 'skip$
+ if$
+ label extra.label * 'label :=
+ label width$ longest.label.width >
+ { label 'longest.label :=
+ label width$ 'longest.label.width :=
+ }
+ 'skip$
+ if$
+ extra.label 'next.extra :=
+}
+
+EXECUTE {initialize.longest.label}
+
+ITERATE {forward.pass}
+
+REVERSE {reverse.pass}
+
+FUNCTION {begin.bib}
+
+{ et.al.char.used
+ { "\newcommand{\etalchar}[1]{$^{#1}$}" write$ newline$ }
+ 'skip$
+ if$
+ preamble$ empty$
+
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+
+ "\begin{thebibliography}{" "}" * write$ newline$
+
+}
+
+EXECUTE {begin.bib}
+
+EXECUTE {init.state.consts}
+
+ITERATE {call.type$}
+
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+
+EXECUTE {end.bib}
+
diff --git a/papers/acl-06/colacl06.sty b/papers/acl-06/colacl06.sty
new file mode 100644
index 0000000..ebc0598
--- /dev/null
+++ b/papers/acl-06/colacl06.sty
@@ -0,0 +1,368 @@
+% File colacl06.sty
+% This is the LaTeX style file for COLING/ACL 2006. It is identical to the style file for EACL 2006.
+
+% File eacl2006.sty
+% September 19, 2005
+% Contact: e.agirre at ehu.es or Sergi.Balari at uab.es
+
+% This is the LaTeX style file for EACL 2006. It is nearly identical to the
+% style files for ACL2005, ACL 2002, ACL 2001, ACL 2000, EACL 95 and EACL
+% 99.
+%
+% Changes made include: adapt layout to A4 and centimeters, widden abstract
+
+% This is the LaTeX style file for ACL 2000. It is nearly identical to the
+% style files for EACL 95 and EACL 99. Minor changes include editing the
+% instructions to reflect use of \documentclass rather than \documentstyle
+% and removing the white space before the title on the first page
+% -- John Chen, June 29, 2000
+
+% To convert from submissions prepared using the style file aclsub.sty
+% prepared for the ACL 2000 conference, proceed as follows:
+% 1) Remove submission-specific information: \whichsession, \id,
+% \wordcount, \otherconferences, \area, \keywords
+% 2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% 3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% 4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+% 5) Change the style reference from aclsub to acl2000, and be sure
+% this style file is in your TeX search path
+
+
+% This is the LaTeX style file for EACL-95. It is identical to the
+% style file for ANLP '94 except that the margins are adjusted for A4
+% paper. -- abney 13 Dec 94
+
+% The ANLP '94 style file is a slightly modified
+% version of the style used for AAAI and IJCAI, using some changes
+% prepared by Fernando Pereira and others and some minor changes
+% by Paul Jacobs.
+
+% Papers prepared using the aclsub.sty file and acl.bst bibtex style
+% should be easily converted to final format using this style.
+% (1) Submission information (\wordcount, \subject, and \makeidpage)
+% should be removed.
+% (2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% (between \begin{abstract} and \end{abstract}).
+% (3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% (4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+
+% Place this in a file called aclap.sty in the TeX search path.
+% (Placing it in the same directory as the paper should also work.)
+
+% Prepared by Peter F. Patel-Schneider, liberally using the ideas of
+% other style hackers, including Barbara Beeton.
+% This style is NOT guaranteed to work. It is provided in the hope
+% that it will make the preparation of papers easier.
+%
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at research.att.com
+
+% Papers are to be prepared using the ``acl'' bibliography style,
+% as follows:
+% \documentclass[11pt]{article}
+% \usepackage{acl2000}
+% \title{Title}
+% \author{Author 1 \and Author 2 \\ Address line \\ Address line \And
+% Author 3 \\ Address line \\ Address line}
+% \begin{document}
+% ...
+% \bibliography{bibliography-file}
+% \bibliographystyle{acl}
+% \end{document}
+
+% Author information can be set in various styles:
+% For several authors from the same institution:
+% \author{Author 1 \and ... \and Author n \\
+% Address line \\ ... \\ Address line}
+% if the names do not fit well on one line use
+% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
+% For authors from different institutions:
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \And ... \And
+% Author n \\ Address line \\ ... \\ Address line}
+% To start a seperate ``row'' of authors use \AND, as in
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \AND
+% Author 2 \\ Address line \\ ... \\ Address line \And
+% Author 3 \\ Address line \\ ... \\ Address line}
+
+% If the title and author information does not fit in the area allocated,
+% place \setlength\titlebox{<new height>} right after
+% \usepackage{acl2000}
+% where <new height> can be something larger than 2.25in
+
+% \typeout{Conference Style for ACL 2000 -- released June 20, 2000}
+\typeout{Conference Style for ACL 2005 -- released Octobe 11, 2004}
+
+% NOTE: Some laser printers have a serious problem printing TeX output.
+% These printing devices, commonly known as ``write-white'' laser
+% printers, tend to make characters too light. To get around this
+% problem, a darker set of fonts must be created for these devices.
+%
+
+%% % Physical page layout - slightly modified from IJCAI by pj
+%% \setlength\topmargin{0.0in} \setlength\oddsidemargin{-0.0in}
+%% \setlength\textheight{9.0in} \setlength\textwidth{6.5in}
+%% \setlength\columnsep{0.2in}
+%% \newlength\titlebox
+%% \setlength\titlebox{2.25in}
+%% \setlength\headheight{0pt} \setlength\headsep{0pt}
+%% %\setlength\footheight{0pt}
+%% \setlength\footskip{0pt}
+%% \thispagestyle{empty} \pagestyle{empty}
+%% \flushbottom \twocolumn \sloppy
+
+%% Original A4 version of page layout
+%% \setlength\topmargin{-0.45cm} % changed by Rz -1.4
+%% \setlength\oddsidemargin{.8mm} % was -0cm, changed by Rz
+%% \setlength\textheight{23.5cm}
+%% \setlength\textwidth{15.8cm}
+%% \setlength\columnsep{0.6cm}
+%% \newlength\titlebox
+%% \setlength\titlebox{2.00in}
+%% \setlength\headheight{5pt}
+%% \setlength\headsep{0pt}
+%% \setlength\footheight{0pt}
+%% \setlength\footskip{0pt}
+%% \thispagestyle{empty}
+%% \pagestyle{empty}
+
+% A4 modified by Eneko
+\setlength{\paperwidth}{21cm} % A4
+\setlength{\paperheight}{29.7cm}% A4
+\setlength\topmargin{-0.5cm}
+\setlength\oddsidemargin{0cm}
+\setlength\textheight{24.7cm}
+\setlength\textwidth{16.0cm}
+\setlength\columnsep{0.6cm}
+\newlength\titlebox
+\setlength\titlebox{2.00in}
+\setlength\headheight{5pt}
+\setlength\headsep{0pt}
+\thispagestyle{empty}
+\pagestyle{empty}
+
+
+\flushbottom \twocolumn \sloppy
+
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip at sandia-2
+\def\addcontentsline#1#2#3{}
+
+% Title stuff, taken from deproc.
+\def\maketitle{\par
+ \begingroup
+ \def\thefootnote{\fnsymbol{footnote}}
+ \def\@makefnmark{\hbox to 0pt{$^{\@thefnmark}$\hss}}
+ \twocolumn[\@maketitle] \@thanks
+ \endgroup
+ \setcounter{footnote}{0}
+ \let\maketitle\relax \let\@maketitle\relax
+ \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax}
+\def\@maketitle{\vbox to \titlebox{\hsize\textwidth
+ \linewidth\hsize \vskip 0.125in minus 0.125in \centering
+ {\Large\bf \@title \par} \vskip 0.2in plus 1fil minus 0.1in
+ {\def\and{\unskip\enspace{\rm and}\enspace}%
+ \def\And{\end{tabular}\hss \egroup \hskip 1in plus 2fil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}%
+ \def\AND{\end{tabular}\hss\egroup \hfil\hfil\egroup
+ \vskip 0.25in plus 1fil minus 0.125in
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf\@author
+ \end{tabular}\hss\egroup
+ \hfil\hfil\egroup}
+ \vskip 0.3in plus 2fil minus 0.1in
+}}
+
+% margins for abstract
+\renewenvironment{abstract}%
+ {\centerline{\large\bf Abstract}%
+ \begin{list}{}%
+ {\setlength{\rightmargin}{0.6cm}%
+ \setlength{\leftmargin}{0.6cm}}%
+ \item[]\ignorespaces}%
+ {\unskip\end{list}}
+
+%\renewenvironment{abstract}{\centerline{\large\bf
+% Abstract}\vspace{0.5ex}\begin{quote}}{\par\end{quote}\vskip 1ex}
+
+
+% bibliography
+
+\def\thebibliography#1{\section*{References}
+ \global\def\@listi{\leftmargin\leftmargini
+ \labelwidth\leftmargini \advance\labelwidth-\labelsep
+ \topsep 1pt plus 2pt minus 1pt
+ \parsep 0.25ex plus 1pt \itemsep 0.25ex plus 1pt}
+ \list {[\arabic{enumi}]}{\settowidth\labelwidth{[#1]}\leftmargin\labelwidth
+ \advance\leftmargin\labelsep\usecounter{enumi}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy
+ \sfcode`\.=1000\relax}
+
+\def\@up#1{\raise.2ex\hbox{#1}}
+
+% most of cite format is from aclsub.sty by SMS
+
+% don't box citations, separate with ; and a space
+% also, make the penalty between citations negative: a good place to break
+% changed comma back to semicolon pj 2/1/90
+% \def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+% \def\@citea{}\@cite{\@for\@citeb:=#2\do
+% {\@citea\def\@citea{;\penalty\@citeseppen\ }\@ifundefined
+% {b@\@citeb}{{\bf ?}\@warning
+% {Citation `\@citeb' on page \thepage \space undefined}}%
+% {\csname b@\@citeb\endcsname}}}{#1}}
+
+% don't box citations, separate with ; and a space
+% Replaced for multiple citations (pj)
+% don't box citations and also add space, semicolon between multiple citations
+\def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@cite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{; }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+ {\csname b@\@citeb\endcsname}}}{#1}}
+
+% Allow short (name-less) citations, when used in
+% conjunction with a bibliography style that creates labels like
+% \citename{<names>, }<year>
+%
+\let\@internalcite\cite
+\def\cite{\def\citename##1{##1, }\@internalcite}
+\def\shortcite{\def\citename##1{}\@internalcite}
+\def\newcite{\def\citename##1{{\frenchspacing##1} (}\@internalciteb}
+
+% Macros for \newcite, which leaves name in running text, and is
+% otherwise like \shortcite.
+\def\@citexb[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@newcite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{;\penalty\@m\ }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+{\csname b@\@citeb\endcsname}}}{#1}}
+\def\@internalciteb{\@ifnextchar [{\@tempswatrue\@citexb}{\@tempswafalse\@citexb[]}}
+
+\def\@newcite#1#2{{#1\if at tempswa, #2\fi)}}
+
+\def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+
+%%% More changes made by SMS (originals in latex.tex)
+% Use parentheses instead of square brackets in the text.
+\def\@cite#1#2{({#1\if at tempswa , #2\fi})}
+
+% Don't put a label in the bibliography at all. Just use the unlabeled format
+% instead.
+\def\thebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{References\@mkboth
+ {References}{References}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthebibliography=\endlist
+
+% Allow for a bibliography of sources of attested examples
+\def\thesourcebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{Sources of Attested Examples\@mkboth
+ {Sources of Attested Examples}{Sources of Attested Examples}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthesourcebibliography=\endlist
+
+\def\@lbibitem[#1]#2{\item[]\if at filesw
+ { \def\protect##1{\string ##1\space}\immediate
+ \write\@auxout{\string\bibcite{#2}{#1}}\fi\ignorespaces}}
+
+\def\@bibitem#1{\item\if at filesw \immediate\write\@auxout
+ {\string\bibcite{#1}{\the\c at enumi}}\fi\ignorespaces}
+
+% sections with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+ -0.5ex minus -.2ex}{1.5ex plus 0.3ex minus .2ex}{\large\bf\raggedright}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-1.8ex plus
+ -0.5ex minus -.2ex}{0.8ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{0.5ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\paragraph{\@startsection{paragraph}{4}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+\def\subparagraph{\@startsection{subparagraph}{5}{\parindent}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+
+% Footnotes
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+
+% Lists and paragraphs
+\parindent 1em
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 2pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+
+\leftmargin 2em \leftmargini\leftmargin \leftmarginii 2em
+\leftmarginiii 1.5em \leftmarginiv 1.0em \leftmarginv .5em \leftmarginvi .5em
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+ \labelwidth\leftmarginii\advance\labelwidth-\labelsep
+ \topsep 2pt plus 1pt minus 0.5pt
+ \parsep 1pt plus 0.5pt minus 0.5pt
+ \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+ \labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+ \topsep 1pt plus 0.5pt minus 0.5pt
+ \parsep \z@ \partopsep 0.5pt plus 0pt minus 0.5pt
+ \itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+ \labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+ \labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+ \labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\@normalsize{\@setsize\normalsize{11pt}\xpt\@xpt}
+\def\small{\@setsize\small{10pt}\ixpt\@ixpt}
+\def\footnotesize{\@setsize\footnotesize{10pt}\ixpt\@ixpt}
+\def\scriptsize{\@setsize\scriptsize{8pt}\viipt\@viipt}
+\def\tiny{\@setsize\tiny{7pt}\vipt\@vipt}
+\def\large{\@setsize\large{14pt}\xiipt\@xiipt}
+\def\Large{\@setsize\Large{16pt}\xivpt\@xivpt}
+\def\LARGE{\@setsize\LARGE{20pt}\xviipt\@xviipt}
+\def\huge{\@setsize\huge{23pt}\xxpt\@xxpt}
+\def\Huge{\@setsize\Huge{28pt}\xxvpt\@xxvpt}
diff --git a/papers/acl-06/rdparser.eps.gz b/papers/acl-06/rdparser.eps.gz
new file mode 100644
index 0000000..2da2c80
Binary files /dev/null and b/papers/acl-06/rdparser.eps.gz differ
diff --git a/papers/acl-06/srparser.eps.gz b/papers/acl-06/srparser.eps.gz
new file mode 100644
index 0000000..52ae950
Binary files /dev/null and b/papers/acl-06/srparser.eps.gz differ
diff --git a/papers/acl-08/acl-08.bib b/papers/acl-08/acl-08.bib
new file mode 100644
index 0000000..7f81b39
--- /dev/null
+++ b/papers/acl-08/acl-08.bib
@@ -0,0 +1,204 @@
+ at InProceedings{BaldridgeErk08,
+ author = {Jason Baldridge and Katrin Erk},
+ title = {Teaching computational linguistics to a large, diverse student body: courses, tools, and interdepartmental interaction},
+ booktitle = {Proceedings of the Third Workshop on Issues in Teaching Computational Linguistics},
+ year = {2008},
+ publisher = {Association for Computational Linguistics}
+}
+
+ at Article{Barker83,
+ author = {Ricky Barker and E. A. Unger},
+ title = {A predictor for success in an introductory programming class based upon abstract reasoning development},
+ journal = {ACM SIGCSE Bulletin},
+ volume = {15},
+ pages = {154--158},
+ year = 1983
+}
+
+ at InProceedings{Bird05icon,
+ author = {Steven Bird},
+ title = {{NLTK-Lite}: Efficient Scripting for Natural Language Processing},
+ booktitle = {4th International Conference on Natural Language Processing, Kanpur, India},
+ pages = {1--8},
+ year = 2005
+}
+
+ at InProceedings{Bird06,
+ author = {Bird, Steven},
+ title = {{NLTK}: The {Natural} {Language} {Toolkit}},
+ booktitle = {Proceedings of the COLING/ACL 2006 Interactive Presentation Sessions},
+ month = {July},
+ year = {2006},
+ address = {Sydney, Australia},
+ publisher = {Association for Computational Linguistics},
+ pages = {69--72}
+}
+
+ at InProceedings{Bird06nltk,
+ author = {Bird, Steven},
+ title = {{NLTK}: The {Natural} {Language} {Toolkit}},
+ booktitle = {Proceedings of the COLING/ACL 2006 Interactive Presentation Sessions},
+ month = {July},
+ year = {2006},
+ address = {Sydney, Australia},
+ publisher = {Association for Computational Linguistics},
+ pages = {69--72}
+}
+
+ at InProceedings{Bird08curriculum,
+ author = {Bird, Steven},
+ title = {Defining a Core Body of Knowledge for the Introductory
+ Computational Linguistics Curriculum},
+ booktitle = {Proceedings of the Third Workshop on Issues in Teaching Computational Linguistics},
+ year = {2008},
+ publisher = {Association for Computational Linguistics}
+}
+
+ at Misc{BirdKleinLoper08,
+ author = {Steven Bird and Ewan Klein and Edward Loper},
+ title = {{Natural Language Processing in Python}},
+ note = {\url{http://nltk.org/book.html}},
+ year = 2008
+}
+
+ at inproceedings{BirdLoper04,
+ author = {Bird, Steven and Loper, Edward},
+ title = {{NLTK: The Natural Language Toolkit}},
+ booktitle = {Companion Volume to the Proceedings of 42st Annual Meeting of the Association for Computational Linguistics},
+ editor = {},
+ year = 2004,
+ publisher = {Association for Computational Linguistics},
+ pages = {214--217}
+}
+
+ at Book{BonwellEison91,
+ author = {Charles C.\ Bonwell and James A.\ Eison},
+ title = {Active Learning: Creating Excitement in the Classroom},
+ booktitle = {AEHE-ERIC Higher Education Report, No.\ 1},
+ publisher = {Washington, D.C.: Jossey-Bass},
+ pages = {1-6},
+ year = 1991
+}
+
+ at article{Caspersen07,
+ author = {Michael Caspersen and Kasper Larsen and Jens Bennedsen},
+ title = {Mental models and programming aptitude},
+ journal = {SIGCSE Bulletin},
+ volume = {39},
+ year = {2007},
+ pages = {206--210},
+ publisher = {ACM}
+}
+
+ at InProceedings{Hearst05,
+ author = {Hearst, Marti},
+ title = {Teaching Applied Natural Language Processing: Triumphs and Tribulations},
+ booktitle = {Proceedings of the Second ACL Workshop on Effective Tools and Methodologies for Teaching NLP and CL},
+ month = {June},
+ year = {2005},
+ address = {Ann Arbor, Michigan},
+ publisher = {Association for Computational Linguistics},
+ pages = {1--8}
+}
+
+ at InProceedings{Klein06altw,
+ author = {Ewan Klein},
+ title = {Computational semantics in the {Natural Language Toolkit}},
+ booktitle = {Proceedings of the Australasian Language Technology Workshop},
+ pages = {26--33},
+ year = 2006
+}
+
+ at InProceedings{Liddy05,
+ author = {Liddy, Elizabeth and McCracken, Nancy},
+ title = {Hands-On {NLP} for an Interdisciplinary Audience},
+ booktitle = {Proceedings of the Second ACL Workshop on Effective Tools and Methodologies for Teaching NLP and CL},
+ month = {June},
+ year = {2005},
+ address = {Ann Arbor, Michigan},
+ publisher = {Association for Computational Linguistics},
+ pages = {62--68}
+}
+
+ at InProceedings{Loper04,
+ author = {Edward Loper},
+ title = {{NLTK}: Building a Pedagogical Toolkit in {Python}},
+ booktitle = {PyCon DC 2004},
+ year = 2004,
+ publisher = {Python Software Foundation}
+}
+
+ at InProceedings{LoperBird02,
+ author = {Edward Loper and Steven Bird},
+ title = {{NLTK: The Natural Language Toolkit}},
+ booktitle = {Proceedings of the ACL Workshop on Effective Tools and
+ Methodologies for Teaching Natural Language Processing and Computational
+ Linguistics},
+ year = 2002,
+ publisher={Association for Computational Linguistics},
+ pages={62--69}
+}
+
+ at Article{Madnani07,
+ author = {Nitin Madnani},
+ title = {Getting Started on Natural Language Processing with {Python}},
+ journal = {ACM Crossroads},
+ volume = 13,
+ number = 4,
+ year = 2007
+}
+
+ at InProceedings{MadnaniDorr08,
+ author = {Nitin Madnani and Bonnie Dorr},
+ title = {Combining Open-Source with Research to Re-engineer a Hands-on Introductory {NLP} Course},
+ booktitle = {Proceedings of the Third Workshop on Issues in Teaching Computational Linguistics},
+ year = {2008},
+ publisher = {Association for Computational Linguistics}
+}
+
+ at Misc{McCune08,
+ author = {William McCune},
+ title = {Prover9: Automated theorem prover for first-order and equational logic},
+ year = 2008,
+ note = {\url{http://www.cs.unm.edu/~mccune/mace4/manual-examples.html}}
+}
+
+ at Article{Robinson07,
+ author = {Stuart Robinson and Greg Aumann and Steven Bird},
+ title = {Managing fieldwork data with {Toolbox} and the {Natural Language Toolkit}},
+ journal = {Language Documentation and Conservation},
+ year = 2007,
+ volume = 1,
+ pages = {44--57}
+}
+
+ at InProceedings{Shannon03,
+ author = {Christine Shannon},
+ title = {Another breadth-first approach to {CS I} using {Python}},
+ booktitle = {Proceedings of the 34th SIGCSE Technical Symposium on Computer Science Education},
+ pages = {248--251},
+ year = 2003,
+ publisher = {ACM}
+}
+
+ at Book{WittenFrank05,
+ author = {Ian H. Witten and Eibe Frank},
+ title = {Data Mining: Practical machine learning tools and techniques},
+ publisher = {Morgan Kaufmann},
+ year = 2005
+}
+
+ at Misc{matplotlib,
+ author = {Matplotlib},
+ year = 2008,
+ title = {Matplotlib: Python {2D} Plotting Library},
+ note = {\url{http://matplotlib.sourceforge.net/}}
+}
+
+
+ at Misc{numpy,
+ author = {NumPy},
+ year = 2008,
+ title = {{NumPy}: Scientific Computing with {Python}},
+ note = {\url{http://numpy.scipy.org/}}
+}
diff --git a/papers/acl-08/acl-08.tex b/papers/acl-08/acl-08.tex
new file mode 100644
index 0000000..3da8d4d
--- /dev/null
+++ b/papers/acl-08/acl-08.tex
@@ -0,0 +1,749 @@
+% $Rev%
+\documentclass[11pt]{article}
+\usepackage{acl08}
+\usepackage{times}
+\usepackage{latexsym}
+\usepackage{epsfig,url}
+\usepackage{alltt}
+
+\newcommand{\NLP}{\textsc{nlp}}
+\newcommand{\NLTK}{\textsc{nltk}}
+\newcommand{\code}[1]{\texttt{\small #1}}
+
+\pretolerance 250
+\tolerance 500
+\hyphenpenalty 200
+\exhyphenpenalty 100
+\doublehyphendemerits 7500
+\finalhyphendemerits 7500
+\brokenpenalty 10000
+\lefthyphenmin 3
+\righthyphenmin 3
+\looseness 1
+
+\setlength\titlebox{6.5cm} % Expanding the titlebox
+
+\title{Multidisciplinary Instruction with the Natural Language Toolkit}
+
+\author{Steven Bird \\
+ Department of Computer Science\\
+ University of Melbourne\\
+ {\small\tt stevenbird1 at gmail.com} \And
+ Ewan Klein\\
+ School of Informatics\\
+ University of Edinburgh\\
+ {\small\tt ewan at inf.ed.ac.uk} \AND
+ Edward Loper\\
+ Computer and Information Science\\
+ University of Pennsylvania\\
+ {\small\tt edloper at gmail.com} \And
+ Jason Baldridge\\
+ Department of Linguistics\\
+ University of Texas at Austin\\
+ {\small\tt jbaldrid at mail.utexas.edu}
+}
+
+\date{}
+
+\begin{document}
+\maketitle
+
+\begin{abstract}
+ The Natural Language Toolkit (\NLTK) is widely used for teaching
+ natural language processing to students majoring in linguistics or
+ computer science. This paper describes the design of \NLTK, and
+ reports on how it has been used effectively in classes that involve
+ different mixes of
+ linguistics and computer science students. We focus
+ on three key issues: getting started with a course, delivering
+ interactive demonstrations in the classroom,
+ and organizing assignments and projects.
+ In each case, we report on practical experience and make
+ recommendations on how to use \NLTK\ to maximum effect.
+\end{abstract}
+
+\section{Introduction}
+
+It is relatively easy to teach natural language processing (\NLP) in a
+single-disciplinary mode to a uniform cohort of students. Linguists
+can be taught to program, leading to projects where students
+manipulate their own linguistic data. Computer scientists can be
+taught methods for automatic text processing, leading to projects on
+text mining and chatbots. Yet these approaches have almost nothing in
+common, and it is a stretch to call either of these \NLP: more apt
+titles for such courses might be ``linguistic data management'' and
+``text technologies.''
+
+The Natural Language Toolkit, or \NLTK, was
+developed to give a broad range of students access to the core
+knowledge and skills of \NLP\ \cite{LoperBird02}. In particular, \NLTK\ makes it feasible
+to run a course that covers a substantial amount of theory and
+practice with an audience consisting of both linguists and computer
+scientists. \NLTK\ is a suite of Python modules distributed under the
+GPL open source license via \url{nltk.org}. \NLTK\ comes with a large collection of
+corpora, extensive documentation, and hundreds of exercises, making
+\NLTK\ unique in providing a comprehensive framework for students to
+develop a computational understanding of language. \NLTK's code base
+of 100,000 lines of Python code includes support for corpus access,
+tokenizing, stemming, tagging, chunking, parsing, clustering,
+classification, language modeling, semantic interpretation,
+unification, and much else besides. As a measure of its
+impact, \NLTK\ has been used in over 60 university courses in 20
+countries, listed on the \NLTK\ website.
+
+Since its inception in 2001, \NLTK\ has undergone considerable
+evolution, based on the experience gained by teaching courses at
+several universities, and based on feedback from many teachers and
+students.\footnote{\cite{BirdLoper04,Loper04,Bird05icon,Hearst05,Bird06nltk,Klein06altw,Liddy05,Madnani07,MadnaniDorr08,BaldridgeErk08}}
+Over this period, a series of practical online tutorials
+about \NLTK\ has grown up into a comprehensive online book \cite{BirdKleinLoper08}.
+The book has been designed to stay in lock-step
+with the \NLTK\ library, and is intended to facilitate
+``active learning'' \cite{BonwellEison91}.
+
+This paper describes the main features of \NLTK, and reports on how it has
+been used effectively in classes that involve a combination of
+linguists and computer scientists. First we discuss aspects of the
+design of the toolkit that arose from our need to teach computational
+linguistics to a multidisciplinary audience (\S\ref{sec:design}).
+The following sections cover three distinct challenges:
+getting started with a course (\S\ref{sec:getting-started});
+interactive demonstrations (\S\ref{sec:interactive-demonstrations});
+and organizing assignments and projects (\S\ref{sec:projects}).
+
+\section{Design Decisions Affecting Teaching}
+\label{sec:design}
+
+\subsection{Python}
+
+We chose Python\footnote{\url{http://www.python.org/}} as the
+implementation language for \NLTK\ because it has a shallow learning
+curve, its syntax and semantics are transparent, and it has good
+string-handling functionality. As an interpreted language, Python
+facilitates interactive exploration. As an object-oriented language,
+Python permits data and methods to be encapsulated and re-used easily.
+Python comes with an extensive standard library, including tools for
+graphical programming and numerical processing, which means it can be
+used for a wide range of non-trivial applications. Python is ideal in
+a context serving newcomers and experienced programmers
+\cite{Shannon03}.
+
+We have taken the step of incorporating a detailed introduction to
+Python programming in the \NLTK\ book, taking care to motivate
+programming constructs with linguistic examples. Extensive feedback
+from students has been humbling, and revealed that for students with
+no prior programming experience, it is almost impossible to
+over-explain. Despite the difficulty of providing a
+self-contained introduction to Python for linguists, we nevertheless
+have also had very positive feedback, and in combination with the
+teaching techniques described below, have managed to bring a
+large group of non-programmer students rapidly to a point where they
+could carry out interesting and useful exercises in text processing.
+
+In addition to the \NLTK\ book, the code in the \NLTK\ core is richly
+documented, using Python docstrings and
+Epydoc\footnote{\url{http://epydoc.sourceforge.net/}} support
+for API documentation.\footnote{\url{http://nltk.org/doc/api/}} Access
+to the code documentation is available using the Python \code{help()}
+command at the interactive prompt, and this can be especially useful
+for checking the parameters and return type of functions.
+
+Other Python libraries are useful in the \NLP\ context: NumPy
+provides optimized support for linear algebra and sparse
+arrays \cite{numpy} and PyLab provides
+sophisticated facilities for scientific
+visualization \cite{matplotlib}.
+
+\subsection{Coding Requirements}
+
+As discussed in Loper \& Bird~\shortcite{LoperBird02}, the priorities for \NLTK\ code
+focus on its teaching role. When code is readable, a student who
+doesn't understand the maths of {\sc hmm}s, smoothing, and so on may benefit
+from looking at how an algorithm is implemented. Thus consistency,
+simplicity, modularity are all vital features of \NLTK\ code. A
+similar importance is placed on extensibility, since this helps to
+ensure that the code grows as a coherent whole, rather than by
+unpredictable and haphazard additions.
+
+By contrast, although efficiency cannot be ignored, it has
+always taken second place to simplicity and clarity of coding. In a
+similar vein, we have tried to avoid clever programming tricks,
+since these typically hinder intelligibility of the code. Finally,
+comprehensiveness of coverage has never been an overriding concern of
+\NLTK; this leaves open many possibilities for student projects and
+community involvement.
+
+% comment about restricted scope of NLTK omitted given earlier point
+% about 100,000 lines of code which will certainly overwhelm students
+
+\subsection{Naming}
+
+One issue which has absorbed a considerable amount of attention is the
+naming of user-oriented functions in \NLTK. To a large extent, the
+system of naming \emph{is} the user interface to the toolkit, and it is
+important that users should be able to guess what action might be
+performed by a given function. Consequently, naming conventions need
+to be consistent and semantically transparent. At the same time, there is a
+countervailing pressure for relatively succinct names, since excessive verbosity
+can also hinder comprehension and usability. An additional
+complication is that adopting an object-oriented style of programming
+may be well-motivated for a number of reasons but nevertheless
+baffling to the linguist student. For example, although it is
+perfectly respectable to invoke an instance method
+\code{WordPunctTokenizer().tokenize(text)} (for some input
+string \code{text}), a simpler version is also provided:
+\code{wordpunct\_tokenize(text)}.
+
+\subsection{Corpus Access}
+
+The scope of exercises and projects that students can perform is
+greatly increased by the inclusion of a large collection of corpora,
+along with easy-to-use corpus readers. This collection, which
+currently stands at 45 corpora, includes parsed, POS-tagged, plain
+text, categorized text, and lexicons.\footnote{\url{http://nltk.org/corpora.html}}
+
+\begin{figure*}[t]
+{\small
+\begin{alltt}
+\textbf{>>> nltk.corpus.treebank.tagged_words()}
+[('Pierre', 'NNP'), ('Vinken', 'NNP'), (',', ','), ...]
+\textbf{>>> nltk.corpus.brown.tagged_words()}
+[('The', 'AT'), ('Fulton', 'NP-TL'), ...]
+\textbf{>>> nltk.corpus.floresta.tagged_words()}
+[('Um', '>N+art'), ('revivalismo', 'H+n'), ...]
+\textbf{>>> nltk.corpus.cess_esp.tagged_words()}
+[('El', 'da0ms0'), ('grupo', 'ncms000'), ...]
+\textbf{>>> nltk.corpus.alpino.tagged_words()}
+[('De', 'det'), ('verzekeringsmaatschappijen', 'noun'), ...]
+\end{alltt}}
+\caption{Accessing Different Corpora via a Uniform Interface}
+\label{fig:tagged}
+\vspace*{1ex}\hrule
+\end{figure*}
+
+In designing the corpus readers, we emphasized simplicity,
+consistency, and efficiency. \emph{Corpus objects}, such as
+\code{nltk.corpus.brown} and \code{nltk.corpus.treebank}, define
+common methods for reading the corpus contents, abstracting
+away from idiosyncratic file formats to provide a uniform interface.
+See Figure~\ref{fig:tagged} for an example of accessing POS-tagged
+data from different tagged and parsed corpora.
+
+The corpus objects provide methods for loading corpus contents
+in various ways. Common methods include:
+%
+\code{raw()}, for the raw contents of the corpus;
+\code{words()}, for a list of tokenized words;
+\code{sents()}, for the same list grouped into sentences;
+\code{tagged\_words()}, for a list of (\textit{word}, \textit{tag}) pairs;
+\code{tagged\_sents()}, for the same list grouped into sentences;
+and
+\code{parsed\_sents()}, for a list of parse trees.
+%
+Optional parameters can be used to restrict what portion of the corpus
+is returned, e.g., a particular section, or an individual corpus file.
+
+Most corpus reader methods return a \emph{corpus view} which acts as
+a list of text objects, but maintains responsiveness and memory
+efficiency by only loading items from the file on an as-needed basis.
+Thus, when we print a corpus view we only load the first block of the corpus
+into memory, but when we process this object we load the whole corpus:
+
+{\footnotesize
+\begin{alltt}
+\textbf{>>> nltk.corpus.alpino.words()}
+['De', 'verzekeringsmaatschappijen',
+'verhelen', ...]
+\textbf{>>> len(nltk.corpus.alpino.words())}
+139820
+\end{alltt}}
+
+\subsection{Accessing Shoebox Files}
+
+\NLTK\ provides functionality for working with ``Shoebox'' (or ``Toolbox'') data
+\cite{Robinson07}. Shoebox is a system used by many
+documentary linguists to produce lexicons and interlinear glossed text. The
+ability to work straightforwardly with Shoebox data has created a new
+incentive for linguists to learn how to program.
+
+As an example, in the Linguistics Department at the University of
+Texas at Austin, a course has been offered on Python programming and
+working with
+corpora,\footnote{\url{http://comp.ling.utexas.edu/courses/2007/corpora07/}}
+but so far uptake from the target audience of core linguistics
+students has been low. They usually have practical computational needs
+and many of them are intimidated by the very idea of programming.
+We believe that the appeal of this course can be
+enhanced by designing a significant component
+with the goal of helping documentary linguistics students take control of their
+\emph{own} Shoebox data. This will give them skills that are
+useful for their research and also transferable to other activities.
+Although the \NLTK\ Shoebox functionality was not
+originally designed with instruction in mind, its relevance to
+students of documentary linguistics is highly fortuitous and
+may prove appealing for similar linguistics departments.
+
+\section{Getting Started}
+\label{sec:getting-started}
+
+\NLP\ is usually only available as an elective course, and students
+will vote with their feet after attending one or two classes. This
+initial period is important for attracting and retaining students. In
+particular, students need to get a sense of the richness of language
+in general, and \NLP\ in particular, while gaining a realistic
+impression of what will be accomplished during the course and what
+skills they will have by the end. During this time when rapport needs
+to be rapidly established, it is easy for instructors to alienate
+students through the use of linguistic or computational concepts and
+terminology that are foreign to students, or to bore students by
+getting bogged down in defining terms like ``noun phrase'' or ``function''
+which are basic to one audience and new for the other. Thus, we
+believe it is crucial for instructors to understand and shape the
+student's expectations, and to get off to a good start. The best
+overall strategy that we have found is to use succinct nuggets of
+\NLTK\ code to stimulate students' interest in both data and
+processing techniques.
+
+\subsection{Student Expectations}
+
+Computer science students come to \NLP\ expecting to learn about \NLP\
+algorithms and data structures. They typically have enough
+mathematical preparation to be confident in playing with abstract
+formal systems (including systems of linguistic rules). Moreover,
+they are already proficient in multiple programming languages, and
+have little difficulty in learning \NLP\ algorithms by reading and
+manipulating the implementations provided with \NLTK. At the same
+time, they tend to be unfamiliar with the terminology and concepts
+that linguists take for granted, and may struggle to come up with
+reasonable linguistic analyses of data.
+
+Linguistics students, on the other hand, are interested in
+understanding \NLP\ algorithms and data structures only insofar as it helps them
+to use computational tools to perform analytic tasks from ``core linguistics,''
+e.g.\ writing a set of CFG productions to parse some sentences, or
+plugging together \NLP\ components in order to derive the subcategorization
+requirements of verbs in a corpus.
+They are usually not interested in reading significant chunks of code;
+it isn't what they care about and they
+probably lack the confidence to poke around in source files.
+
+In a nutshell, the computer science students typically want to analyze
+the tools and synthesize new implementations, while the linguists
+typically want to use the tools to analyze language and
+synthesize new theories. There is a risk that the former group
+never really gets to grips with natural language, while the latter
+group never really gets to grips with processing. Instead,
+computer science students need to learn that \NLP\ is not just an
+application of techniques from formal language theory and compiler
+construction, and linguistics students need to understand that \NLP\ is not
+just computer-based housekeeping and a solution to the shortcomings of
+office productivity software for managing their data.
+
+In many courses, linguistics students or computer science students
+will dominate the class numerically, simply because the course is only
+listed in one department. In such cases it is usually enough to
+provide additional support in the form of some extra readings,
+tutorials, and exercises in the opening stages of the course. In
+other cases, e.g.\ courses we have taught at the universities of
+Edinburgh, Melbourne, Pennsylvania, and Texas-Austin or in summer
+intensive programs in several countries, there is more of an even
+split, and the challenge of serving both cohorts of students becomes
+acute. It helps to address this issue head-on, with an early
+discussion of the goals of the course.
+
+\subsection{Articulating the Goals}
+
+Despite an instructor's efforts to add a cross-disciplinary angle, students
+easily ``revert to type.'' The pressure of assessment encourages students to emphasize
+what they do well. Students' desire to understand what is expected of them encourages
+instructors to stick to familiar assessment instruments. As a consequence,
+the path of least resistance is for students to remain firmly
+monolingual in their own discipline, while
+acquiring a smattering of words from a foreign language, at a level we might
+call ``survival linguistics'' or ``survival computer science.''
+If they ever get to work in a multidisciplinary team they are likely
+only to play a type-cast role.
+
+Asking computer science students to write their first essay in years,
+or asking linguistics students to write their first ever program,
+leads to stressed students who complain that they don't know what is
+expected of them. Nevertheless, students need to confront the
+challenge of becoming bilingual, of working hard to learn the basics
+of another discipline. In parallel, instructors need to confront the
+challenge of synthesizing material from linguistics and computer
+science into a coherent whole, and devising effective methods for
+teaching, learning, and assessment.
+
+\subsection{Entry Points}
+
+It is possible to identify several distinct pathways into the field of
+Computational Linguistics. \newcite{Bird08curriculum} identifies four;
+each of these are supported by \NLTK, as detailed below:
+
+\textbf{Text Processing First:} \NLTK\ supports variety of approaches to tokenization,
+tagging, evaluation, and language engineering more generally.
+
+\textbf{Programming First:} \NLTK\ is based on Python and
+the documentation teaches the language and provides many examples and
+exercises to test and reinforce student learning.
+
+\textbf{Linguistics First:} Here, students come with a grounding in
+one or more areas of linguistics, and focus on computational
+approaches to that area by working with the relevant
+chapter of the \NLTK\ book in conjunction with learning how to program.
+
+\textbf{Algorithms First:} Here, students come with a grounding in
+one or more areas of computer science, and can use, test and extend
+\NLTK'S reference implementations of standard NLP algorithms.
+
+\subsection{The First Lecture}
+
+It is important that the first lecture is effective at motivating and
+exemplifying \NLP\ to an audience of computer science and linguistics
+students. They need to get an accurate sense of the interesting
+conceptual and technical challenges awaiting them. Fortunately, the
+task is made easier by the simple fact that language technologies, and
+language itself, are intrinsically interesting and appealing to a wide audience.
+Several opening topics appear to work particularly well:
+
+\textbf{The holy grail:}
+A long term challenge, mythologized in science fiction movies, is to
+build machines that understand human language. Current technologies
+that exhibit some basic level of natural language understanding include
+spoken dialogue systems, question answering systems, summarization
+systems, and machine translation systems. These can be demonstrated
+in class without too much difficulty. The Turing test is a linguistic
+test, easily understood by all students, and which helps the computer science
+students to see \NLP\ in relation to the field of Artificial Intelligence.
+The evolution of programming languages has brought them closer to natural language,
+helping students see the essentially linguistic purpose of this central development
+in computer science. The corresponding holy grail in linguistics is full
+understanding of the human language faculty; writing programs and building machines
+surely informs this quest too.
+
+\textbf{The riches of language:}
+It is easy to find examples of the creative richness of language in
+its myriad uses. However, linguists will understand that language
+contains hidden riches that can only be uncovered by careful analysis
+of large quantities of linguistically annotated data, work that
+benefits from suitable computational tools. Moreover, the
+computational needs for exploratory linguistic research often go
+beyond the capabilities of the current tools. Computer scientists
+will appreciate the cognate problem of extracting information from the
+web, and the economic riches associated with state-of-the-art text
+mining technologies.
+
+\textbf{Formal approaches to language:}
+Computer science and linguistics have a shared history in the area of
+philosophical logic and formal language theory. Whether the language
+is natural or artificial, computer scientists and linguists use
+similar logical formalisms for investigating the formal semantics of
+languages, similar grammar formalisms for modeling the syntax of
+languages, and similar finite-state methods for manipulating text.
+Both rely on the recursive, compositional nature of natural and
+artificial languages.
+
+\subsection{First Assignment}
+
+The first coursework assignment can be a significant step forwards in
+helping students get to grips with the material, and is best given out
+early, perhaps even in week 1. We have found it advisable for this
+assignment to include both programming and linguistics content. One
+example is to ask students to carry out NP chunking of some data
+(e.g.\ a section of the Brown Corpus). The \code{nltk.RegexpParser}
+class is initialized with a set of chunking rules expressed in a
+simple, regular expression-oriented syntax, and the resulting chunk
+parser can be run over POS-tagged input text. Given a Gold Standard
+test set like the CoNLL-2000
+data,\footnote{\url{http://www.cnts.ua.ac.be/conll2000/chunking/}}
+precision and recall of the chunk grammar can be easily determined.
+Thus, if students are given an existing, incomplete set of rules as
+their starting point, they just have to modify and test their rules.
+
+There are distinctive outcomes for each set of students: linguistics students
+learn to write grammar fragments that respect the literal-minded
+needs of the computer, and also come to appreciate the noisiness of
+typical \NLP\ corpora (including automatically annotated corpora like
+CoNLL-2000).
+Computer science students become more familiar with parts of speech
+and with typical syntactic structures in English. Both groups learn
+the importance of formal evaluation using precision and recall.
+
+\section{Interactive Demonstrations}
+\label{sec:interactive-demonstrations}
+
+\subsection{Python Demonstrations}
+
+Python fosters a highly interactive style of teaching. It is quite
+natural to build up moderately complex programs in front of a class,
+with the less confident students transcribing it into a Python session on
+their laptop to satisfy themselves it works (but not necessarily
+understanding everything they enter first time), while the stronger
+students quickly grasp the theoretical concepts and algorithms. While
+both groups can be served by the same presentation, they tend to ask
+quite different questions. However, this is addressed by dividing
+them into smaller clusters and having teaching assistants visit them
+separately to discuss issues arising from the content.
+
+The \NLTK\ book contains many examples, and the instructor
+can present an interactive lecture that includes running these examples
+and experimenting with them in response to student questions. In early
+classes, the focus will probably be on learning Python. In later classes,
+the driver for such interactive lessons can be an externally-motivated
+empirical or theoretical question.
+
+As a practical matter, it is important to consider low-level issues
+that may get in the way of students' ability to capture the material
+covered in interactive Python sessions. These include choice of
+appropriate font size for screen display, avoiding the problem of output scrolling the
+command out of view, and distributing a log of the instructor's interactive session
+for students to study in their own time.
+
+
+\subsection{NLTK Demonstrations}
+
+A significant fraction of any \NLP\ syllabus covers fundamental data
+structures and algorithms. These are usually taught with the help of
+formal notations and complex diagrams. Large trees and charts are
+copied onto the board and edited in tedious slow motion, or
+laboriously prepared for presentation slides. It is more effective to
+use live demonstrations in which those diagrams are generated and
+updated automatically. \NLTK\ provides interactive graphical user
+interfaces, making it possible to view program state and to study
+program execution step-by-step. Most \NLTK\ components have a
+demonstration mode, and will perform an interesting task without
+requiring any special input from the user. It is even possible to make
+minor modifications to programs in response to ``what if'' questions. In
+this way, students learn the mechanics of \NLP\ quickly, gain deeper
+insights into the data structures and algorithms, and acquire new
+problem-solving skills.
+
+An example of a particularly effective set of demonstrations are those for
+shift-reduce and recursive descent parsing. These make the difference
+between the algorithms glaringly obvious. More importantly, students
+get a concrete sense of many issues that affect the design of
+algorithms for tasks like parsing. The partial analysis constructed by
+the recursive descent parser bobs up and down as it steps forward and
+backtracks, and students often go wide-eyed as the parser retraces its
+steps and does ``dumb'' things like expanding N to {\it man} when it
+has already tried the rule unsuccessfully (but is now trying to match
+a bare NP rather than an NP with a PP modifier). Linguistics students
+who are extremely knowledgeable about context-free grammars and thus
+understand the representations gain a new appreciation for just how
+naive an algorithm can be. This helps students grasp
+the need for techniques like dynamic programming and motivates
+them to learn how they can be used to solve such problems
+much more efficiently.
+
+Another highly useful aspect of \NLTK\ is the ability to define a
+context-free grammar using a simple format and to display tree
+structures graphically. This can be used to teach context-free
+grammars interactively, where the instructor and the students develop
+a grammar from scratch and check its coverage against a testbed of
+grammatical and ungrammatical sentences. Because it is so easy to
+modify the grammar and check its behavior, students readily
+participate and suggest various solutions. When the grammar produces
+an analysis for an ungrammatical sentence in the testbed, the tree
+structure can be displayed graphically and inspected to see what went
+wrong. Conversely, the parse chart can be inspected to see where the
+grammar failed on grammatical sentences.
+
+\NLTK's easy access to many corpora greatly facilitates classroom
+instruction. It is straightforward to pull in different sections of
+corpora and build programs in class for many different tasks.
+This not only makes it easier to experiment with ideas on the fly,
+but also allows students to replicate the
+exercises outside of class. Graphical displays that show the
+dispersion of terms throughout a text also give students excellent
+examples of how a few simple statistics collected from a corpus can
+provide useful and interesting views on a text---including seeing the
+frequency with which various characters appear in a novel. This can in
+turn be related to other resources like Google Trends, which shows the
+frequency with which a term has been referenced in news reports or
+been used in search terms over several years.
+
+% \subsection{Small Group Interaction}
+%
+% Even the most engaging interactive Python demonstration may only
+% amount to a demonstration of the instructor's Python prowess ('charming python').
+% Student learning is enhanced when they are encouraged to engage actively
+% with the material, responding to a quiz, discussing ...
+
+% animate this with a quiz, presented as a slide or a handout, giving code samples and asking what they do.
+% could be a competition
+% class exercise, e.g. counting uses of ``must'' in a spoken vs written language corpus
+% (deontic vs epistemic uses).
+% chatroom for online discussion: useful during intensive summer program; otherwise couldn't be staffed adequately
+
+\section{Exercises, Assignments and Projects}
+\label{sec:projects}
+
+\subsection{Exercises}
+
+Copious exercises are provided with the \NLTK\ book; these
+have been graded for difficulty relative to the concepts covered in the
+preceding sections of the book. Exercises have the tremendous advantage of
+building on the \NLTK\ infrastructure, both code and
+documentation. The exercises are intended to be suitable both for
+self-paced learning and in formally assigned coursework.
+
+A mixed class of linguistics and computer science students will have a
+diverse range of programming experience, and students with no
+programming experience will typically have different aptitudes for
+programming \cite{Barker83,Caspersen07}. A course which forces all students to
+progress at the same rate will be too difficult for some, and too dull
+for others, and will risk alienating many students. Thus, course
+materials need to accommodate self-paced learning. An effective way
+to do this is to provide students with contexts in which they can test
+and extend their knowledge at their own rate.
+
+One such context is provided by lecture or laboratory sessions in
+which students have a machine in front of them (or one between two),
+and where there is time to work through a series of exercises to
+consolidate what has just been taught from the front, or read from a
+chapter of the book. When this can be done at regular intervals, it is easier for
+students to know which part of the materials to re-read. It also
+encourages them to get into the habit of checking their understanding of a concept by writing
+code.
+
+When exercises are graded for difficulty, it is easier for students to
+understand how much effort is expected, and whether they even have
+time to attempt an exercise. Graded exercises are also good for
+supporting self-evaluation. If a student takes 20 minutes to write a
+solution, they also need to have some idea of whether this was an
+appropriate amount of time.
+
+% well-motivated questions: needs to be transparently obvious why this
+% is a linguistically meaningful thing to do (esp for inexperienced
+% programmers)
+
+% include open-ended questions for the experienced programmers
+
+% For students who are
+% learning to program as part of a computational linguistics course, the
+% parallels between the examples in the documentation and the
+% requirements of the assignments is very helpful. As a result, they
+% are able use \NLTK\ to carry out far more complex tasks than they could
+% otherwise have hoped to do.
+% The availability of online examples that
+% they could try out in interactive Python were a huge help for them.
+
+
+\begin{figure}
+{\footnotesize
+\begin{verbatim}
+nltk.FreqDist(nltk.corpus.brown.words())
+\end{verbatim}
+\medskip
+
+\begin{verbatim}
+fd = nltk.FreqDist()
+for filename in corpus_files:
+ text = open(filename).read()
+ for w in nltk.wordpunct_tokenize(text):
+ fd.inc(w)
+\end{verbatim}
+\medskip
+
+\begin{verbatim}
+counts = {}
+for w in nltk.corpus.brown.words():
+ if w not in counts:
+ counts[w] = 0
+ counts[w] += 1
+\end{verbatim}
+}
+\caption{Three Ways to Build up a Frequency Distribution of Words in the Brown Corpus}
+\label{fig:freqdist}
+\vspace*{1ex}
+\hrule
+\end{figure}
+
+The exercises are also highly adaptable. It is common for instructors
+to take them as a starting point in building homework assignments that
+are tailored to their own students. Some instructors prefer to
+include exercises that do not allow students to take advantage of
+built-in \NLTK\ functionality, e.g.\ using a Python dictionary to
+count word frequencies in the Brown corpus rather than \NLTK 's
+\code{FreqDist} (see Figure~\ref{fig:freqdist}). This is an important
+part of building facility with general text processing in Python,
+since eventually students will have to work outside of the \NLTK\ sandbox.
+Nonetheless, students often use \NLTK\ functionality as part
+of their solutions, e.g., for managing frequencies and
+distributions. Again, this flexibility is a good thing: students learn
+to work with resources they know how to use, and can branch out to new
+exercises from that basis. When course content includes discussion of
+Unix command line utilities for text processing, students can
+furthermore gain a better appreciation of the pros and cons of writing
+their own scripts versus using an appropriate Unix pipeline.
+
+\subsection{Assignments}
+
+\NLTK\ supports assignments of varying difficulty and scope:
+experimenting with existing components to see what happens for
+different inputs or parameter settings;
+modifying existing components and creating systems using existing components;
+leveraging \NLTK's extensible architecture by developing entirely new components;
+or employing \NLTK's interfaces to other toolkits such as Weka \cite{WittenFrank05}
+and Prover9 \cite{McCune08}.
+
+\subsection{Projects}
+
+Group projects involving a mixture of linguists and computer science
+students have an initial appeal, assuming that each kind of student
+can learn from the other. However, there's a complex social dynamic
+in such groups, one effect of which is that the linguistics students
+may opt out of the programming aspects of the task, perhaps with view
+that their contribution would only hurt the chances of achieving a
+good overall project mark. It is difficult to mandate significant
+collaboration across disciplinary boundaries, with the more likely
+outcome being, for example, that a parser is developed by a computer
+science team member, then thrown over the wall to a linguist who will
+develop an appropriate grammar.
+
+Instead, we believe that it is generally more productive in the
+context of a single-semester introductory course to have students work individually
+on their own projects. Distinct projects can be devised for students
+depending on their background, or students can be given a list of
+project topics,\footnote{\url{http://nltk.org/projects.html}} and offered option of
+self-proposing other projects.
+
+% Peer review (including code review) to improve quality of programming, and
+% emphasize the communicative dimension of programming.
+% (Even grade a student on the quality of his/her peer review of another student.)
+
+\section{Conclusion}
+
+We have argued that the distinctive features of \NLTK\ make it an apt
+vehicle for teaching \NLP\ to mixed audiences of linguistic and
+computer science students. On the one hand, complete novices can
+quickly gain confidence in their ability to do interesting and useful things
+with language processing, while the transparency and consistency of
+the implementation also makes it easy for experienced programmers to
+learn about natural language and to explore more challenging
+tasks. The success of this recipe is borne out by the wide
+uptake of the toolkit, not only within tertiary education but more
+broadly by users who just want try their hand at \NLP. We also have
+encouraging results in presenting \NLTK\ in classrooms at the
+secondary level, thereby trying to inspire the computational linguists of the
+future!
+
+Finally, we believe that \NLTK\ has gained much by participating
+in the Open Source software movement, specifically from
+the infrastructure provided by
+\url{SourceForge.net} and from the
+invaluable contributions of a wide range of people, including many
+students.
+
+\section{Acknowledgments}
+
+We are grateful to the members of the \NLTK\ community for their
+helpful feedback on the toolkit and their many contributions.
+We thank the anonymous reviewers for their feedback on an earlier
+version of this paper.
+
+\clearpage
+\bibliographystyle{acl}
+\bibliography{acl-08}
+
+\end{document}
diff --git a/papers/acl-08/acl08.sty b/papers/acl-08/acl08.sty
new file mode 100644
index 0000000..86092b5
--- /dev/null
+++ b/papers/acl-08/acl08.sty
@@ -0,0 +1,344 @@
+% File acl2005.sty
+% October 11, 2004
+% Contact: oflazer at sabanciuniv.edu
+
+% This is the LaTeX style file for ACL 2005. It is nearly identical to the
+% style files for ACL 2002, ACL 2001, ACL 2000, EACL 95 and EACL
+% 99.
+%
+
+% This is the LaTeX style file for ACL 2000. It is nearly identical to the
+% style files for EACL 95 and EACL 99. Minor changes include editing the
+% instructions to reflect use of \documentclass rather than \documentstyle
+% and removing the white space before the title on the first page
+% -- John Chen, June 29, 2000
+
+% To convert from submissions prepared using the style file aclsub.sty
+% prepared for the ACL 2000 conference, proceed as follows:
+% 1) Remove submission-specific information: \whichsession, \id,
+% \wordcount, \otherconferences, \area, \keywords
+% 2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% 3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% 4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+% 5) Change the style reference from aclsub to acl2000, and be sure
+% this style file is in your TeX search path
+
+
+% This is the LaTeX style file for EACL-95. It is identical to the
+% style file for ANLP '94 except that the margins are adjusted for A4
+% paper. -- abney 13 Dec 94
+
+% The ANLP '94 style file is a slightly modified
+% version of the style used for AAAI and IJCAI, using some changes
+% prepared by Fernando Pereira and others and some minor changes
+% by Paul Jacobs.
+
+% Papers prepared using the aclsub.sty file and acl.bst bibtex style
+% should be easily converted to final format using this style.
+% (1) Submission information (\wordcount, \subject, and \makeidpage)
+% should be removed.
+% (2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% (between \begin{abstract} and \end{abstract}).
+% (3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% (4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+
+% Place this in a file called aclap.sty in the TeX search path.
+% (Placing it in the same directory as the paper should also work.)
+
+% Prepared by Peter F. Patel-Schneider, liberally using the ideas of
+% other style hackers, including Barbara Beeton.
+% This style is NOT guaranteed to work. It is provided in the hope
+% that it will make the preparation of papers easier.
+%
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at research.att.com
+
+% Papers are to be prepared using the ``acl'' bibliography style,
+% as follows:
+% \documentclass[11pt]{article}
+% \usepackage{acl2000}
+% \title{Title}
+% \author{Author 1 \and Author 2 \\ Address line \\ Address line \And
+% Author 3 \\ Address line \\ Address line}
+% \begin{document}
+% ...
+% \bibliography{bibliography-file}
+% \bibliographystyle{acl}
+% \end{document}
+
+% Author information can be set in various styles:
+% For several authors from the same institution:
+% \author{Author 1 \and ... \and Author n \\
+% Address line \\ ... \\ Address line}
+% if the names do not fit well on one line use
+% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
+% For authors from different institutions:
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \And ... \And
+% Author n \\ Address line \\ ... \\ Address line}
+% To start a seperate ``row'' of authors use \AND, as in
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \AND
+% Author 2 \\ Address line \\ ... \\ Address line \And
+% Author 3 \\ Address line \\ ... \\ Address line}
+
+% If the title and author information does not fit in the area allocated,
+% place \setlength\titlebox{<new height>} right after
+% \usepackage{acl2000}
+% where <new height> can be something larger than 2.25in
+
+% \typeout{Conference Style for ACL 2000 -- released June 20, 2000}
+\typeout{Conference Style for ACL 2005 -- released Octobe 11, 2004}
+
+% NOTE: Some laser printers have a serious problem printing TeX output.
+% These printing devices, commonly known as ``write-white'' laser
+% printers, tend to make characters too light. To get around this
+% problem, a darker set of fonts must be created for these devices.
+%
+
+% Physical page layout - slightly modified from IJCAI by pj
+\setlength\topmargin{0.0in} \setlength\oddsidemargin{-0.0in}
+\setlength\textheight{9.0in} \setlength\textwidth{6.5in}
+\setlength\columnsep{0.2in}
+\newlength\titlebox
+\setlength\titlebox{2.25in}
+\setlength\headheight{0pt} \setlength\headsep{0pt}
+%\setlength\footheight{0pt}
+\setlength\footskip{0pt}
+\thispagestyle{empty} \pagestyle{empty}
+\flushbottom \twocolumn \sloppy
+
+%% A4 version of page layout
+%\setlength\topmargin{-0.45cm} % changed by Rz -1.4
+%\setlength\oddsidemargin{.8mm} % was -0cm, changed by Rz
+%\setlength\textheight{23.5cm}
+%\setlength\textwidth{15.8cm}
+%\setlength\columnsep{0.6cm}
+%\newlength\titlebox
+%\setlength\titlebox{2.00in}
+%\setlength\headheight{5pt}
+%\setlength\headsep{0pt}
+%%\setlength\footheight{0pt}
+%\setlength\footskip{0pt}
+%\thispagestyle{empty}
+%\pagestyle{empty}
+
+\flushbottom \twocolumn \sloppy
+
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip at sandia-2
+\def\addcontentsline#1#2#3{}
+
+% Title stuff, taken from deproc.
+\def\maketitle{\par
+ \begingroup
+ \def\thefootnote{\fnsymbol{footnote}}
+ \def\@makefnmark{\hbox to 0pt{$^{\@thefnmark}$\hss}}
+ \twocolumn[\@maketitle] \@thanks
+ \endgroup
+ \setcounter{footnote}{0}
+ \let\maketitle\relax \let\@maketitle\relax
+ \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax}
+\def\@maketitle{\vbox to \titlebox{\hsize\textwidth
+ \linewidth\hsize \vskip 0.125in minus 0.125in \centering
+ {\Large\bf \@title \par} \vskip 0.2in plus 1fil minus 0.1in
+ {\def\and{\unskip\enspace{\rm and}\enspace}%
+ \def\And{\end{tabular}\hss \egroup \hskip 1in plus 2fil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}%
+ \def\AND{\end{tabular}\hss\egroup \hfil\hfil\egroup
+ \vskip 0.25in plus 1fil minus 0.125in
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf\@author
+ \end{tabular}\hss\egroup
+ \hfil\hfil\egroup}
+ \vskip 0.3in plus 2fil minus 0.1in
+}}
+\renewenvironment{abstract}{\centerline{\large\bf
+ Abstract}\vspace{0.5ex}\begin{quote} \small}{\par\end{quote}\vskip 1ex}
+
+
+% bibliography
+
+\def\thebibliography#1{\section*{References}
+ \global\def\@listi{\leftmargin\leftmargini
+ \labelwidth\leftmargini \advance\labelwidth-\labelsep
+ \topsep 1pt plus 2pt minus 1pt
+ \parsep 0.25ex plus 1pt \itemsep 0.25ex plus 1pt}
+ \list {[\arabic{enumi}]}{\settowidth\labelwidth{[#1]}\leftmargin\labelwidth
+ \advance\leftmargin\labelsep\usecounter{enumi}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy
+ \sfcode`\.=1000\relax}
+
+\def\@up#1{\raise.2ex\hbox{#1}}
+
+% most of cite format is from aclsub.sty by SMS
+
+% don't box citations, separate with ; and a space
+% also, make the penalty between citations negative: a good place to break
+% changed comma back to semicolon pj 2/1/90
+% \def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+% \def\@citea{}\@cite{\@for\@citeb:=#2\do
+% {\@citea\def\@citea{;\penalty\@citeseppen\ }\@ifundefined
+% {b@\@citeb}{{\bf ?}\@warning
+% {Citation `\@citeb' on page \thepage \space undefined}}%
+% {\csname b@\@citeb\endcsname}}}{#1}}
+
+% don't box citations, separate with ; and a space
+% Replaced for multiple citations (pj)
+% don't box citations and also add space, semicolon between multiple citations
+\def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@cite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{; }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+ {\csname b@\@citeb\endcsname}}}{#1}}
+
+% Allow short (name-less) citations, when used in
+% conjunction with a bibliography style that creates labels like
+% \citename{<names>, }<year>
+%
+\let\@internalcite\cite
+\def\cite{\def\citename##1{##1, }\@internalcite}
+\def\shortcite{\def\citename##1{}\@internalcite}
+\def\newcite{\def\citename##1{{\frenchspacing##1} (}\@internalciteb}
+
+% Macros for \newcite, which leaves name in running text, and is
+% otherwise like \shortcite.
+\def\@citexb[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@newcite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{;\penalty\@m\ }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+{\csname b@\@citeb\endcsname}}}{#1}}
+\def\@internalciteb{\@ifnextchar [{\@tempswatrue\@citexb}{\@tempswafalse\@citexb[]}}
+
+\def\@newcite#1#2{{#1\if at tempswa, #2\fi)}}
+
+\def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+
+%%% More changes made by SMS (originals in latex.tex)
+% Use parentheses instead of square brackets in the text.
+\def\@cite#1#2{({#1\if at tempswa , #2\fi})}
+
+% Don't put a label in the bibliography at all. Just use the unlabeled format
+% instead.
+\def\thebibliography#1{\small\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{References\@mkboth
+ {References}{References}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemsep}{-1ex}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthebibliography=\endlist
+
+% Allow for a bibliography of sources of attested examples
+\def\thesourcebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{Sources of Attested Examples\@mkboth
+ {Sources of Attested Examples}{Sources of Attested Examples}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthesourcebibliography=\endlist
+
+\def\@lbibitem[#1]#2{\item[]\if at filesw
+ { \def\protect##1{\string ##1\space}\immediate
+ \write\@auxout{\string\bibcite{#2}{#1}}\fi\ignorespaces}}
+
+\def\@bibitem#1{\item\if at filesw \immediate\write\@auxout
+ {\string\bibcite{#1}{\the\c at enumi}}\fi\ignorespaces}
+
+% sections with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+ -0.5ex minus -.2ex}{1.5ex plus 0.3ex minus .2ex}{\large\bf\raggedright}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-1.8ex plus
+ -0.5ex minus -.2ex}{0.8ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{0.5ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\paragraph{\@startsection{paragraph}{4}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+\def\subparagraph{\@startsection{subparagraph}{5}{\parindent}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+
+% Footnotes
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+
+% Lists and paragraphs
+\parindent 1em
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 2pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+
+\leftmargin 2em \leftmargini\leftmargin \leftmarginii 2em
+\leftmarginiii 1.5em \leftmarginiv 1.0em \leftmarginv .5em \leftmarginvi .5em
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+ \labelwidth\leftmarginii\advance\labelwidth-\labelsep
+ \topsep 2pt plus 1pt minus 0.5pt
+ \parsep 1pt plus 0.5pt minus 0.5pt
+ \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+ \labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+ \topsep 1pt plus 0.5pt minus 0.5pt
+ \parsep \z@ \partopsep 0.5pt plus 0pt minus 0.5pt
+ \itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+ \labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+ \labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+ \labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+%\def\@normalsize{\@setsize\normalsize{11pt}\xpt\@xpt}
+%\def\small{\@setsize\small{10pt}\ixpt\@ixpt}
+%\def\footnotesize{\@setsize\footnotesize{10pt}\ixpt\@ixpt}
+%\def\scriptsize{\@setsize\scriptsize{8pt}\viipt\@viipt}
+%\def\tiny{\@setsize\tiny{7pt}\vipt\@vipt}
+%\def\large{\@setsize\large{14pt}\xiipt\@xiipt}
+%\def\Large{\@setsize\Large{16pt}\xivpt\@xivpt}
+%\def\LARGE{\@setsize\LARGE{20pt}\xviipt\@xviipt}
+%\def\huge{\@setsize\huge{23pt}\xxpt\@xxpt}
+%\def\Huge{\@setsize\Huge{28pt}\xxvpt\@xxvpt}
+
+\let\@@makecaption\@makecaption
+\renewcommand{\@makecaption}[1]{\@@makecaption{\small #1}}
+
+\newcommand{\Thanks}[1]{\thanks{\ #1}}
\ No newline at end of file
diff --git a/papers/acl-08/grammar1.py b/papers/acl-08/grammar1.py
new file mode 100644
index 0000000..7d2a08d
--- /dev/null
+++ b/papers/acl-08/grammar1.py
@@ -0,0 +1,22 @@
+import nltk
+
+def parse(sent, grammar):
+ gr = nltk.cfg.parse_cfg(grammar)
+ parser = nltk.ChartParser(gr, nltk.parse.TD_STRATEGY)
+ trees = parser.nbest_parse(sent.split())
+ nltk.draw.draw_trees(*trees)
+
+grammar = """
+ S -> NP VP
+ VP -> V NP | VP PP
+ NP -> Det N | NP PP
+ PP -> P NP
+ NP -> 'I'
+ Det -> 'the' | 'a' | 'my'
+ N -> 'elephant' | 'pajamas' | 'man' | 'park' | 'telescope'
+ V -> 'shot' | 'saw'
+ P -> 'in' | 'on' | 'with'
+"""
+
+sent = 'I shot the elephant in my pajamas'
+parse(sent, grammar)
diff --git a/papers/acl-08/grammar2.py b/papers/acl-08/grammar2.py
new file mode 100644
index 0000000..f537c7c
--- /dev/null
+++ b/papers/acl-08/grammar2.py
@@ -0,0 +1,22 @@
+import nltk
+
+def parse(sent, grammar):
+ gr = nltk.cfg.parse_cfg(grammar)
+ parser = nltk.ChartParser(gr, nltk.parse.TD_STRATEGY)
+ trees = parser.nbest_parse(sent.split())
+ nltk.draw.draw_trees(*trees)
+
+grammar = """
+ S -> NP VP
+ VP -> V NP | VP PP
+ NP -> Det N | NP PP
+ PP -> P NP
+ NP -> 'I'
+ Det -> 'the' | 'a' | 'my'
+ N -> 'elephant' | 'pajamas' | 'man' | 'park' | 'telescope'
+ V -> 'shot' | 'saw'
+ P -> 'in' | 'on' | 'with'
+"""
+
+sent = 'I saw the man in the park with a telescope'
+parse(sent, grammar)
diff --git a/papers/acl-08/police.py b/papers/acl-08/police.py
new file mode 100644
index 0000000..683a888
--- /dev/null
+++ b/papers/acl-08/police.py
@@ -0,0 +1,19 @@
+import nltk
+
+def parse(sent, grammar):
+ gr = nltk.cfg.parse_cfg(grammar)
+ parser = nltk.ChartParser(gr, nltk.parse.TD_STRATEGY)
+ trees = parser.nbest_parse(sent.split())
+ nltk.draw.draw_trees(*trees)
+
+grammar = """
+ S -> NP V NP
+ NP -> NP Sbar
+ Sbar -> NP V
+ NP -> 'fish' | 'police'
+ V -> 'fish' | 'police'
+"""
+
+sent = 'police police police police police police police police police'
+parse(sent, grammar)
+
diff --git a/papers/altw-06/acl.bst b/papers/altw-06/acl.bst
new file mode 100644
index 0000000..b95ec04
--- /dev/null
+++ b/papers/altw-06/acl.bst
@@ -0,0 +1,1322 @@
+
+% BibTeX `acl' style file for BibTeX version 0.99c, LaTeX version 2.09
+% This version was made by modifying `aaai-named' format based on the master
+% file by Oren Patashnik (PATASHNIK at SCORE.STANFORD.EDU)
+
+% Copyright (C) 1985, all rights reserved.
+% Modifications Copyright 1988, Peter F. Patel-Schneider
+% Further modifictions by Stuart Shieber, 1991, and Fernando Pereira, 1992.
+% Copying of this file is authorized only if either
+% (1) you make absolutely no changes to your copy, including name, or
+% (2) if you do make changes, you name it something other than
+% btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst.
+% This restriction helps ensure that all standard styles are identical.
+
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at spar.slb.com
+
+% Citation format: [author-last-name, year]
+% [author-last-name and author-last-name, year]
+% [author-last-name {\em et al.}, year]
+%
+% Reference list ordering: alphabetical by author or whatever passes
+% for author in the absence of one.
+%
+% This BibTeX style has support for short (year only) citations. This
+% is done by having the citations actually look like
+% \citename{name-info, }year
+% The LaTeX style has to have the following
+% \let\@internalcite\cite
+% \def\cite{\def\citename##1{##1}\@internalcite}
+% \def\shortcite{\def\citename##1{}\@internalcite}
+% \def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+% which makes \shortcite the macro for short citations.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Changes made by SMS for thesis style
+% no emphasis on "et al."
+% "Ph.D." includes periods (not "PhD")
+% moved year to immediately after author's name
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ENTRY
+ { address
+ author
+ booktitle
+ chapter
+ edition
+ editor
+ howpublished
+ institution
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ publisher
+ school
+ series
+ title
+ type
+ volume
+ year
+ }
+ {}
+ { label extra.label sort.label }
+
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+
+STRINGS { s t }
+
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { add.period$ " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+
+ "\bibitem[" write$
+ label write$
+ "]{" write$
+
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+
+FUNCTION {new.block.checka}
+{ empty$
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.sentence.checka}
+{ empty$
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {new.sentence.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+
+FUNCTION {emphasize}
+{ duplicate$ empty$
+ { pop$ "" }
+ { "{\em " swap$ * "}" * }
+ if$
+}
+
+INTEGERS { nameptr namesleft numnames }
+
+FUNCTION {format.names}
+{ 's :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+
+ { s nameptr "{ff~}{vv~}{ll}{, jj}" format.name$ 't :=
+
+ nameptr #1 >
+ { namesleft #1 >
+ { ", " * t * }
+ { numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ t "others" =
+ { " et~al." * }
+ { " and " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {format.authors}
+{ author empty$
+ { "" }
+ { author format.names }
+ if$
+}
+
+FUNCTION {format.editors}
+{ editor empty$
+ { "" }
+ { editor format.names
+ editor num.names$ #1 >
+ { ", editors" * }
+ { ", editor" * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title empty$
+ { "" }
+
+ { title "t" change.case$ }
+
+ if$
+}
+
+FUNCTION {n.dashify}
+{ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {format.date}
+{ year empty$
+ { month empty$
+ { "" }
+ { "there's a month but no year in " cite$ * warning$
+ month
+ }
+ if$
+ }
+ { month empty$
+ { "" }
+ { month }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.btitle}
+{ title emphasize
+}
+
+FUNCTION {tie.or.space.connect}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$ * *
+}
+
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { "volume" volume tie.or.space.connect
+ series empty$
+ 'skip$
+ { " of " * series emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { output.state mid.sentence =
+ { "number" }
+ { "Number" }
+ if$
+ number tie.or.space.connect
+ series empty$
+ { "there's a number but no series in " cite$ * warning$ }
+ { " in " * series * }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition empty$
+ { "" }
+ { output.state mid.sentence =
+ { edition "l" change.case$ " edition" * }
+ { edition "t" change.case$ " edition" * }
+ if$
+ }
+ if$
+}
+
+INTEGERS { multiresult }
+
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+
+FUNCTION {format.pages}
+{ pages empty$
+ { "" }
+ { pages multi.page.check
+ { "pages" pages n.dashify tie.or.space.connect }
+ { "page" pages tie.or.space.connect }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.year.label}
+{ year extra.label *
+}
+
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ number empty$
+ 'skip$
+ { "(" number * ")" * *
+ volume empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ }
+ if$
+ pages empty$
+ 'skip$
+ { duplicate$ empty$
+ { pop$ format.pages }
+ { ":" * pages n.dashify * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { "chapter" }
+ { type "l" change.case$ }
+ if$
+ chapter tie.or.space.connect
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.in.ed.booktitle}
+{ booktitle empty$
+ { "" }
+ { editor empty$
+ { "In " booktitle emphasize * }
+ { "In " format.editors * ", " * booktitle emphasize * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {empty.misc.check}
+{ author empty$ title empty$ howpublished empty$
+ month empty$ year empty$ note empty$
+ and and and and and
+
+ key empty$ not and
+
+ { "all relevant fields are empty in " cite$ * warning$ }
+ 'skip$
+ if$
+}
+
+FUNCTION {format.thesis.type}
+{ type empty$
+ 'skip$
+ { pop$
+ type "t" change.case$
+ }
+ if$
+}
+
+FUNCTION {format.tr.number}
+{ type empty$
+ { "Technical Report" }
+ 'type
+ if$
+ number empty$
+ { "t" change.case$ }
+ { number tie.or.space.connect }
+ if$
+}
+
+FUNCTION {format.article.crossref}
+{ key empty$
+ { journal empty$
+ { "need key or journal for " cite$ * " to crossref " * crossref *
+ warning$
+ ""
+ }
+ { "In {\em " journal * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.crossref.editor}
+{ editor #1 "{vv~}{ll}" format.name$
+ editor num.names$ duplicate$
+ #2 >
+ { pop$ " et~al." * }
+ { #2 <
+ 'skip$
+ { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" =
+ { " et~al." * }
+ { " and " * editor #2 "{vv~}{ll}" format.name$ * }
+ if$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.book.crossref}
+{ volume empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ "In "
+ }
+ { "Volume" volume tie.or.space.connect
+ " of " *
+ }
+ if$
+ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { series empty$
+ { "need editor, key, or series for " cite$ * " to crossref " *
+ crossref * warning$
+ "" *
+ }
+ { "{\em " * series * "\/}" * }
+ if$
+ }
+ { key * }
+ if$
+ }
+ { format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.incoll.inproc.crossref}
+{ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { booktitle empty$
+ { "need editor, key, or booktitle for " cite$ * " to crossref " *
+ crossref * warning$
+ ""
+ }
+ { "In {\em " booktitle * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ }
+ { "In " format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { journal emphasize "journal" output.check
+ format.vol.num.pages output
+ format.date output
+ }
+ { format.article.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {booklet}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ howpublished address new.block.checkb
+ howpublished output
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.chapter.pages output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ format.edition output
+ format.date output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.chapter.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.pages output
+ address empty$
+ { organization publisher new.sentence.checkb
+ organization output
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ organization output
+ publisher output
+ }
+ if$
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {conference} { inproceedings }
+
+FUNCTION {manual}
+{ output.bibitem
+ author empty$
+ { organization empty$
+ 'skip$
+ { organization output.nonnull
+ address output
+ }
+ if$
+ }
+ { format.authors output.nonnull }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ author empty$
+ { organization empty$
+ { address new.block.checka
+ address output
+ }
+ 'skip$
+ if$
+ }
+ { organization address new.block.checkb
+ organization output
+ address output
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ "Master's thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label output
+ new.block
+ title howpublished new.block.checkb
+ format.title output
+ howpublished new.block.checka
+ howpublished output
+ format.date output
+ new.block
+ note output
+ fin.entry
+ empty.misc.check
+}
+
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ new.block
+ "{Ph.D.} thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ editor empty$
+ { organization output }
+ { format.editors output.nonnull }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ format.bvolume output
+ format.number.series output
+ address empty$
+ { editor empty$
+ { publisher new.sentence.checka }
+ { organization publisher new.sentence.checkb
+ organization output
+ }
+ if$
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ editor empty$
+ 'skip$
+ { organization output }
+ if$
+ publisher output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ format.tr.number output.nonnull
+ institution "institution" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ note "note" output.check
+ format.date output
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+
+MACRO {jan} {"January"}
+
+MACRO {feb} {"February"}
+
+MACRO {mar} {"March"}
+
+MACRO {apr} {"April"}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"August"}
+
+MACRO {sep} {"September"}
+
+MACRO {oct} {"October"}
+
+MACRO {nov} {"November"}
+
+MACRO {dec} {"December"}
+
+MACRO {acmcs} {"ACM Computing Surveys"}
+
+MACRO {acta} {"Acta Informatica"}
+
+MACRO {cacm} {"Communications of the ACM"}
+
+MACRO {ibmjrd} {"IBM Journal of Research and Development"}
+
+MACRO {ibmsj} {"IBM Systems Journal"}
+
+MACRO {ieeese} {"IEEE Transactions on Software Engineering"}
+
+MACRO {ieeetc} {"IEEE Transactions on Computers"}
+
+MACRO {ieeetcad}
+ {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"}
+
+MACRO {ipl} {"Information Processing Letters"}
+
+MACRO {jacm} {"Journal of the ACM"}
+
+MACRO {jcss} {"Journal of Computer and System Sciences"}
+
+MACRO {scp} {"Science of Computer Programming"}
+
+MACRO {sicomp} {"SIAM Journal on Computing"}
+
+MACRO {tocs} {"ACM Transactions on Computer Systems"}
+
+MACRO {tods} {"ACM Transactions on Database Systems"}
+
+MACRO {tog} {"ACM Transactions on Graphics"}
+
+MACRO {toms} {"ACM Transactions on Mathematical Software"}
+
+MACRO {toois} {"ACM Transactions on Office Information Systems"}
+
+MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"}
+
+MACRO {tcs} {"Theoretical Computer Science"}
+
+READ
+
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+
+INTEGERS { len }
+
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+
+INTEGERS { et.al.char.used }
+
+FUNCTION {initialize.et.al.char.used}
+{ #0 'et.al.char.used :=
+}
+
+EXECUTE {initialize.et.al.char.used}
+
+FUNCTION {format.lab.names}
+{ 's :=
+ s num.names$ 'numnames :=
+
+ numnames #1 =
+ { s #1 "{vv }{ll}" format.name$ }
+ { numnames #2 =
+ { s #1 "{vv }{ll }and " format.name$ s #2 "{vv }{ll}" format.name$ *
+ }
+ { s #1 "{vv }{ll }\bgroup et al.\egroup " format.name$ }
+ if$
+ }
+ if$
+
+}
+
+FUNCTION {author.key.label}
+{ author empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.editor.key.label}
+{ author empty$
+ { editor empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.key.organization.label}
+{ author empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {editor.key.organization.label}
+{ editor empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+}
+
+FUNCTION {calc.label}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.label
+ { type$ "proceedings" =
+ 'editor.key.organization.label
+ { type$ "manual" =
+ 'author.key.organization.label
+ 'author.key.label
+ if$
+ }
+ if$
+ }
+ if$
+ duplicate$
+
+ "\protect\citename{" swap$ * "}" *
+ year field.or.null purify$ *
+ 'label :=
+ year field.or.null purify$ *
+
+ sortify 'sort.label :=
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { nameptr #1 >
+ { " " * }
+ 'skip$
+ if$
+
+ s nameptr "{vv{ } }{ll{ }}{ ff{ }}{ jj{ }}" format.name$ 't :=
+
+ nameptr numnames = t "others" = and
+ { "et al" * }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.organization.sort}
+{ author empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need author, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {editor.organization.sort}
+{ editor empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need editor, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+
+FUNCTION {presort}
+
+{ calc.label
+ sort.label
+ " "
+ *
+ type$ "book" =
+
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.organization.sort
+ { type$ "manual" =
+ 'author.organization.sort
+ 'author.sort
+ if$
+ }
+ if$
+ }
+ if$
+
+ *
+
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+
+SORT
+
+STRINGS { longest.label last.sort.label next.extra }
+
+INTEGERS { longest.label.width last.extra.num }
+
+FUNCTION {initialize.longest.label}
+{ "" 'longest.label :=
+ #0 int.to.chr$ 'last.sort.label :=
+ "" 'next.extra :=
+ #0 'longest.label.width :=
+ #0 'last.extra.num :=
+}
+
+FUNCTION {forward.pass}
+{ last.sort.label sort.label =
+ { last.extra.num #1 + 'last.extra.num :=
+ last.extra.num int.to.chr$ 'extra.label :=
+ }
+ { "a" chr.to.int$ 'last.extra.num :=
+ "" 'extra.label :=
+ sort.label 'last.sort.label :=
+ }
+ if$
+}
+
+FUNCTION {reverse.pass}
+{ next.extra "b" =
+ { "a" 'extra.label := }
+ 'skip$
+ if$
+ label extra.label * 'label :=
+ label width$ longest.label.width >
+ { label 'longest.label :=
+ label width$ 'longest.label.width :=
+ }
+ 'skip$
+ if$
+ extra.label 'next.extra :=
+}
+
+EXECUTE {initialize.longest.label}
+
+ITERATE {forward.pass}
+
+REVERSE {reverse.pass}
+
+FUNCTION {begin.bib}
+
+{ et.al.char.used
+ { "\newcommand{\etalchar}[1]{$^{#1}$}" write$ newline$ }
+ 'skip$
+ if$
+ preamble$ empty$
+
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+
+ "\begin{thebibliography}{" "}" * write$ newline$
+
+}
+
+EXECUTE {begin.bib}
+
+EXECUTE {init.state.consts}
+
+ITERATE {call.type$}
+
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+
+EXECUTE {end.bib}
+
diff --git a/papers/altw-06/altw-06.bib b/papers/altw-06/altw-06.bib
new file mode 100644
index 0000000..7e2fa7b
--- /dev/null
+++ b/papers/altw-06/altw-06.bib
@@ -0,0 +1,53 @@
+ at Book{Blackburn:2005:RINL,
+ author = {Patrick Blackburn and Johan Bos},
+ title = {Representation and Inference for Natural Language: A First Course in Computational Semantics},
+ publisher = {CSLI Publications},
+ year = 2005}
+
+
+ at InCollection{Montague:1974:PTQ,
+ author = {Richard Montague},
+ title = {The Proper Treatment of Quantification in Ordinary {E}nglish},
+ booktitle = {Formal Philosphy: Selected Papers of Richard Montague},
+ pages = {247--270},
+ publisher = {Yale University Press},
+ year = 1974,
+ editor = {R. H. Thomason},
+ address = {New Haven}}
+
+ @Book{Dowty:1981:IMS,
+ author = {D. R. Dowty and R. E. Wall and S. Peters},
+ title = {Introduction to {M}ontague {S}emantics},
+ publisher = {Reidel},
+ year = 1981,
+ series = {Studies in Linguistics and Philosophy},
+ address = {Dordrecht}}
+
+ at InProceedings{Bird:2005:NES,
+ author = {Steven Bird},
+ title = {{NLTK-Lite}: Efficient Scripting for Natural Language Processing},
+ booktitle = {Proceedings of the 4th International Conference on Natural Language Processing (ICON)},
+ pages = {11--18},
+ year = 2005,
+ address = {New Delhi},
+ month = {December},
+ publisher = {Allied Publishers}}
+
+
+ at Book{vanRossum:2006:PT,
+ author = {Guido van Rossum},
+ title = {Python Tutorial},
+ year = 2006,
+ month = {March},
+ note = {Release 2.4.3},
+ url = {http://docs.python.org/tut/tut.html}
+}
+
+
+ at Book{Russell:2003:AIMA,
+ author = {Stuart Russell and Peter Norvig},
+ title = {Artifical Intelligence: A Modern Approach},
+ publisher = {Prentice Hall},
+ year = 2003,
+ note = {2nd edition}}
+
diff --git a/papers/altw-06/altw-06.tex b/papers/altw-06/altw-06.tex
new file mode 100644
index 0000000..fa806b3
--- /dev/null
+++ b/papers/altw-06/altw-06.tex
@@ -0,0 +1,864 @@
+\documentclass[11pt]{article}
+\usepackage{colacl06}
+\usepackage{times}
+\usepackage{latexsym}
+
+\usepackage{url}
+\usepackage{fancyvrb}
+\usepackage{relsize}
+%\usepackage{float}
+% \restylefloat{table*}
+% \usepackage{placeins}
+
+\usepackage{examples}
+\exampleindent=\leftmargini
+
+\usepackage{fixltx2e}
+
+\newcommand{\code}[1]{\texttt{#1}}
+\newcommand{\ling}[1]{\textit{#1}}
+\newcommand{\prog}[1]{\textsf{#1}}
+\newcommand{\scare}[1]{`#1'}
+
+\newcommand{\bnb}{B\&B}
+\newcommand{\nltk}{\textsc{nltk}}
+\newcommand{\NLP}{\textsc{nlp}}
+
+\RecustomVerbatimEnvironment
+{Verbatim}{Verbatim}
+{frame=lines,fontfamily=courier}
+
+\DefineVerbatimEnvironment%
+{SmVerbatim}{Verbatim}
+{fontsize=\relsize{-2}, frame=lines}
+
+\DefineVerbatimEnvironment%
+{SVerbatim}{Verbatim}
+{fontsize=\relsize{-2}}
+
+\setlength\titlebox{6.5cm} % Expanding the titlebox
+
+\title{Computational Semantics in the \textit{Natural Language Toolkit}}
+
+\author{Ewan Klein\\
+ School of Informatics\\
+ University of Edinburgh\\
+ Scotland, UK\\
+ {\tt ewan at inf.ed.ac.uk} }
+
+\date{\today}
+
+\begin{document}
+\maketitle
+\begin{abstract}
+ \nltk, the Natural Language Toolkit, is an open source project whose
+ goals include providing students with software and language
+ resources that will help them to learn basic \NLP. Until now, the
+ program modules in \nltk\ have covered such topics as tagging,
+ chunking, and parsing, but have not incorporated any aspect of
+ semantic interpretation. This paper describes recent work on
+ building a new semantics package for \nltk. This currently allows
+ semantic representations to be built compositionally as a part of
+ sentence parsing, and for the representations to be evaluated by a
+ model checker. We present the main components of this work, and
+ consider comparisons between the Python implementation and the
+ Prolog approach developed by Blackburn and Bos
+ \shortcite{Blackburn:2005:RINL}.
+\end{abstract}
+
+%\section{Credits}
+
+
+
+\section{Introduction}
+
+\nltk, the Natural Language
+Toolkit,\footnote{\url{http://nltk.sourceforge.net/}} is an open
+source project whose goals include providing students with software
+and language resources that will help them to learn basic \NLP. \nltk\
+is implemented in Python, and provides a set of modules (grouped into
+packages) which can be imported into the user's Python programs.
+
+Up till now, the modules in \nltk\ have covered such topics as
+tagging, chunking, and parsing, but have not incorporated any aspect
+of semantic interpretation. Over the last year, I have been working
+on remedying this lack, and in this paper I will describe progress to
+date. In combination with the \nltk\ \prog{parse} package, \nltk 's
+\prog{semantics} package currently allow semantic representations to
+be built compositionally within a feature-based chart parser, and
+allows the representations to be evaluated by a model checker.
+
+One source of inspiration for this work came from
+Blackburn and Bos's \shortcite{Blackburn:2005:RINL} landmark book
+\textit{Representation and Inference for Natural Language} (henceforth
+referred to as \bnb). The two primary goals set forth by \bnb\ are (i)
+automating the association of semantic representations with
+expressions of natural language, and (ii) using logical
+representations of natural language to automate the process of drawing
+inferences. I will be focussing on (i), and the related issue of
+defining satisfaction in a model for the semantic representations. By
+contrast, the important topic of (ii) will not be covered---as yet,
+there are no theorem provers in \nltk. That said, as pointed out by
+\bnb, for many inference problems in \NLP\ it is desirable to call
+external and highly sophisticated first-order theorem provers.
+
+One notable feature of \bnb\ is the use of Prolog as the language of
+implementation. It is not hard to defend the use of Prolog in defining
+logical representations, given the presence of first-order clauses in
+Prolog and the fundamental role of resolution in Prolog's model of
+computation. Nevertheless, in some circumstances it may be helpful to
+offer students access to an alternative framework, such as the Python
+implementation presented here. I also hope that the existence of work
+in both programming paradigms will turn out to be mutually beneficial,
+and will lead to a broader community of upcoming researchers becoming
+involved in the area of computational semantics.
+
+% In this paper, I review my experience of attempting to implement
+% something similar to the \cite{Blackburn:2005:RINL} first-order model
+% checker within Python as part of the broader \nltk\ initiative.%
+% \footnote{\url{http://nltk.sourceforge.net/}. See also
+% \cite{Bird:2005:NES}.}
+% One motivation for integrating some formal
+% semantics into \nltk\ is that the latter already provides a wealth of
+% tools for computational linguistics, including a variety of parsers,
+% and this makes it extremely easy for students to anchor their
+% understanding of computational semantics within a broader framework of
+% language processing activities.
+
+
+% Since my primary interest is in the contrast between Prolog and
+% Python, I shall not attempt to consider respects in which Python
+% differs from other procedural languages such as Perl or
+% Java.\footnote{However, see
+% \url{http://nltk.sourceforge.net/lite/doc/en/preface.html} for an
+% illustration of how Python compares with a number of other
+% languages, including Prolog, in performing a simple \NLP\ task.} It
+% should also be noted, of course, that both Python and Prolog offer a
+% wide variety of techniques for implementing concepts from
+% computational semantics, so that a specific approach may say more
+% about the programmer than the language \textit{per se}.
+
+
+\section{Building Semantic Representations}
+
+The initial question that we faced in \nltk\ was how to induce
+semantic representations for English sentences. Earlier efforts by
+Edward Loper and Rob Speer had led to the construction of a chart
+parser for (untyped) feature-based grammars, and we therefore decided
+to introduce a \code{sem} feature to hold the semantics in a parse
+tree node. However, rather than representing the value of \code{sem}
+as a feature structure, we opted for a more traditional (and more
+succinct) logical formalism. Since the $\lambda$ calculus was the
+pedagogically obvious choice of \scare{glue} language for combining
+the semantic representations of subconstituents in a sentence, we
+opted to build on \prog{church.py},%
+\footnote{\url{http://www.alcyone.com/pyos/church/}.} an independent
+implementation of the untyped $\lambda$ calculus due to Erik Max
+Francis. The \nltk\ module \prog{semantics.logic} extends
+\prog{church.py} to bring it closer to first-order logic, though the
+resulting language is still untyped. (\ref{ex:logic1}) illustrates a
+representative formula, translating \ling{A dog barks}. From a Python
+point of view, (\ref{ex:logic1}) is just a string, and has to be
+parsed into an instance of the
+\code{Expression} class from \prog{semantics.logic}.
+\begin{examples}
+ \item \verb!some x.(and (dog x) (bark x))! \label{ex:logic1}
+\end{examples}
+The string \code{(dog x)} is analyzed as a function application. A
+statement such as \ling{Suzie chases Fido}, involving a binary
+relation $\mathit{chase}$, will be translated as another function
+application: \code{((chase fido) suzie)}, or equivalently \code{(chase
+ fido suzie)}. So in this case, \code{chase} is taken to denote a
+function which, when applied to an argument yields the second function
+denoted by \code{(chase fido)}. Boolean connectives are also parsed
+as functors, as indicated by \code{and} in (\ref{ex:logic1}). However, infix
+notation for Boolean connectives is accepted as input and can also be
+displayed.
+
+
+For comparison, the Prolog counterpart of (\ref{ex:logic1}) on \bnb 's
+approach is shown in (\ref{ex:logic2}).
+\begin{examples}
+ \item \verb!some(X,(and(dog(X),bark(X))! \label{ex:logic2}
+\end{examples}
+(\ref{ex:logic2}) is a Prolog term and does not require any
+additional parsing machinery; first-order variables are treated as
+Prolog variables.
+
+(\ref{ex:logic3}) illustrates a $\lambda$ term from
+\prog{semantics.logic} that represents the determiner \ling{a}.
+\begin{examples}
+\item \verb!\Q P.some x.(and (Q x) (P x))! \label{ex:logic3}
+\end{examples}
+\verb!\Q! is
+the ascii rendering of $\lambda Q$, and \verb!\Q P! is
+shorthand for $\lambda Q \lambda P$.
+
+For comparison, (\ref{ex:logic4})
+illustrates the Prolog counterpart of (\ref{ex:logic3}) in \bnb.
+\begin{examples}
+\item \label{ex:logic4}
+\verb!lam(Q,lam(P,some(X,!
+\verb! and(app(Q,X),app(P,X)))))!
+\end{examples}
+Note that \verb!app! is used in \bnb\ to signal the application of a
+$\lambda$ term to an argument. The right-branching structure for
+$\lambda$ terms shown in the Prolog rendering can become fairly
+unreadable when there are multiple bindings. Given that readability is
+a design goal in \nltk, the additional overhead of invoking a
+specialized parser for logical representations is arguable a cost
+worth paying.
+
+
+Figure~\ref{fig:gram} presents a minimal grammar exhibiting the most
+important aspects of the grammar formalism extended with the
+\code{sem} feature.
+\begin{figure*}[tb]
+ \centering
+\begin{Verbatim}
+S[sem = <app(?subj,?vp)>] -> NP[sem=?subj] VP[sem=?vp]
+VP[sem=?v] -> IV[sem=?v]
+NP[sem=<app(?det,?n)>] -> Det[sem=?det] N[sem=?n]
+
+Det[sem=<\Q P. some x. ((Q x) and (P x))>] -> 'a'
+N[sem=<dog>] -> 'dog'
+IV[sem=<\x.(bark x)>] -> 'barks'
+\end{Verbatim}
+ \caption{Minimal Grammar with Semantics}
+ \label{fig:gram}
+\end{figure*}
+Since the values of the \code{sem} feature have to handed off to a
+separate processor, we have adopted the convention of enclosing the values
+in angle brackets, except in the case of variables (e.g., \code{?subj}
+and \code{?vp}), which undergo unification in the usual way. The
+\code{app} relation corresponds to function application;
+
+% Figure~\ref{fig:tree} gives a parse tree produced by the \nltk\ module
+% \prog{parse.featurechart}.
+% \begin{figure*}
+% \begin{tabular}{p{5in}}
+% \begin{Verbatim}
+% ([INIT][]:
+% (S[ sem = ApplicationExpression('(\Q P.some x.(and (Q x) (P x)) dog)', 'bark') ]:
+% (NP[ sem = ApplicationExpression('\Q P.some x.(and (Q x) (P x))', 'dog') ]:
+% (Det[ sem = LambdaExpression('Q', '\P.some x.(and (Q x) (P x))') ]: 'a')
+% (N[ sem = VariableExpression('dog') ]: 'dog'))
+% (VP[ sem = VariableExpression('bark') ]:
+% (IV[ sem = VariableExpression('bark') ]: 'barks'))))
+% \end{Verbatim}
+% \end{tabular}
+% \caption{Parse tree for \ling{a dog barks}}
+% \label{fig:tree}
+% \end{figure*}
+
+In Figure~\ref{fig:tree}, we show a trace produced by the \nltk\ module
+\prog{parse.featurechart}.
+\begin{figure*}[tb]
+{\small
+\begin{Verbatim}
+Predictor |> . . .| S[sem='(?subj ?vp)'] -> * NP[sem=?subj] VP[sem=?vp]
+Predictor |> . . .| NP[sem='(?det ?n)'] -> * Det[sem=?det] N[sem=?n]
+Scanner |[-] . .| [0:1] 'a'
+Completer |[-> . .| NP[sem='(\\Q P.some x.(and (Q x) (P x)) ?n)']
+ -> Det[sem='\\Q P.some x.(and (Q x) (P x))'] * N[sem=?n]
+Scanner |. [-] .| [1:2] 'dog'
+Completer |[---] .| NP[sem='(\\Q P.some x.(and (Q x) (P x)) dog)']
+ -> Det[sem='\\Q P.some x.(and (Q x) (P x))'] N[sem='dog'] *
+Completer |[---> .| S[sem='(\\Q P.some x.(and (Q x) (P x)) dog ?vp)']
+ -> NP[sem='(\\Q P.some x.(and (Q x) (P x)) dog)'] * VP[sem=?vp]
+Predictor |. . > .| VP[sem=?v] -> * V[sem=?v]
+Scanner |. . [-]| [2:3] 'barks'
+Completer |. . [-]| VP[sem='bark'] -> V[sem='bark'] *
+Completer |[=====]| S[sem='(\\Q P.some x.(and (Q x) (P x)) dog bark)']
+ -> NP[sem='(\\Q P.some x.(and (Q x) (P x)) dog)'] VP[sem='bark'] *
+Completer |[=====]| [INIT] -> S *
+\end{Verbatim}
+}
+ \caption{Parse tree for \ling{a dog barks}}
+ \label{fig:tree}
+\end{figure*}
+This illustrates how variable values of the \code{sem} feature are
+instantiated when completed edges are added to the chart. At present,
+$\beta$ reduction is not carried out as the \code{sem} values are
+constructed, but has to be invoked after the parse has completed.
+
+The following example of a session with the Python interactive
+interpreter illustrates how a grammar and a sentence are processed by
+a parser to produce an object \code{tree}; the semantics is extracted
+from the root node of the latter and bound to the variable \code{e},
+which can then be displayed in various ways.
+\begin{SVerbatim}
+>>> gram = GrammarFile.read_file('sem1.cfg')
+>>> s = 'a dog barks'
+>>> tokens = list(tokenize.whitespace(s))
+>>> parser = gram.earley_parser()
+>>> tree = parser.parse(tokens)
+>>> e = root_semrep(tree)
+>>> print e
+(\Q P.some x.(and (Q x) (P x)) dog \x.(bark x))
+>>> print e.simplify()
+some x.(and (dog x) (bark x))
+>>> print e.simplify().infixify()
+some x.((dog x) and (bark x))
+\end{SVerbatim}
+
+
+
+% Although Blackburn and Bos \shortcite{Blackburn:2005:RINL} commence
+% their presentation by a discussion of how to represent first-order
+% formulas in Prolog, I shall defer this discussion, and instead start
+% out by looking at Python data structures which seem to lend themselves
+% to use in building first-order models.
+
+Apart from the pragmatic reasons for choosing a functional language as
+our starting point, there are also theoretical attractions. It helps
+introduce students to the tradition of Montague Grammar
+\cite{Montague:1974:PTQ,Dowty:1981:IMS}, which in turn provides an
+elegant correspondence between binary syntax trees and semantic
+composition rules, in the style celebrated by categorial grammar. In
+the next part of the paper, I will turn to the issue of how to
+represent models for the logical representations.
+
+\section{Representing Models in Python}
+
+Although our logical language is untyped, we will interpret it as
+though it were typed. In particular, expressions which are
+intended to translate unary predicates will be interpreted as
+functions of type $e \rightarrow \{0, 1\}$ (from individuals to truth
+values) and expressions corresponding to binary predicates will be
+interpreted as though they were of
+type $e \rightarrow (e \rightarrow \{0, 1\})$. We will start
+out by looking at data structures which can be used to
+provide denotations for such expressions.
+
+\subsection{Dictionaries and Boolean Types}
+
+The standard mapping type in Python is the dictionary, which
+associates keys with arbitrary values. Dictionaries are the obvious
+choice for representing various kinds of functions, and can be
+specialized by user-defined classes. This means that it is possible to
+benefit from the standard Python operations on dictionaries, while
+adding additional features and constraints, or in some cases
+overriding the standard operations. Since we are assuming that our
+logical language is based on function application, we can readily
+construct the interpretation of $n$-ary relations in terms of
+dictionaries-as-functions.
+
+Characteristic functions (i.e., functions that correspond to sets)
+are dictionaries with Boolean values:
+\begin{Verbatim}
+cf = {'d1': True,
+ 'd2': True,
+ 'd3': False}
+\end{Verbatim}
+\code{cf} corresponds to the set $\{d_1, d_2\}$. Since functions are
+being implemented as dictionaries, function application is implemented
+as indexing (e.g., \code{cf['d1']} applies \code{cf} to argument
+\code{'d1'}). Note that \code{True} and \code{False} are
+instances of the Python built-in \code{bool} type, and can be used
+in any Boolean context. Since Python also includes \code{and} and
+\code{not}, we can make statements (here, using the Python interactive
+interpreter) such as the following:
+\begin{Verbatim}
+>>> cf['d1'] and not cf['d3']
+True
+\end{Verbatim}
+As mentioned earlier, relations of higher arity are also modeled as
+functions. For example, a binary relation will be a function from
+entities to a characteristic function; we can call these `curryed
+characteristic functions'.
+\begin{Verbatim}
+cf2 = {'d2': {'d1': True},
+ 'd3': {'d2': True}}
+\end{Verbatim}
+\code{cf2} corresponds to the relation $\{(d_1, d_2), (d_2, d_3)\}$,
+on two assumptions. First, we are allowed to omit values terminating
+in \code{False}, since arguments that are missing the function will
+be taken to yield \code{False}. Second, as in Montague Grammar, the
+`object' argument of a binary relation is consumed before the
+`subject' argument. Thus we write $((\mathit{love}\; m)\; j)$ in place
+of $\mathit{love}(j, m)$. Recall that we also allow the abbreviated form
+$(\mathit{love}\; m\; j)$
+
+Once we have curryed characteristic functions in place, it is
+straightforward to implement the valuation of non-logical constants as a
+another dictionary-based class \code{Valuation}, where constants are
+the keys and the values are functions (or entities in the case of
+individual constants).
+
+While variable assignments could be treated as a list of
+variable-value pairs, as in \bnb, an alternative
+is again to use a dictionary-based class. This approach makes it
+relatively easy to impose further restrictions on assignments, such as
+only assigning values to strings of the form \code{x}, \code{y},
+\code{z}, \code{x0}, \code{x1}, \ldots.
+
+
+\subsection{Sets}
+
+Python provides support for sets, including standard operations such as
+intersection and subset relationships. Sets are useful in a wide
+variety of contexts. For example, instances of the class
+\code{Valuation} can be given a property \code{domain}, consisting
+of the \code{set} of entities that act as keys in curryed
+characteristic functions; then a condition on objects in the
+\code{Model} class is that the \code{domain} of some model
+\code{m} is a superset
+of \code{m}'s \code{valuation.domain}:
+\begin{Verbatim}
+m.domain.issuperset
+ (m.valuation.domain)
+\end{Verbatim}
+
+For convenience, \code{Valuation} objects
+have a \code{read} method which allows $n$-ary predicates to be
+specified as relations (i.e., sets of tuples) rather than functions.
+In the following example, \verb!rel! is a set consisting of the
+pairs \code{('d1', 'd2')} and \code{('d2', 'd3')}.
+\begin{Verbatim}
+val = Valuation()
+rel = set([('d1', 'd2'),('d2', 'd3')])
+val.read([('love', rel)])
+\end{Verbatim}
+\code{read} converts \code{rel} internally to the curryed
+characteristic function \code{cf2} defined earlier.
+
+\section{Key Concepts}
+
+\subsection{Satisfaction}
+
+
+The definition of satisfaction presupposes that we have defined a
+first-order language, and that we have a way of parsing that language
+so that satisfaction can be stated recursively. In the interests of
+modularity, it seems desirable to make the relationship between
+language and interpretation less tightly coupled than it is on the
+approach of \bnb; for example, we would like to be able apply similar
+evaluation techniques to different logical representations.
+In the current \nltk\ implementation, the
+\verb!nltk_lite.semantics.evaluate! module imports a second module
+\code{logic}, and calls a \code{parse} method from this module to
+determine whether a given Python string can be analysed as first-order
+formula. However, \code{evaluate} tries to make relatively weak
+assumptions about the resulting parse structure. Specifically, given a
+parsed expression, it tries to match the structure with one of the
+following three kinds of pattern:
+\begin{Verbatim}
+(binder, body)
+(op, arg_list)
+(fun, arg)
+\end{Verbatim}
+Any string which cannot be decomposed is taken to be a
+primitive (that is, a non-logical constant or individual variable).
+
+A \code{binder} can be a $\lambda$ or a quantifier (existential or
+universal); an \code{op} can be a Boolean connective or the equality
+symbol. Any other paired expression is assumed to be a function
+application. In principle, it should be possible to interface the
+\code{evaluate} module with any parser for first-order formulas which
+can deliver these structures. Although the model checker expects
+predicate-argument structure as function applications, it would be
+straightforward to accept atomic clauses that have been parsed into a
+predicate and a list of arguments.
+
+Following the functional style of interpretation, Boolean connectives
+in \code{evaluate} are interpreted as truth functions; for example,
+the connective \code{and} can be interpreted as the function
+\code{AND}:
+\begin{Verbatim}
+AND = {True: {True: True,
+ False: False},
+ False: {True: False,
+ False: False}}
+\end{Verbatim}
+We define \code{OPS} as a mapping between the Boolean connectives
+and their associated truth functions. Then the simplified clause for
+the satisfaction of Boolean formulas looks as follows:%
+\footnote{In order to simplify presentation, tracing and some error
+ handling code has been omitted from definitions. Object-oriented
+ uses of \code{self} have also been suppressed.}
+\begin{SmVerbatim}
+def satisfy(expr, g):
+ if parsed(expr) == (op, args)
+ if args == (phi, psi):
+ val1 = satisfy(phi, g)
+ val2 = satisfy(psi, g)
+ return OPS[op][val1][val2]
+\end{SmVerbatim}
+% Although Boolean connectives are treated as functions, we also allow
+% infix notation; both of the following formulas are interpretable.
+% \begin{Verbatim}
+% (or (and p q) r)
+% ((p and q) or r)
+% \end{Verbatim}
+% However, since negation is also parsed as an application, we have to
+% be careful to insert brackets properly:
+% \begin{Verbatim}
+% (p or (not p))
+% (not (not p))
+% \end{Verbatim}
+
+% The \code{satisfy} function is defined as a method of the
+% \code{Model} class, and this requires us to use
+% \code{self.satisfy} in recursive calls.
+In this and subsequent clauses for \code{satisfy}, the return value
+is intended to be one of Python's Boolean values, \code{True} or
+\code{False}. (The exceptional case, where the result is undefined,
+is discussed in Section~\ref{sec:partial}.)
+
+An equally viable (and probably more efficient) alternative to logical
+connnectives would be to use the native Python Boolean operators. The
+approach adopted here was chosen on the grounds that it conforms to
+the functional framework adopted elsewhere in the semantic
+representations, and can be expressed succinctly in the satisfaction
+clauses. By contrast, in the \bnb\ Prolog implementation, \code{and}
+and \code{or} each require five clauses in the satisfaction definition
+(one for each combination of Boolean-valued arguments, and a fifth for
+the `undefined' case).
+
+We will defer discussion of the quantifiers to the next section.
+The \code{satisfy} clause for function application is similar to
+that for the connectives. In
+order to handle type errors, application is delegated to a wrapper
+function \code{app} rather than by directly indexing the curryed
+characteristic function as described earlier.
+\begin{SmVerbatim}
+ ...
+ elif parsed(expr) == (fun, arg):
+ funval = satisfy(fun, g)
+ argval = satisfy(psi, g)
+ return app(funval, argval)
+\end{SmVerbatim}
+
+
+\subsection{Quantifers}
+Examples of quantified formulas accepted by the \code{evaluate} module
+are pretty unexceptional. \ling{Some boy loves every girl} is rendered as:
+\begin{Verbatim}
+'some x.((boy x) and
+ all y.((girl y) implies
+ (love y x)))'
+\end{Verbatim}
+
+The first step in interpreting quantified formulas is to define the
+\textit{satisfiers} of a formula that is open in some
+variable. Formally, given an open formula $\phi[x]$ dependent on $x$
+and a model with domain $D$, we define the set $sat(\phi [x], g)$ of
+satisfiers of $\phi[x]$ to be:
+\[
+ \{u \in D: \mathit{satisfy}(\phi [x], g[u/x]) = \mathit{True} \}
+\]
+We use `$g[u/x]$' to mean that assignment which is just like $g$
+except that $g(x) = u$. In Python, we can build the
+set $sat(\phi [x], g)$ with a \code{for} loop.%
+\footnote{The function \code{satisfiers} is an instance method of the
+ \code{Models} class, and \code{domain} is an attribute of that class.}
+\begin{SmVerbatim}
+def satisfiers(expr, var, g):
+ candidates = []
+ if freevar(var, expr):
+ for u in domain:
+ g.add(u, var)
+ if satisfy(expr, g):
+ candidates.append(u)
+ return set(candidates)
+\end{SmVerbatim}
+An existentially quantified formula $\exists x.\phi[x]$ is held to be
+true if and only if $sat(\phi [x], g)$ is nonempty. In Python,
+\code{len} can be used to return the cardinality of a set.
+\begin{SmVerbatim}
+ ...
+ elif parsed(expr) == (binder, body):
+ if binder == ('some', var):
+ sat = satisfiers(body, var, g)
+ return len(sat) > 0
+
+\end{SmVerbatim}
+In other words, a formula $\exists x.\phi[x]$ has the same value in
+model $M$ as the statement that the number of satisfiers in $M$ of
+$\phi[x]$ is greater than $0$.
+
+For comparison, Figure~\ref{fig:bb} shows the two Prolog clauses (one
+for truth and one for falsity) used to evaluate existentially
+quantified formulas in the \bnb\ code \code{modelChecker2.pl}.
+\begin{figure*}[tb]
+ \centering
+{\small
+\begin{Verbatim}
+satisfy(Formula,model(D,F),G,pos):-
+ nonvar(Formula),
+ Formula = some(X,SubFormula),
+ var(X),
+ memberList(V,D),
+ satisfy(SubFormula,model(D,F),[g(X,V)|G],pos).
+
+satisfy(Formula,model(D,F),G,neg):-
+ nonvar(Formula),
+ Formula = some(X,SubFormula),
+ var(X),
+ setof(V,memberList(V,D),All),
+ setof(V,
+ (
+ memberList(V,D),
+ satisfy(SubFormula,model(D,F),[g(X,V)|G],neg)
+ ),
+ All).
+\end{Verbatim}
+}
+ \caption{Prolog Clauses for Existential Quantification}
+ \label{fig:bb}
+\end{figure*}
+One reason why these clauses look more complex than their Python
+counterparts is that they include code for building the list of
+satisfiers by recursion.
+However, in Python we gain bivalency from
+the use of Boolean types as return values, and do not need to
+explicitly mark the polarity of the satisfaction clause. In addition,
+processing of sets and lists is supplied by a built-in Python library
+which avoids the use of predicates such as \code{memberList} and the
+\code{[Head|Tail]} notation.
+
+A universally quantified formula $\forall x.\phi[x]$ is held to be
+true if and only if every $u$ in the model's domain $D$ belongs to
+$sat(\phi [x], g)$. The \code{satisfy} clause above for
+existentials can therefore be extended with the clause:
+\begin{SmVerbatim}
+ ...
+ elif parsed(expr) == (binder, body):
+ ...
+ elif binder == ('all', var):
+ sat = self.satisfiers(body,var,g)
+ return domain.issubset(sat)
+\end{SmVerbatim}
+In other words, a formula $\forall x. \phi[x]$ has the same value
+in model $M$ as the statement that the domain of $M$ is a subset of the
+set of satisfiers in $M$ of $\phi[x]$.
+
+
+\subsection{Partiality}
+\label{sec:partial}
+
+
+As pointed out by \bnb, there are at least two
+cases where we might want the model checker to yield an `Undefined'
+value. The first is when we try to assign a semantic value to an
+unknown vocabulary item (i.e., to an unknown non-logical
+constant). The second arises through the use of partial variable
+assignments, when we try to evaluate $g(x)$ for some variable $x$ that
+is outside $g$'s domain. We adopt the assumption that if any sub-part
+of a complex expression is undefined, then the whole expression is
+undefined.%
+\footnote{This is not the only approach, since one could adopt the
+ position that a tautology such as $p \vee \neg p$ should be true
+ even if $p$ is undefined.}
+This means that an `undefined' value needs to propagate through all
+the recursive clauses of the \code{satisfy} function. This is
+potentially quite tedious to implement, since it means that instead of
+the clauses being able to expect return values to be Boolean, we also
+need to allow some alternative return type, such as a
+string. Fortunately, Python offers a nice solution through its
+exception handling mechanism.
+
+It is possible to create a new class of exceptions, derived from
+Python's \code{Exception} class. The \code{evaluate} module defines
+the class \code{Undefined}, and any function called by
+\code{satisfy} which attempts to interpret unknown vocabulary or
+assign a value to an out-of-domain variable will raise an
+\code{Undefined} exception. A recursive call within \code{satisfy}
+will automatically raise an \code{Undefined} exception to the
+calling function, and this means that an `undefined' value is
+automatically propagated up the stack without any additional
+machinery. At the top level, we wrap \code{satisfy} with a function
+\code{evaluate} which handles the exception by returning the string
+\code{'Undefined'} as value, rather than allowing the exception to
+raise any higher.
+
+
+\textsc{eafp} stands for `Easier to ask for forgiveness than
+permission'. According to van Rossum \shortcite{vanRossum:2006:PT},
+``this common Python coding style assumes the existence of valid keys
+or attributes and catches exceptions if the assumption proves false.''
+It contrasts with \textsc{lbyl} (`Look before you leap'), which explicitly
+tests for pre-conditions (such as type checks) before making calls or
+lookups. To continue with the discussion of partiality, we can see an
+example of \textsc{eafp} in the definition of the \code{i} function, which
+handles the interpretion of non-logical constants and individual
+variables.
+\begin{Verbatim}
+try:
+ return self.valuation[expr]
+except Undefined:
+ return g[expr]
+\end{Verbatim}
+We first try to evaluate \code{expr} as a non-logical constant; if
+\code{valuation} throws an \code{Undefined} exception, we check
+whether \code{g} can assign a value. If the latter also throws an
+\code{Undefined} exception, this will automatically be raised to the
+calling function.
+
+To sum up, an attractive consequence of this approach in Python is
+that no additional stipulations need to be added to the recursive
+clauses for interpreting Boolean connectives. By contrast, in the \bnb\
+\code{modelChecker2.pl} code, the clauses for existential
+quantification shown in Figure~\ref{fig:bb} need to be supplemented
+with a separate clause for the `undefined' case. In addition, as
+remarked earlier, each Boolean connective receives an additional
+clause when undefined.
+
+
+
+\section{Specifying Models}
+
+Models are specified by instantiating the \code{Model} class. At
+initialization, two parameters are called, determining the model's
+domain and valuation function. In Table~\ref{fig:folmodel}, we start
+by creating a \code{Valuation} object \code{val} (line 1), we then
+specify the valuation as a list \code{v} of \textit{constant-value}
+pairs (lines 2--9), using relational notation. For example, the value
+for \code{'adam'} is the individual \code{'d1'} (i.e., a Python
+string); the value for \code{'girl'} is the set consisting of
+individuals \code{'g1'} and \code{'g1'}; and the value for
+\code{'love'} is a set of pairs, as described above. We use the
+\code{parse} method to update \code{val} with this information
+(line 10). As mentioned earlier, a \code{Valuation} object has a
+\code{domain} property (line 11), and in this case \code{dom} will
+evaluate to the set \code{set(['b1', 'b2', 'g1', 'g2', 'd1'])}. It
+is convenient to use this set as the value for the model's domain when
+it is initialized (line 12). We also declare an \code{Assignment}
+object (line 13), specifying that its domain is the same as the model's
+domain.
+\begin{figure*}[tb]
+ \centering
+{\small
+\begin{Verbatim}[numbers=right]
+val = Valuation()
+v = [('adam', 'b1'), ('betty', 'g1'), ('fido', 'd1'),\
+ ('girl', set(['g1', 'g2'])),\
+ ('boy', set(['b1', 'b2'])),\
+ ('dog', set(['d1'])),\
+ ('love', set([('b1', 'g1'),\
+ ('b2', 'g2'),\
+ ('g1', 'b1'),\
+ ('g2', 'b1')]))]
+val.parse(v)
+dom = val.domain
+m = Model(dom, val)
+g = Assignment(dom, {'x': 'b1', 'y': 'g2'})
+\end{Verbatim}
+}
+ \caption{First-order model \code{m}}
+ \label{fig:folmodel}
+\end{figure*}
+
+Given model \code{m} and assignment \code{g}, we can evaluate
+\code{m.satisfiers(formula, g)}, for various values of
+\code{formulas}. This is quite a handy way of getting a feel for how
+connectives and quantifiers interact. A range of cases is illustrated
+in Table~\ref{fig:satisfiers}. As pointed out earlier, all formulas
+are represented as Python strings, and therefore need to be parsed
+before being evaluated.
+\begin{figure*}[htb]
+ \centering
+{\small
+ \begin{tabular}{ll}
+ Formula open in $x$ & Satisfiers \\ \hline
+\code{'(boy x)'} & \code{set(['b1', 'b2'])}\\
+\code{'(x = x)'} & \code{set(['b1', 'b2', 'g2', 'g1', 'd1'])}\\
+\code{'((boy x) or (girl x))'} & \code{set(['b2', 'g2', 'g1', 'b1'])}\\
+\code{'((boy x) and (girl x))'} & \code{set([])}\\
+\code{'(love x adam)'} & \code{set(['g1'])}\\
+\code{'(love adam x)'} & \code{set(['g2', 'g1'])}\\
+\code{'(not (x = adam))'} & \code{set(['b2', 'g2', 'g1', 'd1'])}\\
+\code{'some y.(love x y)'} & \code{set(['g2', 'g1', 'b1'])}\\
+\code{'all y.((girl y) implies (love y x))'} & \code{set([])}\\
+\code{'all y.((girl y) implies (love x y))'} & \code{set(['b1'])}\\
+\code{'((girl x) implies (dog x))'} & \code{set(['b1', 'b2', 'd1'])}\\
+\code{'all y.((dog y) implies (x = y))'} & \code{set(['d1'])}\\
+\code{'(not some y.(love x y))'} & \code{set(['b2', 'd1'])}\\
+\code{'some y.((love y adam) and (love x y))'} & \code{set(['b1'])}\\
+ \end{tabular}
+}
+ \caption{Satisfiers in model \code{m}}
+ \label{fig:satisfiers}
+\end{figure*}
+
+% Tracing of the evaluation process in the \code{evaluate} module is
+% still fairly rudimentary. Table~\ref{tab:trace} illustrates a trace of
+% the model checking carried out on
+% \code{'all x.((boy x) or (girl x))}. Although the final line of the trace
+% specifies a specific assigment, in fact variable assignments are
+% always set to be empty before evaluation commences; in other words,
+% closed formulas are true or false under arbitrary assignments.
+% \begin{table*}
+% \centering
+% \begin{tabular}{p{5in}}
+% \begin{Verbatim}
+% Open formula is '(or (boy x) (girl x))' with assignment g
+% ...trying assignment g[g2/x]
+% value of '(or (boy x) (girl x))' under g[g2/x] is True
+% ...trying assignment g[g1/x]
+% value of '(or (boy x) (girl x))' under g[g1/x] is True
+% ...trying assignment g[d1/x]
+% value of '(or (boy x) (girl x))' under g[d1/x] is False
+% ...trying assignment g[b2/x]
+% value of '(or (boy x) (girl x))' under g[b2/x] is True
+% ...trying assignment g[b1/x]
+% value of '(or (boy x) (girl x))' under g[b1/x] is True
+% '(boy x)' evaluates to False under M, g[d1/x]
+% '(girl x)' evaluates to False under M, g[d1/x]
+% '(or (boy x) (girl x))' evaluates to False under M, g[d1/x]
+% 'all x.((boy x) or (girl x))' evaluates to False under M, g[d1/x]
+% \end{Verbatim}
+% \end{tabular}
+% \caption{Sample Evaluation Trace}
+% \label{tab:trace}
+% \end{table*}
+
+%\FloatBarrier
+\section{Conclusion}
+
+In this paper, I have tried to show how various aspects of Python lend
+themselves well to the task of interpreting first-order formulas,
+following closely in the footsteps of Blackburn and Bos. I argue that
+at least in some cases, the Python implementation compares quite
+favourably to a Prolog-based approach. It will be observed that I have
+not considered efficiency issues. Although these cannot be ignored
+(and are certainly worth exploring), they are not a priority at this
+stage of development. As discussed at the outset, our main goal is
+develop a framework that can be used to communicate key ideas of
+formal semantics to students, rather than to build systems which can
+scale easily to tackle large problems.
+
+Clearly, there are many design choices to be made in any
+implementation, and an alternative framework which overlaps in part
+with what I have presented can be found in the Python code
+supplement to \cite{Russell:2003:AIMA}.%
+\footnote{\url{http://aima.cs.berkeley.edu/python}}
+One important distinction is that the approach adopted here is
+explicitly targeted at students learning computational linguistics,
+rather than being intended for a more general artificial intelligence
+audience.
+
+While I have restricted attention to rather basic topics in semantic
+interpretation, there is no obstacle to addressing more sophisticated
+topics in computational semantics. For example, I have not tried to
+address the crucial issue of quantifier scope ambiguity. However, work
+by Peter Wang (author of the \nltk\ module
+\verb!nltk_lite.contrib.hole!) implements the Hole Semantics of
+\bnb. This module contains a `plugging' algorithm which converts
+underspecified representations into fully-specified first-order logic
+formulas that can be displayed textually or graphically. In future
+work, we plan to extend the \prog{semantics} package in various
+directions, in particular by adding some basic inferencing mechanisms
+to \nltk.
+
+\subsection*{Acknowledgements}
+I am very grateful to Steven Bird, Patrick Blackburn, Alex Lascarides and
+three anonymous reviewers for helpful feedback and comments.
+
+\bibliographystyle{acl}
+\bibliography{altw-06}
+
+
+
+\end{document}
diff --git a/papers/altw-06/colacl06.sty b/papers/altw-06/colacl06.sty
new file mode 100644
index 0000000..ebc0598
--- /dev/null
+++ b/papers/altw-06/colacl06.sty
@@ -0,0 +1,368 @@
+% File colacl06.sty
+% This is the LaTeX style file for COLING/ACL 2006. It is identical to the style file for EACL 2006.
+
+% File eacl2006.sty
+% September 19, 2005
+% Contact: e.agirre at ehu.es or Sergi.Balari at uab.es
+
+% This is the LaTeX style file for EACL 2006. It is nearly identical to the
+% style files for ACL2005, ACL 2002, ACL 2001, ACL 2000, EACL 95 and EACL
+% 99.
+%
+% Changes made include: adapt layout to A4 and centimeters, widden abstract
+
+% This is the LaTeX style file for ACL 2000. It is nearly identical to the
+% style files for EACL 95 and EACL 99. Minor changes include editing the
+% instructions to reflect use of \documentclass rather than \documentstyle
+% and removing the white space before the title on the first page
+% -- John Chen, June 29, 2000
+
+% To convert from submissions prepared using the style file aclsub.sty
+% prepared for the ACL 2000 conference, proceed as follows:
+% 1) Remove submission-specific information: \whichsession, \id,
+% \wordcount, \otherconferences, \area, \keywords
+% 2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% 3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% 4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+% 5) Change the style reference from aclsub to acl2000, and be sure
+% this style file is in your TeX search path
+
+
+% This is the LaTeX style file for EACL-95. It is identical to the
+% style file for ANLP '94 except that the margins are adjusted for A4
+% paper. -- abney 13 Dec 94
+
+% The ANLP '94 style file is a slightly modified
+% version of the style used for AAAI and IJCAI, using some changes
+% prepared by Fernando Pereira and others and some minor changes
+% by Paul Jacobs.
+
+% Papers prepared using the aclsub.sty file and acl.bst bibtex style
+% should be easily converted to final format using this style.
+% (1) Submission information (\wordcount, \subject, and \makeidpage)
+% should be removed.
+% (2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% (between \begin{abstract} and \end{abstract}).
+% (3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% (4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+
+% Place this in a file called aclap.sty in the TeX search path.
+% (Placing it in the same directory as the paper should also work.)
+
+% Prepared by Peter F. Patel-Schneider, liberally using the ideas of
+% other style hackers, including Barbara Beeton.
+% This style is NOT guaranteed to work. It is provided in the hope
+% that it will make the preparation of papers easier.
+%
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at research.att.com
+
+% Papers are to be prepared using the ``acl'' bibliography style,
+% as follows:
+% \documentclass[11pt]{article}
+% \usepackage{acl2000}
+% \title{Title}
+% \author{Author 1 \and Author 2 \\ Address line \\ Address line \And
+% Author 3 \\ Address line \\ Address line}
+% \begin{document}
+% ...
+% \bibliography{bibliography-file}
+% \bibliographystyle{acl}
+% \end{document}
+
+% Author information can be set in various styles:
+% For several authors from the same institution:
+% \author{Author 1 \and ... \and Author n \\
+% Address line \\ ... \\ Address line}
+% if the names do not fit well on one line use
+% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
+% For authors from different institutions:
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \And ... \And
+% Author n \\ Address line \\ ... \\ Address line}
+% To start a seperate ``row'' of authors use \AND, as in
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \AND
+% Author 2 \\ Address line \\ ... \\ Address line \And
+% Author 3 \\ Address line \\ ... \\ Address line}
+
+% If the title and author information does not fit in the area allocated,
+% place \setlength\titlebox{<new height>} right after
+% \usepackage{acl2000}
+% where <new height> can be something larger than 2.25in
+
+% \typeout{Conference Style for ACL 2000 -- released June 20, 2000}
+\typeout{Conference Style for ACL 2005 -- released Octobe 11, 2004}
+
+% NOTE: Some laser printers have a serious problem printing TeX output.
+% These printing devices, commonly known as ``write-white'' laser
+% printers, tend to make characters too light. To get around this
+% problem, a darker set of fonts must be created for these devices.
+%
+
+%% % Physical page layout - slightly modified from IJCAI by pj
+%% \setlength\topmargin{0.0in} \setlength\oddsidemargin{-0.0in}
+%% \setlength\textheight{9.0in} \setlength\textwidth{6.5in}
+%% \setlength\columnsep{0.2in}
+%% \newlength\titlebox
+%% \setlength\titlebox{2.25in}
+%% \setlength\headheight{0pt} \setlength\headsep{0pt}
+%% %\setlength\footheight{0pt}
+%% \setlength\footskip{0pt}
+%% \thispagestyle{empty} \pagestyle{empty}
+%% \flushbottom \twocolumn \sloppy
+
+%% Original A4 version of page layout
+%% \setlength\topmargin{-0.45cm} % changed by Rz -1.4
+%% \setlength\oddsidemargin{.8mm} % was -0cm, changed by Rz
+%% \setlength\textheight{23.5cm}
+%% \setlength\textwidth{15.8cm}
+%% \setlength\columnsep{0.6cm}
+%% \newlength\titlebox
+%% \setlength\titlebox{2.00in}
+%% \setlength\headheight{5pt}
+%% \setlength\headsep{0pt}
+%% \setlength\footheight{0pt}
+%% \setlength\footskip{0pt}
+%% \thispagestyle{empty}
+%% \pagestyle{empty}
+
+% A4 modified by Eneko
+\setlength{\paperwidth}{21cm} % A4
+\setlength{\paperheight}{29.7cm}% A4
+\setlength\topmargin{-0.5cm}
+\setlength\oddsidemargin{0cm}
+\setlength\textheight{24.7cm}
+\setlength\textwidth{16.0cm}
+\setlength\columnsep{0.6cm}
+\newlength\titlebox
+\setlength\titlebox{2.00in}
+\setlength\headheight{5pt}
+\setlength\headsep{0pt}
+\thispagestyle{empty}
+\pagestyle{empty}
+
+
+\flushbottom \twocolumn \sloppy
+
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip at sandia-2
+\def\addcontentsline#1#2#3{}
+
+% Title stuff, taken from deproc.
+\def\maketitle{\par
+ \begingroup
+ \def\thefootnote{\fnsymbol{footnote}}
+ \def\@makefnmark{\hbox to 0pt{$^{\@thefnmark}$\hss}}
+ \twocolumn[\@maketitle] \@thanks
+ \endgroup
+ \setcounter{footnote}{0}
+ \let\maketitle\relax \let\@maketitle\relax
+ \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax}
+\def\@maketitle{\vbox to \titlebox{\hsize\textwidth
+ \linewidth\hsize \vskip 0.125in minus 0.125in \centering
+ {\Large\bf \@title \par} \vskip 0.2in plus 1fil minus 0.1in
+ {\def\and{\unskip\enspace{\rm and}\enspace}%
+ \def\And{\end{tabular}\hss \egroup \hskip 1in plus 2fil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}%
+ \def\AND{\end{tabular}\hss\egroup \hfil\hfil\egroup
+ \vskip 0.25in plus 1fil minus 0.125in
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf\@author
+ \end{tabular}\hss\egroup
+ \hfil\hfil\egroup}
+ \vskip 0.3in plus 2fil minus 0.1in
+}}
+
+% margins for abstract
+\renewenvironment{abstract}%
+ {\centerline{\large\bf Abstract}%
+ \begin{list}{}%
+ {\setlength{\rightmargin}{0.6cm}%
+ \setlength{\leftmargin}{0.6cm}}%
+ \item[]\ignorespaces}%
+ {\unskip\end{list}}
+
+%\renewenvironment{abstract}{\centerline{\large\bf
+% Abstract}\vspace{0.5ex}\begin{quote}}{\par\end{quote}\vskip 1ex}
+
+
+% bibliography
+
+\def\thebibliography#1{\section*{References}
+ \global\def\@listi{\leftmargin\leftmargini
+ \labelwidth\leftmargini \advance\labelwidth-\labelsep
+ \topsep 1pt plus 2pt minus 1pt
+ \parsep 0.25ex plus 1pt \itemsep 0.25ex plus 1pt}
+ \list {[\arabic{enumi}]}{\settowidth\labelwidth{[#1]}\leftmargin\labelwidth
+ \advance\leftmargin\labelsep\usecounter{enumi}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy
+ \sfcode`\.=1000\relax}
+
+\def\@up#1{\raise.2ex\hbox{#1}}
+
+% most of cite format is from aclsub.sty by SMS
+
+% don't box citations, separate with ; and a space
+% also, make the penalty between citations negative: a good place to break
+% changed comma back to semicolon pj 2/1/90
+% \def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+% \def\@citea{}\@cite{\@for\@citeb:=#2\do
+% {\@citea\def\@citea{;\penalty\@citeseppen\ }\@ifundefined
+% {b@\@citeb}{{\bf ?}\@warning
+% {Citation `\@citeb' on page \thepage \space undefined}}%
+% {\csname b@\@citeb\endcsname}}}{#1}}
+
+% don't box citations, separate with ; and a space
+% Replaced for multiple citations (pj)
+% don't box citations and also add space, semicolon between multiple citations
+\def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@cite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{; }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+ {\csname b@\@citeb\endcsname}}}{#1}}
+
+% Allow short (name-less) citations, when used in
+% conjunction with a bibliography style that creates labels like
+% \citename{<names>, }<year>
+%
+\let\@internalcite\cite
+\def\cite{\def\citename##1{##1, }\@internalcite}
+\def\shortcite{\def\citename##1{}\@internalcite}
+\def\newcite{\def\citename##1{{\frenchspacing##1} (}\@internalciteb}
+
+% Macros for \newcite, which leaves name in running text, and is
+% otherwise like \shortcite.
+\def\@citexb[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@newcite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{;\penalty\@m\ }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+{\csname b@\@citeb\endcsname}}}{#1}}
+\def\@internalciteb{\@ifnextchar [{\@tempswatrue\@citexb}{\@tempswafalse\@citexb[]}}
+
+\def\@newcite#1#2{{#1\if at tempswa, #2\fi)}}
+
+\def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+
+%%% More changes made by SMS (originals in latex.tex)
+% Use parentheses instead of square brackets in the text.
+\def\@cite#1#2{({#1\if at tempswa , #2\fi})}
+
+% Don't put a label in the bibliography at all. Just use the unlabeled format
+% instead.
+\def\thebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{References\@mkboth
+ {References}{References}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthebibliography=\endlist
+
+% Allow for a bibliography of sources of attested examples
+\def\thesourcebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{Sources of Attested Examples\@mkboth
+ {Sources of Attested Examples}{Sources of Attested Examples}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthesourcebibliography=\endlist
+
+\def\@lbibitem[#1]#2{\item[]\if at filesw
+ { \def\protect##1{\string ##1\space}\immediate
+ \write\@auxout{\string\bibcite{#2}{#1}}\fi\ignorespaces}}
+
+\def\@bibitem#1{\item\if at filesw \immediate\write\@auxout
+ {\string\bibcite{#1}{\the\c at enumi}}\fi\ignorespaces}
+
+% sections with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+ -0.5ex minus -.2ex}{1.5ex plus 0.3ex minus .2ex}{\large\bf\raggedright}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-1.8ex plus
+ -0.5ex minus -.2ex}{0.8ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{0.5ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\paragraph{\@startsection{paragraph}{4}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+\def\subparagraph{\@startsection{subparagraph}{5}{\parindent}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+
+% Footnotes
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+
+% Lists and paragraphs
+\parindent 1em
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 2pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+
+\leftmargin 2em \leftmargini\leftmargin \leftmarginii 2em
+\leftmarginiii 1.5em \leftmarginiv 1.0em \leftmarginv .5em \leftmarginvi .5em
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+ \labelwidth\leftmarginii\advance\labelwidth-\labelsep
+ \topsep 2pt plus 1pt minus 0.5pt
+ \parsep 1pt plus 0.5pt minus 0.5pt
+ \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+ \labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+ \topsep 1pt plus 0.5pt minus 0.5pt
+ \parsep \z@ \partopsep 0.5pt plus 0pt minus 0.5pt
+ \itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+ \labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+ \labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+ \labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\@normalsize{\@setsize\normalsize{11pt}\xpt\@xpt}
+\def\small{\@setsize\small{10pt}\ixpt\@ixpt}
+\def\footnotesize{\@setsize\footnotesize{10pt}\ixpt\@ixpt}
+\def\scriptsize{\@setsize\scriptsize{8pt}\viipt\@viipt}
+\def\tiny{\@setsize\tiny{7pt}\vipt\@vipt}
+\def\large{\@setsize\large{14pt}\xiipt\@xiipt}
+\def\Large{\@setsize\Large{16pt}\xivpt\@xivpt}
+\def\LARGE{\@setsize\LARGE{20pt}\xviipt\@xviipt}
+\def\huge{\@setsize\huge{23pt}\xxpt\@xxpt}
+\def\Huge{\@setsize\Huge{28pt}\xxvpt\@xxvpt}
diff --git a/papers/icon-05/acl.bst b/papers/icon-05/acl.bst
new file mode 100644
index 0000000..b95ec04
--- /dev/null
+++ b/papers/icon-05/acl.bst
@@ -0,0 +1,1322 @@
+
+% BibTeX `acl' style file for BibTeX version 0.99c, LaTeX version 2.09
+% This version was made by modifying `aaai-named' format based on the master
+% file by Oren Patashnik (PATASHNIK at SCORE.STANFORD.EDU)
+
+% Copyright (C) 1985, all rights reserved.
+% Modifications Copyright 1988, Peter F. Patel-Schneider
+% Further modifictions by Stuart Shieber, 1991, and Fernando Pereira, 1992.
+% Copying of this file is authorized only if either
+% (1) you make absolutely no changes to your copy, including name, or
+% (2) if you do make changes, you name it something other than
+% btxbst.doc, plain.bst, unsrt.bst, alpha.bst, and abbrv.bst.
+% This restriction helps ensure that all standard styles are identical.
+
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at spar.slb.com
+
+% Citation format: [author-last-name, year]
+% [author-last-name and author-last-name, year]
+% [author-last-name {\em et al.}, year]
+%
+% Reference list ordering: alphabetical by author or whatever passes
+% for author in the absence of one.
+%
+% This BibTeX style has support for short (year only) citations. This
+% is done by having the citations actually look like
+% \citename{name-info, }year
+% The LaTeX style has to have the following
+% \let\@internalcite\cite
+% \def\cite{\def\citename##1{##1}\@internalcite}
+% \def\shortcite{\def\citename##1{}\@internalcite}
+% \def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+% which makes \shortcite the macro for short citations.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Changes made by SMS for thesis style
+% no emphasis on "et al."
+% "Ph.D." includes periods (not "PhD")
+% moved year to immediately after author's name
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ENTRY
+ { address
+ author
+ booktitle
+ chapter
+ edition
+ editor
+ howpublished
+ institution
+ journal
+ key
+ month
+ note
+ number
+ organization
+ pages
+ publisher
+ school
+ series
+ title
+ type
+ volume
+ year
+ }
+ {}
+ { label extra.label sort.label }
+
+INTEGERS { output.state before.all mid.sentence after.sentence after.block }
+
+FUNCTION {init.state.consts}
+{ #0 'before.all :=
+ #1 'mid.sentence :=
+ #2 'after.sentence :=
+ #3 'after.block :=
+}
+
+STRINGS { s t }
+
+FUNCTION {output.nonnull}
+{ 's :=
+ output.state mid.sentence =
+ { ", " * write$ }
+ { output.state after.block =
+ { add.period$ write$
+ newline$
+ "\newblock " write$
+ }
+ { output.state before.all =
+ 'write$
+ { add.period$ " " * write$ }
+ if$
+ }
+ if$
+ mid.sentence 'output.state :=
+ }
+ if$
+ s
+}
+
+FUNCTION {output}
+{ duplicate$ empty$
+ 'pop$
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.check}
+{ 't :=
+ duplicate$ empty$
+ { pop$ "empty " t * " in " * cite$ * warning$ }
+ 'output.nonnull
+ if$
+}
+
+FUNCTION {output.bibitem}
+{ newline$
+
+ "\bibitem[" write$
+ label write$
+ "]{" write$
+
+ cite$ write$
+ "}" write$
+ newline$
+ ""
+ before.all 'output.state :=
+}
+
+FUNCTION {fin.entry}
+{ add.period$
+ write$
+ newline$
+}
+
+FUNCTION {new.block}
+{ output.state before.all =
+ 'skip$
+ { after.block 'output.state := }
+ if$
+}
+
+FUNCTION {new.sentence}
+{ output.state after.block =
+ 'skip$
+ { output.state before.all =
+ 'skip$
+ { after.sentence 'output.state := }
+ if$
+ }
+ if$
+}
+
+FUNCTION {not}
+{ { #0 }
+ { #1 }
+ if$
+}
+
+FUNCTION {and}
+{ 'skip$
+ { pop$ #0 }
+ if$
+}
+
+FUNCTION {or}
+{ { pop$ #1 }
+ 'skip$
+ if$
+}
+
+FUNCTION {new.block.checka}
+{ empty$
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.block.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.block
+ if$
+}
+
+FUNCTION {new.sentence.checka}
+{ empty$
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {new.sentence.checkb}
+{ empty$
+ swap$ empty$
+ and
+ 'skip$
+ 'new.sentence
+ if$
+}
+
+FUNCTION {field.or.null}
+{ duplicate$ empty$
+ { pop$ "" }
+ 'skip$
+ if$
+}
+
+FUNCTION {emphasize}
+{ duplicate$ empty$
+ { pop$ "" }
+ { "{\em " swap$ * "}" * }
+ if$
+}
+
+INTEGERS { nameptr namesleft numnames }
+
+FUNCTION {format.names}
+{ 's :=
+ #1 'nameptr :=
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+
+ { s nameptr "{ff~}{vv~}{ll}{, jj}" format.name$ 't :=
+
+ nameptr #1 >
+ { namesleft #1 >
+ { ", " * t * }
+ { numnames #2 >
+ { "," * }
+ 'skip$
+ if$
+ t "others" =
+ { " et~al." * }
+ { " and " * t * }
+ if$
+ }
+ if$
+ }
+ 't
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {format.authors}
+{ author empty$
+ { "" }
+ { author format.names }
+ if$
+}
+
+FUNCTION {format.editors}
+{ editor empty$
+ { "" }
+ { editor format.names
+ editor num.names$ #1 >
+ { ", editors" * }
+ { ", editor" * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.title}
+{ title empty$
+ { "" }
+
+ { title "t" change.case$ }
+
+ if$
+}
+
+FUNCTION {n.dashify}
+{ 't :=
+ ""
+ { t empty$ not }
+ { t #1 #1 substring$ "-" =
+ { t #1 #2 substring$ "--" = not
+ { "--" *
+ t #2 global.max$ substring$ 't :=
+ }
+ { { t #1 #1 substring$ "-" = }
+ { "-" *
+ t #2 global.max$ substring$ 't :=
+ }
+ while$
+ }
+ if$
+ }
+ { t #1 #1 substring$ *
+ t #2 global.max$ substring$ 't :=
+ }
+ if$
+ }
+ while$
+}
+
+FUNCTION {format.date}
+{ year empty$
+ { month empty$
+ { "" }
+ { "there's a month but no year in " cite$ * warning$
+ month
+ }
+ if$
+ }
+ { month empty$
+ { "" }
+ { month }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.btitle}
+{ title emphasize
+}
+
+FUNCTION {tie.or.space.connect}
+{ duplicate$ text.length$ #3 <
+ { "~" }
+ { " " }
+ if$
+ swap$ * *
+}
+
+FUNCTION {either.or.check}
+{ empty$
+ 'pop$
+ { "can't use both " swap$ * " fields in " * cite$ * warning$ }
+ if$
+}
+
+FUNCTION {format.bvolume}
+{ volume empty$
+ { "" }
+ { "volume" volume tie.or.space.connect
+ series empty$
+ 'skip$
+ { " of " * series emphasize * }
+ if$
+ "volume and number" number either.or.check
+ }
+ if$
+}
+
+FUNCTION {format.number.series}
+{ volume empty$
+ { number empty$
+ { series field.or.null }
+ { output.state mid.sentence =
+ { "number" }
+ { "Number" }
+ if$
+ number tie.or.space.connect
+ series empty$
+ { "there's a number but no series in " cite$ * warning$ }
+ { " in " * series * }
+ if$
+ }
+ if$
+ }
+ { "" }
+ if$
+}
+
+FUNCTION {format.edition}
+{ edition empty$
+ { "" }
+ { output.state mid.sentence =
+ { edition "l" change.case$ " edition" * }
+ { edition "t" change.case$ " edition" * }
+ if$
+ }
+ if$
+}
+
+INTEGERS { multiresult }
+
+FUNCTION {multi.page.check}
+{ 't :=
+ #0 'multiresult :=
+ { multiresult not
+ t empty$ not
+ and
+ }
+ { t #1 #1 substring$
+ duplicate$ "-" =
+ swap$ duplicate$ "," =
+ swap$ "+" =
+ or or
+ { #1 'multiresult := }
+ { t #2 global.max$ substring$ 't := }
+ if$
+ }
+ while$
+ multiresult
+}
+
+FUNCTION {format.pages}
+{ pages empty$
+ { "" }
+ { pages multi.page.check
+ { "pages" pages n.dashify tie.or.space.connect }
+ { "page" pages tie.or.space.connect }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.year.label}
+{ year extra.label *
+}
+
+FUNCTION {format.vol.num.pages}
+{ volume field.or.null
+ number empty$
+ 'skip$
+ { "(" number * ")" * *
+ volume empty$
+ { "there's a number but no volume in " cite$ * warning$ }
+ 'skip$
+ if$
+ }
+ if$
+ pages empty$
+ 'skip$
+ { duplicate$ empty$
+ { pop$ format.pages }
+ { ":" * pages n.dashify * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.chapter.pages}
+{ chapter empty$
+ 'format.pages
+ { type empty$
+ { "chapter" }
+ { type "l" change.case$ }
+ if$
+ chapter tie.or.space.connect
+ pages empty$
+ 'skip$
+ { ", " * format.pages * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.in.ed.booktitle}
+{ booktitle empty$
+ { "" }
+ { editor empty$
+ { "In " booktitle emphasize * }
+ { "In " format.editors * ", " * booktitle emphasize * }
+ if$
+ }
+ if$
+}
+
+FUNCTION {empty.misc.check}
+{ author empty$ title empty$ howpublished empty$
+ month empty$ year empty$ note empty$
+ and and and and and
+
+ key empty$ not and
+
+ { "all relevant fields are empty in " cite$ * warning$ }
+ 'skip$
+ if$
+}
+
+FUNCTION {format.thesis.type}
+{ type empty$
+ 'skip$
+ { pop$
+ type "t" change.case$
+ }
+ if$
+}
+
+FUNCTION {format.tr.number}
+{ type empty$
+ { "Technical Report" }
+ 'type
+ if$
+ number empty$
+ { "t" change.case$ }
+ { number tie.or.space.connect }
+ if$
+}
+
+FUNCTION {format.article.crossref}
+{ key empty$
+ { journal empty$
+ { "need key or journal for " cite$ * " to crossref " * crossref *
+ warning$
+ ""
+ }
+ { "In {\em " journal * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.crossref.editor}
+{ editor #1 "{vv~}{ll}" format.name$
+ editor num.names$ duplicate$
+ #2 >
+ { pop$ " et~al." * }
+ { #2 <
+ 'skip$
+ { editor #2 "{ff }{vv }{ll}{ jj}" format.name$ "others" =
+ { " et~al." * }
+ { " and " * editor #2 "{vv~}{ll}" format.name$ * }
+ if$
+ }
+ if$
+ }
+ if$
+}
+
+FUNCTION {format.book.crossref}
+{ volume empty$
+ { "empty volume in " cite$ * "'s crossref of " * crossref * warning$
+ "In "
+ }
+ { "Volume" volume tie.or.space.connect
+ " of " *
+ }
+ if$
+ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { series empty$
+ { "need editor, key, or series for " cite$ * " to crossref " *
+ crossref * warning$
+ "" *
+ }
+ { "{\em " * series * "\/}" * }
+ if$
+ }
+ { key * }
+ if$
+ }
+ { format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {format.incoll.inproc.crossref}
+{ editor empty$
+ editor field.or.null author field.or.null =
+ or
+ { key empty$
+ { booktitle empty$
+ { "need editor, key, or booktitle for " cite$ * " to crossref " *
+ crossref * warning$
+ ""
+ }
+ { "In {\em " booktitle * "\/}" * }
+ if$
+ }
+ { "In " key * }
+ if$
+ }
+ { "In " format.crossref.editor * }
+ if$
+ " \cite{" * crossref * "}" *
+}
+
+FUNCTION {article}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { journal emphasize "journal" output.check
+ format.vol.num.pages output
+ format.date output
+ }
+ { format.article.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {book}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {booklet}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ howpublished address new.block.checkb
+ howpublished output
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inbook}
+{ output.bibitem
+ author empty$
+ { format.editors "author and editor" output.check }
+ { format.authors output.nonnull
+ crossref missing$
+ { "author and editor" editor either.or.check }
+ 'skip$
+ if$
+ }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ crossref missing$
+ { format.bvolume output
+ format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.number.series output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ }
+ { format.chapter.pages "chapter and pages" output.check
+ new.block
+ format.book.crossref output.nonnull
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {incollection}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.chapter.pages output
+ new.sentence
+ publisher "publisher" output.check
+ address output
+ format.edition output
+ format.date output
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.chapter.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {inproceedings}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ crossref missing$
+ { format.in.ed.booktitle "booktitle" output.check
+ format.bvolume output
+ format.number.series output
+ format.pages output
+ address empty$
+ { organization publisher new.sentence.checkb
+ organization output
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ organization output
+ publisher output
+ }
+ if$
+ }
+ { format.incoll.inproc.crossref output.nonnull
+ format.pages output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {conference} { inproceedings }
+
+FUNCTION {manual}
+{ output.bibitem
+ author empty$
+ { organization empty$
+ 'skip$
+ { organization output.nonnull
+ address output
+ }
+ if$
+ }
+ { format.authors output.nonnull }
+ if$
+ format.year.label "year" output.check
+ new.block
+ new.block
+ format.btitle "title" output.check
+ author empty$
+ { organization empty$
+ { address new.block.checka
+ address output
+ }
+ 'skip$
+ if$
+ }
+ { organization address new.block.checkb
+ organization output
+ address output
+ }
+ if$
+ format.edition output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {mastersthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ "Master's thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {misc}
+{ output.bibitem
+ format.authors output
+ new.block
+ format.year.label output
+ new.block
+ title howpublished new.block.checkb
+ format.title output
+ howpublished new.block.checka
+ howpublished output
+ format.date output
+ new.block
+ note output
+ fin.entry
+ empty.misc.check
+}
+
+FUNCTION {phdthesis}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ new.block
+ "{Ph.D.} thesis" format.thesis.type output.nonnull
+ school "school" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {proceedings}
+{ output.bibitem
+ editor empty$
+ { organization output }
+ { format.editors output.nonnull }
+ if$
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.btitle "title" output.check
+ format.bvolume output
+ format.number.series output
+ address empty$
+ { editor empty$
+ { publisher new.sentence.checka }
+ { organization publisher new.sentence.checkb
+ organization output
+ }
+ if$
+ publisher output
+ format.date output
+ }
+ { address output.nonnull
+ format.date output
+ new.sentence
+ editor empty$
+ 'skip$
+ { organization output }
+ if$
+ publisher output
+ }
+ if$
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {techreport}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ format.tr.number output.nonnull
+ institution "institution" output.check
+ address output
+ format.date output
+ new.block
+ note output
+ fin.entry
+}
+
+FUNCTION {unpublished}
+{ output.bibitem
+ format.authors "author" output.check
+ new.block
+ format.year.label "year" output.check
+ new.block
+ format.title "title" output.check
+ new.block
+ note "note" output.check
+ format.date output
+ fin.entry
+}
+
+FUNCTION {default.type} { misc }
+
+MACRO {jan} {"January"}
+
+MACRO {feb} {"February"}
+
+MACRO {mar} {"March"}
+
+MACRO {apr} {"April"}
+
+MACRO {may} {"May"}
+
+MACRO {jun} {"June"}
+
+MACRO {jul} {"July"}
+
+MACRO {aug} {"August"}
+
+MACRO {sep} {"September"}
+
+MACRO {oct} {"October"}
+
+MACRO {nov} {"November"}
+
+MACRO {dec} {"December"}
+
+MACRO {acmcs} {"ACM Computing Surveys"}
+
+MACRO {acta} {"Acta Informatica"}
+
+MACRO {cacm} {"Communications of the ACM"}
+
+MACRO {ibmjrd} {"IBM Journal of Research and Development"}
+
+MACRO {ibmsj} {"IBM Systems Journal"}
+
+MACRO {ieeese} {"IEEE Transactions on Software Engineering"}
+
+MACRO {ieeetc} {"IEEE Transactions on Computers"}
+
+MACRO {ieeetcad}
+ {"IEEE Transactions on Computer-Aided Design of Integrated Circuits"}
+
+MACRO {ipl} {"Information Processing Letters"}
+
+MACRO {jacm} {"Journal of the ACM"}
+
+MACRO {jcss} {"Journal of Computer and System Sciences"}
+
+MACRO {scp} {"Science of Computer Programming"}
+
+MACRO {sicomp} {"SIAM Journal on Computing"}
+
+MACRO {tocs} {"ACM Transactions on Computer Systems"}
+
+MACRO {tods} {"ACM Transactions on Database Systems"}
+
+MACRO {tog} {"ACM Transactions on Graphics"}
+
+MACRO {toms} {"ACM Transactions on Mathematical Software"}
+
+MACRO {toois} {"ACM Transactions on Office Information Systems"}
+
+MACRO {toplas} {"ACM Transactions on Programming Languages and Systems"}
+
+MACRO {tcs} {"Theoretical Computer Science"}
+
+READ
+
+FUNCTION {sortify}
+{ purify$
+ "l" change.case$
+}
+
+INTEGERS { len }
+
+FUNCTION {chop.word}
+{ 's :=
+ 'len :=
+ s #1 len substring$ =
+ { s len #1 + global.max$ substring$ }
+ 's
+ if$
+}
+
+INTEGERS { et.al.char.used }
+
+FUNCTION {initialize.et.al.char.used}
+{ #0 'et.al.char.used :=
+}
+
+EXECUTE {initialize.et.al.char.used}
+
+FUNCTION {format.lab.names}
+{ 's :=
+ s num.names$ 'numnames :=
+
+ numnames #1 =
+ { s #1 "{vv }{ll}" format.name$ }
+ { numnames #2 =
+ { s #1 "{vv }{ll }and " format.name$ s #2 "{vv }{ll}" format.name$ *
+ }
+ { s #1 "{vv }{ll }\bgroup et al.\egroup " format.name$ }
+ if$
+ }
+ if$
+
+}
+
+FUNCTION {author.key.label}
+{ author empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.editor.key.label}
+{ author empty$
+ { editor empty$
+ { key empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {author.key.organization.label}
+{ author empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { author format.lab.names }
+ if$
+}
+
+FUNCTION {editor.key.organization.label}
+{ editor empty$
+ { key empty$
+ { organization empty$
+
+ { cite$ #1 #3 substring$ }
+
+ { "The " #4 organization chop.word #3 text.prefix$ }
+ if$
+ }
+ { key #3 text.prefix$ }
+ if$
+ }
+ { editor format.lab.names }
+ if$
+}
+
+FUNCTION {calc.label}
+{ type$ "book" =
+ type$ "inbook" =
+ or
+ 'author.editor.key.label
+ { type$ "proceedings" =
+ 'editor.key.organization.label
+ { type$ "manual" =
+ 'author.key.organization.label
+ 'author.key.label
+ if$
+ }
+ if$
+ }
+ if$
+ duplicate$
+
+ "\protect\citename{" swap$ * "}" *
+ year field.or.null purify$ *
+ 'label :=
+ year field.or.null purify$ *
+
+ sortify 'sort.label :=
+}
+
+FUNCTION {sort.format.names}
+{ 's :=
+ #1 'nameptr :=
+ ""
+ s num.names$ 'numnames :=
+ numnames 'namesleft :=
+ { namesleft #0 > }
+ { nameptr #1 >
+ { " " * }
+ 'skip$
+ if$
+
+ s nameptr "{vv{ } }{ll{ }}{ ff{ }}{ jj{ }}" format.name$ 't :=
+
+ nameptr numnames = t "others" = and
+ { "et al" * }
+ { t sortify * }
+ if$
+ nameptr #1 + 'nameptr :=
+ namesleft #1 - 'namesleft :=
+ }
+ while$
+}
+
+FUNCTION {sort.format.title}
+{ 't :=
+ "A " #2
+ "An " #3
+ "The " #4 t chop.word
+ chop.word
+ chop.word
+ sortify
+ #1 global.max$ substring$
+}
+
+FUNCTION {author.sort}
+{ author empty$
+ { key empty$
+ { "to sort, need author or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.editor.sort}
+{ author empty$
+ { editor empty$
+ { key empty$
+ { "to sort, need author, editor, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {author.organization.sort}
+{ author empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need author, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { author sort.format.names }
+ if$
+}
+
+FUNCTION {editor.organization.sort}
+{ editor empty$
+ { organization empty$
+ { key empty$
+ { "to sort, need editor, organization, or key in " cite$ * warning$
+ ""
+ }
+ { key sortify }
+ if$
+ }
+ { "The " #4 organization chop.word sortify }
+ if$
+ }
+ { editor sort.format.names }
+ if$
+}
+
+FUNCTION {presort}
+
+{ calc.label
+ sort.label
+ " "
+ *
+ type$ "book" =
+
+ type$ "inbook" =
+ or
+ 'author.editor.sort
+ { type$ "proceedings" =
+ 'editor.organization.sort
+ { type$ "manual" =
+ 'author.organization.sort
+ 'author.sort
+ if$
+ }
+ if$
+ }
+ if$
+
+ *
+
+ " "
+ *
+ year field.or.null sortify
+ *
+ " "
+ *
+ title field.or.null
+ sort.format.title
+ *
+ #1 entry.max$ substring$
+ 'sort.key$ :=
+}
+
+ITERATE {presort}
+
+SORT
+
+STRINGS { longest.label last.sort.label next.extra }
+
+INTEGERS { longest.label.width last.extra.num }
+
+FUNCTION {initialize.longest.label}
+{ "" 'longest.label :=
+ #0 int.to.chr$ 'last.sort.label :=
+ "" 'next.extra :=
+ #0 'longest.label.width :=
+ #0 'last.extra.num :=
+}
+
+FUNCTION {forward.pass}
+{ last.sort.label sort.label =
+ { last.extra.num #1 + 'last.extra.num :=
+ last.extra.num int.to.chr$ 'extra.label :=
+ }
+ { "a" chr.to.int$ 'last.extra.num :=
+ "" 'extra.label :=
+ sort.label 'last.sort.label :=
+ }
+ if$
+}
+
+FUNCTION {reverse.pass}
+{ next.extra "b" =
+ { "a" 'extra.label := }
+ 'skip$
+ if$
+ label extra.label * 'label :=
+ label width$ longest.label.width >
+ { label 'longest.label :=
+ label width$ 'longest.label.width :=
+ }
+ 'skip$
+ if$
+ extra.label 'next.extra :=
+}
+
+EXECUTE {initialize.longest.label}
+
+ITERATE {forward.pass}
+
+REVERSE {reverse.pass}
+
+FUNCTION {begin.bib}
+
+{ et.al.char.used
+ { "\newcommand{\etalchar}[1]{$^{#1}$}" write$ newline$ }
+ 'skip$
+ if$
+ preamble$ empty$
+
+ 'skip$
+ { preamble$ write$ newline$ }
+ if$
+
+ "\begin{thebibliography}{" "}" * write$ newline$
+
+}
+
+EXECUTE {begin.bib}
+
+EXECUTE {init.state.consts}
+
+ITERATE {call.type$}
+
+FUNCTION {end.bib}
+{ newline$
+ "\end{thebibliography}" write$ newline$
+}
+
+EXECUTE {end.bib}
+
diff --git a/papers/icon-05/acl2005.sty b/papers/icon-05/acl2005.sty
new file mode 100644
index 0000000..bc91c6c
--- /dev/null
+++ b/papers/icon-05/acl2005.sty
@@ -0,0 +1,338 @@
+% File acl2005.sty
+% October 11, 2004
+% Contact: oflazer at sabanciuniv.edu
+
+% This is the LaTeX style file for ACL 2005. It is nearly identical to the
+% style files for ACL 2002, ACL 2001, ACL 2000, EACL 95 and EACL
+% 99.
+%
+
+% This is the LaTeX style file for ACL 2000. It is nearly identical to the
+% style files for EACL 95 and EACL 99. Minor changes include editing the
+% instructions to reflect use of \documentclass rather than \documentstyle
+% and removing the white space before the title on the first page
+% -- John Chen, June 29, 2000
+
+% To convert from submissions prepared using the style file aclsub.sty
+% prepared for the ACL 2000 conference, proceed as follows:
+% 1) Remove submission-specific information: \whichsession, \id,
+% \wordcount, \otherconferences, \area, \keywords
+% 2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% 3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% 4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+% 5) Change the style reference from aclsub to acl2000, and be sure
+% this style file is in your TeX search path
+
+
+% This is the LaTeX style file for EACL-95. It is identical to the
+% style file for ANLP '94 except that the margins are adjusted for A4
+% paper. -- abney 13 Dec 94
+
+% The ANLP '94 style file is a slightly modified
+% version of the style used for AAAI and IJCAI, using some changes
+% prepared by Fernando Pereira and others and some minor changes
+% by Paul Jacobs.
+
+% Papers prepared using the aclsub.sty file and acl.bst bibtex style
+% should be easily converted to final format using this style.
+% (1) Submission information (\wordcount, \subject, and \makeidpage)
+% should be removed.
+% (2) \summary should be removed. The summary material should come
+% after \maketitle and should be in the ``abstract'' environment
+% (between \begin{abstract} and \end{abstract}).
+% (3) Check all citations. This style should handle citations correctly
+% and also allows multiple citations separated by semicolons.
+% (4) Check figures and examples. Because the final format is double-
+% column, some adjustments may have to be made to fit text in the column
+% or to choose full-width (\figure*} figures.
+
+% Place this in a file called aclap.sty in the TeX search path.
+% (Placing it in the same directory as the paper should also work.)
+
+% Prepared by Peter F. Patel-Schneider, liberally using the ideas of
+% other style hackers, including Barbara Beeton.
+% This style is NOT guaranteed to work. It is provided in the hope
+% that it will make the preparation of papers easier.
+%
+% There are undoubtably bugs in this style. If you make bug fixes,
+% improvements, etc. please let me know. My e-mail address is:
+% pfps at research.att.com
+
+% Papers are to be prepared using the ``acl'' bibliography style,
+% as follows:
+% \documentclass[11pt]{article}
+% \usepackage{acl2000}
+% \title{Title}
+% \author{Author 1 \and Author 2 \\ Address line \\ Address line \And
+% Author 3 \\ Address line \\ Address line}
+% \begin{document}
+% ...
+% \bibliography{bibliography-file}
+% \bibliographystyle{acl}
+% \end{document}
+
+% Author information can be set in various styles:
+% For several authors from the same institution:
+% \author{Author 1 \and ... \and Author n \\
+% Address line \\ ... \\ Address line}
+% if the names do not fit well on one line use
+% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
+% For authors from different institutions:
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \And ... \And
+% Author n \\ Address line \\ ... \\ Address line}
+% To start a seperate ``row'' of authors use \AND, as in
+% \author{Author 1 \\ Address line \\ ... \\ Address line
+% \AND
+% Author 2 \\ Address line \\ ... \\ Address line \And
+% Author 3 \\ Address line \\ ... \\ Address line}
+
+% If the title and author information does not fit in the area allocated,
+% place \setlength\titlebox{<new height>} right after
+% \usepackage{acl2000}
+% where <new height> can be something larger than 2.25in
+
+% \typeout{Conference Style for ACL 2000 -- released June 20, 2000}
+\typeout{Conference Style for ACL 2005 -- released Octobe 11, 2004}
+
+% NOTE: Some laser printers have a serious problem printing TeX output.
+% These printing devices, commonly known as ``write-white'' laser
+% printers, tend to make characters too light. To get around this
+% problem, a darker set of fonts must be created for these devices.
+%
+
+% Physical page layout - slightly modified from IJCAI by pj
+\setlength\topmargin{0.0in} \setlength\oddsidemargin{-0.0in}
+\setlength\textheight{9.0in} \setlength\textwidth{6.5in}
+\setlength\columnsep{0.2in}
+\newlength\titlebox
+\setlength\titlebox{2.25in}
+\setlength\headheight{0pt} \setlength\headsep{0pt}
+%\setlength\footheight{0pt}
+\setlength\footskip{0pt}
+\thispagestyle{empty} \pagestyle{empty}
+\flushbottom \twocolumn \sloppy
+
+%% A4 version of page layout
+%\setlength\topmargin{-0.45cm} % changed by Rz -1.4
+%\setlength\oddsidemargin{.8mm} % was -0cm, changed by Rz
+%\setlength\textheight{23.5cm}
+%\setlength\textwidth{15.8cm}
+%\setlength\columnsep{0.6cm}
+%\newlength\titlebox
+%\setlength\titlebox{2.00in}
+%\setlength\headheight{5pt}
+%\setlength\headsep{0pt}
+%%\setlength\footheight{0pt}
+%\setlength\footskip{0pt}
+%\thispagestyle{empty}
+%\pagestyle{empty}
+
+\flushbottom \twocolumn \sloppy
+
+% We're never going to need a table of contents, so just flush it to
+% save space --- suggested by drstrip at sandia-2
+\def\addcontentsline#1#2#3{}
+
+% Title stuff, taken from deproc.
+\def\maketitle{\par
+ \begingroup
+ \def\thefootnote{\fnsymbol{footnote}}
+ \def\@makefnmark{\hbox to 0pt{$^{\@thefnmark}$\hss}}
+ \twocolumn[\@maketitle] \@thanks
+ \endgroup
+ \setcounter{footnote}{0}
+ \let\maketitle\relax \let\@maketitle\relax
+ \gdef\@thanks{}\gdef\@author{}\gdef\@title{}\let\thanks\relax}
+\def\@maketitle{\vbox to \titlebox{\hsize\textwidth
+ \linewidth\hsize \vskip 0.125in minus 0.125in \centering
+ {\Large\bf \@title \par} \vskip 0.2in plus 1fil minus 0.1in
+ {\def\and{\unskip\enspace{\rm and}\enspace}%
+ \def\And{\end{tabular}\hss \egroup \hskip 1in plus 2fil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}%
+ \def\AND{\end{tabular}\hss\egroup \hfil\hfil\egroup
+ \vskip 0.25in plus 1fil minus 0.125in
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf}
+ \hbox to \linewidth\bgroup\large \hfil\hfil
+ \hbox to 0pt\bgroup\hss \begin{tabular}[t]{c}\bf\@author
+ \end{tabular}\hss\egroup
+ \hfil\hfil\egroup}
+ \vskip 0.3in plus 2fil minus 0.1in
+}}
+\renewenvironment{abstract}{\centerline{\large\bf
+ Abstract}\vspace{0.5ex}\begin{quote}}{\par\end{quote}\vskip 1ex}
+
+
+% bibliography
+
+\def\thebibliography#1{\section*{References}
+ \global\def\@listi{\leftmargin\leftmargini
+ \labelwidth\leftmargini \advance\labelwidth-\labelsep
+ \topsep 1pt plus 2pt minus 1pt
+ \parsep 0.25ex plus 1pt \itemsep 0.25ex plus 1pt}
+ \list {[\arabic{enumi}]}{\settowidth\labelwidth{[#1]}\leftmargin\labelwidth
+ \advance\leftmargin\labelsep\usecounter{enumi}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy
+ \sfcode`\.=1000\relax}
+
+\def\@up#1{\raise.2ex\hbox{#1}}
+
+% most of cite format is from aclsub.sty by SMS
+
+% don't box citations, separate with ; and a space
+% also, make the penalty between citations negative: a good place to break
+% changed comma back to semicolon pj 2/1/90
+% \def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+% \def\@citea{}\@cite{\@for\@citeb:=#2\do
+% {\@citea\def\@citea{;\penalty\@citeseppen\ }\@ifundefined
+% {b@\@citeb}{{\bf ?}\@warning
+% {Citation `\@citeb' on page \thepage \space undefined}}%
+% {\csname b@\@citeb\endcsname}}}{#1}}
+
+% don't box citations, separate with ; and a space
+% Replaced for multiple citations (pj)
+% don't box citations and also add space, semicolon between multiple citations
+\def\@citex[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@cite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{; }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+ {\csname b@\@citeb\endcsname}}}{#1}}
+
+% Allow short (name-less) citations, when used in
+% conjunction with a bibliography style that creates labels like
+% \citename{<names>, }<year>
+%
+\let\@internalcite\cite
+\def\cite{\def\citename##1{##1, }\@internalcite}
+\def\shortcite{\def\citename##1{}\@internalcite}
+\def\newcite{\def\citename##1{{\frenchspacing##1} (}\@internalciteb}
+
+% Macros for \newcite, which leaves name in running text, and is
+% otherwise like \shortcite.
+\def\@citexb[#1]#2{\if at filesw\immediate\write\@auxout{\string\citation{#2}}\fi
+ \def\@citea{}\@newcite{\@for\@citeb:=#2\do
+ {\@citea\def\@citea{;\penalty\@m\ }\@ifundefined
+ {b@\@citeb}{{\bf ?}\@warning
+ {Citation `\@citeb' on page \thepage \space undefined}}%
+{\csname b@\@citeb\endcsname}}}{#1}}
+\def\@internalciteb{\@ifnextchar [{\@tempswatrue\@citexb}{\@tempswafalse\@citexb[]}}
+
+\def\@newcite#1#2{{#1\if at tempswa, #2\fi)}}
+
+\def\@biblabel#1{\def\citename##1{##1}[#1]\hfill}
+
+%%% More changes made by SMS (originals in latex.tex)
+% Use parentheses instead of square brackets in the text.
+\def\@cite#1#2{({#1\if at tempswa , #2\fi})}
+
+% Don't put a label in the bibliography at all. Just use the unlabeled format
+% instead.
+\def\thebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{References\@mkboth
+ {References}{References}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthebibliography=\endlist
+
+% Allow for a bibliography of sources of attested examples
+\def\thesourcebibliography#1{\vskip\parskip%
+\vskip\baselineskip%
+\def\baselinestretch{1}%
+\ifx\@currsize\normalsize\@normalsize\else\@currsize\fi%
+\vskip-\parskip%
+\vskip-\baselineskip%
+\section*{Sources of Attested Examples\@mkboth
+ {Sources of Attested Examples}{Sources of Attested Examples}}\list
+ {}{\setlength{\labelwidth}{0pt}\setlength{\leftmargin}{\parindent}
+ \setlength{\itemindent}{-\parindent}}
+ \def\newblock{\hskip .11em plus .33em minus -.07em}
+ \sloppy\clubpenalty4000\widowpenalty4000
+ \sfcode`\.=1000\relax}
+\let\endthesourcebibliography=\endlist
+
+\def\@lbibitem[#1]#2{\item[]\if at filesw
+ { \def\protect##1{\string ##1\space}\immediate
+ \write\@auxout{\string\bibcite{#2}{#1}}\fi\ignorespaces}}
+
+\def\@bibitem#1{\item\if at filesw \immediate\write\@auxout
+ {\string\bibcite{#1}{\the\c at enumi}}\fi\ignorespaces}
+
+% sections with less space
+\def\section{\@startsection {section}{1}{\z@}{-2.0ex plus
+ -0.5ex minus -.2ex}{1.5ex plus 0.3ex minus .2ex}{\large\bf\raggedright}}
+\def\subsection{\@startsection{subsection}{2}{\z@}{-1.8ex plus
+ -0.5ex minus -.2ex}{0.8ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\subsubsection{\@startsection{subsubsection}{3}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{0.5ex plus .2ex}{\normalsize\bf\raggedright}}
+\def\paragraph{\@startsection{paragraph}{4}{\z@}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+\def\subparagraph{\@startsection{subparagraph}{5}{\parindent}{1.5ex plus
+ 0.5ex minus .2ex}{-1em}{\normalsize\bf}}
+
+% Footnotes
+\footnotesep 6.65pt %
+\skip\footins 9pt plus 4pt minus 2pt
+\def\footnoterule{\kern-3pt \hrule width 5pc \kern 2.6pt }
+\setcounter{footnote}{0}
+
+% Lists and paragraphs
+\parindent 1em
+\topsep 4pt plus 1pt minus 2pt
+\partopsep 1pt plus 0.5pt minus 0.5pt
+\itemsep 2pt plus 1pt minus 0.5pt
+\parsep 2pt plus 1pt minus 0.5pt
+
+\leftmargin 2em \leftmargini\leftmargin \leftmarginii 2em
+\leftmarginiii 1.5em \leftmarginiv 1.0em \leftmarginv .5em \leftmarginvi .5em
+\labelwidth\leftmargini\advance\labelwidth-\labelsep \labelsep 5pt
+
+\def\@listi{\leftmargin\leftmargini}
+\def\@listii{\leftmargin\leftmarginii
+ \labelwidth\leftmarginii\advance\labelwidth-\labelsep
+ \topsep 2pt plus 1pt minus 0.5pt
+ \parsep 1pt plus 0.5pt minus 0.5pt
+ \itemsep \parsep}
+\def\@listiii{\leftmargin\leftmarginiii
+ \labelwidth\leftmarginiii\advance\labelwidth-\labelsep
+ \topsep 1pt plus 0.5pt minus 0.5pt
+ \parsep \z@ \partopsep 0.5pt plus 0pt minus 0.5pt
+ \itemsep \topsep}
+\def\@listiv{\leftmargin\leftmarginiv
+ \labelwidth\leftmarginiv\advance\labelwidth-\labelsep}
+\def\@listv{\leftmargin\leftmarginv
+ \labelwidth\leftmarginv\advance\labelwidth-\labelsep}
+\def\@listvi{\leftmargin\leftmarginvi
+ \labelwidth\leftmarginvi\advance\labelwidth-\labelsep}
+
+\abovedisplayskip 7pt plus2pt minus5pt%
+\belowdisplayskip \abovedisplayskip
+\abovedisplayshortskip 0pt plus3pt%
+\belowdisplayshortskip 4pt plus3pt minus3pt%
+
+% Less leading in most fonts (due to the narrow columns)
+% The choices were between 1-pt and 1.5-pt leading
+\def\@normalsize{\@setsize\normalsize{11pt}\xpt\@xpt}
+\def\small{\@setsize\small{10pt}\ixpt\@ixpt}
+\def\footnotesize{\@setsize\footnotesize{10pt}\ixpt\@ixpt}
+\def\scriptsize{\@setsize\scriptsize{8pt}\viipt\@viipt}
+\def\tiny{\@setsize\tiny{7pt}\vipt\@vipt}
+\def\large{\@setsize\large{14pt}\xiipt\@xiipt}
+\def\Large{\@setsize\Large{16pt}\xivpt\@xivpt}
+\def\LARGE{\@setsize\LARGE{20pt}\xviipt\@xviipt}
+\def\huge{\@setsize\huge{23pt}\xxpt\@xxpt}
+\def\Huge{\@setsize\Huge{28pt}\xxvpt\@xxvpt}
diff --git a/papers/icon-05/icon-05.tex b/papers/icon-05/icon-05.tex
new file mode 100644
index 0000000..64ee22c
--- /dev/null
+++ b/papers/icon-05/icon-05.tex
@@ -0,0 +1,657 @@
+\documentclass[11pt]{article}
+\usepackage{acl2005}
+\usepackage{times,url}
+\usepackage{latexsym}
+\usepackage{hyphen}
+\setlength\titlebox{6.5cm} % Expanding the titlebox
+
+\title{NLTK-Lite: Efficient Scripting for Natural Language Processing}
+
+\author{Steven Bird\\[.5ex]
+ Department of Computer Science and Software Engineering\\
+ University of Melbourne, Victoria 3010, AUSTRALIA\\[.5ex]
+ Linguistic Data Consortium, University of Pennsylvania,\\
+ Philadelphia PA 19104-2653, USA}
+\date{}
+
+\begin{document}
+\maketitle
+\begin{abstract}
+ The Natural Language Toolkit is a suite of program modules, data
+ sets, tutorials and exercises covering symbolic and statistical
+ natural language processing. NLTK is popular in teaching and
+ research, and has been adopted in dozens of NLP courses. NLTK is
+ written in Python and distributed under the GPL open source license.
+ Over the past year the toolkit has been completely rewritten,
+ simplifying many linguistic data structures and taking advantage of
+ recent enhancements in the Python language. This paper reports on
+ the resulting, simplified toolkit, NLTK-Lite, and shows how it is
+ used to support efficient scripting for natural language processing.
+\end{abstract}
+
+\section{Introduction}
+
+NLTK, the Natural Language Toolkit, is a suite of Python libraries and
+programs for symbolic and statistical natural language processing
+\cite{LoperBird02,Loper04}.
+NLTK includes graphical demonstrations and sample data. It is
+accompanied by extensive documentation, including tutorials that
+explain the underlying concepts behind the language processing tasks
+supported by the toolkit.
+
+NLTK is ideally suited to students who are learning NLP (natural
+language processing) or conducting research in NLP or closely related
+areas, including empirical linguistics, cognitive science, artificial
+intelligence, information retrieval, and machine learning. NLTK has
+been used successfully as a teaching tool, as an individual study
+tool, and as a platform for prototyping and building research systems
+\cite{Liddy05,Satre05}.
+
+We chose Python because it has a shallow learning curve, its syntax
+and semantics are transparent, and it has good string-handling
+functionality. As an interpreted language, Python facilitates
+interactive exploration. As an object-oriented language, Python
+permits data and methods to be encapsulated and re-used easily.
+Python comes with an extensive standard library, including tools for
+graphical programming and numerical processing
+\cite{Rossum03intro,Rossum03ref}.
+
+Over the past four years the toolkit grew rapidly and the data
+structures became significantly more complex. Each new processing
+task brought with it new requirements on input and output
+representations. It was not clear how to generalize tasks so they
+could be applied independently of each other. As a simple example,
+consider the independent tasks of tagging and stemming, which both
+operate on sequences of tokens. If stemming is done first, we lose
+information required for tagging. If tagging is done first, the
+stemming must be able to skip over the tags. If both are done independently,
+we need to be able to align the results. As task combinations multiply,
+managing the data becomes extremely difficult.
+
+To address this problem, NLTK 1.4 introduced a new architecture for
+tokens based on Python's native dictionary data type. Tokens could have
+an arbitrary number of named properties, like \texttt{TAG}, and \texttt{STEM}.
+Whole sentences, and even whole documents, were represented as single tokens
+having a \texttt{SUBTOKENS} attribute to hold sequences of smaller tokens.
+Parse trees were likewise tokens, with a special \texttt{CHILDREN} property.
+The advantage of this token architecture was that it unified many different
+data types, and permitted distinct tasks to be run independently.
+Unfortunately this architecture also came with a significant overhead for
+programmers, who had to keep track of a growing set of property names,
+and who were often forced to use ``rather awkward code structures'' \cite{Hearst05}.
+It was clear that the re-engineering done in NLTK 1.4 mainly got in the way of
+efficient authoring of NLP scripts.
+
+This paper presents a new, simplified toolkit called NLTK-Lite. This paper
+presents a brief overview and tutorial on NLTK-Lite, and identifies
+some areas where more contributions would be welcome.
+
+\section{Overview of NLTK-Lite}
+
+NLTK-Lite is a suite of Python packages providing a range of standard
+NLP data types, interface definitions and processing tasks, corpus
+samples and readers, together with animated algorithms, extensive
+tutorials, and problem sets. Data types include: tokens, tags,
+chunks, trees, and feature structures. Interface definitions and
+reference implementations are provided for tokenizers, stemmers,
+taggers (regexp, ngram, Brill), chunkers, parsers (recursive-descent,
+shift-reduce, chart, probabilistic). Corpus samples and readers
+include: Brown Corpus, CoNLL-2000 Chunking Corpus, CMU pronunciation
+dictionary, NIST Information Extraction and Entity Recognition Corpus,
+Ratnaparkhi's Prepositional Phrase Attachment Corpus, Penn Treebank,
+and the SIL Shoebox corpus format.
+
+NLTK-Lite differs from NLTK in the following key respects: fundamental
+representations are kept as simple as possible (e.g.\ strings, tuples,
+trees); all streaming tasks are implemented as iterators instead of
+lists in order to limit memory usage and to ensure that
+data-intensive tasks produce output as early as possible; the default
+pipeline processing paradigm leads to more transparent code; taggers
+incorporate backoff leading to much smaller models and faster
+operation; method names are shorter (e.g.\
+\texttt{tokenizer.RegexpTokenizer} becomes \texttt{tokenize.regexp},
+and the barrier to entry for contributed software is removed now that
+there is no requirement to support the special NLTK token
+architecture.
+
+\section{Simple Processing Tasks}
+
+In this section we review some simple NLP processing tasks, and show how
+they are performed in NLTK-Lite.
+
+\subsection{Tokenization and Stemming}
+
+The following three-line program imports the \texttt{tokenize}
+package, defines a text string, and then tokenizes the string on
+whitespace to create a list of tokens. (Note that "\url{>>>}" is
+Python's interactive prompt; "\url{...}" is the second-level prompt.)
+
+{\small\begin{verbatim}
+>>> from nltk_lite import tokenize
+>>> text = 'Hello world. This is a test.'
+>>> list(tokenize.whitespace(text))
+['Hello', 'world.', 'This', 'is', 'a', 'test']
+\end{verbatim}}
+
+\noindent
+Several other useful tokenizers are provided. We can stem the output of tokenization
+using the Porter Stemmer as follows:
+
+{\small\begin{verbatim}
+>>> text = 'stemming can be fun and exciting'
+>>> tokens = tokenize.whitespace(text)
+>>> porter = tokenize.PorterStemmer()
+>>> for token in tokens:
+... print porter.stem(token),
+stem can be fun and excit
+\end{verbatim}}
+
+The corpora included with NLTK-Lite come supplied with corpus readers that understand
+the file structure of the corpus, and load the data into Python data structures.
+For example, the following code reads the first sentence of part a of the Brown Corpus.
+It prints a list of tuples, where each tuple consists of a word and its tag.
+
+{\small\begin{verbatim}
+>>> from nltk_lite.corpora\
+... import brown, extract
+>>> print extract(0, brown.tagged('a'))
+[('The', 'at'), ('Fulton', 'np-tl'),
+('County', 'nn-tl'), ('Grand', 'jj-tl'),
+('Jury', 'nn-tl'), ('said', 'vbd'), ...]
+\end{verbatim}}
+
+\begin{figure*}[t]
+{\small\begin{verbatim}
+>>> cfdist = ConditionalFreqDist()
+>>> for genre in brown.items: # each genre
+... for sent in brown.tagged(genre): # each sentence
+... for (word,tag) in sent: # each tagged token
+... if tag == 'md': # found a modal
+... cfdist[genre].inc(word.lower())
+>>> modals = ['can', 'could', 'may', 'might', 'must', 'will']
+>>> print "%-40s" % 'Genre', ' '.join([("%6s" % m) for m in modals])
+>>> for genre in cfdist.conditions(): # generate rows
+... print "%-40s" % brown.item_name[genre],
+... for modal in modals:
+... print "%6d" % cfdist[genre].count(modal),
+... print
+Genre can could may might must will
+press: reportage 94 86 66 36 50 387
+press: reviews 44 40 45 26 18 56
+press: editorial 122 56 74 37 53 225
+skill and hobbies 273 59 130 22 83 259
+religion 84 59 79 12 54 64
+belles-lettres 249 216 213 113 169 222
+popular lore 168 142 165 45 95 163
+miscellaneous: government & house organs 115 37 152 13 99 237
+fiction: general 39 168 8 42 55 50
+learned 366 159 325 126 202 330
+fiction: science 16 49 4 12 8 16
+fiction: mystery 44 145 13 57 31 17
+fiction: adventure 48 154 6 58 27 48
+fiction: romance 79 195 11 51 46 43
+humor 17 33 8 8 9 13
+\end{verbatim}}
+\caption{Program to Generate a Table of Modals and their Frequency of Use in Different Genres\label{fig:genre}}
+\vspace{1ex}\hrule
+\end{figure*}
+
+NLTK-Lite provides support for conditional frequency distributions, making it easy
+to count up items of interest in specified contexts. The code sample and output in
+Figure~\ref{fig:genre} counts the usage of modal verbs in the Brown Corpus, displaying
+them by genre. Such information may be useful for studies in stylistics, and also in
+text categorization.
+
+\subsection{Tagging}
+
+The simplest possible tagger assigns the same tag to each token
+regardless of the token's text. The DefaultTagger class implements
+this kind of tagger. In the following program, we create a tagger
+called \url{my_tagger} which tags everything as a noun.
+
+{\small\begin{verbatim}
+>>> from nltk_lite import tag
+>>> my_tagger = tag.Default('nn')
+>>> list(my_tagger.tag(tokens))
+[('John', 'nn'), ('saw', 'nn'),
+ ('3', 'nn'), ('polar', 'nn'),
+ ('bears', 'nn'), ('.', 'nn')]
+\end{verbatim}}
+
+This is a simple algorithm, and it performs poorly when used on its own.
+On a typical corpus, it will tag only 20--30\% of the tokens correctly.
+However, it is a very reasonable tagger to use as a default, if a more
+advanced tagger fails to determine a token's tag.
+
+The regular expression tagger assigns tags to tokens on the basis of
+matching patterns in the token's text. For instance, the following
+tagger assigns \texttt{cd} to cardinal numbers, and \texttt{nn} to
+everything else:
+
+{\small\begin{verbatim}
+>>> patterns = [
+... (r'^[0-9]+(.[0-9]+)?$', 'cd'),
+... (r'.*', 'nn')]
+>>> nn_cd_tagger = tag.Regexp(patterns)
+>>> list(nn_cd_tagger.tag(tokens))
+[('John', 'nn'), ('saw', 'nn'),
+ ('3', 'cd'), ('polar', 'nn'),
+ ('bears', 'nn'), ('.', 'nn')]
+\end{verbatim}}
+
+The \texttt{UnigramTagger} class implements a simple statistical tagging
+algorithm: for each token, it assigns the tag that is most likely
+for that token's text. For example, it will assign the tag \texttt{jj} to
+any occurrence of the word \textit{frequent}, since \textit{frequent} is used as an
+adjective (e.g.\ \textit{a frequent word}) more often than it is used as a
+verb (e.g.\ \textit{I frequent this cafe}).
+Before a unigram tagger can be used, it must be trained on
+a corpus, as shown below for the first section of the Brown Corpus.
+
+{\small\begin{verbatim}
+>>> from nltk_lite.corpora import brown
+>>> unigram_tagger = tag.Unigram()
+>>> unigram_tagger.train(brown('a'))
+\end{verbatim}}
+
+\noindent
+Once a unigram tagger has been trained, it can be used to tag new text.
+Note that it assigns the default tag \texttt{None} to any token that was not
+encountered during training.
+
+{\small\begin{verbatim}
+>>> text = "John saw the book on the table"
+>>> tokens = list(tokenize.whitespace(text))
+>>> list(unigram_tagger.tag(tokens))
+[('John', 'np'), ('saw', 'vbd'),
+ ('the', 'at'), ('book', None),
+ ('on', 'in'), ('the', 'at'),
+ ('table', None)]
+\end{verbatim}}
+
+\noindent
+We can instruct the unigram tagger to back off to our default
+\url{nn_cd_tagger} when it cannot assign a tag itself. Now all
+the words are guaranteed to be tagged:
+
+{\small\begin{verbatim}
+>>> unigram_tagger =
+... tag.Unigram(backoff=nn_cd_tagger)
+>>> unigram_tagger.train(train_sents)
+>>> list(unigram_tagger.tag(tokens))
+[('John', 'np'), ('saw', 'vbd'),
+ ('the', 'at'), ('book', 'nn'),
+ ('on', 'in'), ('the', 'at'),
+ ('table', 'nn')]
+\end{verbatim}}
+
+\noindent
+We can go on to define and train a bigram tagger, as shown below:
+
+{\small\begin{verbatim}
+>>> bigram_tagger =\
+... tag.Bigram(backoff=unigram_tagger)
+>>> bigram_tagger.train(brown.tagged('a'))
+\end{verbatim}}
+
+\noindent
+We can easily evaluate this tagger against some gold-standard tagged
+text, using the \url{tag.accuracy()} function.
+
+NLTK-Lite also includes a Brill tagger (contributed by Christopher
+Maloof) and an HMM tagger (contributed by Trevor Cohn).
+
+\section{Chunking and Parsing}
+
+\subsection{Chunking}
+
+Chunking is a technique for shallow syntactic analysis of (tagged)
+text. Chunk data can be loaded from files that use the bracket notation
+(e.g.\ Penn Treebank) or the IOB (INSIDE/OUTSIDE/BEGIN) notation
+(e.g.\ CoNLL-2000).
+
+{\small\begin{verbatim}
+>>> from nltk_lite import parse
+>>> text = """
+... he PRP B-NP
+... accepted VBD B-VP
+... the DT B-NP
+... position NN I-NP
+... of IN B-PP
+... vice NN B-NP
+... chairman NN I-NP
+... of IN B-PP
+... Carlyle NNP B-NP
+... Group NNP I-NP
+... , , O
+... a DT B-NP
+... merchant NN I-NP
+... banking NN I-NP
+... concern NN I-NP
+... . . O
+... """
+>>> sent = parse.conll_chunk(text)
+\end{verbatim}}
+
+\noindent
+Internally, the data is stored in a tree structure. We can
+print this as a nested bracketting, and we can conveniently access
+its children using indexes:
+
+{\small\begin{verbatim}
+>>> print sent
+(S:
+ (NP: ('he', 'PRP'))
+ ('accepted', 'VBD')
+ (NP: ('the', 'DT') ('position', 'NN'))
+ ('of', 'IN')
+ (NP: ('vice', 'NN') ('chairman', 'NN'))
+ ('of', 'IN')
+ (NP: ('Carlyle', 'NNP') ('Group', 'NNP'))
+ (',', ',')
+ (NP:
+ ('a', 'DT')
+ ('merchant', 'NN')
+ ('banking', 'NN')
+ ('concern', 'NN'))
+ ('.', '.'))
+>>> print sent[2]
+(NP: ('the', 'DT') ('position', 'NN'))
+>>> print sent[2][1]
+('position', 'NN')
+\end{verbatim}}
+
+We can define a regular-expression based chunk parser for use in chunking tagged
+text, as shown in Figure~\ref{fig:chunking}. NLTK-Lite also supports several other
+operations on chunks, such as merging, splitting, and chinking. Corpus readers
+for chunked data in Penn Treebank and CoNLL-2000 are provided, along with comprehensive
+support for evaluation and error analysis.
+
+\begin{figure*}[t]
+{\small\begin{verbatim}
+>>> sent = tag.string2tags("the/DT little/JJ cat/NN sat/VBD on/IN the/DT mat/NN")
+>>> rule1 = parse.ChunkRule('<DT><JJ><NN>', 'Chunk det+adj+noun')
+>>> rule2 = parse.ChunkRule('<DT|NN>+', 'Chunk sequences of NN and DT')
+>>> chunkparser = parse.RegexpChunk([rule1, rule2], chunk_node='NP', top_node='S')
+>>> chunk_tree = chunkparser.parse(sent, trace=1)
+Input:
+ <DT> <JJ> <NN> <VBD> <IN> <DT> <NN>
+Chunk det+adj+noun:
+ {<DT> <JJ> <NN>} <VBD> <IN> <DT> <NN>
+Chunk sequences of NN and DT:
+ {<DT> <JJ> <NN>} <VBD> <IN> {<DT> <NN>}
+>>> print chunk_tree
+(S:
+ (NP: ('the', 'DT') ('little', 'JJ') ('cat', 'NN'))
+ ('sat', 'VBD')
+ ('on', 'IN')
+ (NP: ('the', 'DT') ('mat', 'NN')))
+\end{verbatim}}
+\caption{Regular-Expression based Chunk Parser\label{fig:chunking}}
+\vspace{1ex}\hrule
+\end{figure*}
+
+\subsection{Simple Parsers}
+
+NLTK-Lite provides several parsers for context-free phrase-structure
+grammars. Grammars can be defined using a series of productions as follows:
+
+{\small\begin{verbatim}
+>>> from nltk_lite.parse import cfg
+>>> grammar = cfg.parse_grammar('''
+... S -> NP VP
+... VP -> V NP | V NP PP
+... V -> "saw" | "ate"
+... NP -> "John" | Det N | Det N PP
+... Det -> "a" | "an" | "the" | "my"
+... N -> "dog" | "cat" | "ball"
+... PP -> P NP
+... P -> "on" | "by" | "with"
+... ''')
+\end{verbatim}}
+
+\noindent
+Now we can tokenize and parse a sentence. Here we use a recursive
+descent parser. Note that we have had to avoid using left-recursive
+productions in the above grammar, so that this parser does not end up
+in an infinite loop.
+
+{\small\begin{verbatim}
+>>> text = "John saw a cat with my ball"
+>>> sent = list(tokenize.whitespace(text))
+>>> rd = parse.RecursiveDescent(grammar)
+\end{verbatim}}
+
+The recursive descent parser \texttt{rd} can be used many times over.
+Here we apply it to our sentence, and iterate over all the parses that
+it generates. Observe that two parses are possible, thanks to
+prepositional phrase attachment ambiguity.
+
+{\small\begin{verbatim}
+>>> for p in rd.get_parse_list(sent):
+... print p
+(S:
+ (NP: 'John')
+ (VP:
+ (V: 'saw')
+ (NP:
+ (Det: 'a')
+ (N: 'cat')
+ (PP: (P: 'with')
+ (NP: (Det: 'my') (N: 'ball'))))))
+(S:
+ (NP: 'John')
+ (VP:
+ (V: 'saw')
+ (NP: (Det: 'a') (N: 'cat'))
+ (PP: (P: 'with')
+ (NP: (Det: 'my') (N: 'ball')))))
+\end{verbatim}}
+
+\noindent
+The same sentence can be parsed using a grammar with left-recursive
+productions, so long as we use a chart parser. Here we use the
+bottom-up rule-invocation strategy \url{BU_STRATEGY}.
+
+{\small\begin{verbatim}
+>>> from nltk_lite.parse import cfg, chart
+>>> grammar = cfg.parse_grammar('''
+... S -> NP VP
+... VP -> V NP | VP PP
+... V -> "saw" | "ate"
+... NP -> "John" | Det N | NP PP
+... Det -> "a" | "an" | "the" | "my"
+... N -> "dog" | "cat" | "ball"
+... PP -> P NP
+... P -> "on" | "by" | "with"
+... ''')
+>>> parser = chart.ChartParse(grammar,
+... chart.BU_STRATEGY)
+>>> for tree in parser.get_parse_list(sent):
+... print tree
+(S:
+ (NP: 'John')
+ (VP:
+ (VP: (V: 'saw')
+ (NP: (Det: 'a') (N: 'cat')))
+ (PP: (P: 'with')
+ (NP: (Det: 'my') (N: 'ball')))))
+(S:
+ (NP: 'John')
+ (VP:
+ (V: 'saw')
+ (NP:
+ (NP: (Det: 'a') (N: 'cat'))
+ (PP: (P: 'with')
+ (NP: (Det: 'my') (N: 'ball'))))))
+\end{verbatim}}
+
+Tracing can be turned on, to display each step of the parsing process,
+as shown in Figure~\ref{fig:chart}. Each row represents an edge
+of the chart, with a specified span, together with a (possibly
+incomplete) dotted grammar production.
+
+\begin{figure*}[p]
+{\small\begin{verbatim}
+>>> parser = ChartParse(grammar, BU_STRATEGY, trace=2)
+>>> parser.get_parse(sent)
+|. John. saw . a . cat . with. my .ball .|
+Bottom Up Init Rule:
+|[-----] . . . . . .| [0:1] 'John'
+|. [-----] . . . . .| [1:2] 'saw'
+|. . [-----] . . . .| [2:3] 'a'
+|. . . [-----] . . .| [3:4] 'cat'
+|. . . . [-----] . .| [4:5] 'with'
+|. . . . . [-----] .| [5:6] 'my'
+|. . . . . . [-----]| [6:7] 'ball'
+Bottom Up Predict Rule:
+|> . . . . . . .| [0:0] NP -> * 'John'
+|. > . . . . . .| [1:1] V -> * 'saw'
+|. . > . . . . .| [2:2] Det -> * 'a'
+|. . . > . . . .| [3:3] N -> * 'cat'
+|. . . . > . . .| [4:4] P -> * 'with'
+|. . . . . > . .| [5:5] Det -> * 'my'
+|. . . . . . > .| [6:6] N -> * 'ball'
+Fundamental Rule:
+|[-----] . . . . . .| [0:1] NP -> 'John' *
+|. [-----] . . . . .| [1:2] V -> 'saw' *
+|. . [-----] . . . .| [2:3] Det -> 'a' *
+|. . . [-----] . . .| [3:4] N -> 'cat' *
+|. . . . [-----] . .| [4:5] P -> 'with' *
+|. . . . . [-----] .| [5:6] Det -> 'my' *
+|. . . . . . [-----]| [6:7] N -> 'ball' *
+Bottom Up Predict Rule:
+|> . . . . . . .| [0:0] S -> * NP VP
+|> . . . . . . .| [0:0] NP -> * NP PP
+|. > . . . . . .| [1:1] VP -> * V NP
+|. . > . . . . .| [2:2] NP -> * Det N
+|. . . . > . . .| [4:4] PP -> * P NP
+|. . . . . > . .| [5:5] NP -> * Det N
+Fundamental Rule:
+|[-----> . . . . . .| [0:1] S -> NP * VP
+|[-----> . . . . . .| [0:1] NP -> NP * PP
+|. [-----> . . . . .| [1:2] VP -> V * NP
+|. . [-----> . . . .| [2:3] NP -> Det * N
+|. . [-----------] . . .| [2:4] NP -> Det N *
+|. . . . [-----> . .| [4:5] PP -> P * NP
+|. . . . . [-----> .| [5:6] NP -> Det * N
+|. . . . . [-----------]| [5:7] NP -> Det N *
+|. [-----------------] . . .| [1:4] VP -> V NP *
+|. . . . [-----------------]| [4:7] PP -> P NP *
+|[-----------------------] . . .| [0:4] S -> NP VP *
+Bottom Up Predict Rule:
+|. . > . . . . .| [2:2] S -> * NP VP
+|. . > . . . . .| [2:2] NP -> * NP PP
+|. . . . . > . .| [5:5] S -> * NP VP
+|. . . . . > . .| [5:5] NP -> * NP PP
+|. > . . . . . .| [1:1] VP -> * VP PP
+Fundamental Rule:
+|. . [-----------> . . .| [2:4] S -> NP * VP
+|. . [-----------> . . .| [2:4] NP -> NP * PP
+|. . . . . [----------->| [5:7] S -> NP * VP
+|. . . . . [----------->| [5:7] NP -> NP * PP
+|. [-----------------> . . .| [1:4] VP -> VP * PP
+|. . [-----------------------------]| [2:7] NP -> NP PP *
+|. [-----------------------------------]| [1:7] VP -> VP PP *
+|. . [----------------------------->| [2:7] S -> NP * VP
+|. . [----------------------------->| [2:7] NP -> NP * PP
+|. [----------------------------------->| [1:7] VP -> VP * PP
+|. [-----------------------------------]| [1:7] VP -> V NP *
+|[=========================================]| [0:7] S -> NP VP *
+|[=========================================]| [0:7] S -> NP VP *
+|. [----------------------------------->| [1:7] VP -> VP * PP
+(S: (NP: 'John') (VP: (VP: (V: 'saw') (NP: (Det: 'a') (N: 'cat')))
+(PP: (P: 'with') (NP: (Det: 'my') (N: 'ball')))))
+\end{verbatim}}
+\caption{Trace of Edges Created by the Bottom-Up Chart Parser\label{fig:chart}}
+\end{figure*}
+
+Other rule-invocation strategies are top-down, alternating
+top-down/bottom-up, and Earley (contributed by Jean Mark Gawron).
+
+\subsection{Probabilistic Parsing}
+
+A probabilistic context free grammar (or PCFG) is a context free
+grammar that associates a probability with each production. It
+generates the same set of parses for a text that the corresponding
+context free grammar does, and it assigns a probability to each parse.
+The probability of a parse generated by a PCFG is simply the product
+of the probabilities of the productions used to generate it.
+NLTK-Lite provides a Viterbi-style PCFG parser, together with a suite
+of bottom-up probabilistic chart parsers.
+
+\section{Contributing to NLTK-Lite}
+
+NLTK-Lite includes a variety of other modules supporting natural
+language processing tasks. Many more are in the planning stages,
+including fieldwork analysis tools, a concordancer, feature-based
+grammars, a cascaded chunk parser, semantic interpretation via the
+lambda-calculus, and several others.
+
+NLTK-Lite is an open source project, being developed by a community of
+NLP researchers and teachers. It is continually being expanded and
+improved, with the help of interested members of the community.
+
+There are several ways to contribute. Many users have suggested new
+features, reported bugs, or contributed patches via the Sourceforge
+site \url{nltk.sourceforge.net}. Several teachers and students have
+submitted NLTK-based projects for inclusion in the contrib directory,
+and in some cases these have made it into the core toolkit. The
+tutorials are continually being expanded and refined, with the help of
+input from users. The tutorials are being translated into Portuguese
+(by Tiago Tresoldi), and we hope to find translators for other
+languages.
+
+\section{Teaching with NLTK-Lite}
+
+NLTK-Lite provides ready-to-use courseware and a flexible framework
+for project work. Students augment and replace existing components,
+learn structured programming by example, and manipulate sophisticated
+models from the outset. Tutorials describe each component of the
+toolkit, and include a wide variety of student exercises and project
+ideas.
+
+NLTK-Lite can be used to create student assignments of varying
+difficulty and scope. In the simplest assignments, students experiment
+with an existing module. The wide variety of existing modules provide
+many opportunities for creating these simple assignments. Once
+students become more familiar with the toolkit, they can be asked to
+make minor changes or extensions to an existing module. A more
+challenging task is to develop a new module. Here, NLTK-Lite provides some
+useful starting points: predefined interfaces and data structures, and
+existing modules that implement the same interface.
+
+NLTK-Lite provides animated algorithms that can be used in class
+demonstrations. These interactive tools can be used to display
+relevant data structures and to show the step-by-step execution of
+algorithms. Both data structures and control flow can be easily
+modified during the demonstration, in response to questions from the
+class. Since these graphical tools are included with the toolkit, they
+can also be used by students. This allows students to experiment at
+home with the algorithms that they have seen presented in class.
+
+\section{Conclusion}
+
+Python is a particularly convenient language to use for writing
+scripts to perform natural language processing tasks. NLTK-Lite
+provides ready access to standard corpora, along with representations
+for common linguistic data structures, reference implementations for
+many NLP tasks, and extensive documentation including tutorials and
+library reference.
+
+\section*{Acknowledgements}
+
+The work reported here has been supported by the US National Science
+Foundation, the Australian Research Council, and NICTA Victoria
+Laboratory. Much of the original NLTK implementation and
+documentation work was done in close collaboration with Edward Loper.
+Ewan Klein has provided substantial input to several of the tutorials.
+I am grateful to James Curran for persuading me of the need to develop
+this lightweight version of NLTK. Dozens of others have provided
+valuable contributions and feedback; they are named on the NLTK
+contributors page, linked from \url{nltk.sourceforge.net}.
+
+\bibliographystyle{acl}
+\bibliography{general}
+
+\end{document}
diff --git a/papers/iwcs-08/drs.png b/papers/iwcs-08/drs.png
new file mode 100644
index 0000000..bfc309f
Binary files /dev/null and b/papers/iwcs-08/drs.png differ
diff --git a/papers/iwcs-08/garrette-klein.tar.gz b/papers/iwcs-08/garrette-klein.tar.gz
new file mode 100644
index 0000000..34e039b
Binary files /dev/null and b/papers/iwcs-08/garrette-klein.tar.gz differ
diff --git a/papers/iwcs-08/iwcs.doctest b/papers/iwcs-08/iwcs.doctest
new file mode 100644
index 0000000..a27aa5d
--- /dev/null
+++ b/papers/iwcs-08/iwcs.doctest
@@ -0,0 +1,169 @@
+Done
+ >>> from nltk.sem import logic
+ >>> lp = logic.LogicParser()
+ >>> e = lp.parse('all x.(girl(x) -> exists y.(dog(y) & chase(x,y)))')
+ >>> e
+ <AllExpression all x.(girl(x) -> exists y.(dog(y) & chase(x,y)))>
+
+Done
+ >>> from nltk.sem import Variable
+ >>> e1 = lp.parse(r'\x.P(x)(y)')
+ >>> print e1.simplify()
+ P(y)
+ >>> e2 = lp.parse('all x.P(x,a,b)')
+ >>> print e2
+ all x.P(x,a,b)
+ >>> print e2.free()
+ set([Variable('a'), Variable('b')])
+ >>> print e2.alpha_convert(Variable('z'))
+ all z.P(z,a,b)
+ >>> e3 = lp.parse('x')
+ >>> print e2.replace(Variable('b'), e3)
+ all z1.P(z1,a,x)
+
+Done
+ >>> from nltk.sem import parse_valuation, Model, Assignment
+ >>> v = """
+ ... suzie => s
+ ... fido => f
+ ... rover => r
+ ... girl => {s}
+ ... chase => {(f, s), (r, s), (s, f)}
+ ... """
+ >>> val = parse_valuation(v) #create a Valuation
+ >>> m = Model(val.domain, val) #initialize a Model
+ >>> g = Assignment(val.domain) #initialize an Assignment
+ >>> e4 = lp.parse('exists y. (girl(y) & chase(x, y))')
+ >>> m.satisfiers(e4, 'x', g) #check satisfiers of e4 wrt to x
+ set(['r', 'f'])
+
+Typed Logic
+ >>> tlp = logic.LogicParser(type_check=True)
+ >>> a = tlp.parse(r'\x y.see(x,y)')
+ >>> b = tlp.parse(r'\x.man(x)')
+ >>> a.type, b.type
+ (<e,<e,t>>, <e,t>)
+ >>> tlp.parse(r'\x y.see(x,y)(\x.man(x))')
+ Traceback (most recent call last):
+ . . .
+ TypeException: The function '\x y.see(x,y)' is of type '<e,<e,t>>' and cannot be applied to '\x.man(x)' of type '<e,t>'. Its argument must be of type 'e'.
+
+Done
+ >>> from nltk.sem import drt
+ >>> dp = drt.DrtParser()
+ >>> d1 = dp.parse('([x][walk(x)]) + ([y][run(y)])')
+ >>> print d1
+ (([x],[walk(x)]) + ([y],[run(y)]))
+ >>> print d1.simplify()
+ ([x,y],[walk(x), run(y)])
+ >>> d2 = dp.parse('([x,y][Bill(x), Fred(y)])')
+ >>> d3 = dp.parse("""([][([u][Porsche(u), own(x,u)])
+ ... -> ([v][Ferrari(v),own(y,u)])])""")
+ >>> d4 = d2 + d3
+ >>> print d4.simplify()
+ ([x,y],[Bill(x), Fred(y), (([u],[Porsche(u), own(x,u)]) -> ([v],[Ferrari(v), own(y,u)]))])
+
+Done
+ >>> print d1.toFol()
+ (exists x.walk(x) & exists y.run(y))
+ >>> #d4.simplify().draw()
+
+Done
+ >>> from nltk.parse import load_earley
+ >>> parser = load_earley('grammars/sem1.fcfg', trace=0)
+ >>> tokens = 'a dog barks'.split()
+ >>> trees = parser.nbest_parse(tokens)
+ >>> print trees[0].node['sem'].simplify()
+ exists x.(dog(x) & bark(x))
+
+Done
+ >>> from nltk.sem import hole
+ >>> readings = hole.hole_readings('every girl chases a dog')
+ >>> for r in readings: print r
+ exists z3.(dog(z3) & all z8.(girl(z8) -> chase(z3,z8)))
+ all z8.(girl(z8) -> exists z3.(dog(z3) & chase(z3,z8)))
+
+Done
+ >>> from nltk.sem.glue import GlueFormula
+ >>> john = GlueFormula(r'john', 'g')
+ >>> walks = GlueFormula(r'\x.walk(x)', '(g -o f)')
+ >>> john_walks = walks.applyto(john)
+ >>> print john_walks.meaning.simplify()
+ walk(john)
+
+Done
+ >>> from nltk.sem.glue import GlueFormula, Glue
+ >>> a = GlueFormula(r'\Q.all x.(girl(x) -> Q(x))', '((g -o G) -o G)')
+ >>> b = GlueFormula(r'\x y.chase(x,y)', '(g -o (h -o f))')
+ >>> c = GlueFormula(r'\Q.exists x.(dog(x)&Q(x))', '((h -o H) -o H)')
+ >>> glue = Glue()
+ >>> for reading in glue.get_readings(glue.gfl_to_compiled([a,b,c])):
+ ... print reading.simplify()
+ exists x.(dog(x) & all z13.(girl(z13) -> chase(z13,x)))
+ all x.(girl(x) -> exists z14.(dog(z14) & chase(x,z14)))
+
+Done
+ >>> from nltk import inference
+ >>> a = lp.parse('all x.(dog(x) -> bark(x))')
+ >>> b = lp.parse('dog(rover)')
+ >>> c = lp.parse('bark(rover)')
+ >>> prover = inference.get_prover(c, [a,b])
+ >>> prover.prove()
+ True
+
+Done
+ >>> a = lp.parse('all x.walk(x)')
+ >>> b = lp.parse('all y.walk(y)')
+ >>> a == b
+ True
+ >>> c = lp.parse('-(P(x) & Q(x))')
+ >>> d = lp.parse('-P(x) | -Q(x)')
+ >>> c == d
+ False
+ >>> c.tp_equals(d)
+ True
+
+ >>> from nltk.inference.discourse import DiscourseTester as DT
+ >>> dt = DT(['A student dances', 'Every student is a person'])
+ >>> dt.readings()
+ <BLANKLINE>
+ s0 readings:
+ ------------------------------
+ s0-r0: exists x.(student(x) & dance(x))
+ <BLANKLINE>
+ s1 readings:
+ ------------------------------
+ s1-r0: all x.(student(x) -> person(x))
+ >>> dt.add_sentence('No person dances', consistchk=True)
+ Inconsistent discourse d0 ['s0-r0', 's1-r0', 's2-r0']:
+ s0-r0: exists x.(student(x) & dance(x))
+ s1-r0: all x.(student(x) -> person(x))
+ s2-r0: -exists x.(person(x) & dance(x))
+ <BLANKLINE>
+ >>> dt.retract_sentence('No person dances', quiet=False)
+ Current sentences are
+ s0: A student dances
+ s1: Every student is a person
+ >>> dt.add_sentence('A person dances', informchk=True)
+ Sentence 'A person dances' under reading 'exists x.(person(x) & dance(x))':
+ Not informative relative to thread 'd0'
+
+
+Discourse in DRT
+ >>> from nltk.inference.discourse import DrtGlueReadingCommand as RC
+ >>> dt = DT(['Every dog chases a boy', 'He runs'], RC())
+ >>> dt.readings()
+ <BLANKLINE>
+ s0 readings:
+ ------------------------------
+ s0-r0: ([],[(([x],[dog(x)]) -> ([z15],[boy(z15), chase(x,z15)]))])
+ s0-r1: ([z16],[boy(z16), (([x],[dog(x)]) -> ([],[chase(x,z16)]))])
+ <BLANKLINE>
+ s1 readings:
+ ------------------------------
+ s1-r0: ([x],[PRO(x), run(x)])
+ >>> dt.readings(show_thread_readings=True)
+ d0: ['s0-r0', 's1-r0'] : INVALID: AnaphoraResolutionException
+ d1: ['s0-r1', 's1-r0'] : ([z20,z24],[boy(z20), (([x],[dog(x)]) -> ([],[chase(x,z20)])), (z24 = z20), run(z24)])
+ >>> dt.readings(filter=True, show_thread_readings=True)
+ d1: ['s0-r1', 's1-r0'] : ([z26,z29],[boy(z26), (([x],[dog(x)]) -> ([],[chase(x,z26)])), (z29 = z26), run(z29)])
diff --git a/papers/iwcs-08/lingmacros.sty b/papers/iwcs-08/lingmacros.sty
new file mode 100755
index 0000000..4bba509
--- /dev/null
+++ b/papers/iwcs-08/lingmacros.sty
@@ -0,0 +1,262 @@
+% Lingmacros
+% include
+% \enumsentence, \eenumsentence, \ex
+% \smalltree
+% \shortex, \shortexnt (\shortexdt no longer needed)
+% \clap, \ollap, \orlap, \oclap on analogy with \rlap and \llap
+% \outerfs
+
+% this file created 1/23/89 by Emma Pease, CSLI
+% modified 4/1/91 by Emma Pease
+
+%Note (for those who use jbmacros):
+% (a) the enumsentence macro is very similar to the \example command
+% in jbmacros; however, it does not have the annoying habit of leaving
+% the example number on one page and the example on the next (some
+% jbmacros don't have this problem). The eenumsentence macro is very
+% similar to the \examples macro.
+% (b) \ex may be different
+
+% the enumsentence macro.
+%
+% \enumsentence{This is an example}
+% produces
+% (1) This is an example
+% while
+% \enumsentence[(a)]{This is another example}
+% produces
+% (a) This is another example
+%
+% A related macros is
+% \eenumsentence{\item[a.] A third example
+% \item[b.] A fourth example}
+% which produces
+% (2) a. A third example
+% b. A fourth example
+%
+% other macro is \ex{1} which produces the number of the following
+% enumsentence (\ex{0} produces number of preceeding enumsentence and
+% so on.)
+%
+% The standard \label command also works within \enumsentence and
+% \eenumsentence.
+%
+% the \enumsentence counter. Add [chapter] if using report style
+\newcounter{enums}
+
+% \widelabel is defined for use when the width of the enumsentence or
+% eenumsentence number is wider than 20pt (default labelwidth). As a
+% rule of thumb if your enumerated sentences are only 1 or 2 digit
+% you don't need to worry. At 3 digits, set \widelabel to 2.78pt at 4
+% digits set to 7.78pt.
+
+\newdimen\widelabel
+\widelabel=0pt
+
+
+\def\enumsentence{\@ifnextchar[{\@enumsentence}% %]
+{\refstepcounter{enums}\@enumsentence[(\theenums)]}}
+
+\long\def\@enumsentence[#1]#2{\begin{list}{}{%
+\advance\leftmargin by\widelabel \advance\labelwidth by \widelabel}
+\item[#1] #2
+\end{list}}
+
+% The \ex definition
+\newcounter{tempcnt}
+
+\newcommand{\ex}[1]{\setcounter{tempcnt}{\value{enums}}%
+\addtocounter{tempcnt}{#1}%
+\arabic{tempcnt}}
+
+% modified \@item command \unhbox\@tempboxa replaces \makelabel{#1}
+% Used to get \eenumsentence to work correctly
+\def\@item[#1]{\if at noparitem \@donoparitem
+ \else \if at inlabel \indent \par \fi
+ \ifhmode \unskip\unskip \par \fi
+ \if at newlist \if at nobreak \@nbitem \else
+ \addpenalty\@beginparpenalty
+ \addvspace\@topsep \addvspace{-\parskip}\fi
+ \else \addpenalty\@itempenalty \addvspace\itemsep
+ \fi
+ \global\@inlabeltrue
+\fi
+\everypar{\global\@minipagefalse\global\@newlistfalse
+ \if at inlabel\global\@inlabelfalse \hskip -\parindent \box\@labels
+ \penalty\z@ \fi
+ \everypar{}}\global\@nobreakfalse
+\if at noitemarg \@noitemargfalse \if at nmbrlist \refstepcounter{\@listctr}\fi \fi
+\setbox\@tempboxa\hbox{\makelabel{#1}}%
+\global\setbox\@labels
+ \hbox{\unhbox\@labels \hskip \itemindent
+ \hskip -\labelwidth \hskip -\labelsep
+ \ifdim \wd\@tempboxa >\labelwidth
+ \box\@tempboxa
+ \else \hbox to\labelwidth {\unhbox\@tempboxa}\fi
+ \hskip \labelsep}\ignorespaces}
+
+% for enumerated enumsentences. Internal enumeration is alpha if not
+% otherwise defined.
+
+% counter for items within \eenumsentence. (might use enumi instead?)
+\newcounter{enumsi}
+
+% \eenumsentence{\item[a.] A third \label{bar}example \toplabel{foo}
+% \item[b.] A fourth \label{baz}example}
+% Testing references \ref{foo}, \ref{bar}, and \ref{baz}.
+% which produces
+% (3) a. A third example
+% b. A fourth example
+% Testing references 3, 3a, and 3b.
+
+% To ensure that labels in \eenumsentence come out right.
+%
+\def\theenumsi{\theenums\alph{enumsi}}
+\newdimen\eeindent
+\eeindent=15pt
+% changes \leftmargin increased by \eeindent
+% \labelwidth increased by \eeindent
+% .. .. .leftmargin.. .. .. .. ..
+% oldlabelwidth eeindent labelsep actual entry
+% 20pt 15pt 5pt
+% (enumlabel) eelabel
+% eelabel
+
+\def\@mklab#1{\hfil#1}
+\def\enummklab#1{\hfil(\eelabel)\hbox to \eeindent{\hfil#1}}
+\def\enummakelabel#1{\enummklab{#1}\global\let\makelabel=\@mklab}
+\def\toplabel#1{{\edef\@currentlabel{\p at enums\theenums}\label{#1}}}
+
+\def\eenumsentence{\@ifnextchar[{\@eenumsentence}% %]
+{\refstepcounter{enums}\@eenumsentence[\theenums]}}
+
+\long\def\@eenumsentence[#1]#2{\def\eelabel{#1}\let\holdlabel\makelabel%
+\begin{list}{\alph{enumsi}.}{\usecounter{enumsi}%
+\advance\leftmargin by \eeindent \advance\leftmargin by \widelabel%
+\advance\labelwidth by \eeindent \advance\labelwidth by \widelabel%
+\let\makelabel=\enummakelabel}
+#2
+\end{list}\let\makelabel\holdlabel}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% smalltree
+% use
+% \smalltree{& &a\\
+% &b& &c\\
+% d& &e& &f}
+% produces
+% a
+% b c
+% d e f
+
+
+\long\def\smalltree#1{\leavevmode{\def\\{\cr\noalign{\vskip12pt}}%
+\def\mc##1##2{\multispan{##1}{\hfil##2\hfil}}%
+\tabskip=1em%
+\hbox{\vtop{\halign{&\hfil##\hfil\cr
+#1\crcr}}}}}
+
+% modsmalltree
+% use
+% \modsmalltree{3}{&&a\\ &b&&c\\ d&&e&&f}
+% and produces the same output
+% \mc{n}{item} within either will produce item centered across n
+% number of columns
+% \modsmalltree[arraystretch]{num of cols}{argument}
+%
+\def\modsmalltree{\@ifnextchar[{\@modsmalltree}{\@modsmalltree[2]}}
+
+\long\def\@modsmalltree[#1]#2#3{{\def\mc##1##2{%
+\multicolumn{##1}{c}{\def\arraystretch{1}##2}}%
+\def\arraystretch{#1}%
+\def\ns{\def\arraystretch{1}}%
+\setbox0=\hbox{\begin{tabular}[t]{@{}*{#2}{c}@{}}
+#3
+\end{tabular}}%
+\dimen0=\ht0
+\advance\dimen0 by -\arraystretch \ht\strutbox
+\advance\dimen0 by \ht\strutbox
+\ht0=\dimen0
+\dimen0=\dp0
+\advance\dimen0 by -\arraystretch \dp\strutbox
+\advance\dimen0 by \dp\strutbox
+\dp0=\dimen0
+\leavevmode\box0}}
+
+% center lap creates a box of 0 width centered on the point
+\def\clap#1{\hbox to 0pt{\hss#1\hss}}
+%The first argument in both of these commands is the distance above the
+%baseline desired. The second is what is the actual text.
+% over right lap
+\def\orlap#1#2{\vbox to0pt{\vss\hbox to 0pt{#2\hss}\hbox{\vrule height#1
+width0pt depth0pt}}}
+% over left lap
+\def\ollap#1#2{\vbox to0pt{\vss\hbox to 0pt{\hss#2}\hbox{\vrule height#1
+width0pt depth0pt}}}
+% over center lap
+\def\oclap#1#2{\vbox to0pt{\vss\hbox to 0pt{\hss#2\hss}\hbox{\vrule height#1
+width0pt depth0pt}}}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% need to be modified?
+
+% Format
+% \shortex{4}{a & b & c & d}
+% {z & y & x & w}
+% {mnop}
+% produces
+% a b c d
+% z y x w
+% mnop
+%
+% \shortexdt takes two gloss lines
+% \shortexnt takes no gloss lines
+
+\def\shortex#1#2#3#4{\begin{tabular}[t]{@{}*{#1}{l@{\ }}}
+#2\\ #3\\ \multicolumn{#1}{@{}l@{}}{\parbox{\linewidth}{#4}}
+\end{tabular}}
+
+%\def\shortexdt#1#2#3#4#5{\begin{tabular}[t]{@{}*{#1}{l@{\ }}}
+%#2\\ #3\\ \multicolumn{#1}{@{}l@{}}{#4}\\
+%\multicolumn{#1}{@{}l@{}}{#5}
+%\end{tabular}}
+
+\def\shortexnt#1#2#3{\begin{tabular}[t]{@{}*{#1}{l@{\ }}}
+#2\\ #3
+\end{tabular}}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% need to create equivalent of AVM structures.
+
+% Structure is
+% \outerfs{alpha & beta\\ gamma & delta}
+% produces
+% __ __
+% | alpha beta |
+% | gamma delta |
+% -- --
+%
+% \outerfs can be nested.
+% inside of enumsentence use with \evnup[2pt]{\outerfs{. . .}}
+% to ensure that top of fs structure lines up with the number
+%
+% Note that you can use \\[1ex] to increase the space between
+% individual lines
+
+\def\outerfs#1{$\left[\begin{tabular}{ll}#1\end{tabular}\right]$}
+
+% \evnup is used to line up the enumsentence number and an entry along
+% the top. It can take an argument to improve lining up.
+\def\evnup{\@ifnextchar[{\@evnup}{\@evnup[0pt]}}
+
+\def\@evnup[#1]#2{\setbox1=\hbox{#2}%
+\dimen1=\ht1 \advance\dimen1 by -.5\baselineskip%
+\advance\dimen1 by -#1%
+\leavevmode\lower\dimen1\box1}
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% End of main lingmacros
+%
\ No newline at end of file
diff --git a/papers/iwcs-08/modules.graffle b/papers/iwcs-08/modules.graffle
new file mode 100644
index 0000000..b41d0fa
Binary files /dev/null and b/papers/iwcs-08/modules.graffle differ
diff --git a/papers/iwcs-08/modules.pdf b/papers/iwcs-08/modules.pdf
new file mode 100644
index 0000000..ff7c556
Binary files /dev/null and b/papers/iwcs-08/modules.pdf differ
diff --git a/papers/iwcs-08/nltk_iwcs_09.bib b/papers/iwcs-08/nltk_iwcs_09.bib
new file mode 100644
index 0000000..d78d86d
--- /dev/null
+++ b/papers/iwcs-08/nltk_iwcs_09.bib
@@ -0,0 +1,72 @@
+ at book{Dalrymple2001,
+ author = {Mary Dalrymple},
+ title = {Lexical Functional Grammar},
+ series = {Syntax and Semantics},
+ volume = {34},
+ publisher = {Academic Press},
+ address = {New York},
+ year = {2001}
+}
+
+
+ at InCollection{Dalrymple:1999:RRB,
+ author = {Mary Dalrymple and V. Gupta and John Lamping and V. Saraswat},
+ title = {Relating resource-based
+semantics to categorial semantics},
+ booktitle = {Semantics and syntax in {Lexical Functional Grammar}: the resource
+logic approach},
+ pages = { 261--280},
+ publisher = {MIT Press},
+ year = 1999,
+ editor = {Mary Dalrymple},
+ address = {Cambridge, MA}}
+
+
+
+
+ at book{BB,
+ author = {Patrick Blackburn and Johan Bos},
+ title = {Representation and Inference for Natural Language: A First Course in Computational Semantics},
+ publisher = {CSLI Publications},
+ address = {New York},
+ year = {2005}
+}
+
+ at book{KampReyle,
+ author = {Hans Kamp and Uwe Reyle},
+ title = {From Discourse to the Lexicon: Introduction to Modeltheoretic Semantics of Natural Language, Formal Logic and Discourse Representation Theory},
+ publisher = {Kluwer Academic Publishers},
+ year = {1993}
+}
+
+ at inproceedings{Multidisciplinary,
+ author = {Steven Bird and Ewan Klein and Edward Loper and Jason Baldridge},
+ title = {Multidisciplinary instruction with the {Natural Language Toolkit}},
+ booktitle = {Proceedings of the Third Workshop on Issues in Teaching Computational Linguistics},
+ address = {Columbus, Ohio, USA},
+ month = {June},
+ year = {2008}
+}
+
+ at Misc{McCune,
+ author = {William McCune},
+ title = {Prover9: Automated theorem prover for first-order and equational logic},
+ year = 2008,
+ note = {\url{http://www.cs.unm.edu/~mccune/mace4/manual-examples.html}}
+}
+
+ at inproceedings{BosRTE,
+ author = {Johan Bos and Katja Markert},
+ title = {Recognising textual entailment with logical inference},
+ booktitle = {Proceedings of the conference on Human Language Technology and Empirical Methods in Natural Language Processing},
+ address = {Vancouver, British Columbia, Canada},
+ year = {2005}
+}
+
+ at InProceedings{Klein06altw,
+ author = {Ewan Klein},
+ title = {Computational semantics in the {Natural Language Toolkit}},
+ booktitle = {Proceedings of the Australasian Language Technology Workshop},
+ pages = {26--33},
+ year = 2006
+}
diff --git a/papers/iwcs-08/nltk_iwcs_09.tex b/papers/iwcs-08/nltk_iwcs_09.tex
new file mode 100755
index 0000000..6daa0d0
--- /dev/null
+++ b/papers/iwcs-08/nltk_iwcs_09.tex
@@ -0,0 +1,708 @@
+\documentclass[11pt, a4paper]{article}
+
+\usepackage[pdftex,colorlinks=true,
+ pdfstartview=FitV,
+ linkcolor=blue,
+ citecolor=blue,
+ urlcolor=blue
+ ]{hyperref}
+
+\usepackage{amssymb}
+\usepackage{amsmath}
+\usepackage{graphicx}
+\usepackage{url}
+\usepackage{wrapfig}
+\usepackage{fancyvrb}
+\fvset{fontsize=\small, frame=lines}
+
+\newcommand{\BB}{\textsc{B{\small\&}B}}
+\newcommand{\DRS}{\textsc{drs}}
+\newcommand{\DRT}{\textsc{drt}}
+\newcommand{\FOL}{\textsc{fol}}
+\newcommand{\LF}{\textsc{lf}}
+\newcommand{\NLP}{\textsc{nlp}}
+\newcommand{\NLTK}{\textsc{nltk}}
+\newcommand{\RTE}{\textsc{rte}}
+
+
+\pdfinfo{
+ /Author (Dan Garrette and Ewan Klein)
+ /Title (An Extensible Toolkit for Computational Semantics)
+ /Subject (Computational Semantics)
+ /Keywords (Natural Language Processing;Computational Semantics;Natural Language Toolkit;NLTK)
+}
+
+\usepackage[round]{natbib}
+% \bibpunct{[}{]}{;}{a}{,}{,}
+\bibliographystyle{plainnat}
+
+\usepackage{lingmacros}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% Copied from covington.sty by Michael A. Covington
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\newcommand{\dhgdrs}[2]
+{
+ {
+ \it
+ \begin{tabular}{|l|}
+ \hline
+ ~ \vspace{-2ex} \\
+ #1
+ \\
+ ~ \vspace{-2ex} \\
+ \hline
+ ~ \vspace{-2ex} \\
+ #2
+ \\
+ ~ \\ % can't vspace here or the line will come out wrong
+ \hline
+ \end{tabular}
+ }
+}
+\newcommand{\dhgsdrs}[3]
+{\begin{tabular}{l}
+\mbox{\it #1} \\
+~ \\
+\dhgdrs{#2}{#3}
+\end{tabular}}\newcommand{\dhgifdrs}[4]
+{
+ \mbox{\dhgdrs{#1}{#2}~~{\large $\Rightarrow$}~\dhgdrs{#3}{#4}}
+}
+\newcommand{\dhgalifdrs}[4]
+{
+ \mbox{$\!\!\!$\dhgdrs{#1}{#2}~~{\large $\Rightarrow$}~\dhgdrs{#3}{#4}}
+}
+\newcommand{\dhgnegdrs}[2]
+{
+ \mbox{{\large $\neg$}\dhgdrs{#1}{#2}}
+}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+% END covington.sty
+%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+
+\begin{document}
+
+
+\title{An Extensible Toolkit for Computational Semantics}
+\author{Dan Garrette \and Ewan Klein}
+\date{\today}
+
+\maketitle
+
+\section{Introduction}
+
+In this paper we focus on the software for computational semantics provided
+by the Python-based Natural Language Toolkit (\NLTK). The semantics
+modules in \NLTK\ are
+inspired in large part by the approach developed in \citet{BB}
+(henceforth referred to as \BB).
+Since Blackburn and Bos have also provided a software suite to
+accompany their excellent textbook, one might ask what the
+justification is for the \NLTK\ offering, which is similarly slanted
+towards teaching computational semantics.
+
+This question can be answered in a number of ways. First, we believe
+there is intrinsic merit in the availability of different software
+tools for semantic analysis, even when there is some duplication of
+coverage; and this will become more true as computational semantics
+starts to be as widely studied as computational syntax. For example,
+one rarely hears the objection that too many implementations of
+syntactic parsers are available. Moreover, the \NLTK\ software
+significantly goes beyond \BB\ in providing an implementation of
+Glue Semantics.
+
+Second, whatever the relative merits of Prolog vs.\ Python as
+programming languages, there is surely an advantage in offering
+students and instructors a choice in this respect. Given that many
+students have either already been exposed to Java, or else have had no
+programming experience at all, Python offers them the option of
+accomplishing interesting results with only a shallow
+learning curve.
+
+Third, \NLTK\ is a rapidly developing, open source
+project\footnote{See \url{http://nltk.org}} with a broad coverage of
+natural language processing (\NLP) tools; see \citet{Multidisciplinary} for
+a recent overview. This wide functionality has a number of benefits,
+most notably that lexical, syntactic and semantic processing can be
+carried out within a uniform computational framework. As a result,
+\NLTK\ makes it much easier to include some computational semantics
+subject matter in
+a broad course on natural language analysis, rather than having to
+devote a whole course exclusively to the topic.
+
+Fourth, \NLTK\ is accompanied by a substantial collection of corpora,
+plus easy-to-use corpus readers. This collection, which currently
+stands at over 50 corpora and trained models, includes parsed, POS-tagged, plain text,
+categorized text, and
+lexicons. The
+availability of corpora can help encourage students to go beyond
+writing toy grammars, and instead to start grappling with the
+complexities of semantically analysing realistic bodies of text.
+
+Fifth, \NLTK\ is not just for students. Although Python is slower than
+languages like Java and C++, its suitability for rapid prototyping
+makes it an attractive addition to the researcher's inventory of
+resources. Building an experimental set-up in \NLTK\ to test a
+hypothesis or explore some data is straightforward and quick, and the
+rich variety of existing \NLP\ components in the toolkit allows rapid
+assembly of quite sophisticated processing pipelines.
+
+
+\section{Overview}
+\label{sec:overview}
+
+
+\begin{wrapfigure}{r}{3in}
+%\label{modules}
+ \centering
+\includegraphics[scale=.6]{modules}
+% \caption{Overview of semantic processing in NLTK}
+\end{wrapfigure}
+
+Like \BB, we assume that one of the most important tasks for
+the teacher is to ground students in the basic concepts of first order
+logic and the lambda calculus, model-theoretic interpretation and
+inference. This provides a basis for exploring more modern approaches
+like Discourse Representation Theory (\DRT; \citet{KampReyle}) and
+underspecification.
+
+In the accompanying figure, we give a diagrammatic overview of the
+main semantics-related functionality that is currently available in
+\NLTK. Logical forms (\LF s) can be induced as result of syntactic
+parsing, using either feature-based grammars that are processed with
+an Earley chart parser, or else by associating \LF s with the output
+of a broad-coverage dependency parser. Our basic \LF s are expressions
+of first order logic, supplemented with the lambda operator. However,
+we also admit Discourse Representation Structures (\DRS s) as \LF s,
+and underspecified \LF s can be built using either Hole Semantics
+\citep{BB} or Glue Semantics
+\citep{Dalrymple:1999:RRB}.
+%\citep{Dalrymple:1999:RRB,Dalrymple2001}.
+Once we have constructed \LF
+s, they can be evaluated in a first order model \citep{Klein06altw},
+tested for equivalence and validity in a variety of theorem provers,
+or tested for consistency in a model builder. The latter two tasks are
+aided by \NLTK\
+ interfaces to third-party inference tools, currently Prover9
+and Mace4 \citep{McCune}.
+
+We do not have space in this paper to discuss all of these components,
+but will try to present some of the key aspects, and along the way
+noting certain points of difference \textit{vis-\`a-vis} \BB.
+
+
+\section{Logical Form}
+
+\subsection{First Order Predicate Logic with Lambda Calculus}
+From a pedagogical point of view, it is usually important to ensure
+that students have some grasp of the language of first order predicate
+logic (\FOL), and can also manipulate $\lambda$-abstraction. The
+\texttt{nltk.sem.logic} module contains an object-oriented approach
+to representing \FOL\ plus
+$\lambda$-abstraction. Logical formulas are typically fed to the
+\texttt{logic} parser as strings, and then represented as instances of
+various subclasses of \texttt{Expression}, as we will see shortly.
+
+An attractive feature of Python is its interactive interpreter,
+which allows the user to enter Python expressions and statements for
+evaluation. In the example below and subsequently, \verb!>>>! is the
+Python interpreter's prompt.
+\begin{Verbatim}[numbers=left]
+>>> from nltk.sem import LogicParser
+>>> lp = LogicParser()
+>>> e = lp.parse('all x.(girl(x) -> exists y.(dog(y) & chase(x,y)))')
+>>> e
+<AllExpression all x.(girl(x) -> exists y.(dog(y) & chase(x,y)))>
+\end{Verbatim}
+As illustrated, the result of parsing the formula at line~3 is an object
+\texttt{e} belonging to the class \texttt{AllExpression}, itself a
+subclass of \texttt{Expression}. All such subclasses have numerous
+methods that implement standard logical operations. For
+example, the \texttt{simplify()} method carries out
+$\beta$-conversion; the \texttt{free()} method finds
+all the free variables in an expression; and for quantified expressions
+(such as \texttt{AllExpression}s), there is an \texttt{alpha\_convert()}
+method. The \texttt{logic}
+module will $\alpha$-convert automatically when appropriate to
+avoid name-clashes in the \texttt{replace()} method. Let's illustrate
+these methods
+with a formula involving $\lambda$-abstraction, namely
+\verb!\x.P(x)(y)!; we use \protect{\verb!\!} to represent
+$\lambda$. (Since \verb!\! is a special character in Python,
+we add the \texttt{r} prefix to strings containing it to preclude
+additional escape characters.)
+\begin{Verbatim}
+>>> from nltk.sem import Variable
+>>> e1 = lp.parse(r'\x.P(x)(y)')
+>>> print e1.simplify()
+P(y)
+>>> e2 = lp.parse('all x.P(x,a,b)')
+>>> print e2.free()
+set([<Variable('a'), Variable('b')])
+>>> print e2.alpha_convert(Variable('z'))
+all z.P(z,a,b)
+>>> e3 = lp.parse('x')
+>>> print e2.replace(Variable('b'), e3)
+all z1.P(z1,a,x)
+\end{Verbatim}
+Allowing students to build simple first order models, and evaluate
+expressions in those models, can be useful for helping them clarify
+their intuitions about quantification. In the next example, we show
+one of the available methods in \NLTK\ for specifying a model and
+using it to determine the set of satisfiers of the open formula
+$\exists x.(\mathit{girl}(y) \wedge
+\mathit{chase}(x,y))$.\footnote{The triple quotes \texttt{"""} in
+ Python allow us to break a logical line across several physical
+ lines.},
+\footnote{Given a valuation \texttt{val}, the property
+ \texttt{val.domain} returns the set of all domain individuals
+ specified in the valuation.}
+\begin{Verbatim}
+>>> from nltk.sem import parse_valuation, Model, Assignment
+>>> v = """
+... suzie => s
+... fido => f
+... rover => r
+... girl => {s}
+... chase => {(f, s), (r, s), (s, f)}
+... """
+>>> val = parse_valuation(v) #create a Valuation
+>>> m = Model(val.domain, val) #initialize a Model
+>>> g = Assignment(val.domain) #initialize an Assignment
+>>> e4 = lp.parse('exists y. (girl(y) & chase(x, y))')
+>>> m.satisfiers(e4, 'x', g) #check satisfiers of e4 wrt to x
+set(['r', 'f'])
+\end{Verbatim}
+
+In \BB, $\lambda$-abstracts are second-class citizens, used
+exclusively as a `glue' mechanism for composing meaning
+representations. Although we use $\lambda$-abstracts as glue too,
+abstracts over individual variables are semantically interpreted in \NLTK, namely as
+characteristic functions.
+
+\texttt{Expression}s in \NLTK\ are typed (using Montague-style
+types). The type can be accessed with the \texttt{type}
+property. A type checking procedure can be invoked with the
+\texttt{typecheck()} method. \texttt{typecheck()} will return
+a dictionary of all variables and their types if the expression
+is well typed; for non-well typed expressions, an exception will
+be raised. Additionally, type checking will be automatically
+invoked by the \texttt{LogicParser} if the parameter
+\texttt{type\_check=True} is set.
+\begin{Verbatim}
+>>> a = lp.parse(r'\x.(man(x) & tall(x))')
+>>> a.type
+<e,t>
+>>> a.typecheck()
+{'x': e, 'man': <e,t>, 'tall': <e,t>}
+>>> tlp = LogicParser(type_check=True)
+>>> tlp.parse(r'\x y.-see(x,y)(\x.(man(x) & tall(x)))')
+Traceback (most recent call last):
+ . . .
+TypeException: The function '\x y.-see(x,y)' is of type '<e,<e,t>>'
+and cannot be applied to '\x.(man(x) & tall(x))' of type '<e,t>'.
+Its argument must match type 'e'.
+\end{Verbatim}
+% Fix for code colorization: >>>>
+
+\subsection{Discourse Representation Theory}
+As mentioned earlier, \NLTK\ contains an extension to the
+\texttt{logic} module for working with Discourse Representation Theory
+(\DRT) \citep{KampReyle}. The \texttt{nltk.sem.drt} module introduces
+a classes for working with Discourse Representation Structures (\DRS s).
+In \NLTK , a \DRS\ is represented as a pair consisting of a list of
+discourse of referents and a list of \DRS\ conditions:
+
+\enumsentence{\label{drt3} \texttt{([j,d],[John(j), dog(d),
+ sees(j,d)])}}
+
+On top of the functionality available for \FOL\
+expressions, \DRT\ expressions have a `\DRS-concatenation' operator,
+represented as the \texttt{+} symbol. The concatenation of two \DRS s
+is a single \DRS\ containing the merged discourse referents and the
+conditions from both arguments. \DRS-concatenation automatically
+$\alpha$-converts bound variables to avoid name-clashes. The
+\texttt{+} symbol is overloaded so that \DRT\ expressions can be added
+together easily. The \texttt{DrtParser} allows \DRS s to be
+specified succinctly as strings.
+\begin{Verbatim}
+>>> from nltk.sem.drt import DrtParser
+>>> dp = DrtParser()
+>>> d1 = dp.parse('([x],[walk(x)]) + ([y],[run(y)])')
+>>> print d1
+(([x],[walk(x)]) + ([y],[run(y)]))
+>>> print d1.simplify()
+([x,y],[walk(x), run(y)])
+>>> d2 = dp.parse('([x,y],[Bill(x), Fred(y)])')
+>>> d3 = dp.parse("""([],[([u],[Porsche(u), own(x,u)])
+... -> ([v],[Ferrari(v), own(y,u)])])""")
+>>> d4 = d2 + d3
+>>> print d4.simplify()
+([x,y],[Bill(x), Fred(y),
+(([u],[Porsche(u), own(x,u)]) -> ([v],[Ferrari(v), own(y,u)]))])
+\end{Verbatim}
+
+\noindent
+\DRT\ expressions can be converted to their first order predicate
+logic equivalents using the \texttt{fol()} method and can be
+graphically rendered on screen with the \texttt{draw()} method.
+
+\begin{wrapfigure}{r}{0.2\textheight}
+\vspace{-16ex}
+\begin{center}
+ \includegraphics[scale=.5]{drs.png}
+ \end{center}
+\vspace{-4ex}
+\caption{\small DRS Screenshot}
+\vspace{-10ex}
+\end{wrapfigure}
+
+\begin{Verbatim}[frame=none]
+>>> print d1.fol()
+(exists x.walk(x) & exists y.run(y))
+>>> d4.simplify().draw()
+\end{Verbatim}
+
+Since the $\lambda$ operator can be combined with \DRT\ expressions,
+the \texttt{nltk.sem.drt} module can be used as a plug-in replacement for
+\texttt{nltk.sem.logic} in building compositional semantics.
+
+
+\section{Scope Ambiguity and Underspecification}
+
+Two key questions in introducing students to computational semantics are:
+\begin{enumerate}
+\item[Q1:] How are semantic representations constructed from input
+ sentences?
+\vspace{-2ex}
+\item[Q2:] What is scope ambiguity and how is it captured?
+\end{enumerate}
+A standard pedagogical approach is to address (Q1) with a simple
+syntax-driven induction of logical forms which fails to deal with
+scope ambiguity, while (Q2) is addressed by introducing underspecified
+representations which are resolved to produce different readings of
+ambiguous sentences.
+
+\NLTK\ includes a suite of parsing tools, amongst which is a chart
+parser for context free grammars augmented with feature structures. A
+`semantics' feature \texttt{sem} allows us to compose the
+contributions of constituents to build a logical form for a complete
+sentence. To illustrate, the following minimal grammar
+\texttt{sem1.fcfg} handles quantification and intransitive verbs
+(where values such as \texttt{?subj} and \texttt{?vp} are unification
+variables, while \texttt{P} and \texttt{Q} are $\lambda$-bound object
+language variables):
+\begin{Verbatim}
+S[sem = <?subj(?vp)>] -> NP[sem=?subj] VP[sem=?vp]
+VP[sem=?v] -> IV[sem=?v]
+NP[sem=<?det(?n)>] -> Det[sem=?det] N[sem=?n]
+Det[sem=<\P.\Q.exists x.(P(x) & Q(x))>] -> 'a'
+N[sem=<\x.dog(x)>] -> 'dog'
+IV[sem=<\x.bark(x)>] -> 'barks'
+\end{Verbatim}
+Using \texttt{sem1.fcfg}, we can parse \textit{A dog barks} and view
+its semantics.
+The \texttt{load\_earley()} method
+takes an optional parameter \texttt{logic\_parser} which specifies the
+logic-parser for processing the value of the \texttt{sem} feature, thus
+allowing different kinds of logical forms to be constructed.
+\begin{Verbatim}
+>>> from nltk.parse import load_earley
+>>> parser = load_earley('grammars/sem1.fcfg', trace=0)
+>>> trees = parser.nbest_parse('a dog barks'.split())
+>>> print trees[0].node['sem'].simplify()
+exists x.(dog(x) & bark(x))
+\end{Verbatim}
+
+Underspecified logical forms allow us to loosen the relation between
+syntactic and semantic representations. We consider two approaches to
+underspecification, namely Hole
+Semantics and Glue Semantics. Since the former will be familiar from
+\BB, we devote most of our attention to presenting Glue
+Semantics.
+
+\subsection{Hole Semantics}
+
+Hole Semantics in \NLTK\ is handled by the
+\texttt{nltk.sem.hole} module, which uses a context free grammar to
+generate an underspecified logical form. Since the latter is itself a
+formula of first order logic, we can continue to use the \texttt{sem} feature
+in the context free grammar:
+\begin{Verbatim}[frame=none,fontsize=\small]
+N[sem=<\x h l.(PRED(l,dog,x) & LEQ(l,h) & HOLE(h) & LABEL(l))>] -> 'dog'
+\end{Verbatim}
+The Hole Semantics module uses a standard plugging algorithm to derive the
+sentence's readings from the underspecified \LF.
+\begin{Verbatim}
+>>> from nltk.sem.hole import hole_readings
+>>> readings = hole_readings('every girl chases a dog')
+>>> for r in readings: print r
+exists z1.(dog(z1) & all z2.(girl(z2) -> chase(z1,z2)))
+all z2.(girl(z2) -> exists z1.(dog(z1) & chase(z1,z2)))
+\end{Verbatim}
+
+
+\subsection{Glue Semantics}
+Glue Semantics
+ \citep{Dalrymple:1999:RRB}, or Glue for
+short, is an approach to compositionality that tries to handle
+semantic ambiguity by using resource-sensitive logic to assemble
+meaning expressions.
+The approach builds proofs over `meaning constructors'; these are of the
+form $\cal{M}: \cal{G}$, where $\cal{M}$ is a meaning representation and
+$\cal{G}$ is a term of linear logic. The linear logic term $\cal{G}$
+dictates how the meaning expression $\cal{M}$ can be combined. Each
+distinct proof that can be derived reflects a different semantic
+reading of the entire sentence.
+
+The variant of linear logic that we use has \emph{(linear)
+ implication} (i.e., $\multimap$) as its
+only operator, so the primary operation during the proof is Modus
+Ponens. Linear logic is an appropriate logic to serve as `glue'
+because it is resource-sensitive. This means that when Modus Ponens
+combines two terms to create a new one, the two original
+terms are `consumed', and cannot be used again in the proof;
+cf.\ (\ref{glue1}) vs.\ (\ref{glue2}).
+Additionally, every premise must be used for the proof to be valid;
+cf.\ (\ref{glue3}).
+This resource-sensitivity dictates that each word contributes its
+meaning exactly once to the meaning of the whole.
+\enumsentence{\label{glue1} $A, (A \multimap B) \vdash B$}
+\enumsentence{\label{glue2} $A, (A \multimap B) \nvdash A, B$}
+\enumsentence{\label{glue3} $A, A, (A \multimap B) \nvdash B$}
+\NLTK's \texttt{nltk.sem.linearlogic} module
+contains an implementation of linear logic.
+
+The primary rule for composing Glue formulas is (\ref{glue4}).
+Function-argument application of meaning expressions is reflected (\textit{via}
+the Curry-Howard isomorphism) by the application of Modus Ponens in a
+linear logic proof. Note that $A$ and $B$ are meta-variables over
+constants of linear logic; these constants represent `attachment
+points' for meaning expressions in some kind of syntactically-derived
+representation (such as an LFG \textit{f}-structure). It is
+(\ref{glue4}) which allows Glue to guide the construction of complex
+meaning expressions.
+\vspace{-3ex}
+\enumsentence{\label{glue4} $\alpha : A,\;
+ \gamma : (A \multimap B) \vdash \gamma(\alpha) : B$}
+
+The \NLTK\ module \texttt{sem.glue} implements Glue for
+\FOL\ and \DRT\ meaning expressions.%
+\footnote{See
+ \url{http://nltk.googlecode.com/svn/trunk/doc/contrib/sem/index.html}
+for more details.}
+The following example shows how Glue formulas are
+created and combined to derive a logical form for \textit{John walks}:
+
+\begin{Verbatim}
+>>> from nltk.sem.glue import GlueFormula
+>>> john = GlueFormula('john', 'g')
+>>> walks = GlueFormula(r'\x.walk(x)', '(g -o f)')
+>>> john_walks = walks.applyto(john)
+>>> print john_walks.meaning.simplify()
+walk(john)
+\end{Verbatim}
+Thus, the non-logical constant \textit{john} is associated with the
+Glue term $g$, while the meaning expression $\lambda x.walk(x)$ is
+associated with $(g \multimap f)$ since it is a function that
+takes $g$ as input and returns the meaning expression $f$,
+corresponding to the whole
+sentence. Consequently, a proof of $f$ from the premises is a derivation
+of a meaning representation for the sentence.
+
+Scope ambiguity, resulting, for example, from quantifiers, requires the
+use of \textit{variables} in the Glue terms. Such variables may be
+instantiated to any linear logic constant, so long as this is carried
+out uniformly. Let's assume that the quantified noun phrase
+\textit{every girl} has the meaning constructor (\ref{glue5}) (where
+$G$ is a linear logic variable):
+
+\enumsentence{\label{glue5} $\lambda Q.\forall x.(girl(x) \rightarrow
+ Q(x)) : ((g \multimap G) \multimap G)$}
+Then the Glue derivation shown below correctly
+generates two readings for the sentence \textit{Every girl chases a dog}:
+\begin{Verbatim}
+>>> from nltk.sem.glue import GlueFormula, Glue
+>>> a = GlueFormula(r'\Q.all x.(girl(x) -> Q(x))', '((g -o G) -o G)')
+>>> b = GlueFormula(r'\x y.chase(x,y)', '(g -o (h -o f))')
+>>> c = GlueFormula(r'\Q.exists x.(dog(x)&Q(x))', '((h -o H) -o H)')
+>>> glue = Glue()
+>>> for reading in glue.get_readings(glue.gfl_to_compiled([a,b,c])):
+... print reading.simplify()
+exists x.(dog(x) & all z13.(girl(z13) -> chase(z13,x)))
+all x.(girl(x) -> exists z14.(dog(z14) & chase(x,z14)))
+\end{Verbatim}
+
+
+\section{Inference tools}
+In order to perform inference over semantic representations, \NLTK\
+can call both theorem provers and model builders.
+The library includes a pure Python tableau-based first order theorem prover;
+this is intended to allow students to study
+tableau methods for theorem proving, and provides an
+opportunity for experimentation. In addition, \NLTK\ provides
+interfaces to two off-the-shelf tools, namely the theorem prover Prover9,
+and the model builder Mace4 \citep{McCune}. % Both of these are
+% powerful enough for research.
+
+The \verb!Prover9! class represents the interface to Prover9. It has a
+method \verb!prove(G, A)! and takes as parameters a proof goal
+\texttt{G} and a list \texttt{A} of assumptions.
+Here, we verify that if every dog barks, and Rover is a dog,
+then it is true that Rover barks:
+\begin{Verbatim}
+>>> from nltk.inference import Prover9
+>>> a = lp.parse('all x.(dog(x) -> bark(x))')
+>>> b = lp.parse('dog(rover)')
+>>> c = lp.parse('bark(rover)')
+>>> prover = Prover9()
+>>> prover.prove(c, [a,b])
+True
+\end{Verbatim}
+
+A theorem prover can also be used to check the logical equivalence of
+expressions. For two expressions $A$ and $B$, we can pass $(A\iff B)$
+into a theorem prover and know that the theorem will be proved if and
+only if the expressions are logically equivalent. \NLTK's standard
+equality operator for \texttt{Expression}s (\texttt{==}) is able to
+handle situations where two expressions are identical up to
+$\alpha$-conversion. However, it would be impractical for \NLTK\ to
+invoke a wider range of logic rules every time we checked for equality
+of two expressions. Consequently, both the \texttt{logic} and
+\texttt{drt} modules in \NLTK\
+have a separate method, \texttt{tp\_equals}, for checking `equality'
+up to logical equivalence.
+\begin{Verbatim}
+>>> a = lp.parse('all x.walk(x)')
+>>> b = lp.parse('all y.walk(y)')
+>>> a == b
+True
+>>> c = lp.parse('-(P(x) & Q(x))')
+>>> d = lp.parse('-P(x) | -Q(x)')
+>>> c == d
+False
+>>> c.tp_equals(d)
+True
+\end{Verbatim}
+
+\section{Discourse Processing}
+
+\NLTK\ contains a discourse processing module,
+\texttt{nltk.inference.discourse}, similar to the \textsc{curt} program
+presented in \BB. This module processes sentences incrementally,
+keeping track of all possible threads when there is ambiguity. For
+simplicity, the following example ignores scope ambiguity.
+\begin{Verbatim}[baselinestretch=.5]
+
+>>> from nltk.inference.discourse import DiscourseTester as DT
+>>> dt = DT(['A student dances', 'Every student is a person'])
+>>> dt.readings()
+
+s0 readings:
+
+s0-r0: exists x.(student(x) & dance(x))
+
+s1 readings:
+
+s1-r0: all x.(student(x) -> person(x))
+\end{Verbatim}
+When a new sentence is added to the current discourse, setting the
+parameter \texttt{consistchk=True} causes consistency to be checked
+by invoking the model checker for each `thread', i.e., discourse sequence of
+admissible readings. In this case, the user has the option
+of retracting the sentence in question.
+\newpage
+\begin{Verbatim}
+>>> dt.add_sentence('No person dances', consistchk=True)
+Inconsistent discourse d0 ['s0-r0', 's1-r0', 's2-r0']:
+ s0-r0: exists x.(student(x) & dance(x))
+ s1-r0: all x.(student(x) -> person(x))
+ s2-r0: -exists x.(person(x) & dance(x))
+>>> dt.retract_sentence('No person dances')
+Current sentences are
+s0: A student dances
+s1: Every student is a person
+\end{Verbatim}
+In a similar manner, we use \texttt{informchk=True} to check whether
+the new sentence is informative relative to the current discourse (by
+asking the theorem prover to derive it from the discourse).
+\begin{Verbatim}
+>>> dt.add_sentence('A person dances', informchk=True)
+Sentence 'A person dances' under reading 'exists x.(person(x) & dance(x))':
+Not informative relative to thread 'd0'
+\end{Verbatim}
+It is also possible to pass in an additional set of assumptions as
+background knowledge and use these to filter out inconsistent readings.
+
+The \texttt{discourse} module can accommodate semantic
+ambiguity and filter out readings that are not admissable.
+By invoking both Glue Semantics and \DRT, the following example processes the
+two-sentence discourse \textit{Every dog chases a boy. He runs}. As
+shown, the first sentence has two possible readings, while
+the second sentence contains an anaphoric pronoun, indicated as \texttt{PRO(x)}.
+\begin{Verbatim}[baselinestretch=.5]
+
+>>> from nltk.inference.discourse import DrtGlueReadingCommand as RC
+>>> dt = DT(['Every dog chases a boy', 'He runs'], RC())
+>>> dt.readings()
+
+s0 readings:
+
+s0-r0: ([],[(([x],[dog(x)]) -> ([z15],[boy(z15), chase(x,z15)]))])
+s0-r1: ([z16],[boy(z16), (([x],[dog(x)]) -> ([],[chase(x,z16)]))])
+
+s1 readings:
+
+s1-r0: ([x],[PRO(x), run(x)])
+\end{Verbatim}
+When we examine the two threads \texttt{d0} and \texttt{d1}, we see
+that that reading \texttt{s0-r0}, where \textit{every dog} out-scopes
+\texttt{a boy}, is deemed inadmissable because the pronoun in the
+second sentence cannot be resolved. By contrast, in thread \texttt{d1} the
+pronoun (relettered to \texttt{z24}) has been bound \textit{via} the
+equation \texttt{(z24 = z20)}. % Inadmissable readings are filtered out
+\begin{Verbatim}
+>>> dt.readings(show_thread_readings=True)
+d0: ['s0-r0', 's1-r0'] : INVALID: AnaphoraResolutionException
+d1: ['s0-r1', 's1-r0'] : ([z20,z24],[boy(z20), (([x],[dog(x)]) ->
+([],[chase(x,z20)])), (z24 = z20), run(z24)])
+\end{Verbatim}
+
+\section{Conclusions and Future Work}
+\NLTK's semantics functionality has been written with extensibility in
+mind. The \texttt{logic} module's \texttt{LogicParser}
+employs a basic parsing template and contains hooks that an extending
+module can use to supplement or substitute functionality. Moreover, the
+base \texttt{Expression} class in \texttt{logic}, as well as any
+derived classes, can be extended, allowing variants to reuse the
+existing functionality. For example, the \DRT\ and linear logic modules
+are implemented as extensions to \texttt{logic.py}.
+
+The theorem prover and model builder code has also been carefully
+archi\-tected to allow extensions and the \texttt{nltk.inference.api}
+library exposes the framework for the inference architecture. The
+library therefore provides a good starting point for creating
+interfaces with other theorem provers and model builders in addition
+to Prover9, Mace4, and the tableau prover.
+
+\NLTK\ already includes the beginnings of a framework for `recognizing
+textual entailment'; access to the \RTE\ data sets is provided and we
+are in the course of developing a few simple modules to demonstrate
+\RTE\ techniques. For example, a Logical Entailment \RTE\ tagger
+based on \cite{BosRTE} begins by building a semantic
+representation of both the text and the hypothesis in \DRT. It then
+runs a theorem prover with the text as the assumption and the
+hypothesis as the goal in order to check whether the text entails the
+hypothesis.The tagger is also capable of adding background knowledge
+\textit{via} an interface to the WordNet dictionary in
+\texttt{nltk.wordnet} as a first step in making the entailment
+checking more robust.
+
+
+{\small
+\bibliography{nltk_iwcs_09}
+}
+
+\end{document}
diff --git a/pip-req.txt b/pip-req.txt
new file mode 100644
index 0000000..0e634a6
--- /dev/null
+++ b/pip-req.txt
@@ -0,0 +1,8 @@
+nose>=1.3.0
+tox>=1.6.1
+coverage>=3.7.1
+pylint>=1.1.0
+numpy>=1.8.0
+scipy>=0.13.2
+matplotlib>=1.3.1
+scikit-learn>=0.14.1
diff --git a/setup.cfg b/setup.cfg
new file mode 100644
index 0000000..e69de29
diff --git a/setup.py b/setup.py
new file mode 100644
index 0000000..3ec0949
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+#
+# Setup script for the Natural Language Toolkit
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Steven Bird <stevenbird1 at gmail.com>
+# Edward Loper <edloper at gmail.com>
+# Ewan Klein <ewan at inf.ed.ac.uk>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# Work around mbcs bug in distutils.
+# http://bugs.python.org/issue10945
+import codecs
+try:
+ codecs.lookup('mbcs')
+except LookupError:
+ ascii = codecs.lookup('ascii')
+ func = lambda name, enc=ascii: {True: enc}.get(name=='mbcs')
+ codecs.register(func)
+
+import os
+
+# Use the VERSION file to get NLTK version
+version_file = os.path.join(os.path.dirname(__file__), 'nltk', 'VERSION')
+with open(version_file) as fh:
+ nltk_version = fh.read().strip()
+
+# setuptools
+from setuptools import setup, find_packages
+
+#
+# Prevent setuptools from trying to add extra files to the source code
+# manifest by scanning the version control system for its contents.
+#
+from setuptools.command import sdist
+del sdist.finders[:]
+
+setup(
+ name = "nltk",
+ description = "Natural Language Toolkit",
+ version = nltk_version,
+ url = "http://nltk.org/",
+ long_description = """\
+The Natural Language Toolkit (NLTK) is a Python package for
+natural language processing. NLTK requires Python 2.6, 2.7, or 3.2+.""",
+ license = "Apache License, Version 2.0",
+ keywords = ['NLP', 'CL', 'natural language processing',
+ 'computational linguistics', 'parsing', 'tagging',
+ 'tokenizing', 'syntax', 'linguistics', 'language',
+ 'natural language', 'text analytics'],
+ maintainer = "Steven Bird",
+ maintainer_email = "stevenbird1 at gmail.com",
+ author = "Steven Bird",
+ author_email = "stevenbird1 at gmail.com",
+ classifiers = [
+ 'Development Status :: 5 - Production/Stable',
+ 'Intended Audience :: Developers',
+ 'Intended Audience :: Education',
+ 'Intended Audience :: Information Technology',
+ 'Intended Audience :: Science/Research',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Operating System :: OS Independent',
+ 'Programming Language :: Python :: 2.6',
+ 'Programming Language :: Python :: 2.7',
+ 'Programming Language :: Python :: 3.2',
+ 'Programming Language :: Python :: 3.3',
+ 'Programming Language :: Python :: 3.4',
+ 'Topic :: Scientific/Engineering',
+ 'Topic :: Scientific/Engineering :: Artificial Intelligence',
+ 'Topic :: Scientific/Engineering :: Human Machine Interfaces',
+ 'Topic :: Scientific/Engineering :: Information Analysis',
+ 'Topic :: Text Processing',
+ 'Topic :: Text Processing :: Filters',
+ 'Topic :: Text Processing :: General',
+ 'Topic :: Text Processing :: Indexing',
+ 'Topic :: Text Processing :: Linguistic',
+ ],
+ package_data = {'nltk': ['test/*.doctest', 'VERSION']},
+ packages = find_packages(),
+ zip_safe=False, # since normal files will be present too?
+ )
diff --git a/tools/find_deprecated.py b/tools/find_deprecated.py
new file mode 100755
index 0000000..43cd2a3
--- /dev/null
+++ b/tools/find_deprecated.py
@@ -0,0 +1,226 @@
+#!/usr/bin/env python
+#
+## Natural Language Toolkit: Deprecated Function & Class Finder
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+from __future__ import print_function
+
+"""
+This command-line tool takes a list of python files or directories,
+and searches them for calls to deprecated NLTK functions, or uses of
+deprecated NLTK classes. For each use of a deprecated object it
+finds, it will print out a warning containing the offending line, as
+well as its line number and containing file name. If the terminal has
+color support (and if epydoc is installed), then the offending
+identifier will be highlighted in red.
+"""
+
+######################################################################
+# Imports
+######################################################################
+
+import os, re, sys, tokenize, textwrap
+import nltk, nltk.corpus
+from doctest import DocTestParser, register_optionflag
+from cStringIO import StringIO
+from nltk import defaultdict
+
+######################################################################
+# Regexps
+######################################################################
+
+#: A little over-simplified, but it'll do.
+STRING_PAT = (r'\s*[ur]{0,2}(?:'
+ '"""[\s\S]*?"""|' '"[^"\n]+?"|'
+ "'''[\s\S]*?'''|" "'[^'\n]+?'" ")\s*")
+STRING_RE = re.compile(STRING_PAT)
+
+STRINGS_PAT = '%s(?:[+]?%s)*' % (STRING_PAT, STRING_PAT)
+STRINGS_RE = re.compile(STRINGS_PAT)
+
+# Define a regexp to search for deprecated definitions.
+DEPRECATED_DEF_PAT = (
+ r'^\s*@deprecated\s*\(\s*(%s)\s*\)\s*\n+' % STRINGS_PAT +
+ r'\s*def\s*(\w+).*' +
+ r'|' +
+ r'^\s*class\s+(\w+)\s*\(.*Deprecated.*\):\s*')
+DEPRECATED_DEF_RE = re.compile(DEPRECATED_DEF_PAT, re.MULTILINE)
+
+CORPUS_READ_METHOD_RE = re.compile(
+ '(%s)\.read\(' % ('|'.join(re.escape(n) for n in dir(nltk.corpus))))
+
+CLASS_DEF_RE = re.compile('^\s*class\s+(\w+)\s*[:\(]')
+
+######################################################################
+# Globals
+######################################################################
+# Yes, it's bad programming practice, but this is a little hack
+# script. :) These get initialized by find_deprecated_defs.
+
+deprecated_funcs = defaultdict(set)
+deprecated_classes = defaultdict(set)
+deprecated_methods = defaultdict(set)
+
+try:
+ from epydoc.cli import TerminalController
+except ImportError:
+ class TerminalController:
+ def __getattr__(self, attr): return ''
+
+term = TerminalController()
+
+######################################################################
+# Code
+######################################################################
+
+# If we're using py24, then ignore the +SKIP directive.
+if sys.version_info[:2] < (2,5): register_optionflag('SKIP')
+
+def strip_quotes(s):
+ s = s.strip()
+ while (s and (s[0] in "ur") and (s[-1] in "'\"")):
+ s = s[1:]
+ while (s and (s[0] in "'\"" and (s[0] == s[-1]))):
+ s = s[1:-1]
+ s = s.strip()
+ return s
+
+def find_class(s, index):
+ lines = s[:index].split('\n')
+ while lines:
+ m = CLASS_DEF_RE.match(lines[-1])
+ if m: return m.group(1)+'.'
+ lines.pop()
+ return '?.'
+
+def find_deprecated_defs(pkg_dir):
+ """
+ Return a list of all functions marked with the @deprecated
+ decorator, and classes with an immediate Deprecated base class, in
+ all Python files in the given directory.
+ """
+ # Walk through the directory, finding python files.
+ for root, dirs, files in os.walk(pkg_dir):
+ for filename in files:
+ if filename.endswith('.py'):
+ # Search the file for any deprecated definitions.
+ s = open(os.path.join(root, filename)).read()
+ for m in DEPRECATED_DEF_RE.finditer(s):
+ if m.group(2):
+ name = m.group(2)
+ msg = ' '.join(strip_quotes(s) for s in
+ STRING_RE.findall(m.group(1)))
+ msg = ' '.join(msg.split())
+ if m.group()[0] in ' \t':
+ cls = find_class(s, m.start())
+ deprecated_methods[name].add( (msg, cls, '()') )
+ else:
+ deprecated_funcs[name].add( (msg, '', '()') )
+ else:
+ name = m.group(3)
+ m2 = STRING_RE.match(s, m.end())
+ if m2: msg = strip_quotes(m2.group())
+ else: msg = ''
+ msg = ' '.join(msg.split())
+ deprecated_classes[name].add( (msg, '', ''))
+
+def print_deprecated_uses(paths):
+ dep_names = set()
+ dep_files = set()
+ for path in sorted(paths):
+ if os.path.isdir(path):
+ dep_names.update(print_deprecated_uses(
+ [os.path.join(path,f) for f in os.listdir(path)]))
+ elif path.endswith('.py'):
+ print_deprecated_uses_in(open(path).readline, path,
+ dep_files, dep_names, 0)
+ elif path.endswith('.doctest') or path.endswith('.txt'):
+ for example in DocTestParser().get_examples(open(path).read()):
+ ex = StringIO(example.source)
+ try:
+ print_deprecated_uses_in(ex.readline, path, dep_files,
+ dep_names, example.lineno)
+ except tokenize.TokenError:
+ print(term.RED + 'Caught TokenError -- '
+ 'malformatted doctest?' + term.NORMAL)
+ return dep_names
+
+def print_deprecated_uses_in(readline, path, dep_files, dep_names,
+ lineno_offset):
+ tokiter = tokenize.generate_tokens(readline)
+ context = ['']
+ for (typ, tok, start, end, line) in tokiter:
+ # Remember the previous line -- it might contain
+ # the @deprecated decorator.
+ if line is not context[-1]:
+ context.append(line)
+ if len(context) > 10: del context[0]
+ esctok = re.escape(tok)
+ # Ignore all tokens except deprecated names.
+ if not (tok in deprecated_classes or
+ (tok in deprecated_funcs and
+ re.search(r'\b%s\s*\(' % esctok, line)) or
+ (tok in deprecated_methods and
+ re.search(r'(?!<\bself)[.]\s*%s\s*\(' % esctok, line))):
+ continue
+ # Hack: only complain about read if it's used after a corpus.
+ if tok == 'read' and not CORPUS_READ_METHOD_RE.search(line):
+ continue
+ # Ignore deprecated definitions:
+ if DEPRECATED_DEF_RE.search(''.join(context)):
+ continue
+ # Print a header for the first use in a file:
+ if path not in dep_files:
+ print('\n'+term.BOLD + path + term.NORMAL)
+ print(' %slinenum%s' % (term.YELLOW, term.NORMAL))
+ dep_files.add(path)
+ # Mark the offending token.
+ dep_names.add(tok)
+ if term.RED: sub = term.RED+tok+term.NORMAL
+ elif term.BOLD: sub = term.BOLD+tok+term.NORMAL
+ else: sub = '<<'+tok+'>>'
+ line = re.sub(r'\b%s\b' % esctok, sub, line)
+ # Print the offending line.
+ print(' %s[%5d]%s %s' % (term.YELLOW, start[0]+lineno_offset,
+ term.NORMAL, line.rstrip()))
+
+
+def main():
+ paths = sys.argv[1:] or ['.']
+
+ print('Importing nltk...')
+ try:
+ import nltk
+ except ImportError:
+ print('Unable to import nltk -- check your PYTHONPATH.')
+ sys.exit(-1)
+
+ print('Finding definitions of deprecated funtions & classes in nltk...')
+ find_deprecated_defs(nltk.__path__[0])
+
+ print('Looking for possible uses of deprecated funcs & classes...')
+ dep_names = print_deprecated_uses(paths)
+
+ if not dep_names:
+ print('No deprecated funcs or classes found!')
+ else:
+ print("\n"+term.BOLD+"What you should use instead:"+term.NORMAL)
+ for name in sorted(dep_names):
+ msgs = deprecated_funcs[name].union(
+ deprecated_classes[name]).union(
+ deprecated_methods[name])
+ for msg, prefix, suffix in msgs:
+ print(textwrap.fill(term.RED+prefix+name+suffix+
+ term.NORMAL+': '+msg,
+ width=75, initial_indent=' '*2,
+ subsequent_indent=' '*6))
+
+
+
+if __name__ == '__main__':
+ main()
+
diff --git a/tools/global_replace.py b/tools/global_replace.py
new file mode 100755
index 0000000..ff60782
--- /dev/null
+++ b/tools/global_replace.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+#
+## Natural Language Toolkit: substitute a pattern with a replacement in every file
+#
+# Copyright (C) 2001-2014 NLTK Project
+# Author: Edward Loper <edloper at gmail.com>
+# Steven Bird <stevenbird1 at gmail.com>
+# URL: <http://nltk.org/>
+# For license information, see LICENSE.TXT
+
+# NB Should work on all platforms, http://www.python.org/doc/2.5.2/lib/os-file-dir.html
+
+import os, stat, sys
+
+def update(file, pattern, replacement, verbose=False):
+ if verbose:
+ print("Updating:", file)
+
+ # make sure we can write the file
+ old_perm = os.stat(file)[0]
+ if not os.access(file, os.W_OK):
+ os.chmod(file, old_perm | stat.S_IWRITE)
+
+ # write the file
+ s = open(file, 'rb').read()
+ t = s.replace(pattern, replacement)
+ out = open(file, 'wb')
+ out.write(t)
+ out.close()
+
+ # restore permissions
+ os.chmod(file, old_perm)
+
+ return s != t
+
+if __name__ == '__main__':
+
+ if len(sys.argv) != 3:
+ exit("Usage: %s <pattern> <replacement>" % sys.argv[0])
+
+ pattern = sys.argv[1]
+ replacement = sys.argv[2]
+ count = 0
+
+ for root, dirs, files in os.walk('.'):
+ if not ('/.git' in root or '/.tox' in root):
+ for file in files:
+ path = os.path.join(root, file)
+ if update(path, pattern, replacement):
+ print("Updated:", path)
+ count += 1
+
+ print("Updated %d files" % count)
diff --git a/tools/nltk_term_index.py b/tools/nltk_term_index.py
new file mode 100755
index 0000000..6255c3b
--- /dev/null
+++ b/tools/nltk_term_index.py
@@ -0,0 +1,102 @@
+from __future__ import print_function
+
+import re, sys
+import nltk
+import epydoc.docbuilder, epydoc.cli
+from epydoc import log
+
+STOPLIST = '../../tools/nltk_term_index.stoplist'
+FILENAMES = ['ch%02d.xml' % n for n in range(13)]
+TARGET_DIR = 'nlp/'
+#FILENAMES = ['../doc/book/ll.xml']
+
+logger = epydoc.cli.ConsoleLogger(0)
+logger._verbosity = 5
+log.register_logger(logger)
+
+def find_all_names(stoplist):
+ ROOT = ['nltk']
+ logger._verbosity = 0
+ docindex = epydoc.docbuilder.build_doc_index(ROOT, add_submodules=True)
+ valdocs = sorted(docindex.reachable_valdocs(
+ imports=False,
+ #packages=False, bases=False, submodules=False,
+ #subclasses=False,
+ private=False))
+ logger._verbosity = 5
+ names = nltk.defaultdict(list)
+ n = 0
+ for valdoc in valdocs:
+ name = valdoc.canonical_name
+ if (name is not epydoc.apidoc.UNKNOWN and
+ name is not None and name[0] == 'nltk'):
+ n += 1
+ for i in range(len(name)):
+ key = str(name[i:])
+ if len(key) == 1: continue
+ if key in stoplist: continue
+ names[key].append(valdoc)
+
+ log.info('Found %s names from %s objects' % (len(names), n))
+
+ return names
+
+SCAN_RE1 = "<programlisting>[\s\S]*?</programlisting>"
+SCAN_RE2 = "<literal>[\s\S]*?</literal>"
+SCAN_RE = re.compile("(%s)|(%s)" % (SCAN_RE1, SCAN_RE2))
+
+TOKEN_RE = re.compile('[\w\.]+')
+
+LINE_RE = re.compile('.*')
+
+INDEXTERM = '<indexterm type="nltk"><primary>%s</primary></indexterm>'
+
+def scan_xml(filenames, names):
+ fdist = nltk.FreqDist()
+
+ def linesub(match):
+ line = match.group()
+ for token in TOKEN_RE.findall(line):
+ if token in names:
+ targets = names[token]
+ fdist.inc(token)
+ if len(targets) > 1:
+ log.warning('%s is ambiguous: %s' % (
+ token, ', '.join(str(v.canonical_name)
+ for v in names[token])))
+ line += INDEXTERM % token
+ #line += INDEXTERM % names[token][0].canonical_name
+ return line
+
+ def scansub(match):
+ return LINE_RE.sub(linesub, match.group())
+
+ for filename in filenames:
+ log.info(' %s' % filename)
+ src = open(filename, 'rb').read()
+ src = SCAN_RE.sub(scansub, src)
+# out = open(filename[:-4]+'.li.xml', 'wb')
+ out = open(TARGET_DIR + filename, 'wb')
+ out.write(src)
+ out.close()
+
+ for word in fdist:
+ namestr = ('\n'+38*' ').join([str(v.canonical_name[:-1])
+ for v in names[word][:1]])
+ print('[%3d] %-30s %s' % (fdist[word], word, namestr))
+ sys.stdout.flush()
+
+
+def main():
+ log.info('Loading stoplist...')
+ stoplist = open(STOPLIST).read().split()
+ log.info(' Stoplist contains %d words' % len(stoplist))
+
+ log.info('Running epydoc to build a name index...')
+ names = find_all_names(stoplist)
+
+ log.info('Scanning xml files...')
+ scan_xml(FILENAMES, names)
+
+main()
+
diff --git a/tools/nltk_term_index.stoplist b/tools/nltk_term_index.stoplist
new file mode 100644
index 0000000..c17d653
--- /dev/null
+++ b/tools/nltk_term_index.stoplist
@@ -0,0 +1,106 @@
+__init__
+Comment
+Plot
+about
+add
+all
+analysis
+args
+book
+bubble
+categories
+close
+concatenate
+contains
+copy
+coverage
+defaultdict
+demo
+describe
+dict
+discourse
+doctype
+documents
+dump
+end
+ends
+fileids
+files
+find
+first
+free
+goal
+groups
+help
+incorrect
+insert
+instances
+items
+join
+key
+labels
+lhs
+line
+lines
+list
+lookup
+matches
+max
+means
+min
+missed
+name
+next
+nltk
+nltk.book
+open
+pairs
+play
+plot
+pop
+pos
+pp
+pprint
+prev
+process
+purge
+put
+quick
+raw
+read
+reader
+readings
+readme
+repr
+rhs
+root
+run
+second
+see
+select
+sentences
+sents
+set
+simple
+size
+sorted
+span
+start
+step
+stop
+str
+table
+test
+text
+texts
+trace
+type
+update
+verbs
+view
+vocab
+walk
+wav
+width
+words
+write
diff --git a/tools/run_doctests.py b/tools/run_doctests.py
new file mode 100755
index 0000000..b47ad69
--- /dev/null
+++ b/tools/run_doctests.py
@@ -0,0 +1,16 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+run doctests
+"""
+
+from __future__ import print_function
+import sys, subprocess, os
+
+for root, dirs, filenames in os.walk('.'):
+ for filename in filenames:
+ if filename.endswith('.py'):
+ path = os.path.join(root, filename)
+ for pyver in ["python2.6", "python2.7", "python3.2"]:
+ print(pyver, filename, file=sys.stderr)
+ subprocess.call([pyver, "-m", "doctest", path])
diff --git a/tools/svnmime.py b/tools/svnmime.py
new file mode 100755
index 0000000..ea7133b
--- /dev/null
+++ b/tools/svnmime.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+# NB, this wouldn't be needed if everyone had .subversion/config
+# configured to automatically set mime types
+# http://code.google.com/p/support/wiki/FAQ
+
+from __future__ import print_function
+
+import os
+import sys
+
+types_map = {
+ 'ai': 'application/postscript',
+ 'coverage': 'text/plain',
+ 'css': 'text/css',
+ 'eps': 'application/postscript',
+ 'exe': 'application/octet-stream',
+ 'errs': 'text/plain',
+ 'gif': 'image/gif',
+ 'htm': 'text/html',
+ 'html': 'text/html',
+ 'jpeg': 'image/jpeg',
+ 'jpg': 'image/jpeg',
+ 'js': 'application/x-javascript',
+ 'pbm': 'image/x-portable-bitmap',
+ 'pdf': 'application/pdf',
+ 'pgm': 'image/x-portable-graymap',
+ 'pnm': 'image/x-portable-anymap',
+ 'png': 'image/png',
+ 'ppm': 'image/x-portable-pixmap',
+ 'py': 'text/x-python',
+ 'ps': 'application/postscript',
+ 'rst': 'text/plain',
+ 'tex': 'application/x-tex',
+ 'txt': 'text/plain',
+ 'xml': 'text/xml',
+ 'xsl': 'text/plain',
+ 'zip': 'application/zip',
+ }
+
+def usage():
+ exit("Usage: svnmime files")
+
+for file in sys.argv[1:]:
+ if "." in file:
+ extension = file.rsplit('.', 1)[1]
+ if extension in types_map:
+ os.system("svn propset svn:mime-type %s %s" % (types_map[extension], file))
+ else:
+ print("Unrecognized extension", extension)
diff --git a/tox.ini b/tox.ini
new file mode 100644
index 0000000..c4a87b0
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,101 @@
+[tox]
+envlist = py26,py27,py32,py33,pypy,py26-nodeps,py27-nodeps,py32-nodeps,py33-nodeps,py26-jenkins,py32-jenkins
+
+[testenv]
+
+; simplify numpy installation
+setenv =
+ LAPACK=
+ ATLAS=None
+
+deps =
+ numpy
+ nose >= 1.2.1
+ coverage
+ text-unidecode
+
+changedir = nltk/test
+commands =
+ ; scipy and scikit-learn requires numpy even to run setup.py so
+ ; they can't be installed in one command
+
+ pip install --download-cache={toxworkdir}/_download scipy scikit-learn
+ ; python runtests.py --with-coverage --cover-inclusive --cover-package=nltk --cover-html --cover-html-dir={envdir}/docs []
+ python runtests.py []
+
+[testenv:pypy]
+; numpy is bundled with pypy; coverage is extra slow and
+; the coverage results are not that different from CPython.
+deps =
+ nose >= 1.2.1
+
+commands =
+ python runtests.py []
+
+[testenv:py32]
+deps =
+ numpy
+ nose >= 1.2.1
+ coverage
+
+commands =
+ ; scipy and scikit-learn requires numpy even to run setup.py so
+ ; they can't be installed in one command
+
+ ; scikit-learn installation fails so scipy & scikit-learn are temporary disabled
+ ; pip install --download-cache={toxworkdir}/_download scipy scikit-learn
+
+ ; python runtests.py --with-coverage --cover-inclusive --cover-package=nltk --cover-html --cover-html-dir={envdir}/docs []
+ python runtests.py []
+
+[testenv:py33]
+deps =
+ numpy
+ nose >= 1.2.1
+ coverage
+ text-unidecode
+
+commands =
+ ; scipy and scikit-learn requires numpy even to run setup.py so
+ ; they can't be installed in one command
+ pip install --download-cache={toxworkdir}/_download scipy scikit-learn
+
+ ; python runtests.py --with-coverage --cover-inclusive --cover-package=nltk --cover-html --cover-html-dir={envdir}/docs []
+ python runtests.py []
+
+
+[testenv:py26-nodeps]
+basepython = python2.6
+deps = nose >= 1.2.1
+commands = python runtests.py []
+
+[testenv:py27-nodeps]
+basepython = python2.7
+deps = nose >= 1.2.1
+commands = python runtests.py []
+
+[testenv:py32-nodeps]
+basepython = python3.2
+deps = nose >= 1.2.1
+commands = python runtests.py []
+
+[testenv:py33-nodeps]
+basepython = python3.3
+deps = nose >= 1.2.1
+commands = python runtests.py []
+
+[testenv:py26-jenkins]
+basepython = python2.6
+commands = {toxinidir}/jenkins.sh
+setenv =
+ STANFORD_MODELS = {homedir}/third/stanford-parser/
+ STANFORD_PARSER = {homedir}/third/stanford-parser/
+ STANFORD_POSTAGGER = {homedir}/third/stanford-postagger/
+
+[testenv:py32-jenkins]
+basepython = python3.2
+commands = {toxinidir}/jenkins.sh
+setenv =
+ STANFORD_MODELS = {homedir}/third/stanford-parser/
+ STANFORD_PARSER = {homedir}/third/stanford-parser/
+ STANFORD_POSTAGGER = {homedir}/third/stanford-postagger/
diff --git a/web/Makefile b/web/Makefile
new file mode 100644
index 0000000..972cb37
--- /dev/null
+++ b/web/Makefile
@@ -0,0 +1,161 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build
+PAPER =
+BUILDDIR = _build
+WEB = ../../nltk.github.com
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+web: clean_api
+ sphinx-apidoc -o api ../nltk
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(WEB)
+ @echo
+ @echo "Build finished. The HTML pages are in $(WEB)."
+
+without_api: clean_api
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(WEB)
+
+clean_api:
+ rm -f api/modules.rst api/nltk.*.rst
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " singlehtml to make a single large HTML file"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " devhelp to make HTML files and a Devhelp project"
+ @echo " epub to make an epub"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " latexpdf to make LaTeX files and run them through pdflatex"
+ @echo " text to make text files"
+ @echo " man to make manual pages"
+ @echo " texinfo to make Texinfo files"
+ @echo " info to make Texinfo files and run them through makeinfo"
+ @echo " gettext to make PO message catalogs"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+ -rm -rf $(BUILDDIR)/* # $(WEB)/*
+
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+ $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+ @echo
+ @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/NLTK.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/NLTK.qhc"
+
+devhelp:
+ $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+ @echo
+ @echo "Build finished."
+ @echo "To view the help file:"
+ @echo "# mkdir -p $$HOME/.local/share/devhelp/NLTK"
+ @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/NLTK"
+ @echo "# devhelp"
+
+epub:
+ $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+ @echo
+ @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+ @echo "Run \`make' in that directory to run these through (pdf)latex" \
+ "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through pdflatex..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+ $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+ @echo
+ @echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+ $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+ @echo
+ @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo
+ @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+ @echo "Run \`make' in that directory to run these through makeinfo" \
+ "(use \`make info' here to do that automatically)."
+
+info:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo "Running Texinfo files through makeinfo..."
+ make -C $(BUILDDIR)/texinfo info
+ @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+ $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+ @echo
+ @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+ @echo
+ @echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+ $(SPHINXBUILD) -b nltk-doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in $(BUILDDIR)/doctest/output.txt."
diff --git a/web/api/nltk.rst b/web/api/nltk.rst
new file mode 100644
index 0000000..9c485f7
--- /dev/null
+++ b/web/api/nltk.rst
@@ -0,0 +1,142 @@
+.. manually constructed -- removed several low-level packages
+
+nltk Package
+============
+
+:mod:`nltk` Package
+-------------------
+
+.. automodule:: nltk.__init__
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`align` Module
+-------------------
+
+.. automodule:: nltk.align
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`collocations` Module
+--------------------------
+
+.. automodule:: nltk.collocations
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`data` Module
+------------------
+
+.. automodule:: nltk.data
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`downloader` Module
+------------------------
+
+.. automodule:: nltk.downloader
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`featstruct` Module
+------------------------
+
+.. automodule:: nltk.featstruct
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`grammar` Module
+---------------------
+
+.. automodule:: nltk.grammar
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`help` Module
+------------------
+
+.. automodule:: nltk.help
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`probability` Module
+-------------------------
+
+.. automodule:: nltk.probability
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`text` Module
+------------------
+
+.. automodule:: nltk.text
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`toolbox` Module
+---------------------
+
+.. automodule:: nltk.toolbox
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`tree` Module
+------------------
+
+.. automodule:: nltk.tree
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`treetransforms` Module
+----------------------------
+
+.. automodule:: nltk.treetransforms
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+:mod:`util` Module
+------------------
+
+.. automodule:: nltk.util
+ :members:
+ :undoc-members:
+ :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+
+ nltk.app
+ nltk.ccg
+ nltk.chat
+ nltk.chunk
+ nltk.classify
+ nltk.cluster
+ nltk.corpus
+ nltk.draw
+ nltk.examples
+ nltk.inference
+ nltk.metrics
+ nltk.misc
+ nltk.model
+ nltk.parse
+ nltk.sem
+ nltk.stem
+ nltk.tag
+ nltk.test
+ nltk.tokenize
+
diff --git a/web/conf.py b/web/conf.py
new file mode 100644
index 0000000..f0967f0
--- /dev/null
+++ b/web/conf.py
@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+#
+# NLTK documentation build configuration file, created by
+# sphinx-quickstart on Wed Nov 2 17:02:59 2011.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+# build docs using nltk from the upper dir, not the installed version
+sys.path.insert(0, os.path.abspath('..'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.pngmath',
+ 'sphinx.ext.viewcode',]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'NLTK'
+copyright = '2013, NLTK Project'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+version = '3.0'
+# The full version, including alpha/beta/rc tags.
+release = '3.0'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = ['_build', 'api/modules.rst', 'dev/*.rst']
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+modindex_common_prefix = ['nltk.']
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = 'agogo'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {
+ 'textalign': 'left'
+}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+#html_short_title = None
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+#html_show_sourcelink = True
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'NLTKdoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+#'papersize': 'letterpaper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+ ('index', 'NLTK.tex', 'NLTK Documentation',
+ 'Steven Bird', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ ('index', 'nltk', 'NLTK Documentation',
+ ['Steven Bird'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ ('index', 'NLTK', 'NLTK Documentation', 'Steven Bird',
+ 'NLTK', 'One line description of project.', 'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
diff --git a/web/contribute.rst b/web/contribute.rst
new file mode 100644
index 0000000..43d45a8
--- /dev/null
+++ b/web/contribute.rst
@@ -0,0 +1,37 @@
+Contribute to NLTK
+==================
+
+The Natural Language Toolkit exists thanks to the efforts of dozens
+of voluntary developers who have contributed functionality and
+bugfixes since the project began in 2000 (`contributors <https://github.com/nltk/nltk#contributing>`_).
+
+In 2014 we are especially keen to improve NLTK coverage for:
+`dependency parsing <https://github.com/nltk/nltk/wiki/Dependency-Parsing>`_,
+`machine translation <https://github.com/nltk/nltk/wiki/Machine-Translation>`_,
+`sentiment analysis <https://github.com/nltk/nltk/wiki/Sentiment-Analysis>`_,
+`twitter processing <https://github.com/nltk/nltk/wiki/Twitter-Processing>`_.
+
+New material in these areas will be covered in the second edition of
+the NLTK book, anticipated in 2015.
+
+* `desired enhancements <https://github.com/nltk/nltk/issues?labels=enhancement&page=1&state=open>`_
+* `contribute a corpus <https://github.com/nltk/nltk/wiki/Adding-a-Corpus>`_
+* `nltk-dev mailing list <http://groups.google.com/group/nltk-dev>`_
+* `GitHub Project <https://github.com/nltk/nltk>`_
+
+NLTK Core Developers
+--------------------
+
+The NLTK project is led by `Steven Bird, Ewan Klein, and Edward Loper <mailto:stevenbird1 at gmail.com,ewan.klein at gmail.com,edloper at gmail.com>`_.
+Individual packages are maintained by the following people:
+
+:Semantics: `Dan Garrette <http://www.cs.utexas.edu/~dhg/>`_, Austin, USA (``nltk.sem, nltk.inference``)
+:Parsing: `Peter Ljunglöf <http://www.cse.chalmers.se/~peb/>`_, Gothenburg, Sweden (``nltk.parse, nltk.featstruct``)
+:Metrics: `Joel Nothman <http://joelnothman.com/>`_, Sydney, Australia (``nltk.metrics, nltk.tokenize.punkt``)
+:Python 3: `Mikhail Korobov <http://kmike.ru/>`_, Ekaterinburg, Russia
+:Integration: `Morten Minde Neergaard <http://8d.no/>`_, Oslo, Norway
+:Releases: `Steven Bird <http://estive.net>`_, Melbourne, Australia
+
+
+
+
diff --git a/web/data.rst b/web/data.rst
new file mode 100644
index 0000000..5b1d0ff
--- /dev/null
+++ b/web/data.rst
@@ -0,0 +1,49 @@
+Installing NLTK Data
+====================
+
+NLTK comes with many corpora, toy grammars, trained models, etc. A complete list is posted at: http://nltk.org/nltk_data/
+
+To install the data, first install NLTK (see http://nltk.org/install.html), then use NLTK's data downloader as described below.
+
+Apart from individual data packages, you can download the entire collection (using "all"), or just the data required for the examples and exercises in the book (using "book"), or just the corpora and no grammars or trained models (using "all-corpora").
+
+Interactive installer
+---------------------
+
+*For central installation on a multi-user machine, do the following from an administrator account.*
+
+Run the Python interpreter and type the commands:
+
+ >>> import nltk
+ >>> nltk.download()
+
+A new window should open, showing the NLTK Downloader. Click on the File menu and select Change Download Directory. For central installation, set this to ``C:\nltk_data`` (Windows), or ``/usr/share/nltk_data`` (Mac, Unix). Next, select the packages or collections you want to download.
+
+If you did not install the data to one of the above central locations, you will need to set the ``NLTK_DATA`` environment variable to specify the location of the data. (On a Windows machine, right click on "My Computer" then select ``Properties > Advanced > Environment Variables > User Variables > New...``)
+
+Test that the data has been installed as follows. (This assumes you downloaded the Brown Corpus):
+
+ >>> from nltk.corpus import brown
+ >>> brown.words()
+ ['The', 'Fulton', 'County', 'Grand', 'Jury', 'said', ...]
+
+Installing via a proxy web server
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If your web connection uses a proxy server, you should specify the proxy address as follows. In the case of an authenticating proxy, specify a username and password. If the proxy is set to None then this function will attempt to detect the system proxy.
+
+ >>> nltk.set_proxy('http://proxy.example.com:3128', ('USERNAME', 'PASSWORD'))
+ >>> nltk.download()
+
+Command line installation
+-------------------------
+
+The downloader will search for an existing ``nltk_data`` directory to install NLTK data. If one does not exist it will attempt to create one in a central location (when using an administrator account) or otherwise in the user's filespace. If necessary, run the download command from an administrator account, or using sudo. The default system location on Windows is ``C:\nltk_data``; and on Mac and Unix is ``/usr/share/nltk_data``. You can use the ``-d`` flag to specify a different loca [...]
+
+Python 2.5-2.7: Run the command ``python -m nltk.downloader all``. To ensure central installation, run the command ``sudo python -m nltk.downloader -d /usr/share/nltk_data all``.
+
+Windows: Use the "Run..." option on the Start menu. Windows Vista users need to first turn on this option, using ``Start -> Properties -> Customize`` to check the box to activate the "Run..." option.
+
+Test the installation: Check that the user environment and privileges are set correctly by logging in to a user account,
+starting the Python interpreter, and accessing the Brown Corpus (see the previous section).
+
diff --git a/web/dev/jenkins.rst b/web/dev/jenkins.rst
new file mode 100644
index 0000000..e5b3025
--- /dev/null
+++ b/web/dev/jenkins.rst
@@ -0,0 +1,113 @@
+NLTK c-i setup
+==============
+
+This is an overview of how our `continuous integration`_ setup works. It
+includes a quick introduction to the tasks it runs, and the later sections
+detail the process of setting up these tasks.
+
+Our continuous integration is currently hosted at `Shining Panda`_, free thanks
+to their FLOSS program. The setup is not specific to their solutions, it could
+be moved to any `Jenkins`_ instance. The URL of our current instance is
+https://jenkins.shiningpanda.com/nltk/
+
+.. _`continuous integration`: http://en.wikipedia.org/wiki/Continuous_integration
+.. _`Shining Panda`: http://shiningpanda.com
+.. _`Jenkins`: http://jenkins-ci.org
+
+
+Base tasks
+----------
+
+The base tasks of the c-i instance is as follows:
+
+* Check out the NLTK project when VCS changes occur
+* Build the project using setup.py
+* Run our test suite
+* Make packages for all platforms
+* Build these web pages
+
+Because the NLTK build environment is highly customized, we only run tests on
+one configuration - the lowest version supported. NLTK 2 supports python down
+to version 2.5, so all tests are run using a python2.5 virtualenv. The
+virtualenv configuration is slightly simplified on ShiningPanda machines by
+their having compiled all relevant python versions and making virtualenv use
+these versions in their custom virtualenv builders.
+
+
+VCS setup/integration
+---------------------
+
+All operations are done against the `NLTK repos on Github`_. The Jenkins
+instance on ShiningPanda has a limit to the build time it can use each day.
+Because of this, it only polls the main NLTK repo once a day, using the `Poll
+SCM` option in Jenkins. Against the main code repo it uses public access only,
+and for pushing to the nltk.github.com repo it uses the key of the user
+nltk-webdeploy.
+
+.. _`NLTK repos on Github`: https://github.com/nltk/
+
+
+The base build
+--------------
+
+To build the project, the following tasks are run:
+
+1. Create a VERSION file
+ A VERSION file is created using
+ ``git describe --tags --match '*.*.*' > nltk/VERSION``.
+ This makes the most recent VCS tag available in nltk.__version__ etc.
+2. ``python setup.py build``
+ This essentially copies the files that are required to run NLTK into build/
+
+
+The test suite
+--------------
+
+The tests require that all dependencies be installed. These have all been
+installed beforehand, and to make them run a series of extra environment
+variables are initialized. These dependencies will not be detailed until the
+last section.
+
+The test suite itself consists of doctests. These are found in each module as
+docstrings, and in all the .doctest files under the test folder in the nltk
+repo. We run these tests using nose_, find code coverage using `coverage.py`_
+and check for `PEP-8`_ etc. standard violations using `pylint`_.
+
+All these tools are easily installable through pip your favourite OS' software
+packaging system. For testing, only nose_ is really needed. This is also the
+only software that does not work properly out of the box. To use the options
++ELLIPSIS and +NORMALIZE_WHITESPACE in our doctests, we have installed nose
+from source with `a patch that allows this`_ applied.
+
+The results of these programs are parsed and published by the jenkins instance,
+giving us pretty graphs :)
+
+.. _nose: http://readthedocs.org/docs/nose/
+.. _`coverage.py`: http://nedbatchelder.com/code/coverage/
+.. _`PEP-8`: http://www.python.org/dev/peps/pep-0008/
+.. _`pylint`: http://www.logilab.org/project/pylint
+.. _`a patch that allows this`: https://github.com/nose-devs/nose/issues/7
+
+
+The builds
+----------
+
+The packages are built using ``make dist``. The outputted builds are all placed
+`in our jenkins workspace`_ and should be safe to distribute. Builds
+specifically for mac are not available. File names are made based on the
+``__version__`` string, so they change every build.
+
+.. _`in our jenkins workspace`: http://example.com/
+
+
+Web page builder
+----------------
+
+The web page is built using Sphinx_. It fetches all code documentation directly
+from the code's docstrings. After building the page using ``make web`` it
+pushes it to the `nltk.github.com repo on github`_. To push it, it needs access
+to the repo – because this cannot be done using a deploy key, it has the ssh
+key of the ``nltk-webdeploy`` user.
+
+.. _Sphinx: http://sphinx.pocoo.org
+.. _`nltk.github.com repo on github`: https://github.com/nltk/nltk.github.com
diff --git a/web/dev/local_testing.rst b/web/dev/local_testing.rst
new file mode 100644
index 0000000..3088679
--- /dev/null
+++ b/web/dev/local_testing.rst
@@ -0,0 +1,167 @@
+NLTK testing
+============
+
+1. Obtain nltk source code;
+2. install virtualenv and tox::
+
+ pip install virtualenv
+ pip install tox
+
+3. make sure python2.6, python2.7, python3.2, python3.3
+ and pypy executables are in system PATH. It is OK not to have all the
+ executables, tests will be executed for available interpreters.
+
+4. Make sure all NLTK data is downloaded (see ``nltk.download()``);
+
+5. run 'tox' command from the root nltk folder. It will install dependencies
+ and run ``nltk/test/runtests.py`` script for all available interpreters.
+ You may pass any options to runtests.py script separating them by '--'.
+
+It may take a long time at first run, but the subsequent runs will
+be much faster.
+
+Please consult http://tox.testrun.org/ for more info about the tox tool.
+
+Examples
+--------
+
+Run tests for python 2.7 in verbose mode; executing only tests
+that failed in the last test run::
+
+ tox -e py27 -- -v --failed
+
+
+Run tree doctests for all available interpreters::
+
+ tox -- tree.doctest
+
+Run a selected unit test for the Python 3.2::
+
+ tox -e py32 -- -v nltk.test.unit.test_seekable_unicode_stream_reader
+
+By default, numpy, scipy and scikit-learn are installed in tox virtualenvs.
+This is slow, requires working build toolchain and is not always feasible.
+In order to skip numpy & friends, use ``..-nodeps`` environments::
+
+ tox -e py26-nodeps,py27-nodeps,py32-nodeps,py33-nodeps,pypy
+
+It is also possible to run tests without tox. This way NLTK would be tested
+only under single interpreter, but it may be easier to have numpy and other
+libraries installed this way. In order to run tests without tox, make sure
+``nose >= 1.2.1`` is installed and execute runtests.py script::
+
+ nltk/test/runtests.py
+
+
+Writing tests
+-------------
+
+Unlike most open-source projects, NLTK test suite is doctest-based.
+This format is very expressive, and doctests are usually read
+as documentation. We don't want to rewrite them to unittests;
+if you're contributing code to NLTK please prefer doctests
+for testing.
+
+Doctests are located at ``nltk/test/*.doctest`` text files and
+in docstrings for modules, classes, methods and functions.
+
+That said, doctests have their limitations and sometimes it is better to use
+unittests. Test should be written as unittest if some of the following apply:
+
+* test deals with non-ascii unicode and Python 2.x support is required;
+* test is a regression test that is not necessary for documentational purposes.
+
+Unittests currently reside in ``nltk/test/unit/test_*.py`` files; nose
+is used for test running.
+
+If a test should be written as unittest but also has a documentational value
+then it should be duplicated as doctest, but with a "# doctest: +SKIP" option.
+
+There are some gotchas with NLTK doctests (and with doctests in general):
+
+* Use ``print("foo")``, not ``print "foo"``: NLTK doctests act
+ like ``from __future__ import print_functions`` is in use.
+
+* Don't write ``+ELLIPSIS``, ``+NORMALIZE_WHITESPACE``,
+ ``+IGNORE_EXCEPTION_DETAIL`` flags (they are already ON by default in NLTK).
+
+* Do not write doctests that has non-ascii output (they are not supported in
+ Python 2.x). Incorrect::
+
+ >>> greeting
+ u'Привет'
+
+ The proper way is to rewrite such doctest as unittest.
+
+* For better Python 2.x - 3.x compatibility, for NLTK the following
+ tests are the same::
+
+ >>> x
+ [u'foo', u'bar']
+
+ >>> x
+ ['foo', 'bar']
+
+ Feel free to write or omit 'u' letters in output unicode constants.
+
+* In order to conditionally skip a doctest in a separate
+ ``nltk/test/foo.doctest`` file, create ``nltk.test/foo_fixt.py``
+ file from the following template::
+
+ # <a comment describing why should the test be skipped>
+
+ def setup_module(module):
+ from nose import SkipTest
+
+ if some_condition:
+ raise SkipTest("foo.doctest is skipped because <...>")
+
+* In order to conditionally skip all doctests from the module/class/function
+ docstrings, put the following function in a top-level module namespace::
+
+ # <a comment describing why should the tests from this module be skipped>
+
+ def setup_module(module):
+ from nose import SkipTest
+
+ if some_condition:
+ raise SkipTest("doctests from nltk.<foo>.<bar> are skipped because <...>")
+
+ A good idea is to define ``__all__`` in such module and omit
+ ``setup_module`` from ``__all__``.
+
+ It is not possible to conditionally skip only some doctests from a module.
+
+* Do not expect the exact float output; this may fail on some machines::
+
+ >>> some_float_constant
+ 0.867
+
+ Use ellipsis in this case to make the test robust (or compare the values)::
+
+ >>> some_float_constant
+ 0.867...
+
+ >>> abs(some_float_constant - 0.867) < 1e-6
+ True
+
+* Do not rely on dictionary or set item order. Incorrect::
+
+ >>> some_dict
+ {"x": 10, "y": 20}
+
+ The proper way is to sort the items and print them::
+
+ >>> for key, value in sorted(some_dict.items()):
+ ... print(key, value)
+ x 10
+ y 20
+
+If the code requires some external dependencies, then
+
+* tests for this code should be skipped if the dependencies are not available:
+ use ``setup_module`` for doctests (as described above) and
+ ``nltk.test.unit.utils.skip / skipIf`` decorators or ``nose.SkipTest``
+ exception for unittests;
+* if the dependency is a Python package, it should be added to tox.ini
+ (but not to ..-nodeps environments).
diff --git a/web/dev/python3porting.rst b/web/dev/python3porting.rst
new file mode 100644
index 0000000..3e1676a
--- /dev/null
+++ b/web/dev/python3porting.rst
@@ -0,0 +1,269 @@
+NLTK Python 3 support
+=====================
+
+The following text is not a general comprehensive Python 3.x porting guide;
+it provides some information about the approach using for NLTK Python 3 port.
+
+Porting Strategy
+----------------
+
+NLTK is being ported to Python 3 using single codebase strategy:
+NLTK should work from a single codebase in Python 2.x and 3.x.
+
+Python 2.5 compatibility is dropped in order to take advantage of
+new ``__future__`` imports, ``b`` bytestring marker, new
+``except Exception as e`` syntax and better standard library compatibility.
+
+General notes
+^^^^^^^^^^^^^
+
+There are good existing guides for writing Python 2.x - 3.x compatible
+code, e.g.
+
+* http://docs.python.org/dev/howto/pyporting.html
+* http://python3porting.com/
+* https://docs.djangoproject.com/en/dev/topics/python3/
+
+Take a look at them to have an idea how the approach works and what
+is changed in Python 3.
+
+nltk.compat
+^^^^^^^^^^^
+
+``nltk.compat`` module is loosely based on a great `six`_ library.
+It provides simple utilities for wrapping over differences
+between Python 2 and Python 3. Moved imports, removed/renamed builtins,
+type names differences and support functions are there.
+
+.. note::
+
+ We don't use `six`_ directly because it didn't work well
+ bundled at the time the port was started (this was a bug in six that
+ is fixed now), and NLTK needs extra custom 2+3 helpers anyway.
+
+.. _six: http://packages.python.org/six/
+
+
+map vs imap, items vs iteritems, ...
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A number of Python builtins and builtin methods returns lists under
+Python 2.x and iterators under Python 3.x. There are 3 possible ways
+to workaround this:
+
+1) use non-iterator versions of functions and methods under Python 3.x
+ (e.g. cast ``zip`` result to list);
+2) convert Python 2.x code to iterator versions (e.g. replace ``zip``
+ with ``itertools.izip`` when possible);
+3) let the code behave different under Python 2.x and Python 3.x.
+
+In this NLTK port (1) and (3) methods are used; (3) is preferred.
+This way there are no breaking interface changes for Python 2.x code
+and Python 3.x code remains idiomatic (it is surprising for a dict
+subclass ``items`` method to return a list under Python 3.x).
+
+Existing code that uses NLTK will have to be ported from
+Python 2.x to 3.x anyway so I think such interface changes are acceptable.
+
+Unicode support
+---------------
+
+Fixing corpora readers
+^^^^^^^^^^^^^^^^^^^^^^
+
+Previously, many corpora readers returned byte strings. In a Python 3.x
+branch the correct encodings are provided for all corpora and all corpora
+readers are now returning unicode.
+
+``__repr__`` and ``__str__``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Under Python 2.x ``obj.__repr__`` and ``obj.__str__`` must return
+byte strings, while under Python 3.x they must return unicode strings.
+
+To make things worse, terminals are tricky, and under Python 2.x
+there is no hassle-free encoding that ``obj.__repr__`` and ``obj.__str__``
+may use except for plain 7 bit ASCII.
+
+..
+
+ Should I link a blog post
+ (http://kmike.ru/python-with-strings-attached/) or extract
+ some text from it to make the statement about encodings more reasoned?
+
+In NLTK most classes with custom ``__repr__`` and/or ``__str__`` should use
+``nltk.compat.python_2_unicode_compatible`` decorator. It works this way:
+
+1) Class should be defined with ``__repr__`` and ``__str__`` methods
+ returning unicode (that's Python 3.x semantics);
+2) under Python 2.x the decorator fixes ``__repr__`` and ``__str__``
+ to return bytestrings;
+3) under both Python 2.x and 3.x the decorator creates
+ ``__unicode__`` method (which is an original ``__str__``)
+ and ``unicode_repr`` method (which is an original ``__repr__``).
+
+Under Python 2.x ``__repr__`` method returns an escaped version
+of the unicode value and ``__str__`` returns a transliterated version.
+For transliteration `Unidecode <http://pypi.python.org/pypi/Unidecode>`_,
+`text-unidecode <http://pypi.python.org/pypi/text-unidecode/0.1>`_
+or a basic "accent remover" may be used, depending on what
+packages are installed.
+
+In order to write unicode-aware ``__str__`` and ``__repr__``, the following
+approach may be used:
+
+1) ``from __future__ import unicode_literals`` is added to a top of file;
+2) ``str(something)`` should be replaced with ``"%s" % something``
+ when used (maybe indirectly) inside ``__str__`` or ``__repr__``;
+3) ``repr(something)`` and ``"%r" % something`` should be replaced with
+ ``unicode_repr(something)`` and ``"%s" % unicode_repr(something)``
+ when used (maybe indirectly) inside ``__str__`` or ``__repr__``.
+
+Doctests porting notes
+----------------------
+
+NLTK test suite is mostly doctest-based. Most usual rules apply to
+porting doctests code. But ther are some issues that make the
+process harder, so in order to make doctests work under
+Python 2.x and Python 3.x extra tricks are needed.
+
+``__future__`` imports
+^^^^^^^^^^^^^^^^^^^^^^
+
+Python's doctest runner doesn't support ``__future__`` imports.
+They are executed but has no effect in doctests' code.
+These imports are quite useful for making code Python 2.x + 3.x
+compatible so there are some methods to overcome the limitation.
+
+* ``from __future__ import print_function``: it may seem the import works
+ because ``print(foo)`` works under python 2.x; but it works only because
+ (foo) == foo; ``print(foo, bar)`` prints tuple; ``print(foo, sep=' ')``
+ raises an exception. In order to make print() work this future import
+ is injected to all doctests' globals within NLTK test suite
+ (implementation: ``nltk.test.doctest_nose_plugin.DoctestPluginHelper``).
+ So NLTK's doctests shouldn't import print_function but they should
+ assume this import is in effect.
+
+* ``from __future__ import unicode_literals``: there is no sane way to
+ use non-ascii constants in doctests under python 2.x
+ (see http://bugs.python.org/issue1293741 ); doctests with non-ascii
+ constants should be better rewritten as unittests or as doctests
+ without non-ascii constants.
+
+ Tests may use variables with unicode values though. In order to print
+ such values and have the same output under python 2 and python 3 the
+ following trick may be used::
+
+ >>> print(unicode_value.encode('unicode-escape').decode('ascii'))
+
+ But it may be a better idea to avoid this trick and rewrite the test to
+ unittest format instead.
+
+* ``from __future__ import division``: it is usually not hard to cast
+ results to int or float to have the same semantics under python 2 and 3.
+
+
+Unicode strings __repr__
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Representation of unicode strings is different in Python 2.x and Python 3.x
+even if they contain only ascii characters.
+
+Python 2.x::
+
+ >>> x = b'foo'.decode('ascii')
+ >>> x
+ u'foo'
+
+Python 3.x::
+
+ >>> x = b'foo'.decode('ascii')
+ >>> x
+ 'foo'
+
+(Note the missing 'u' in Python 3 example).
+
+In order to simplify things NLTK's custom doctest runner
+(see ``nltk.test.doctest_nose_plugin.DoctestPluginHelper``) doesn't
+take 'u''s into account; it considers u'foo' and 'foo' equal;
+developer is free to write u'foo' or 'foo'.
+
+This is not absolutely correct but if this distinction is important
+then doctest should be converted to unittest.
+
+There are other possible fixes for this issue but they
+all make doctests less readable. For example, for single variables
+``print`` may be used. Python 2.x::
+
+ >>> print(x)
+ foo
+
+Python 3.x::
+
+ >>> print(x)
+ foo
+
+This won't help with container types. Python 2.x::
+
+ >>> print([x, x])
+ [u'foo', u'foo']
+
+Possible fixes for lists are::
+
+ >>> for txt in [x, x]:
+ ... print(x)
+ foo
+ foo
+
+or::
+
+ >>> print(", ".join([x, x]))
+ foo, foo
+
+
+Float values representation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The exact representation of float values may vary across Python interpreters
+(this is not only a Python 3.x - specific issue). So instead of this::
+
+ >>> recall
+ 0.8888888888889
+
+write this::
+
+ >>> print(recall)
+ 0.88888888888...
+
+Porting tools
+-------------
+
+python-modernize
+^^^^^^^^^^^^^^^^
+
+`python-modernize <https://github.com/mitsuhiko/python-modernize>`_ script
+was used for tedious parts of python3 porting. Take a look at the docs for
+more information. The process was:
+
+* Run NLTK test suite under Python 2.x;
+* fix one specific aspect of NLTK by running one of python-modernize fixers
+ on NLTK source code;
+* take a look at changes python-modernize proposes, fix stupid things;
+* run NLTK test suite again under Python 2.x and make sure there are no
+ regressions.
+
+After python-modernize code wouldn't be necessary Python 3.x compatible but
+further porting would be easier and there shouldn't be 2.x regressions.
+
+2to3
+^^^^
+
+Doctest porting may be tedious, there is a lot of search/replace work
+(e.g. ``print foo`` -> ``print(foo)`` or
+``raise Exception, e`` -> ``raise Exception as e``). In order to overcome
+this 2to3 utility was used, e.g.::
+
+ $ 2to3 -d -f print nltk/test/*.doctest
+
+Fixers were applied one-by-one, test suite was executed before and after
+fixing.
diff --git a/web/images/book.gif b/web/images/book.gif
new file mode 100644
index 0000000..76ab487
Binary files /dev/null and b/web/images/book.gif differ
diff --git a/web/images/tree.gif b/web/images/tree.gif
new file mode 100644
index 0000000..d7d1c5f
Binary files /dev/null and b/web/images/tree.gif differ
diff --git a/web/index.rst b/web/index.rst
new file mode 100644
index 0000000..ac325c8
--- /dev/null
+++ b/web/index.rst
@@ -0,0 +1,88 @@
+Natural Language Toolkit
+========================
+
+NLTK is a leading platform for building Python programs to work with human language data.
+It provides easy-to-use interfaces to `over 50 corpora and lexical
+resources <http://nltk.org/nltk_data/>`_ such as WordNet,
+along with a suite of text processing libraries for classification, tokenization, stemming, tagging, parsing, and semantic reasoning,
+and an active `discussion forum <http://groups.google.com/group/nltk-users>`_.
+
+Thanks to a hands-on guide introducing programming fundamentals alongside topics in computational linguistics,
+NLTK is suitable for linguists, engineers, students, educators, researchers, and industry users alike.
+NLTK is available for Windows, Mac OS X, and Linux. Best of all, NLTK is a free, open source, community-driven project.
+
+NLTK has been called "a wonderful tool for teaching, and working in, computational linguistics using Python,"
+and "an amazing library to play with natural language."
+
+`Natural Language Processing with Python <http://nltk.org/book>`_ provides a practical
+introduction to programming for language processing.
+Written by the creators of NLTK, it guides the reader through the fundamentals
+of writing Python programs, working with corpora, categorizing text, analyzing linguistic structure,
+and more.
+A `new version <http://nltk.org/book3>`_ with updates for Python 3 and NLTK 3 is in preparation.
+
+Some simple things you can do with NLTK
+---------------------------------------
+
+Tokenize and tag some text:
+
+ >>> import nltk
+ >>> sentence = """At eight o'clock on Thursday morning
+ ... Arthur didn't feel very good."""
+ >>> tokens = nltk.word_tokenize(sentence)
+ >>> tokens
+ ['At', 'eight', "o'clock", 'on', 'Thursday', 'morning',
+ 'Arthur', 'did', "n't", 'feel', 'very', 'good', '.']
+ >>> tagged = nltk.pos_tag(tokens)
+ >>> tagged[0:6]
+ [('At', 'IN'), ('eight', 'CD'), ("o'clock", 'JJ'), ('on', 'IN'),
+ ('Thursday', 'NNP'), ('morning', 'NN')]
+
+Identify named entities:
+
+ >>> entities = nltk.chunk.ne_chunk(tagged)
+ >>> entities
+ Tree('S', [('At', 'IN'), ('eight', 'CD'), ("o'clock", 'JJ'),
+ ('on', 'IN'), ('Thursday', 'NNP'), ('morning', 'NN'),
+ Tree('PERSON', [('Arthur', 'NNP')]),
+ ('did', 'VBD'), ("n't", 'RB'), ('feel', 'VB'),
+ ('very', 'RB'), ('good', 'JJ'), ('.', '.')])
+
+Display a parse tree:
+
+ >>> from nltk.corpus import treebank
+ >>> t = treebank.parsed_sents('wsj_0001.mrg')[0]
+ >>> t.draw()
+
+.. image:: images/tree.gif
+
+NB. If you publish work that uses NLTK, please cite the NLTK book as
+follows:
+
+ Bird, Steven, Edward Loper and Ewan Klein (2009), *Natural Language Processing with Python*. O'Reilly Media Inc.
+
+Next Steps
+----------
+
+* `sign up for release announcements <http://groups.google.com/group/nltk>`_
+* `join in the discussion <http://groups.google.com/group/nltk-users>`_
+
+Contents
+========
+
+
+.. toctree::
+ :maxdepth: 1
+
+ news
+ install
+ data
+ contribute
+ FAQ <https://github.com/nltk/nltk/wiki/FAQ>
+ Wiki <https://github.com/nltk/nltk/wiki>
+ API <api/nltk>
+ HOWTO <http://www.nltk.org/howto>
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
diff --git a/web/install.rst b/web/install.rst
new file mode 100644
index 0000000..c85286c
--- /dev/null
+++ b/web/install.rst
@@ -0,0 +1,27 @@
+Installing NLTK
+===============
+
+NLTK requires Python versions 2.6-2.7 or 3.2+
+
+Mac/Unix
+--------
+
+#. Install Setuptools: http://pypi.python.org/pypi/setuptools
+#. Install Pip: run ``sudo easy_install pip``
+#. Install Numpy (optional): run ``sudo pip install -U numpy``
+#. Install NLTK: run ``sudo pip install -U nltk``
+#. Test installation: run ``python`` then type ``import nltk``
+
+Windows
+-------
+
+These instructions assume that you do not already have Python installed on your machine.
+
+32-bit binary installation
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+#. Install Python: http://www.python.org/download/releases/3.4.1/ (avoid the 64-bit versions)
+#. Install Numpy (optional): http://sourceforge.net/projects/numpy/files/NumPy/1.8.1/numpy-1.8.1-win32-superpack-python3.4.exe
+#. Install NLTK: http://pypi.python.org/pypi/nltk
+#. Test installation: ``Start>Python34``, then type ``import nltk``
+
diff --git a/web/news.rst b/web/news.rst
new file mode 100644
index 0000000..310817f
--- /dev/null
+++ b/web/news.rst
@@ -0,0 +1,236 @@
+NLTK News
+=========
+
+NLTK 3.0b1 released : July 2014
+ FrameNet, SentiWordNet, universal tagset, misc efficiency improvements and bugfixes
+ Several API changes, see https://github.com/nltk/nltk/wiki/Porting-your-code-to-NLTK-3.0
+ For full details see:
+ https://raw.github.com/nltk/nltk/master/ChangeLog
+
+NLTK 3.0a4 released : June 2014
+ FrameNet, universal tagset, misc efficiency improvements and bugfixes
+ Several API changes, see https://github.com/nltk/nltk/wiki/Porting-your-code-to-NLTK-3.0
+ For full details see:
+ https://raw.github.com/nltk/nltk/master/ChangeLog
+ http://nltk.org/nltk3-alpha/
+
+NLTK Book Updates : October 2013
+ We are updating the NLTK book for Python 3 and NLTK 3; please see
+ http://nltk.org/book3/
+
+NLTK 3.0a2 released : July 2013
+ Misc efficiency improvements and bugfixes; for details see
+ https://raw.github.com/nltk/nltk/master/ChangeLog
+ http://nltk.org/nltk3-alpha/
+
+NLTK 3.0a1 released : February 2013
+ This version adds support for NLTK's graphical user interfaces.
+ http://nltk.org/nltk3-alpha/
+
+NLTK 3.0a0 released : January 2013
+ The first alpha release of NLTK 3.0 is now available for testing. This version of NLTK works with Python 2.6, 2.7, and Python 3.
+ http://nltk.org/nltk3-alpha/
+
+2012
+----
+
+Python Grant : November 2012
+ The Python Software Foundation is sponsoring Mikhail Korborov's work on porting NLTK to Python 3.
+ http://pyfound.blogspot.hu/2012/11/grants-to-assist-kivy-nltk-in-porting.html
+
+NLTK 2.0.4 released : November 2012
+ Minor fix to remove numpy dependency.
+
+NLTK 2.0.3 released : September 2012
+ This release contains minor improvements and bugfixes. This is the final release compatible with Python 2.5. For details see https://raw.github.com/nltk/nltk/master/ChangeLog
+
+NLTK 2.0.2 released : July 2012
+ This release contains minor improvements and bugfixes. For details see https://raw.github.com/nltk/nltk/master/ChangeLog
+
+NLTK 2.0.1 released : May 2012
+ The final release of NLTK 2. For details see https://raw.github.com/nltk/nltk/master/ChangeLog
+
+NLTK 2.0.1rc4 released : February 2012
+ The fourth release candidate for NLTK 2.
+
+NLTK 2.0.1rc3 released : January 2012
+ The third release candidate for NLTK 2.
+
+2011
+----
+
+NLTK 2.0.1rc2 released : December 2011
+ The second release candidate for NLTK 2. For full details see the ChangeLog.
+
+NLTK development moved to GitHub : October 2011
+ The development site for NLTK has moved from GoogleCode to GitHub: http://github.com/nltk
+
+NLTK 2.0.1rc1 released : April 2011
+ The first release candidate for NLTK 2. For full details see the ChangeLog.
+
+2010
+----
+
+Python Text Processing with NLTK 2.0 Cookbook : December 2010
+ Jacob Perkins has written a 250-page cookbook full of great recipes for text processing using Python and NLTK, published by Packt Publishing. Some of the royalties are being donated to the NLTK project.
+
+Japanese translation of NLTK book : November 2010
+ Masato Hagiwara has translated the NLTK book into Japanese, along with an extra chapter on particular issues with Japanese language process. See http://www.oreilly.co.jp/books/9784873114705/.
+
+NLTK 2.0b9 released : July 2010
+ The last beta release before 2.0 final. For full details see the ChangeLog.
+
+NLTK in Ubuntu 10.4 (Lucid Lynx) : February 2010
+ NLTK is now in the latest LTS version of Ubuntu, thanks to the efforts of Robin Munn. See http://packages.ubuntu.com/lucid/python/python-nltk
+
+NLTK 2.0b? released : June 2009 - February 2010
+ Bugfix releases in preparation for 2.0 final. For full details see the ChangeLog.
+
+2009
+----
+
+NLTK Book in second printing : December 2009
+ The second print run of Natural Language Processing with Python will go on sale in January. We've taken the opportunity to make about 40 minor corrections. The online version has been updated.
+
+NLTK Book published : June 2009
+ Natural Language Processing with Python, by Steven Bird, Ewan Klein and Edward Loper, has been published by O'Reilly Media Inc. It can be purchased in hardcopy, ebook, PDF or for online access, at http://oreilly.com/catalog/9780596516499/. For information about sellers and prices, see https://isbndb.com/d/book/natural_language_processing_with_python/prices.html.
+
+Version 0.9.9 released : May 2009
+ This version finalizes NLTK's API ahead of the 2.0 release and the publication of the NLTK book. There have been dozens of minor enhancements and bugfixes. Many names of the form nltk.foo.Bar are now available as nltk.Bar. There is expanded functionality in the decision tree, collocations, and Toolbox modules. A new translation toy nltk.misc.babelfish has been added. A new module nltk.help gives access to tagset documentation. Fixed imports so NLTK will build and install withou [...]
+
+Version 0.9.8 released : February 2009
+ This version contains a new off-the-shelf tokenizer, POS tagger, and named-entity tagger. A new metrics package includes inter-annotator agreement scores and various distance and word association measures (Tom Lippincott and Joel Nothman). There's a new collocations package (Joel Nothman). There are many improvements to the WordNet package and browser (Steven Bethard, Jordan Boyd-Graber, Paul Bone), and to the semantics and inference packages (Dan Garrette). The NLTK corpus colle [...]
+
+2008
+----
+
+Version 0.9.7 released : December 2008
+ This version contains fixes to the corpus downloader (see instructions) enabling NLTK corpora to be released independently of the software, and to be stored in compressed format. There are improvements in the grammars, chart parsers, probability distributions, sentence segmenter, text classifiers and RTE classifier. There are many further improvements to the book. For full details see the ChangeLog.
+
+Version 0.9.6 released : December 2008
+ This version has an incremental corpus downloader (see instructions) enabling NLTK corpora to be released independently of the software. A new WordNet interface has been developed by Steven Bethard (details). NLTK now has support for dependency parsing, developed by Jason Narad (sponsored by Google Summer of Code). There are many enhancements to the semantics and inference packages, contributed by Dan Garrette. The frequency distribution classes have new support for tabulation a [...]
+
+The NLTK Project has moved : November 2008
+ The NLTK project has moved to Google Sites, Google Code and Google Groups. Content for users and the nltk.org domain is hosted on Google Sites. The home of NLTK development is now Google Code. All discussion lists are at Google Groups. Our old site at nltk.sourceforge.net will continue to be available while we complete this transition. Old releases are still available via our SourceForge release page. We're grateful to SourceForge for hosting our project since its inception in 2001.
+
+Version 0.9.5 released : August 2008
+ This version contains several low-level changes to facilitate installation, plus updates to several NLTK-Contrib projects. A new text module gives easy access to text corpora for newcomers to NLP. For full details see the ChangeLog.
+
+Version 0.9.4 released : August 2008
+ This version contains a substantially expanded semantics package contributed by Dan Garrette, improvements to the chunk, tag, wordnet, tree and feature-structure modules, Mallet interface, ngram language modeling, new GUI tools (WordNet? browser, chunking, POS-concordance). The data distribution includes the new NPS Chat Corpus. NLTK-Contrib includes the following new packages (still undergoing active development) NLG package (Petro Verkhogliad), dependency parsers (Jason Narad), cor [...]
+NLTK presented at ACL conference : June 2008
+ A paper on teaching courses using NLTK will be presented at the ACL conference: Multidisciplinary Instruction with the Natural Language Toolkit
+
+Version 0.9.3 released : June 2008
+ This version contains an improved WordNet? similarity module using pre-built information content files (included in the corpus distribution), new/improved interfaces to Weka, MEGAM and Prover9/Mace4 toolkits, improved Unicode support for corpus readers, a BNC corpus reader, and a rewrite of the Punkt sentence segmenter contributed by Joel Nothman. NLTK-Contrib includes an implementation of incremental algorithm for generating referring expression contributed by Margaret Mitchell. For [...]
+
+NLTK presented at LinuxFest Northwest : April 2008
+ Sean Boisen presented NLTK at LinuxFest Northwest, which took place in Bellingham, Washington. His presentation slides are available at: http://semanticbible.com/other/talks/2008/nltk/main.html
+
+NLTK in Google Summer of Code : April 2008
+ Google Summer of Code will sponsor two NLTK projects. Jason Narad won funding for a project on dependency parsers in NLTK (mentored by Sebastian Riedel and Jason Baldridge). Petro Verkhogliad won funding for a project on natural language generation in NLTK (mentored by Robert Dale and Edward Loper).
+
+Python Software Foundation adopts NLTK for Google Summer of Code application : March 2008
+ The Python Software Foundation has listed NLTK projects for sponsorship from the 2008 Google Summer of Code program. For details please see http://wiki.python.org/moin/SummerOfCode.
+
+Version 0.9.2 released : March 2008
+ This version contains a new inference module linked to the Prover9/Mace4 theorem-prover and model checker (Dan Garrette, Ewan Klein). It also includes the VerbNet? and PropBank? corpora along with corpus readers. A bug in the Reuters corpus reader has been fixed. NLTK-Contrib includes new work on the WordNet? browser (Jussi Salmela). For full details see the ChangeLog
+
+Youtube video about NLTK : January 2008
+ The video from of the NLTK talk at the Bay Area Python Interest Group last July has been posted at http://www.youtube.com/watch?v=keXW_5-llD0 (1h15m)
+
+Version 0.9.1 released : January 2008
+ This version contains new support for accessing text categorization corpora, along with several corpora categorized for topic, genre, question type, or sentiment. It includes several new corpora: Question classification data (Li & Roth), Reuters 21578 Corpus, Movie Reviews corpus (Pang & Lee), Recognising Textual Entailment (RTE) Challenges. NLTK-Contrib includes expanded support for semantics (Dan Garrette), readability scoring (Thomas Jakobsen, Thomas Skardal), and SIL Toolbox (Gre [...]
+
+2007
+----
+
+NLTK-Lite 0.9 released : October 2007
+ This version is substantially revised and expanded from version 0.8. The entire toolkit can be accessed via a single import statement "import nltk", and there is a more convenient naming scheme. Calling deprecated functions generates messages that help programmers update their code. The corpus, tagger, and classifier modules have been redesigned. All functionality of the old NLTK 1.4.3 is now covered by NLTK-Lite 0.9. The book has been revised and expanded. A new data package incorpo [...]
+
+NLTK-Lite 0.9b2 released : September 2007
+ This version is substantially revised and expanded from version 0.8. The entire toolkit can be accessed via a single import statement "import nltk", and many common NLP functions accessed directly, e.g. nltk.PorterStemmer?, nltk.ShiftReduceParser?. The corpus, tagger, and classifier modules have been redesigned. The book has been revised and expanded, and the chapters have been reordered. NLTK has a new data package incorporating the existing corpus collection and adding new sections [...]
+
+NLTK-Lite 0.9b1 released : August 2007
+ This version is substantially revised and expanded from version 0.8. The entire toolkit can be accessed via a single import statement "import nltk", and many common NLP functions accessed directly, e.g. nltk.PorterStemmer?, nltk.ShiftReduceParser?. The corpus, tagger, and classifier modules have been redesigned. The book has been revised and expanded, and the chapters have been reordered. NLTK has a new data package incorporating the existing corpus collection and adding new sections [...]
+
+NLTK talks in São Paulo : August 2007
+ Steven Bird will present NLTK in a series of talks at the First Brazilian School on Computational Linguistics, at the University of São Paulo in the first week of September.
+
+NLTK talk in Bay Area : July 2007
+ Steven Bird, Ewan Klein, and Edward Loper will present NLTK at the Bay Area Python Interest Group, at Google on Thursday 12 July.
+
+NLTK-Lite 0.8 released : July 2007
+ This version is substantially revised and expanded from version 0.7. The code now includes improved interfaces to corpora, chunkers, grammars, frequency distributions, full integration with WordNet? 3.0 and WordNet? similarity measures. The book contains substantial revision of Part I (tokenization, tagging, chunking) and Part II (grammars and parsing). NLTK has several new corpora including the Switchboard Telephone Speech Corpus transcript sample (Talkbank Project), CMU Problem Rep [...]
+
+NLTK features in Language Documentation and Conservation article : July 2007
+ An article Managing Fieldwork Data with Toolbox and the Natural Language Toolkit by Stuart Robinson, Greg Aumann, and Steven Bird appears in the inaugural issue of ''Language Documentation and Conservation''. It discusses several small Python programs for manipulating field data.
+
+NLTK features in ACM Crossroads article : May 2007
+ An article Getting Started on Natural Language Processing with Python by Nitin Madnani will appear in ''ACM Crossroads'', the ACM Student Journal. It discusses NLTK in detail, and provides several helpful examples including an entertaining free word association program.
+
+NLTK-Lite 0.7.5 released : May 2007
+ This version contains improved interfaces for WordNet 3.0 and WordNet-Similarity, the Lancaster Stemmer (contributed by Steven Tomcavage), and several new corpora including the Switchboard Telephone Speech Corpus transcript sample (Talkbank Project), CMU Problem Reports Corpus sample, CONLL2002 POS+NER data, Patient Information Leaflet corpus sample and WordNet 3.0 data files. With this distribution WordNet no longer needs to be separately installed.
+
+NLTK-Lite 0.7.4 released : May 2007
+ This release contains new corpora and corpus readers for Indian POS-Tagged data (Bangla, Hindi, Marathi, Telugu), and the Sinica Treebank, and substantial revision of Part II of the book on structured programming, grammars and parsing.
+
+NLTK-Lite 0.7.3 released : April 2007
+ This release contains improved chunker and PCFG interfaces, the Shakespeare XML corpus sample and corpus reader, improved tutorials and improved formatting of code samples, and categorization of problem sets by difficulty.
+
+NLTK-Lite 0.7.2 released : March 2007
+ This release contains new text classifiers (Cosine, NaiveBayes?, Spearman), contributed by Sam Huston, simple feature detectors, the UDHR corpus with text samples in 300+ languages and a corpus interface; improved tutorials (340 pages in total); additions to contrib area including Kimmo finite-state morphology system, Lambek calculus system, and a demonstration of text classifiers for language identification.
+
+NLTK-Lite 0.7.1 released : January 2007
+ This release contains bugfixes in the WordNet? and HMM modules.
+
+2006
+----
+
+NLTK-Lite 0.7 released : December 2006
+ This release contains: new semantic interpretation package (Ewan Klein), new support for SIL Toolbox format (Greg Aumann), new chunking package including cascaded chunking (Steven Bird), new interface to WordNet? 2.1 and Wordnet similarity measures (David Ormiston Smith), new support for Penn Treebank format (Yoav Goldberg), bringing the codebase to 48,000 lines; substantial new chapters on semantic interpretation and chunking, and substantial revisions to several other chapters, bri [...]
+
+NLTK-Lite 0.7b1 released : December 2006
+ This release contains: new semantic interpretation package (Ewan Klein), new support for SIL Toolbox format (Greg Aumann), new chunking package including cascaded chunking, wordnet package updated for version 2.1 of Wordnet, and prototype wordnet similarity measures (David Ormiston Smith), bringing the codebase to 48,000 lines; substantial new chapters on semantic interpretation and chunking, and substantial revisions to several other chapters, bringing the textbook documentation to [...]
+
+NLTK-Lite 0.6.6 released : October 2006
+ This release contains bugfixes, improvements to Shoebox file format support, and expanded tutorial discussions of programming and feature-based grammars.
+
+NLTK-Lite 0.6.5 released : July 2006
+ This release contains improvements to Shoebox file format support (by Stuart Robinson and Greg Aumann); an implementation of hole semantics (by Peter Wang); improvements to lambda calculus and semantic interpretation modules (by Ewan Klein); a new corpus (Sinica Treebank sample); and expanded tutorial discussions of trees, feature-based grammar, unification, PCFGs, and more exercises.
+
+NLTK-Lite passes 10k download milestone : May 2006
+ We have now had 10,000 downloads of NLTK-Lite in the nine months since it was first released.
+
+NLTK-Lite 0.6.4 released : April 2006
+ This release contains new corpora (Senseval 2, TIMIT sample), a clusterer, cascaded chunker, and several substantially revised tutorials.
+
+2005
+----
+
+NLTK 1.4 no longer supported : December 2005
+ The main development has switched to NLTK-Lite. The latest version of NLTK can still be downloaded; see the installation page for instructions.
+
+NLTK-Lite 0.6 released : November 2005
+ contains bug-fixes, PDF versions of tutorials, expanded fieldwork tutorial, PCFG grammar induction (by Nathan Bodenstab), and prototype concordance and paradigm display tools (by Peter Spiller and Will Hardy).
+
+NLTK-Lite 0.5 released : September 2005
+ contains bug-fixes, improved tutorials, more project suggestions, and a pronunciation dictionary.
+
+NLTK-Lite 0.4 released : September 2005
+ contains bug-fixes, improved tutorials, more project suggestions, and probabilistic parsers.
+
+NLTK-Lite 0.3 released : August 2005
+ contains bug-fixes, documentation clean-up, project suggestions, and the chart parser demos including one for Earley parsing by Jean Mark Gawron.
+
+NLTK-Lite 0.2 released : July 2005
+ contains bug-fixes, documentation clean-up, and some translations of tutorials into Brazilian Portuguese by Tiago Tresoldi.
+
+NLTK-Lite 0.1 released : July 2005
+ substantially simplified and streamlined version of NLTK has been released
+
+Brazilian Portuguese Translation : April 2005
+ top-level pages of this website have been translated into Brazilian Portuguese by Tiago Tresoldi; translations of the tutorials are in preparation http://hermes.sourceforge.net/nltk-br/
+
+1.4.3 Release : February 2005
+ NLTK 1.4.3 has been released; this is the first version which is compatible with Python 2.4.
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/nltk.git
More information about the debian-science-commits
mailing list