[hfst] 01/02: Imported Upstream version 3.8.1~r4088
Tino Didriksen
tinodidriksen-guest at moszumanska.debian.org
Tue Nov 4 13:00:19 UTC 2014
This is an automated email from the git hooks/post-receive script.
tinodidriksen-guest pushed a commit to branch master
in repository hfst.
commit abe69df5dfd972b0e2f8664216de1c16e292b786
Author: Tino Didriksen <mail at tinodidriksen.com>
Date: Tue Nov 4 12:56:33 2014 +0000
Imported Upstream version 3.8.1~r4088
---
ChangeLog | 535 ++++++++++++++++++
ChangeLog.old | 619 +++++++++++++++++++++
NEWS | 6 +-
back-ends/openfst/src/include/fst/accumulator.h | 2 +-
back-ends/openfst/src/include/fst/arc-map.h | 2 +-
back-ends/openfst/src/include/fst/determinize.h | 2 +-
back-ends/openfst/src/include/fst/encode.h | 2 +-
back-ends/openfst/src/include/fst/epsnormalize.h | 2 +-
back-ends/openfst/src/include/fst/equivalent.h | 2 +-
back-ends/openfst/src/include/fst/factor-weight.h | 2 +-
back-ends/openfst/src/include/fst/interval-set.h | 8 +-
.../openfst/src/include/fst/label-reachable.h | 2 +-
back-ends/openfst/src/include/fst/relabel.h | 2 +-
back-ends/openfst/src/include/fst/replace-util.h | 4 +-
back-ends/openfst/src/include/fst/replace.h | 2 +-
back-ends/openfst/src/include/fst/rmepsilon.h | 2 +-
back-ends/openfst/src/include/fst/rmfinalepsilon.h | 2 +-
.../openfst/src/include/fst/sparse-tuple-weight.h | 4 +-
back-ends/openfst/src/include/fst/state-map.h | 2 +-
.../openfst/src/include/fst/symbol-table-ops.h | 2 +-
back-ends/openfst/src/include/fst/synchronize.h | 4 +-
.../openfst/src/include/fst/test-properties.h | 2 +-
back-ends/openfst/src/include/fst/util.h | 4 +-
configure.ac | 28 +-
libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc | 13 +-
libhfst/src/HfstInputStream.h | 2 +-
libhfst/src/Makefile.am | 2 +-
.../src/implementations/ConvertTransducerFormat.h | 1 +
libhfst/src/implementations/FomaTransducer.h | 1 +
libhfst/src/implementations/HfstTransitionGraph.h | 511 ++++++++++++++++-
.../HfstTropicalTransducerTransitionData.h | 180 +++---
.../optimized-lookup/find_epsilon_loops.cc | 128 ++---
.../src/implementations/optimized-lookup/pmatch.cc | 38 +-
.../src/implementations/optimized-lookup/pmatch.h | 4 +
.../implementations/optimized-lookup/transducer.cc | 105 ++--
.../implementations/optimized-lookup/transducer.h | 49 +-
libhfst/src/parsers/XreCompiler.h | 2 +-
libhfst/src/parsers/pmatch_lex.ll | 16 +-
libhfst/src/parsers/pmatch_parse.yy | 504 ++++++++++-------
libhfst/src/parsers/pmatch_utils.cc | 77 ++-
libhfst/src/parsers/pmatch_utils.h | 8 +-
man/Makefile.am | 55 +-
man/{hfst-format.1 => hfst-affix-guessify.1} | 29 +-
man/hfst-apertium-proc.1 | 7 +-
man/hfst-build-tagger.1 | 15 +
man/hfst-calculate.1 | 2 +-
man/hfst-compare.1 | 2 +-
man/hfst-compose-intersect.1 | 6 +-
man/hfst-compose.1 | 2 +-
man/hfst-concatenate.1 | 2 +-
man/hfst-conjunct.1 | 2 +-
man/hfst-determinise.1 | 1 +
man/hfst-determinize.1 | 7 +-
man/hfst-disjunct.1 | 2 +-
man/{hfst-name.1 => hfst-edit-metadata.1} | 21 +-
man/hfst-expand-equivalences.1 | 63 +++
man/hfst-expand.1 | 1 +
man/hfst-foma-wrapper.1 | 11 +
man/hfst-format.1 | 2 +-
man/hfst-fst2fst.1 | 2 +-
man/hfst-fst2strings.1 | 2 +-
man/hfst-fst2txt.1 | 2 +-
man/hfst-grep.1 | 174 ++++++
man/{hfst-txt2fst.1 => hfst-guess.1} | 53 +-
man/{hfst-txt2fst.1 => hfst-guessify.1} | 47 +-
man/hfst-head.1 | 2 +-
man/{hfst-name.1 => hfst-info.1} | 42 +-
man/hfst-intersect.1 | 1 +
man/hfst-invert.1 | 2 +-
man/hfst-lexc-wrapper.1 | 2 +-
man/hfst-lexc.1 | 20 +-
man/hfst-lookup.1 | 5 +-
man/hfst-minimise.1 | 1 +
man/hfst-minimize.1 | 7 +-
man/hfst-minus.1 | 1 +
man/{hfst-project.1 => hfst-multiply.1} | 20 +-
man/hfst-name.1 | 2 +-
man/hfst-open-input-file-for-tagger.1 | 15 +
man/hfst-optimised-lookup.1 | 1 +
man/hfst-optimized-lookup.1 | 58 ++
man/hfst-pair-test.1 | 88 +++
man/{hfst-invert.1 => hfst-pmatch.1} | 29 +-
man/{hfst-name.1 => hfst-pmatch2fst.1} | 40 +-
man/{hfst-invert.1 => hfst-proc2.1} | 26 +-
man/hfst-project.1 | 2 +-
man/{hfst-name.1 => hfst-prune-alphabet.1} | 30 +-
man/hfst-push-weights.1 | 2 +-
man/hfst-regexp2fst.1 | 47 +-
man/hfst-remove-epsilons.1 | 2 +-
man/hfst-repeat.1 | 2 +-
man/hfst-reverse.1 | 2 +-
man/{hfst-reverse.1 => hfst-reweight-tagger.1} | 16 +-
man/hfst-reweight.1 | 93 ++++
man/hfst-sfstpl2fst.1 | 1 +
man/{hfst-disjunct.1 => hfst-shuffle.1} | 20 +-
man/hfst-split.1 | 2 +-
man/hfst-strings2fst.1 | 7 +-
man/hfst-substitute.1 | 2 +-
man/hfst-subtract.1 | 2 +-
man/hfst-summarise.1 | 1 +
man/hfst-summarize.1 | 2 +-
man/{hfst-invert.1 => hfst-tag.1} | 15 +-
man/hfst-tail.1 | 2 +-
man/hfst-train-tagger-loc.1 | 1 +
man/hfst-train-tagger-system.1 | 1 +
man/{hfst-invert.1 => hfst-train-tagger.1} | 15 +-
man/{hfst-determinize.1 => hfst-traverse.1} | 10 +-
man/hfst-twolc-loc.1 | 1 +
man/hfst-twolc-system.1 | 1 +
man/hfst-twolc.1 | 66 +++
man/hfst-txt2fst.1 | 2 +-
man/hfst-union.1 | 1 +
man/hfst-xfst.1 | 5 +-
man/hfst_tagger_compute_data_statistics.py.1 | 15 +
man/htwolcpre1.1 | 15 +
man/htwolcpre2.1 | 15 +
man/htwolcpre3.1 | 15 +
scripts/hfst-fst2tesseract.xfst | 10 +
swig/doc/libhfst.py | 2 +-
swig/hfstBot.py | 2 +-
swig/setup.py | 2 +-
swig/test/test_examples.py | 2 +-
test/libhfst/test_transducer_functions.cc | 9 +-
test/tools/Makefile.am | 3 +
test/tools/optimized-lookup-functionality.sh | 13 +
test/tools/pmatch-tester.sh | 309 +++++++---
test/tools/pmatch-tests.sh | 431 +++++++++-----
tools/src/HfstAlphabet.h | 26 +-
tools/src/hfst-expand-equivalences.cc | 3 +-
tools/src/hfst-guess.cc | 2 +-
tools/src/hfst-lexc-wrapper.cc | 2 +-
tools/src/hfst-optimized-lookup.cc | 2 +-
tools/src/hfst-tagger/src/hfst-reweight-tagger.cc | 29 +-
.../src/hfst-tagger/src/use_model_src/DataTypes.h | 30 +-
.../src/use_model_src/NewLexicalModel.h | 45 +-
.../hfst-twolc/src/commandline_src/CommandLine.cc | 2 +-
tools/src/parsers/XfstCompiler.cc | 197 ++++++-
tools/src/parsers/XfstCompiler.h | 9 +-
tools/src/parsers/xfst-parser.yy | 4 +-
139 files changed, 4267 insertions(+), 1035 deletions(-)
diff --git a/ChangeLog b/ChangeLog
index 982c46a..56e3859 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,538 @@
+2014-10-31 16:44 eaxelson
+
+ * tools/src/hfst-guess.cc: Fixed std::cout into &std::cout in
+ stream pointer comparison.
+
+2014-10-31 14:36 eaxelson
+
+ * back-ends/openfst/src/include/fst/accumulator.h,
+ back-ends/openfst/src/include/fst/arc-map.h,
+ back-ends/openfst/src/include/fst/determinize.h,
+ back-ends/openfst/src/include/fst/encode.h,
+ back-ends/openfst/src/include/fst/epsnormalize.h,
+ back-ends/openfst/src/include/fst/equivalent.h,
+ back-ends/openfst/src/include/fst/factor-weight.h,
+ back-ends/openfst/src/include/fst/label-reachable.h,
+ back-ends/openfst/src/include/fst/relabel.h,
+ back-ends/openfst/src/include/fst/replace-util.h,
+ back-ends/openfst/src/include/fst/replace.h,
+ back-ends/openfst/src/include/fst/rmepsilon.h,
+ back-ends/openfst/src/include/fst/rmfinalepsilon.h,
+ back-ends/openfst/src/include/fst/sparse-tuple-weight.h,
+ back-ends/openfst/src/include/fst/state-map.h,
+ back-ends/openfst/src/include/fst/symbol-table-ops.h,
+ back-ends/openfst/src/include/fst/synchronize.h,
+ back-ends/openfst/src/include/fst/test-properties.h,
+ back-ends/openfst/src/include/fst/util.h, configure.ac,
+ tools/src/hfst-tagger/src/use_model_src/DataTypes.h,
+ tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h: Now
+ using definitions USE_TR1_UNORDERED_(MAP|SET) when defining what
+ unordered maps and sets to use.
+
+2014-10-30 15:29 eaxelson
+
+ * configure.ac: Fixed a typo tr2 -> tr1.
+
+2014-10-29 13:50 eaxelson
+
+ * configure.ac, tools/src/HfstAlphabet.h,
+ tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h:
+ Unordered maps and sets are used from std namespace if
+ -std=gnu++11 is requested.
+
+2014-10-28 11:40 eaxelson
+
+ * swig/setup.py: Forgot to update version number in swig bindings.
+
+2014-10-27 16:40 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy: Various improvements and
+ additions to function syntax,
+ particularly empty args and string-args
+
+2014-10-27 14:48 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h,
+ tools/src/parsers/XfstCompiler.cc: Added functions for merge
+ operation in HfstTransitionGraph.
+
+2014-10-24 11:03 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/implementations/HfstTropicalTransducerTransitionData.h:
+ Modified weight handling in HfstTransitionGraph::intersect.
+
+2014-10-24 08:38 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/transducer.cc: When
+ the alphabet is constructed from a symbol table, set identity to
+ NO_SYM
+ this was supposed to always happen anyway but didn't matter until
+ recently
+
+2014-10-22 13:16 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/implementations/HfstTropicalTransducerTransitionData.h:
+ Modified intersection algorithms in HfstTransitionGraph.
+
+2014-10-22 10:21 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h: Added
+ functions to be used in xerox's merge operation.
+
+2014-10-22 10:20 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/xfst-parser.yy: Small fixes to list definitions
+ in hfst-xfst.
+
+2014-10-17 15:59 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h,
+ tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/XfstCompiler.h: Added an implementation for
+ compile-replace in hfst-xfst, it still needs lot of testing.
+
+2014-10-16 14:36 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h,
+ tools/src/parsers/XfstCompiler.cc: Yet some more functions added
+ to compile-replace.
+
+2014-10-16 11:48 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h,
+ tools/src/parsers/XfstCompiler.cc: Added more functions for
+ compile-replace.
+
+2014-10-15 14:23 eaxelson
+
+ * libhfst/src/implementations/HfstTransitionGraph.h: Tentatively
+ added functions in HfstTransitionGraph to be used in
+ compile-replace.
+
+2014-10-15 13:26 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc: Added function
+ is_well_formed_for_compile_replace to be used in compile-replace
+ command.
+
+2014-10-15 11:30 eaxelson
+
+ * test/libhfst/test_transducer_functions.cc: Forgot to comment out
+ debugging prints in tests.
+
+2014-10-15 11:28 eaxelson
+
+ * back-ends/openfst/src/include/fst/interval-set.h,
+ test/libhfst/test_transducer_functions.cc: Added brackets around
+ member calls 'Interval.end' and 'Interval.begin' to avoid them
+ getting confused with std::end() and std::begin() templates in
+ C++11.
+
+2014-10-15 10:21 eaxelson
+
+ * back-ends/openfst/src/include/fst/interval-set.h: Rolled back
+ earlier revision in interval-set.h
+
+2014-10-15 09:56 eaxelson
+
+ * back-ends/openfst/src/include/fst/interval-set.h,
+ libhfst/src/implementations/ConvertTransducerFormat.h,
+ libhfst/src/implementations/FomaTransducer.h: Made small
+ modifications for better c++11/c++0x support.
+
+2014-10-15 09:32 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/transducer.cc: Fix
+ bug where identity wasn't being set to NO_SYMBOL when absent from
+ alphabet
+
+2014-10-15 08:46 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h: Refuse
+ to enter flag loops more than once
+ fixes bug #250 bug - arguably this could be applied to epsilon
+ loops too,
+ pending discussion
+
+2014-10-15 08:09 eaxelson
+
+ * man/Makefile.am: Fixed a typo in Makefile.
+
+2014-10-14 15:01 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc: Now hfst-xfst gives a warning
+ (or exits) if a binary command tries to access a stack with less
+ than 2 transducers.
+
+2014-10-14 14:51 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/XfstCompiler.h: Now hfst-xfst exits if a
+ command tries to access an empty stack if quit-on-fail is ON and
+ hfst-xfst is not in interactive mode.
+
+2014-10-14 13:44 mpsilfve
+
+ * scripts/hfst-fst2tesseract.xfst: Comment explaining usage of
+ hfst-fst2tesseract.xfst.
+
+2014-10-14 13:40 mpsilfve
+
+ * scripts/hfst-fst2tesseract.xfst: Added script for converting
+ morphological analyzers to Tesseract word models.
+
+2014-10-14 12:27 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h:
+ Improvements to loop finding
+
+2014-10-14 12:08 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h:
+ Further corrections to loop detection
+ slowdown back to ~10x but may be improved from here
+
+2014-10-14 11:51 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h: Fix
+ some cases of overdetecting infinite ambiguity, there's still
+ some left
+
+2014-10-14 10:51 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc:
+ Forgot to keep adding the repeated states in the loop detection
+ phase
+
+2014-10-13 18:15 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h: Only
+ try to catch infinite ambiguity at epsilon arcs
+ This is the big speed win and presumably correct.
+
+2014-10-13 17:22 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc:
+ This order of comparison is a bit faster since sizes never differ
+
+2014-10-13 16:34 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h: Speed
+ up is_lookup_infinitely_ambiguous() somewhat
+
+2014-10-11 04:10 mie
+
+ * tools/src/hfst-optimized-lookup.cc: Try to avoid using negative
+ indexes for arrays
+
+2014-10-11 03:59 mie
+
+ * test/tools/Makefile.am,
+ test/tools/optimized-lookup-functionality.sh: Optimised lookup
+ tests
+
+2014-10-09 17:22 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/transducer.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h:
+ Runtime handling of identity and unknown
+
+2014-10-09 12:30 hardwick
+
+ * libhfst/src/parsers/pmatch_utils.h: Forgot to remove one thing in
+ the last commit
+
+2014-10-09 12:22 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc: Take out our own
+ harmonization hacks now that they're unneeded
+
+2014-10-08 12:45 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h: Don't use delimiters when
+ they're not necessary,
+ also don't insert everything to RTNs anymore and provide the
+ is_special()
+ function the previous commit required
+
+2014-10-08 12:29 hardwick
+
+ * libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc: Treat special
+ pmatch symbols like flag diacritics for harmonization,
+ also after harmonization add all symbols, including flags, to the
+ alphabets
+
+2014-10-08 11:51 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Fix bug where delimiters
+ were shadowing the named transducers' names
+ also remove extraenous parsing path
+
+2014-10-06 21:33 janiemi
+
+ * test/tools/pmatch-tester.sh, test/tools/pmatch-tests.sh: Updated
+ pmatch functionality tests.
+
+ Current pmatch syntax: string literals in {...}, symbols in
+ double quotes.
+ Added tests: Ins maximizing globally; Difference and character
+ sets in
+ named expressions; Named expressions in OptCap, ToUpper; Named
+ expressions
+ in replace; Long input lines; Ins should not throw
+ std::out_of_range;
+ Disjunction of two Ins expressions.
+ Added options: --include-tests, --exclude-tests,
+ --no-number-tests,
+ --truncate-lines, --truncate-log-lines. Minor new features in
+ test runner.
+
+2014-10-06 17:27 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll: Add string literal syntax for
+ standalone %-escaped chars
+ (they used to be considered symbols which now have to be
+ defined or cause an error)
+
+2014-10-06 16:44 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h: Use minimization guards to
+ keep multiple negative contexts separate in disjunctions
+
+2014-10-06 15:36 eaxelson
+
+ * libhfst/src/HfstInputStream.h,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/parsers/XreCompiler.h, swig/doc/libhfst.py,
+ swig/hfstBot.py, swig/test/test_examples.py: Fixed some more
+ spelling errors noticed by lintian.
+
+2014-10-06 14:45 eaxelson
+
+ * tools/src/hfst-lexc-wrapper.cc,
+ tools/src/hfst-twolc/src/commandline_src/CommandLine.cc: Fixed
+ spelling errors found by lintian.
+
+2014-10-06 14:33 eaxelson
+
+ * man/hfst-train-tagger.1: Added again hfst-train-tagger man page
+ which is no more a symlink.
+
+2014-10-06 14:31 eaxelson
+
+ * man/Makefile.am, man/hfst-build-tagger.1,
+ man/hfst-foma-wrapper.1, man/hfst-open-input-file-for-tagger.1,
+ man/hfst-reweight-tagger.1, man/hfst-train-tagger.1,
+ man/hfst-twolc-loc.1, man/hfst-twolc-system.1, man/hfst-twolc.1,
+ man/hfst_tagger_compute_data_statistics.py.1, man/htwolcpre1.1,
+ man/htwolcpre2.1, man/htwolcpre3.1: Added missing man pages.
+
+2014-10-06 14:00 eaxelson
+
+ * tools/src/hfst-tagger/src/hfst-reweight-tagger.cc: Now
+ hfst-reweight-tagger --help returns EXIT_SUCCESS before trying to
+ access uninitialized values.
+
+2014-10-06 13:24 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h: Warn about shadowing
+ definitions
+
+2014-10-06 13:05 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy: Alternate syntaxes regex for
+ Define TOP and .#. for #
+
+2014-10-06 12:47 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Revamp LABEL parsing and
+ introduce curly literal pairs
+
+2014-10-06 11:08 eaxelson
+
+ * man/Makefile.am, man/hfst-affix-guessify.1,
+ man/hfst-apertium-proc.1, man/hfst-calculate.1,
+ man/hfst-compare.1, man/hfst-compose-intersect.1,
+ man/hfst-compose.1, man/hfst-concatenate.1, man/hfst-conjunct.1,
+ man/hfst-determinise.1, man/hfst-determinize.1,
+ man/hfst-disjunct.1, man/hfst-edit-metadata.1,
+ man/hfst-expand-equivalences.1, man/hfst-expand.1,
+ man/hfst-format.1, man/hfst-fst2fst.1, man/hfst-fst2strings.1,
+ man/hfst-fst2txt.1, man/hfst-grep.1, man/hfst-guess.1,
+ man/hfst-guessify.1, man/hfst-head.1, man/hfst-info.1,
+ man/hfst-intersect.1, man/hfst-invert.1, man/hfst-lexc-wrapper.1,
+ man/hfst-lexc.1, man/hfst-lookup.1, man/hfst-minimise.1,
+ man/hfst-minimize.1, man/hfst-minus.1, man/hfst-multiply.1,
+ man/hfst-name.1, man/hfst-open-input-file-for-tagger.1,
+ man/hfst-optimised-lookup.1, man/hfst-optimized-lookup.1,
+ man/hfst-pair-test.1, man/hfst-pmatch.1, man/hfst-pmatch2fst.1,
+ man/hfst-proc2.1, man/hfst-project.1, man/hfst-prune-alphabet.1,
+ man/hfst-push-weights.1, man/hfst-regexp2fst.1,
+ man/hfst-remove-epsilons.1, man/hfst-repeat.1,
+ man/hfst-reverse.1, man/hfst-reweight.1, man/hfst-sfstpl2fst.1,
+ man/hfst-shuffle.1, man/hfst-split.1, man/hfst-strings2fst.1,
+ man/hfst-substitute.1, man/hfst-subtract.1, man/hfst-summarise.1,
+ man/hfst-summarize.1, man/hfst-tag.1, man/hfst-tail.1,
+ man/hfst-train-tagger-loc.1, man/hfst-train-tagger-system.1,
+ man/hfst-train-tagger.1, man/hfst-traverse.1, man/hfst-txt2fst.1,
+ man/hfst-union.1, man/hfst-xfst.1: Updated and added man pages.
+
+2014-10-06 11:06 eaxelson
+
+ * tools/src/hfst-expand-equivalences.cc: Moved option checking
+ after possible returning from program so that option --help will
+ not generate error messages.
+
+2014-10-05 11:06 hardwick
+
+ * libhfst/src/parsers/pmatch_utils.cc: Require backslash character
+ to be escaped as \\ in curly literals
+
+2014-10-05 10:53 hardwick
+
+ * libhfst/src/parsers/pmatch_utils.cc: Fix bug in unescaping
+ function
+
+2014-10-03 12:28 eaxelson
+
+ * ChangeLog, ChangeLog.old, NEWS, configure.ac,
+ libhfst/src/Makefile.am: Ready for release 3.8.0.
+
+2014-10-03 11:55 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc: Added variable
+ 'lexc-rename-flags' to hfst-xfst.
+
+2014-10-03 11:48 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Desperately fiddle with the
+ way the minus operation expands things
+
+2014-10-01 16:29 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy: Add lambda-like anonymous
+ definitions for controlling subexpression boundaries
+
+2014-10-01 12:46 eaxelson
+
+ * tools/src/hfst-lexc-compiler.cc, tools/src/hfst-regexp2fst.cc,
+ tools/src/parsers/XfstCompiler.cc: Changed the flag handling
+ behavior of hfst-xfst and hfst-lexc. Now both tools by default
+ use Xerox's way when composing, i.e. flag diacritics match
+ unknown and identity symbols. This can be controlled with
+ variable 'xerox-composition' (the default is ON) in hfst-xfst and
+ with option '--xerox-composition={ON,OFF}' (the default is also
+ ON) in hfst-lexc. hfst-regexp2fst also has the option
+ '--xerox-composition' which by default is OFF, as it was earlier.
+
+2014-10-01 11:28 eaxelson
+
+ * tools/src/hfst-regexp2fst.cc: Fixed a typo in hfst-regexp2fst
+ option handling.
+
+2014-09-30 15:38 eaxelson
+
+ * libhfst/src/HfstTransducer.cc: Now one-sided flag diacritics are
+ allowed in composition when flag-is-epsilon is used.
+
+2014-09-30 13:54 eaxelson
+
+ * tools/src/hfst-strings2fst.cc: Added option --log10 for 10-based
+ logarithmic weights in hfst-strings2fst.
+
+2014-09-30 13:18 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/parsers/xre_parse.yy,
+ tools/src/parsers/XfstCompiler.cc: Now an error is thrown if
+ flags are not twosided in composition when xerox composition is
+ used.
+
+2014-09-30 13:05 eaxelson
+
+ * libhfst/src/HfstExceptionDefs.cc,
+ libhfst/src/HfstExceptionDefs.h: Added exception class
+ FlagDiacriticsAreNotIdentitesException.
+
+2014-09-29 11:22 hardwick
+
+ * test/tools/Makefile.am, test/tools/pmatch-functionality.sh,
+ test/tools/pmatch-tester.sh, test/tools/pmatch-tests.sh: Add
+ pmatch functionality test suite
+
+2014-09-29 10:49 eaxelson
+
+ * tools/src/hfst-regexp2fst.cc: Added option -X flag-is-epsilon to
+ hfst-regexp2fst.
+
+2014-09-25 10:38 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Fix another symbol-leaking
+ issue
+
+2014-09-25 10:17 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Allow nested logical
+ operations on contexts
+
+2014-09-23 07:15 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/pmatch_utils.h: We need to avoid symbol
+ pollution for more than just special symbols
+ (this doesn't completely resolve pollution issues, just some
+ urgent ones)
+
+2014-09-23 06:47 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: Don't
+ forget to pop the rtn stack when there's nothing matched
+
+2014-09-23 06:36 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Minimize after adding
+ delimiters, not before
+
+2014-09-23 05:14 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Revert bracket-bounding
+ behaviour
+
+2014-09-22 15:47 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Use brackets for extra
+ delimiters to control tag and context boundaries more
+
+2014-09-22 15:28 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy,
+ libhfst/src/parsers/pmatch_utils.cc: add_delimiters was happening
+ in the wrong place since recent syntax changes
+
2014-09-22 12:57 eaxelson
* libhfst/src/implementations/HfstOlTransducer.cc: Fixed a too
diff --git a/ChangeLog.old b/ChangeLog.old
index e4a1202..982c46a 100644
--- a/ChangeLog.old
+++ b/ChangeLog.old
@@ -1,3 +1,622 @@
+2014-09-22 12:57 eaxelson
+
+ * libhfst/src/implementations/HfstOlTransducer.cc: Fixed a too
+ strict assertion.
+
+2014-09-17 08:05 eaxelson
+
+ * libhfst/src/parsers/pmatch_utils.cc: Added a return value for a
+ case that should never happen to make scan-build happy.
+
+2014-09-16 15:03 eaxelson
+
+ * libhfst/src/HfstXeroxRules.cc: Reverted back to the buggy
+ behaviour of function 'getMarkerNumber', since some
+ HfstXeroxRules tests will fail if it works correctly...
+
+2014-09-16 13:50 eaxelson
+
+ * libhfst/src/HfstXeroxRules.cc, libhfst/src/HfstXeroxRules.h:
+ Fixed an error in function 'getMarkerNumber' where istringstream
+ was not properly initialized and returned random values. Also
+ added a print method for class Rule.
+
+2014-09-16 13:43 eaxelson
+
+ * test/tools/Makefile.am: Fixed a typo in filename in EXTRA_DIST.
+
+2014-09-15 09:13 eaxelson
+
+ * libhfst/src/HfstTransducer.cc,
+ tools/src/parsers/test/Makefile.am,
+ tools/src/parsers/test/test.sh,
+ tools/src/parsers/test/xerox_composition.output,
+ tools/src/parsers/test/xerox_composition.xfst: Now alphabets are
+ copied when encoding and decoding flags in composition. Fixes bug
+ #267.
+
+2014-09-11 12:31 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: Oops,
+ forgot about restoring scope state afer entry arcs
+
+2014-09-11 12:24 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h: Add a
+ scope facility for local context boundaries
+
+2014-09-11 09:18 eaxelson
+
+ * libhfst/src/HfstTransducer.cc,
+ libhfst/src/implementations/FomaTransducer.cc,
+ libhfst/src/implementations/HfstOlTransducer.cc,
+ libhfst/src/parsers/lexc-utils.cc, tools/src/hfst-compare.cc,
+ tools/src/hfst-compose.cc, tools/src/hfst-concatenate.cc,
+ tools/src/hfst-conjunct.cc, tools/src/hfst-disjunct.cc,
+ tools/src/hfst-expand-equivalences.cc, tools/src/hfst-guess.cc,
+ tools/src/hfst-lexc-wrapper.cc, tools/src/hfst-shuffle.cc,
+ tools/src/hfst-substitute.cc, tools/src/hfst-subtract.cc,
+ tools/src/hfst-tagger/src/use_model_src/SentenceTagger.cc,
+ tools/src/parsers/xfst-utils.cc: Fixed most scan-build issues
+ other than dead store, memory leak and errors from foma back-end.
+
+2014-09-10 13:52 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ tools/src/parsers/XfstCompiler.cc: Added implementation for
+ twosided flag-diacritics in hfst-xfst.
+
+2014-09-10 12:04 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ tools/src/parsers/XfstCompiler.cc,
+ tools/src/parsers/XfstCompiler.h,
+ tools/src/parsers/xfst-parser.yy: Now harmonize-flags and
+ flag-is-epsilon are by default OFF in hfst-xfst. Also added a
+ new, mostly untested variable xerox-composition that matches
+ flags with unknowns and identities in composition (default is
+ OFF). All flag-is-epsilon functionalitites are moved under
+ HfstTransducer.
+
+2014-09-09 15:45 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Purge commented-out lines
+
+2014-09-09 15:38 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ libhfst/src/parsers/pmatch_lex.ll,
+ libhfst/src/parsers/pmatch_parse.yy: Major syntactic changes and
+ additions (AND & OR) to bring in line with
+ Karttunen's documentation - amazingly doesn't break backwards
+ compatibility (brobably / mostly)! Function syntax still lags and
+ is incomplete.
+
+2014-09-09 07:57 eaxelson
+
+ * back-ends/openfst/src/include/fst/encode.h: Added missing return
+ value to a function in openfst back-end.
+
+2014-09-09 07:32 eaxelson
+
+ * back-ends/openfst/src/include/fst/encode.h,
+ libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc,
+ libhfst/src/HarmonizeUnknownAndIdentitySymbols.h,
+ libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/parsers/XreCompiler.cc,
+ libhfst/src/parsers/XreCompiler.h,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/xre_parse.yy,
+ libhfst/src/parsers/xre_utils.cc,
+ tools/src/parsers/XfstCompiler.cc: Reverted back to version 3992.
+
+2014-09-08 11:50 eaxelson
+
+ * libhfst/src/parsers/pmatch_utils.cc: Instead of NULL, return a
+ new HfstTransducer in a condition that should never occur. This
+ is for some compilers.
+
+2014-09-08 11:41 eaxelson
+
+ * back-ends/openfst/src/include/fst/encode.h,
+ libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc,
+ libhfst/src/HarmonizeUnknownAndIdentitySymbols.h,
+ libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h,
+ libhfst/src/implementations/HfstTransitionGraph.h,
+ libhfst/src/parsers/XreCompiler.cc,
+ libhfst/src/parsers/XreCompiler.h,
+ libhfst/src/parsers/pmatch_utils.cc,
+ libhfst/src/parsers/xre_parse.yy,
+ libhfst/src/parsers/xre_utils.cc,
+ tools/src/parsers/XfstCompiler.cc: Fixed errors reported in bug
+ #265. Also tentatively added a switch --xfst-harmonization to
+ hfst-xfst that treats flags as ordinary symbols in composition.
+
+2014-09-08 11:29 hardwick
+
+ * libhfst/src/implementations/HfstTransitionGraph.h: Resolved
+ another rpmlint issue, hopefully the right way
+ (A HfstTransitionGraph method returning nothing when it seemed to
+ be meant
+ to be returning *this)
+
+2014-09-08 11:24 hardwick
+
+ * libhfst/src/parsers/pmatch_utils.cc: Resolved some issues and
+ nonissues revealeled by rpmlint
+
+2014-09-08 09:49 eaxelson
+
+ * libhfst/src/parsers/xre_lex.ll: Now xre parser compiles
+ expressions of type foo:bar^{0,N} correctly.
+
+2014-08-29 12:20 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: A better
+ way see if we have a better location than before
+
+2014-08-29 11:36 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: It seems to be generally
+ faster to defer minimization at [] boundaries
+
+2014-08-28 12:05 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc: Made 'harmonize-flags == ON'
+ the default for hfst-xfst. Also hfst-xfst's xre parser now
+ harmonizes flags according to variable 'harmonize-flags'.
+
+2014-08-28 10:06 eaxelson
+
+ * libhfst/src/parsers/xre_utils.cc, tools/src/HfstAlphabet.cc,
+ tools/src/hfst-guessify.cc, tools/src/parsers/XfstCompiler.cc:
+ Fixed some issues noticed on c+11 and reported in bug #258: a
+ space between literal and identifier, missing cstdlib header and
+ ostringstream conversions.
+
+2014-08-28 09:12 eaxelson
+
+ * tools/src/hfst-fst2strings.cc: hfst-fst2strings now gives an
+ error message if option --nbest or --random is used with
+ transducers in optimized lookup format.
+
+2014-08-28 07:11 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.h: Apparently
+ older gcc's libstdc++ require operator< to be const
+
+2014-08-27 16:31 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ tools/src/hfst-pmatch.cc, tools/src/hfst-proc2.cc: Added locate
+ mode on the library side; support for multimatch, weight
+ access and various internal changes
+
+2014-08-27 12:05 eaxelson
+
+ * tools/src/hfst-affix-guessify.cc, tools/src/hfst-commandline.cc,
+ tools/src/hfst-commandline.h, tools/src/hfst-compare.cc,
+ tools/src/hfst-compose-intersect.cc, tools/src/hfst-compose.cc,
+ tools/src/hfst-concatenate.cc, tools/src/hfst-conjunct.cc,
+ tools/src/hfst-determinize.cc, tools/src/hfst-disjunct.cc,
+ tools/src/hfst-edit-metadata.cc,
+ tools/src/hfst-expand-equivalences.cc, tools/src/hfst-invert.cc,
+ tools/src/hfst-minimize.cc, tools/src/hfst-project.cc,
+ tools/src/hfst-prune-alphabet.cc,
+ tools/src/hfst-remove-epsilons.cc, tools/src/hfst-repeat.cc,
+ tools/src/hfst-reverse.cc, tools/src/hfst-reweight.cc,
+ tools/src/hfst-shuffle.cc, tools/src/hfst-substitute.cc,
+ tools/src/hfst-subtract.cc: Now all command line tools should
+ give an error message and exit with >0 if they cannot process
+ input in hfst optimized lookup format.
+
+2014-08-27 10:11 eaxelson
+
+ * tools/src/hfst-invert.cc: Now hfst-invert prints an error message
+ if given a transducer in optimized lookup format. This should be
+ fixed in all command line tools.
+
+2014-08-27 08:24 eaxelson
+
+ * NSIS/AddHfstLibrary.nsi, NSIS/README, NSIS/copy_files.sh,
+ NSIS/hfst_python_installer.nsi: Updated NSIS files.
+
+2014-08-27 08:22 eaxelson
+
+ * swig/libhfst.i: Function set_expand_definitions added to the
+ python interface.
+
+2014-08-27 08:21 eaxelson
+
+ * swig/doc/Doxyfile: Updated HFST version number.
+
+2014-08-27 08:17 eaxelson
+
+ * Doxyfile: Updated HFST version number.
+
+2014-08-26 09:14 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/LexcCompiler.h,
+ tools/src/hfst-lexc-compiler.cc: Added option --renameFlags to
+ hfst-lexc for testing purposes.
+
+2014-08-25 13:00 eaxelson
+
+ * tools/src/hfst-determinize.cc: Added option --encode-weights to
+ hfst-determinize.
+
+2014-08-20 09:52 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/LexcCompiler.h,
+ tools/src/hfst-lexc-compiler.cc,
+ tools/src/parsers/XfstCompiler.cc: Now hfst-lexc has options
+ --withFlags and --minimizeFlags and hfst-xfst variables
+ lexc-with-flags and lexc-minimize-flags to control if
+ hyperminimization is used when parsing lexc files. Flag
+ minimization can be the default behaviour when it has been
+ properly tested.
+
+2014-08-20 06:39 moshagen
+
+ * tools/src/hfst-proc2.cc: The Xerox output should have an empty
+ line between each cohort.
+
+2014-08-19 14:45 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc: Now flag minimization in
+ should work in LexcCompiler. Commented the changes out until all
+ expected results from lexc tests are changed accordingly.
+
+2014-08-19 08:39 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Apparently failing to read a
+ stream now throws HfstException so catch that instead
+
+2014-08-19 06:51 moshagen
+
+ * tools/src: Ignore generated binaries.
+
+2014-08-18 13:53 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc: Tentatively added a piece of
+ code that filters out multiple flags in lexc result. Currently
+ commented out.
+
+2014-08-18 12:39 hardwick
+
+ * configure.ac, tools/src/Makefile.am, tools/src/hfst-proc2.cc: A
+ hopefully temporary tool called proc2 for simple tokenization,
+ to become part of something else or be renamed in the future
+
+2014-08-18 12:35 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ swig/hfst_swig_extensions.h, swig/libhfst.i: Some fixes to python
+ wrapping of pmatch and showing original input in pmatch
+
+2014-08-18 11:15 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ tools/src/hfst-pmatch.cc: Make locate() return a simple data
+ structure rather than a string
+
+2014-08-18 08:19 hardwick
+
+ * swig/hfst_swig_extensions.h, swig/libhfst.i: Convenience loader
+ function for pmatch
+
+2014-08-18 08:09 hardwick
+
+ * swig/libhfst.i: Fix outdated prototype of printConnectedness()
+ and add pmatch prototypes
+
+2014-08-14 12:51 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc,
+ libhfst/src/parsers/LexcCompiler.h, test/tools/Makefile.am,
+ test/tools/lexc-compiler-functionality.sh,
+ test/tools/warn.sublexicon-mentioned-but-not-defined.lexc,
+ test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.flag.result,
+ test/tools/warn.sublexicon-mentioned-but-not-defined.lexc.result,
+ tools/src/hfst-lexc-compiler.cc: Added option --Werror to
+ hfst-lexc that treats warnings as errors.
+
+2014-08-14 10:06 eaxelson
+
+ * test/tools/Makefile.am,
+ test/tools/lexc-compiler-functionality.sh,
+ test/tools/no-newline-before-sublexicon.lexc,
+ test/tools/no-newline-before-sublexicon.lexc.flag.result,
+ test/tools/no-newline-before-sublexicon.lexc.result: Added tests
+ for the fixed bug #243 that will pass now.
+
+2014-08-14 10:05 eaxelson
+
+ * libhfst/src/parsers/lexc-lexer.ll: Now keyword LEXICON is allowed
+ without a preceding newline in lexc parser. Should fix bug #243.
+
+2014-08-13 14:07 eaxelson
+
+ * libhfst/src/HfstTransducer.cc, libhfst/src/HfstTransducer.h:
+ Added a variable encode_epsilons to function priority_union whose
+ default value (true) should fix bug #254. Function lenient
+ composition still calls priority_union with a false value of
+ encode_epsilons.
+
+2014-08-06 13:43 eaxelson
+
+ * libhfst/src/parsers/xre_parse.yy: Added support for
+ cross-products such as {foo}:[bar] and foo:[bar] in regexp
+ parser. Tests still needed.
+
+2014-07-30 20:33 mie
+
+ * back-ends/foma/Makefile.am: .NOTPARALLEL for now
+
+2014-07-30 10:48 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: Fix
+ serious bug with epsilons in contexts - they were incorrectly
+ moving the
+ input tape.
+
+2014-07-19 13:15 mie
+
+ * tools/src/hfst-lookup.cc: with pipes
+
+2014-07-19 12:59 mie
+
+ * tools/src/hfst-lookup.cc: A super cool progress bar functionality
+ \o/
+
+2014-07-15 15:42 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: Fix
+ context-related bug
+
+2014-07-05 08:22 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h:
+ Readability refactoring
+
+2014-07-04 21:18 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h: Comment
+ fixes and some more deadwood elimination
+
+2014-07-04 21:12 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.h: Remove
+ more unneeded stuff
+
+2014-07-04 21:07 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.h: Remove
+ some obsolete & commented out bits
+
+2014-07-04 21:04 hardwick
+
+ * tools/src/hfst-pmatch.cc: Locatefy option
+
+2014-07-04 21:04 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/transducer.cc: Don't
+ make noise about almost always harmless result truncation
+
+2014-07-04 21:03 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h: Large
+ refactor of almost everything
+ (mainly to better serve tokenization apps w/tape synchronisation)
+
+2014-06-25 07:14 hardwick
+
+ * libhfst/src/implementations/Makefile.am,
+ libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.cc: Undo
+ previous commit, was confused by polluted automake cache
+
+2014-06-25 06:59 hardwick
+
+ * libhfst/src/implementations/Makefile.am,
+ libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.cc: It
+ seems that newer toolchains don't like c++ classes being defined
+ over
+ multiple .cc files, or something..
+
+2014-06-18 09:10 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: Don't
+ count flag diacritics when locatefying
+
+2014-06-17 21:11 hardwick
+
+ * libhfst/src/parsers/pmatch_lex.ll: Remove obsolete bogus -> rule
+ from when @-> had problems
+
+2014-06-17 21:10 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Handle missing @bin files
+ more nicely
+
+2014-06-17 20:03 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Autoconvert @bin arguments
+ if necessary
+
+2014-06-16 18:00 hardwick
+
+ * README.deps: Remove obsolete dependency readme that hogs the svn
+ trunk page
+
+2014-06-13 20:47 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.cc,
+ libhfst/src/implementations/optimized-lookup/transducer.h: Add
+ constructor option to make diacritic strings blank or not;
+ make pmatch consider them blank
+
+2014-06-13 12:38 eaxelson
+
+ * tools/src/hfst-twolc/test/Makefile.am: Added file 'test' to
+ EXTRA_DIST..
+
+2014-06-13 12:15 mpsilfve
+
+ * tools/src/hfst-twolc/test/Makefile.am: Fixed hfst-twolc test
+ Makefile.am
+
+2014-06-13 10:49 mpsilfve
+
+ * tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc,
+ tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc: Disabled
+ outdated unit tests.
+
+2014-06-12 12:15 eaxelson
+
+ * tools/src/parsers/XfstCompiler.cc: Tentatively implemented
+ variable 'flag-is-epsilon' in hfst-xfst.
+
+2014-06-10 14:34 mpsilfve
+
+ * tools/src/hfst-twolc/src/htwolcpre3.yy,
+ tools/src/hfst-twolc/src/rule_src/ConflictResolvingLeftArrowRule.cc:
+ Fixed issue with left arrow conflicts for contexts with
+ impossible word boundaries.
+
+2014-06-10 13:51 eaxelson
+
+ * test/tools/Makefile.am, test/tools/basic.regexps.lexc,
+ test/tools/basic.regexps.lexc.flag.result,
+ test/tools/basic.regexps.lexc.result,
+ test/tools/lexc-compiler-functionality.sh: Also added tests for
+ regexps with different continuation lexicons in hfst-lexc.
+
+2014-06-10 13:50 eaxelson
+
+ * libhfst/src/parsers/LexcCompiler.cc: Now regexps with different
+ continuation lexicons are allowed in hfst-lexc. Should fix bug
+ #247
+
+2014-06-10 13:45 mpsilfve
+
+ * tools/src/hfst-twolc/src/rule_src/TwolCGrammar.cc,
+ tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.cc,
+ tools/src/hfst-twolc/src/variable_src/RuleSymbolVector.h: Fixed
+ list center rules and conflict resolutions.
+
+2014-06-10 13:41 mpsilfve
+
+ * tools/src/hfst-twolc/test/test62~: Tests for list centers.
+
+2014-06-10 13:40 mpsilfve
+
+ * tools/src/hfst-twolc/test/Makefile.am,
+ tools/src/hfst-twolc/test/test46.txt_fst,
+ tools/src/hfst-twolc/test/test61.txt_fst,
+ tools/src/hfst-twolc/test/test62,
+ tools/src/hfst-twolc/test/test62.txt_fst,
+ tools/src/hfst-twolc/test/test62~: Tests for list centers.
+
+2014-06-08 12:32 hardwick
+
+ * libhfst/src/parsers/pmatch_utils.cc: Exit on parse error
+ Once we've printed a parsing error, exit rather that print
+ intimidating hfst
+ exception messages
+
+2014-06-05 13:13 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h:
+ Pattern-locating mode
+
+2014-06-03 15:37 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h: Some
+ changes to approach to avoiding stack overflow
+ should fix problems with very long input lines
+
+2014-06-03 15:27 mie
+
+ * tools/src/hfst-affix-guessify.cc: Add harmonization
+
+2014-06-03 10:20 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc,
+ libhfst/src/implementations/optimized-lookup/pmatch.h,
+ tools/src/hfst-pmatch.cc: Add extract tags -option
+ (this commit also includes a secret commented-out facility for
+ profiling
+ activity on the input tape by drawing a funky ascii histogram)
+
+2014-06-02 11:40 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/transducer.cc: Fix
+ incorrect check for whether tokenization needs to respect a long
+ symbol
+
+2014-06-02 09:32 hardwick
+
+ * libhfst/src/parsers/pmatch_parse.yy: Allow epsilon-symbol pairs
+
+2014-06-02 09:31 hardwick
+
+ * libhfst/src/parsers/pmatch_utils.cc: Don't print huge data dumps
+ when parsing fails
+
+2014-05-27 12:15 eaxelson
+
+ * libhfst/src/implementations/Makefile.am,
+ libhfst/src/implementations/TropicalWeightTransducer.cc,
+ libhfst/src/implementations/TropicalWeightTransducerMinimize.cc,
+ libhfst/src/implementations/TropicalWeightTransducerPushWeights.cc:
+ Revert back to the original TropicalWeightTransducer.cc and
+ forget splitting it..
+
+2014-05-23 14:42 eaxelson
+
+ * libhfst/src/implementations/Makefile.am,
+ libhfst/src/implementations/TropicalWeightTransducer.cc,
+ libhfst/src/implementations/TropicalWeightTransducer.h,
+ libhfst/src/implementations/TropicalWeightTransducerMinimize.cc,
+ libhfst/src/implementations/TropicalWeightTransducerPushWeights.cc:
+ Moved minimize and push functionalities from
+ TropicalWeightTransducer.cc to separate files. This will make
+ compiling TropicalWeightTransducer faster and prevent 'File too
+ big' errors on Windows.
+
+2014-05-21 11:27 hardwick
+
+ * libhfst/src/implementations/optimized-lookup/pmatch.cc: While
+ scanning for possible first symbols, don't accidentally go beyond
+ the
+ end of the input tape
+
+2014-05-20 10:26 eaxelson
+
+ * ChangeLog, ChangeLog.old, NEWS, configure.ac,
+ libhfst/src/Makefile.am, swig/setup.py: Ready for release 3.7.1.
+
+2014-05-20 09:43 eaxelson
+
+ * tools/src/parsers/test/test.sh: Now removing CR characters from
+ output of hfst-xfst so that tests will pass on windows too.
+
2014-05-16 11:14 eaxelson
* back-ends/sfst/alphabet.cc: Now an error message is thrown in
diff --git a/NEWS b/NEWS
index b23945f..5c1c381 100644
--- a/NEWS
+++ b/NEWS
@@ -5,9 +5,11 @@
This file contains all noteworthy changes in HFST development between releases.
For full listing of changes see ChangeLog.
-Noteworthy changes in 3.8.0
+Noteworthy changes in 3.8.1
---------------------------
+* (the release of version 3.8.0 was delayed, its changes are also listed here)
+
* changes to flag diacritic handling
* now hfst-lexc has options --withFlags, --minimizeFlags and --renameFlags to control
@@ -29,6 +31,8 @@ Noteworthy changes in 3.8.0
* fixes to pmatch and optimized-lookup functionalities
+* improvements to speed issues in optimized-lookup
+
* fixes to twolc functionalities
* fixed some issues found by scan-build
diff --git a/back-ends/openfst/src/include/fst/accumulator.h b/back-ends/openfst/src/include/fst/accumulator.h
index 665c373..0048bad 100644
--- a/back-ends/openfst/src/include/fst/accumulator.h
+++ b/back-ends/openfst/src/include/fst/accumulator.h
@@ -24,7 +24,7 @@
#include <algorithm>
#include <functional>
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/arc-map.h b/back-ends/openfst/src/include/fst/arc-map.h
index 065c0ca..d898bed 100644
--- a/back-ends/openfst/src/include/fst/arc-map.h
+++ b/back-ends/openfst/src/include/fst/arc-map.h
@@ -23,7 +23,7 @@
#ifndef FST_LIB_ARC_MAP_H__
#define FST_LIB_ARC_MAP_H__
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/determinize.h b/back-ends/openfst/src/include/fst/determinize.h
index d045bc8..d5bcafe 100644
--- a/back-ends/openfst/src/include/fst/determinize.h
+++ b/back-ends/openfst/src/include/fst/determinize.h
@@ -25,7 +25,7 @@
#include <algorithm>
#include <climits>
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/encode.h b/back-ends/openfst/src/include/fst/encode.h
index 6593052..0f77535 100644
--- a/back-ends/openfst/src/include/fst/encode.h
+++ b/back-ends/openfst/src/include/fst/encode.h
@@ -23,7 +23,7 @@
#include <climits>
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/epsnormalize.h b/back-ends/openfst/src/include/fst/epsnormalize.h
index 6109b4c..faec821 100644
--- a/back-ends/openfst/src/include/fst/epsnormalize.h
+++ b/back-ends/openfst/src/include/fst/epsnormalize.h
@@ -21,7 +21,7 @@
#ifndef FST_LIB_EPSNORMALIZE_H__
#define FST_LIB_EPSNORMALIZE_H__
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/equivalent.h b/back-ends/openfst/src/include/fst/equivalent.h
index ba3ddf1..e565881 100644
--- a/back-ends/openfst/src/include/fst/equivalent.h
+++ b/back-ends/openfst/src/include/fst/equivalent.h
@@ -24,7 +24,7 @@
#include <algorithm>
#include <deque>
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/factor-weight.h b/back-ends/openfst/src/include/fst/factor-weight.h
index 367b88e..cde0192 100644
--- a/back-ends/openfst/src/include/fst/factor-weight.h
+++ b/back-ends/openfst/src/include/fst/factor-weight.h
@@ -23,7 +23,7 @@
#include <algorithm>
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/interval-set.h b/back-ends/openfst/src/include/fst/interval-set.h
index cf6ac54..ab64020 100644
--- a/back-ends/openfst/src/include/fst/interval-set.h
+++ b/back-ends/openfst/src/include/fst/interval-set.h
@@ -218,7 +218,7 @@ void IntervalSet<T>::Intersect(const IntervalSet<T> &iset,
interval.end = min(it1->end, it2->end);
ointervals->push_back(interval);
oset->count_ += interval.end - interval.begin;
- if (it1->end < it2->end)
+ if ((it1->end) < (it2->end))
++it1;
else
++it2;
@@ -240,14 +240,14 @@ void IntervalSet<T>::Complement(T maxval, IntervalSet<T> *oset) const {
it != intervals_.end();
++it) {
interval.end = min(it->begin, maxval);
- if (interval.begin < interval.end) {
+ if ((interval.begin) < (interval.end)) {
ointervals->push_back(interval);
oset->count_ += interval.end - interval.begin;
}
interval.begin = it->end;
}
interval.end = maxval;
- if (interval.begin < interval.end) {
+ if ((interval.begin) < (interval.end)) {
ointervals->push_back(interval);
oset->count_ += interval.end - interval.begin;
}
@@ -348,7 +348,7 @@ bool IntervalSet<T>::Contains(const IntervalSet<T> &iset) const {
while (it1 != intervals_.end() && it2 != intervals->end()) {
if (it1->end <= it2->begin) { // no overlap - it1 first
++it1;
- } else if (it2->begin < it1->begin || it2->end > it1->end) { // no C
+ } else if ((it2->begin) < (it1->begin) || (it2->end) > (it1->end)) { // no C
return false;
} else if (it2->end == it1->end) {
++it1;
diff --git a/back-ends/openfst/src/include/fst/label-reachable.h b/back-ends/openfst/src/include/fst/label-reachable.h
index 0c205a5..2d69ec3 100644
--- a/back-ends/openfst/src/include/fst/label-reachable.h
+++ b/back-ends/openfst/src/include/fst/label-reachable.h
@@ -23,7 +23,7 @@
#ifndef FST_LIB_LABEL_REACHABLE_H__
#define FST_LIB_LABEL_REACHABLE_H__
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/relabel.h b/back-ends/openfst/src/include/fst/relabel.h
index 162ed5b..8fb78ca 100644
--- a/back-ends/openfst/src/include/fst/relabel.h
+++ b/back-ends/openfst/src/include/fst/relabel.h
@@ -21,7 +21,7 @@
#ifndef FST_LIB_RELABEL_H__
#define FST_LIB_RELABEL_H__
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/replace-util.h b/back-ends/openfst/src/include/fst/replace-util.h
index 8161286..9a5e52a 100644
--- a/back-ends/openfst/src/include/fst/replace-util.h
+++ b/back-ends/openfst/src/include/fst/replace-util.h
@@ -26,7 +26,7 @@
#include <vector>
using std::vector;
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
@@ -36,7 +36,7 @@ using std::unordered_map;
using std::unordered_multimap;
#endif
-#ifdef HAVE_TR1_UNORDERED_SET
+#ifdef USE_TR1_UNORDERED_SET
#include <tr1/unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
diff --git a/back-ends/openfst/src/include/fst/replace.h b/back-ends/openfst/src/include/fst/replace.h
index 03c9c38..d82c53a 100644
--- a/back-ends/openfst/src/include/fst/replace.h
+++ b/back-ends/openfst/src/include/fst/replace.h
@@ -22,7 +22,7 @@
#ifndef FST_LIB_REPLACE_H__
#define FST_LIB_REPLACE_H__
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/rmepsilon.h b/back-ends/openfst/src/include/fst/rmepsilon.h
index 171835f..4f84b27 100644
--- a/back-ends/openfst/src/include/fst/rmepsilon.h
+++ b/back-ends/openfst/src/include/fst/rmepsilon.h
@@ -21,7 +21,7 @@
#ifndef FST_LIB_RMEPSILON_H__
#define FST_LIB_RMEPSILON_H__
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/rmfinalepsilon.h b/back-ends/openfst/src/include/fst/rmfinalepsilon.h
index 8f7f4a4..7af3d63 100644
--- a/back-ends/openfst/src/include/fst/rmfinalepsilon.h
+++ b/back-ends/openfst/src/include/fst/rmfinalepsilon.h
@@ -21,7 +21,7 @@
#ifndef FST_LIB_RMFINALEPSILON_H__
#define FST_LIB_RMFINALEPSILON_H__
-#ifdef HAVE_TR1_UNORDERED_SET
+#ifdef USE_TR1_UNORDERED_SET
#include <tr1/unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
diff --git a/back-ends/openfst/src/include/fst/sparse-tuple-weight.h b/back-ends/openfst/src/include/fst/sparse-tuple-weight.h
index 935a71a..0b2aaaf 100644
--- a/back-ends/openfst/src/include/fst/sparse-tuple-weight.h
+++ b/back-ends/openfst/src/include/fst/sparse-tuple-weight.h
@@ -34,7 +34,7 @@
#include<list>
#include<stack>
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include<tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
@@ -141,7 +141,7 @@ class SparseTupleWeight {
// Assumes H() function exists for the hash of the key value
size_t Hash() const {
uint64 h = 0;
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
std::tr1::hash<K> H;
#else
std::hash<K> H;
diff --git a/back-ends/openfst/src/include/fst/state-map.h b/back-ends/openfst/src/include/fst/state-map.h
index 5da0b74..78c3b76 100644
--- a/back-ends/openfst/src/include/fst/state-map.h
+++ b/back-ends/openfst/src/include/fst/state-map.h
@@ -24,7 +24,7 @@
#include <algorithm>
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
diff --git a/back-ends/openfst/src/include/fst/symbol-table-ops.h b/back-ends/openfst/src/include/fst/symbol-table-ops.h
index edefaf8..34e5dcd 100644
--- a/back-ends/openfst/src/include/fst/symbol-table-ops.h
+++ b/back-ends/openfst/src/include/fst/symbol-table-ops.h
@@ -21,7 +21,7 @@
using std::vector;
#include <string>
-#ifdef HAVE_TR1_UNORDERED_SET
+#ifdef USE_TR1_UNORDERED_SET
#include <tr1/unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
diff --git a/back-ends/openfst/src/include/fst/synchronize.h b/back-ends/openfst/src/include/fst/synchronize.h
index 3d9560e..1ae087d 100644
--- a/back-ends/openfst/src/include/fst/synchronize.h
+++ b/back-ends/openfst/src/include/fst/synchronize.h
@@ -23,7 +23,7 @@
#include <algorithm>
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
@@ -33,7 +33,7 @@ using std::unordered_map;
using std::unordered_multimap;
#endif
-#ifdef HAVE_TR1_UNORDERED_SET
+#ifdef USE_TR1_UNORDERED_SET
#include <tr1/unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
diff --git a/back-ends/openfst/src/include/fst/test-properties.h b/back-ends/openfst/src/include/fst/test-properties.h
index 480c0a6..a6186ee 100644
--- a/back-ends/openfst/src/include/fst/test-properties.h
+++ b/back-ends/openfst/src/include/fst/test-properties.h
@@ -21,7 +21,7 @@
#ifndef FST_LIB_TEST_PROPERTIES_H__
#define FST_LIB_TEST_PROPERTIES_H__
-#ifdef HAVE_TR1_UNORDERED_SET
+#ifdef USE_TR1_UNORDERED_SET
#include <tr1/unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
diff --git a/back-ends/openfst/src/include/fst/util.h b/back-ends/openfst/src/include/fst/util.h
index 0141b5a..9ab16cf 100644
--- a/back-ends/openfst/src/include/fst/util.h
+++ b/back-ends/openfst/src/include/fst/util.h
@@ -21,7 +21,7 @@
#ifndef FST_LIB_UTIL_H__
#define FST_LIB_UTIL_H__
-#ifdef HAVE_TR1_UNORDERED_MAP
+#ifdef USE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
using std::tr1::unordered_map;
using std::tr1::unordered_multimap;
@@ -31,7 +31,7 @@ using std::unordered_map;
using std::unordered_multimap;
#endif
-#ifdef HAVE_TR1_UNORDERED_SET
+#ifdef USE_TR1_UNORDERED_SET
#include <tr1/unordered_set>
using std::tr1::unordered_set;
using std::tr1::unordered_multiset;
diff --git a/configure.ac b/configure.ac
index 9d08f20..9089334 100644
--- a/configure.ac
+++ b/configure.ac
@@ -19,7 +19,7 @@
HFST_NAME=hfst
HFST_MAJOR=3
HFST_MINOR=8
-HFST_EXTENSION=0
+HFST_EXTENSION=1
HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
### When the VERSION is INCREMENTED, REMEMBER to increment the LONGVERSION too.
@@ -28,10 +28,10 @@ HFST_VERSION=$HFST_MAJOR.$HFST_MINOR.$HFST_EXTENSION
LIBHFST_NAME=hfst
LIBHFST_MAJOR=3
LIBHFST_MINOR=8
-LIBHFST_EXTENSION=0
+LIBHFST_EXTENSION=1
LIBHFST_VERSION=$LIBHFST_MAJOR.$LIBHFST_MINOR.$LIBHFST_EXTENSION
-AC_INIT([hfst], [3.8.0], [hfst-bugs at helsinki.fi], [hfst])
+AC_INIT([hfst], [3.8.1], [hfst-bugs at helsinki.fi], [hfst])
AC_CONFIG_AUX_DIR([build-aux])
AM_INIT_AUTOMAKE([-Wall std-options foreign check-news])
@@ -44,8 +44,8 @@ AC_CONFIG_HEADERS([config.h libhfst/src/hfst.hpp])
AC_SUBST([LIBHFST_MAJOR], [3])
AC_SUBST([LIBHFST_MINOR], [8])
-AC_SUBST([LIBHFST_EXTENSION], [0])
-AC_SUBST([LIBHFST_VERSION], [3.8.0])
+AC_SUBST([LIBHFST_EXTENSION], [1])
+AC_SUBST([LIBHFST_VERSION], [3.8.1])
AC_SUBST([LIBHFST_NAME], [hfst])
# long version = version vector cast in base 10000, for automatic comparisons
@@ -56,9 +56,9 @@ AC_SUBST([LIBHFST_NAME], [hfst])
# $LIBHFST_MINOR * 10000 + $LIBHFST_EXTENSION + "L"
# NB! It turned out to be not portable, and can't be used!
-AC_DEFINE([HFST_LONGVERSION], [300080000L],
+AC_DEFINE([HFST_LONGVERSION], [300080001L],
[Define to hfst version vector as long in base 10000])
-AC_DEFINE([HFST_REVISION], ["$Revision: 4029 $"],
+AC_DEFINE([HFST_REVISION], ["$Revision: 4088 $"],
[Automatically substitute to configure.ac revision])
AC_DEFINE_UNQUOTED([HFST_STRING], ["$PACKAGE_STRING"],
[Define to libhfst pretty name for programs to print])
@@ -629,6 +629,9 @@ AC_CHECK_HEADERS([limits.h stdlib.h string.h error.h glob.h locale.h langinfo.h]
AC_CHECK_HEADERS([libxml/tree.h libxml/mem.h])
AC_LANG_PUSH([C++])
+# Whether std::unordered_map and std::unordered_set should be used
+AS_IF([echo "$CXXFLAGS" | grep '\-std' | grep -E '((gnu\+\+)|(c\+\+))((11)|(0x)|(14)|(1y)|(17)|(1z))' > /dev/null 2> /dev/null], [std_xx_11_requested=yes])
+
AC_CHECK_HEADERS([tr1/unordered_map], [], [],
[#ifdef HAVE_TR1_UNORDERED_MAP
#include <tr1/unordered_map>
@@ -651,11 +654,21 @@ AC_CHECK_HEADERS([unordered_set], [], [],
#endif
])
+AS_IF([test "$std_xx_11_requested" = "yes" -a "x$ac_cv_header_unordered_map" = "xno"],
+ [AC_MSG_ERROR(["<unordered_map> not found, it is needed in c++ standard 11"])])
+AS_IF([test "$std_xx_11_requested" = "yes" -a "x$ac_cv_header_unordered_set" = "xno"],
+ [AC_MSG_ERROR(["<unordered_set> not found, it is needed in c++ standard 11"])])
+
AS_IF([test "x$ac_cv_header_tr1_unordered_map" = "xno" -a "x$ac_cv_header_unordered_map" = "xno"],
[AC_MSG_ERROR(["unordered_map not found"])])
AS_IF([test "x$ac_cv_header_tr1_unordered_set" = "xno" -a "x$ac_cv_header_unordered_set" = "xno"],
[AC_MSG_ERROR(["unordered_set not found"])])
+# Define if tr1/unordered_maps and _sets should be used
+AS_IF([test "$std_xx_11_requested" != "yes" -a "x$ac_cv_header_tr1_unordered_map" != "xno"], AC_DEFINE([USE_TR1_UNORDERED_MAP], [1], [Use tr1/unordered_map]))
+AS_IF([test "$std_xx_11_requested" != "yes" -a "x$ac_cv_header_tr1_unordered_set" != "xno"], AC_DEFINE([USE_TR1_UNORDERED_SET], [1], [Use tr1/unordered_set]))
+
+
AC_CHECK_HEADERS([ext/slist])
#AS_IF([test "x$with_sfst" != "xno" -a "x$enable_windows" == "xno"],
@@ -694,6 +707,7 @@ AC_CHECK_DECLS([program_name, program_invocation_name, program_invocation_short_
AC_CHECK_DECLS([rl_completion_suppress_append, rl_completion_matches])
AC_CHECK_FUNCS([floor strchr strdup strerror strncasecmp strcspn strtol strtoul error error_at_line strndup getline getdelim getopt_long strtod xstrdup set_program_name setprogname setlocale nl_langinfo])
+
# Checks for system services
# config files
diff --git a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
index 1fd6093..c666cde 100644
--- a/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
+++ b/libhfst/src/HarmonizeUnknownAndIdentitySymbols.cc
@@ -3,6 +3,7 @@
// --- HfstTransitionGraph.h is enough
#include "implementations/HfstTransitionGraph.h"
#include "HfstFlagDiacritics.h"
+#include "implementations/optimized-lookup/pmatch.h"
#ifndef MAIN_TEST
@@ -37,7 +38,7 @@ static StringSet remove_flags(const StringSet & alpha)
for (StringSet::const_iterator it = alpha.begin();
it != alpha.end(); it++)
{
- if (!FdOperation::is_diacritic(*it))
+ if (!FdOperation::is_diacritic(*it) && !hfst_ol::PmatchAlphabet::is_special(*it))
{
retval.insert(*it);
}
@@ -118,9 +119,13 @@ HarmonizeUnknownAndIdentitySymbols::HarmonizeUnknownAndIdentitySymbols
harmonize_unknown_symbols(t2,t1_symbols_minus_t2_symbols);
// Add new symbols to the alphabets of the transducers.
-
- add_symbols_to_alphabet(t1,t2_symbols_minus_t1_symbols);
- add_symbols_to_alphabet(t2,t1_symbols_minus_t2_symbols);
+ // We used to only add the difference of the alphabets,
+ // but that way we miss the things removed by remove_flags()
+ // which we do want. Since we'd be iterating over them again
+ // to collect them, might as well just add everything.
+ add_symbols_to_alphabet(t1,t2.get_alphabet());
+ add_symbols_to_alphabet(t2,t1.get_alphabet());
+
if (debug_harmonize)
{
diff --git a/libhfst/src/HfstInputStream.h b/libhfst/src/HfstInputStream.h
index a240620..106255e 100644
--- a/libhfst/src/HfstInputStream.h
+++ b/libhfst/src/HfstInputStream.h
@@ -75,7 +75,7 @@ while (not in->is_eof())
exit(1);
}
HfstTransducer t(*in);
- std::cerr << "One transducer succesfully read." << std::endl;
+ std::cerr << "One transducer successfully read." << std::endl;
transducers_read++;
}
diff --git a/libhfst/src/Makefile.am b/libhfst/src/Makefile.am
index 6086af9..93f21d9 100644
--- a/libhfst/src/Makefile.am
+++ b/libhfst/src/Makefile.am
@@ -97,7 +97,7 @@ HFST_HDRS = \
hfstinclude_HEADERS = $(HFST_HDRS)
-libhfst_la_LDFLAGS = -no-undefined -version-info 38:0:0
+libhfst_la_LDFLAGS = -no-undefined -version-info 39:0:0
LIBHFST_TSTS=HfstApply HfstInputStream HfstTransducer \
HfstOutputStream HfstXeroxRules HfstRules HfstSymbolDefs \
diff --git a/libhfst/src/implementations/ConvertTransducerFormat.h b/libhfst/src/implementations/ConvertTransducerFormat.h
index baf5297..7dea891 100644
--- a/libhfst/src/implementations/ConvertTransducerFormat.h
+++ b/libhfst/src/implementations/ConvertTransducerFormat.h
@@ -38,6 +38,7 @@
#ifndef _FOMALIB_H_
#define _FOMALIB_H_
#include <stdbool.h>
+#define _Bool bool
#include "back-ends/foma/fomalib.h"
#endif // _FOMALIB_H_
#endif // HAVE_FOMA
diff --git a/libhfst/src/implementations/FomaTransducer.h b/libhfst/src/implementations/FomaTransducer.h
index 58eedd4..16161bf 100644
--- a/libhfst/src/implementations/FomaTransducer.h
+++ b/libhfst/src/implementations/FomaTransducer.h
@@ -22,6 +22,7 @@
#ifndef _FOMALIB_H_
#define _FOMALIB_H_
+#define _Bool bool
#include "back-ends/foma/fomalib.h"
#endif
diff --git a/libhfst/src/implementations/HfstTransitionGraph.h b/libhfst/src/implementations/HfstTransitionGraph.h
index ae7cb41..24096f2 100644
--- a/libhfst/src/implementations/HfstTransitionGraph.h
+++ b/libhfst/src/implementations/HfstTransitionGraph.h
@@ -51,6 +51,11 @@
/** @brief The number of a state in an HfstTransitionGraph. */
typedef unsigned int HfstState;
+ typedef std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > HfstReplacement;
+ typedef std::vector<HfstReplacement> HfstReplacements;
+ typedef std::map<HfstState, HfstReplacements > HfstReplacementsMap;
+
+
/** @brief A simple transition graph format that consists of
states and transitions between those states.
@@ -1176,7 +1181,7 @@
// Extract input and output symbols, if possible, from prolog arc
// \a str and store them to \a isymbol and \a osymbol.
- // Return whether symbols were succesfully extracted.
+ // Return whether symbols were successfully extracted.
// \a str must be of format "foo":"bar" or "foo"
static bool get_prolog_arc_symbols
(const std::string & str, std::string & isymbol, std::string & osymbol)
@@ -1389,7 +1394,7 @@
}
// Try to get a line from \a is (if \a file == NULL)
- // or from \a file. If succesfull, strip the line from newlines,
+ // or from \a file. If successfull, strip the line from newlines,
// increment \a linecount by one and return the line.
// Else, throw an EndOfStreamException.
static std::string get_stripped_line
@@ -3855,6 +3860,508 @@
}
+ void check_regexp_state_for_cycle(HfstState s, const std::set<HfstState> & states_visited)
+ {
+ if (states_visited.find(s) != states_visited.end())
+ {
+ throw "error: loop detected inside compile-replace regular expression";
+ }
+ }
+
+ // Returns whether tr is "^]":"^]". If tr is not allowed, throws an error message.
+ bool check_regexp_transition_end(const HfstBasicTransition & tr)
+ {
+ std::string istr = tr.get_input_symbol();
+ std::string ostr = tr.get_output_symbol();
+ if (is_special_symbol(istr) || is_special_symbol(ostr))
+ {
+ throw "error: special symbol detected in compile-replace regular expression";
+ }
+ if (("^[" == istr) || ("^[" == ostr))
+ {
+ throw "error: ^[ detected inside compile-replace regular expression";
+ }
+ if (("^]" == istr) || ("^]" == ostr))
+ {
+ if (istr != ostr)
+ {
+ throw "error: ^] detected on only one side of transition inside compile-replace regular expression";
+ }
+ return true;
+ }
+ return false;
+ // weights?
+ // flag diacritics?
+ }
+
+ // If there is a "^[":"^[" transition leading to state \a s from some state
+ // S and state S is included in \a states_visited and \a path and \a full_paths
+ // are empty, this function can be used to find all (sub-)paths of form
+ // [x:y]* "^]" (x and y cannot be "^]" or "^[") starting from state \a s. The resulting
+ // paths are stored in \a full_paths. \a path is used to keep track of each path so
+ // far. Weights are currently ignored.
+ void find_regexp_paths
+ (HfstState s,
+ std::set<HfstState> & states_visited,
+ std::vector<std::pair<std::string, std::string> > & path,
+ HfstReplacements & full_paths)
+ {
+ // no cycles allowed inside "^[" and "^]"
+ check_regexp_state_for_cycle(s, states_visited);
+ states_visited.insert(s);
+
+ // go through all transitions
+ const HfstBasicTransducer::HfstTransitions &transitions
+ = this->operator[](s);
+ for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ = transitions.begin();
+ it != transitions.end(); it++)
+ {
+ // closing bracket..
+ if (check_regexp_transition_end(*it)) // throws error message if *it is not a valid transition
+ {
+ // ..cannot lead to a state already visited..
+ check_regexp_state_for_cycle(it->get_target_state(), states_visited);
+ // ..but else we can add the expression that it ends to the results
+ full_paths.push_back
+ (HfstReplacement(it->get_target_state(), path));
+ }
+ // add transition to path and call function again for its target state
+ else
+ {
+ path.push_back(StringPair(it->get_input_symbol(), it->get_output_symbol()));
+ find_regexp_paths
+ (it->get_target_state(),
+ states_visited,
+ path,
+ full_paths);
+ path.pop_back();
+ }
+ }
+ // all transitions gone through
+ states_visited.erase(s);
+ }
+
+ // For each "^[":"^[" transition in state \a s, find continuing paths of form [x:y]* "^]":"^]"
+ // (where x and y can freely be any symbols except "^]" or "^[") and store those paths in \a full_paths
+ // vector where the first member of each element is the state where the ending "^]":"^]" transition
+ // leads to and the second element is a vector of transitions (i.e. string pairs) without the ending
+ // "^]":"^]" transition.
+ // An error is thrown if mismatched "^[" or "^]" symbols, special symbols (epsilon, unknown, identity),
+ // or loops are encountered on a regexp path. Final states are allowed on regexp paths as they are also
+ // allowed by Xerox tools.
+ // Weights are currently ignored.
+ void find_regexp_paths
+ (HfstState s,
+ std::vector<std::pair<HfstState, std::vector<std::pair<std::string, std::string> > > > & full_paths)
+ {
+ // go through all transitions
+ const HfstBasicTransducer::HfstTransitions &transitions
+ = this->operator[](s);
+ for (HfstBasicTransducer::HfstTransitions::const_iterator it
+ = transitions.begin();
+ it != transitions.end(); it++)
+ {
+ std::string istr = it->get_input_symbol();
+ std::string ostr = it->get_output_symbol();
+ if ("^[" == istr || "^[" == ostr)
+ {
+ if (istr != ostr)
+ {
+ throw "error: ^[ detected on only one side of transition";
+ }
+ std::set<HfstState> states_visited;
+ states_visited.insert(s);
+ std::vector<std::pair<std::string, std::string> > path;
+ find_regexp_paths(it->get_target_state(), states_visited, path, full_paths);
+ fprintf(stderr, "%u regexp paths found for state %u\n", (unsigned int)full_paths.size(), s);
+ }
+ }
+ }
+
+ // Find all subpaths of form "^[" [x:y]* "^]" (x and y cannot be "^[" or "^]") and return them.
+ // retval[start_state] == vector(pair(end_state, vector(pair(isymbol,osymbol) ) ) )
+ // Weights are currently ignored.
+ HfstReplacementsMap find_replacements()
+ {
+ HfstReplacementsMap replacements;
+ unsigned int state = 0;
+ for (iterator it = begin(); it != end(); it++)
+ {
+ fprintf(stderr, "state %u......\n", state);
+ HfstReplacements full_paths;
+ find_regexp_paths(state, full_paths);
+ if (full_paths.size() > 0)
+ {
+ replacements[state] = full_paths;
+ }
+ state++;
+ }
+ return replacements;
+ }
+
+ // Attach a copy of \a graph between states \a state1 and \a state2 with epsilon transitions.
+ // There will be an epsilon transition with weight zero from state \a state1 to the
+ // initial state of \a graph and one epsilon transition from each final state of
+ // \a graph to state \a state2 with a weight of that state's final weight. Final states of
+ // \a graph as well as its initial state are made normal non-final states when copying \a graph.
+ // Todo: copy alphabet? harmonize graphs?
+ void insert_transducer(HfstState state1, HfstState state2, const HfstTransitionGraph & graph)
+ {
+ HfstState offset = add_state();
+ HfstState source_state=0;
+ for (const_iterator it = graph.begin();
+ it != graph.end(); it++)
+ {
+ for (typename HfstTransitions::const_iterator tr_it
+ = it->begin();
+ tr_it != it->end(); tr_it++)
+ {
+ C data = tr_it->get_transition_data();
+
+ HfstTransition <C> transition
+ (tr_it->get_target_state() + offset,
+ data.get_input_symbol(),
+ data.get_output_symbol(),
+ data.get_weight());
+
+ add_transition(source_state + offset, transition);
+ }
+ source_state++;
+ }
+
+ // Epsilon transitions
+ for (typename FinalWeightMap::const_iterator it
+ = graph.final_weight_map.begin();
+ it != graph.final_weight_map.end(); it++)
+ {
+ HfstTransition <C> epsilon_transition
+ (state2, C::get_epsilon(), C::get_epsilon(),
+ it->second);
+ add_transition(it->first + offset, epsilon_transition);
+ }
+
+ // Initial transition
+ HfstTransition <C> epsilon_transition
+ (offset, C::get_epsilon(), C::get_epsilon(), 0);
+ add_transition(state1, epsilon_transition);
+ }
+
+ typedef std::pair<HfstState, HfstState> StatePair;
+ typedef std::map<StatePair, HfstState> StateMap;
+
+ // Find target state corresponding to state pair \a target1, \a target2 in \a state_map and return that state.
+ // If not found, add a new state to \a intersection, add it to \a state_map and return it.
+ // \a was_new_state specifies whether a new state was added.
+ static HfstState find_target_state
+ (HfstState target1, HfstState target2, StateMap & state_map,
+ HfstTransitionGraph & intersection, bool & was_new_state)
+ {
+ StatePair state_pair(target1, target2);
+ StateMap::const_iterator it = state_map.find(state_pair);
+ if (it != state_map.end())
+ {
+ was_new_state=false;
+ return it->second;
+ }
+ HfstState retval = intersection.add_state();
+ state_map[state_pair] = retval;
+ was_new_state=true;
+ return retval;
+ }
+
+ // A function used by find_matches.
+ // Copy matching transition tr1/tr2 to state \a state in \a intersection and return
+ // the target state of that transition. Also make that state final, if needed.
+ static HfstState handle_match(const HfstTransitionGraph & graph1, const HfstTransition <C> & tr1,
+ const HfstTransitionGraph & graph2, const HfstTransition <C> & tr2,
+ HfstTransitionGraph & intersection, HfstState state, StateMap & state_map)
+
+ {
+ HfstState target1 = tr1.get_target_state();
+ HfstState target2 = tr2.get_target_state();
+ bool was_new_state = false;
+ HfstState retval = find_target_state
+ (target1, target2, state_map, intersection, was_new_state);
+ // The sum of weight is copied to the resulting intersection.
+ float transition_weight = tr1.get_weight() + tr2.get_weight();
+ intersection.add_transition
+ (state, HfstTransition <C>
+ (retval, tr1.get_input_symbol(), tr1.get_output_symbol(), transition_weight));
+ // For each new state added, check if the corresponding states in \a graph1 and \a graph2
+ // are final. If they are, make the new state final with the sum of final weights.
+ if (was_new_state && (graph1.is_final_state(target1) && graph2.is_final_state(target2)))
+ {
+ float final_weight = graph1.get_final_weight(target1) + graph2.get_final_weight(target2);
+ intersection.set_final_weight(retval, final_weight);
+ }
+ return retval;
+ }
+
+
+ // A recursive function used by function intersect.
+ //
+ // @param graph1 The first transducer argument of intersection.
+ // @param state1 The current state of \a graph1.
+ // @param graph2 The second transducer argument of intersection.
+ // @param state2 The current state of \a graph2.
+ // @param intersection The intersection of \a graph1 and \a graph2.
+ // @param state The current state of \a intersection.
+ // @param state_map State pairs from \a graph1 and \a graph2 mapped to states in \a intersection.
+ // @param agenda States in \a intersection already handled or scheduled to be handled.
+ //
+ // @pre \a graph1 and \a graph2 must be arc-sorted (via sort_arcs()) to make transition matching faster.
+ // @pre \a graph1 and \a graph2 must be deterministic. (todo: handle equivalent transitions, maybe even epsilons?)
+ static void find_matches
+ (HfstTransitionGraph & graph1, HfstState state1, HfstTransitionGraph & graph2, HfstState state2,
+ HfstTransitionGraph & intersection, HfstState state, StateMap & state_map, std::set<HfstState> & agenda)
+ {
+ agenda.insert(state); // do not handle \a state twice
+ HfstTransitions & tr1 = graph1.state_vector[state1]; // transitions of graph1
+ HfstTransitions & tr2 = graph2.state_vector[state2]; // transitions of graph2
+
+ if (tr1.size() == 0 || tr2.size() == 0)
+ {
+ return; // we cannot proceed as no matches are possible
+ }
+ unsigned int start_search_from=0; // where to start search in tr2
+
+ // *** Go through all transitions of state \a state1 in \a graph1 and try to find a match in
+ // transitions of state \a state2 in \a graph2. As transitions are sorted, we know that
+ // if a match is found for tr1[i] in tr2[j], a match for tr1[i+1] can be searched starting from
+ // tr2[j+1]. If no match is found for tr1[i] in tr2 but tr2[j] is the first element that is bigger
+ // than tr1[i], a match for tr1[i+1] can be searched starting from tr2[j]. ***
+ for (unsigned int i=0; i < tr1.size(); i++)
+ {
+ HfstTransition <C> & transition1 = tr1[i];
+ // Transition data (input and output symbols) to be compared.
+ const C & transition_data1 = transition1.get_transition_data();
+
+ // --- Go through tr2 starting from start_search_from. ---
+ for(unsigned int j=start_search_from; j < tr2.size(); j++)
+ {
+ HfstTransition <C> & transition2 = tr2[j];
+ // Transition data (input and output symbols) to be compared.
+ const C & transition_data2 = transition2.get_transition_data();
+ // todo: input:output duplicates with different weights? (lower weight is chosen always?)
+ if (transition_data2.less_than_ignore_weight(transition_data1))
+ {
+ // no match found, continue searching
+ }
+ else if (transition_data1.less_than_ignore_weight(transition_data2))
+ {
+ // No match can be found, start searching for next item in tr1
+ // starting from current item of tr2.
+ start_search_from=j;
+ break;
+ }
+ else
+ {
+ // Match found, copy transitions and define target state in intersection
+ HfstState target = handle_match(graph1, transition1, graph2, transition2, intersection, state, state_map);
+ // Recursive call for target state, if it is not already on the agenda.
+ if (agenda.find(target) == agenda.end())
+ {
+ find_matches(graph1, transition1.get_target_state(), graph2, transition2.get_target_state(), intersection, target, state_map, agenda);
+ }
+ // Start searching for next item in tr1 starting from next item of tr2.
+ start_search_from=j+1;
+ break;
+ }
+ }
+ // --- A transition in tr1 compared for all possible transitions in tr2, compare next transition. ---
+ }
+ // *** All transitions in tr1 gone through. ***
+ return;
+ }
+
+ static HfstTransitionGraph intersect
+ (HfstTransitionGraph & graph1, HfstTransitionGraph & graph2)
+ {
+ HfstTransitionGraph retval;
+ StateMap state_map;
+ std::set<HfstState> agenda;
+ graph1.sort_arcs();
+ graph2.sort_arcs();
+ state_map[StatePair(0, 0)] = 0; // initial states
+
+ if (graph1.is_final_state(0) && graph2.is_final_state(0))
+ {
+ float final_weight = std::min(graph1.get_final_weight(0), graph2.get_final_weight(0));
+ retval.set_final_weight(0, final_weight);
+ }
+
+ find_matches(graph1, 0, graph2, 0, retval, 0, state_map, agenda);
+
+ return retval;
+ }
+
+
+
+ // A function used by find_matches_for_merge
+ // Copy matching transition graph_tr/merger_tr to state \a result_state in \a result and return
+ // the target state of that transition. Also make that state final, if needed.
+ static HfstState handle_non_list_match(const HfstTransitionGraph & graph, const HfstTransition <C> & graph_transition,
+ const HfstTransitionGraph & merger, HfstState merger_target,
+ HfstTransitionGraph & result, HfstState result_state, StateMap & state_map)
+
+ {
+ HfstState graph_target = graph_transition.get_target_state();
+ bool was_new_state = false;
+ HfstState retval = find_target_state
+ (graph_target, merger_target, state_map, result, was_new_state);
+ result.add_transition
+ (result_state, HfstTransition <C>
+ (retval, graph_transition.get_input_symbol(), graph_transition.get_output_symbol(), graph_transition.get_weight()));
+ // For each new state added, check if the corresponding states in \a graph and \a merger
+ // are final. If they are, make the new state final with the sum of final weights.
+ if (was_new_state && (graph.is_final_state(graph_target) && merger.is_final_state(merger_target)))
+ {
+ float final_weight = graph.get_final_weight(graph_target) + merger.get_final_weight(merger_target);
+ result.set_final_weight(retval, final_weight);
+ }
+ return retval;
+ }
+
+
+ // A function used by find_matches_for_merge
+ // Copy matching transition graph_tr/merger_tr to state \a result_state in \a result and return
+ // the target state of that transition. Also make that state final, if needed.
+ static HfstState handle_list_match(const HfstTransitionGraph & graph, const HfstTransition <C> & graph_transition,
+ const HfstTransitionGraph & merger, const HfstTransition <C> & merger_transition,
+ HfstTransitionGraph & result, HfstState result_state, StateMap & state_map)
+
+ {
+ HfstState graph_target = graph_transition.get_target_state();
+ HfstState merger_target = merger_transition.get_target_state();
+ bool was_new_state = false;
+ HfstState retval = find_target_state
+ (graph_target, merger_target, state_map, result, was_new_state);
+ // The sum of weight is copied to the resulting intersection.
+ float transition_weight = graph_transition.get_weight() + merger_transition.get_weight();
+ result.add_transition
+ (result_state, HfstTransition <C>
+ (retval, merger_transition.get_input_symbol(), merger_transition.get_output_symbol(), transition_weight));
+ // For each new state added, check if the corresponding states in \a graph1 and \a graph2
+ // are final. If they are, make the new state final with the sum of final weights.
+ if (was_new_state && (graph.is_final_state(graph_target) && merger.is_final_state(merger_target)))
+ {
+ float final_weight = graph.get_final_weight(graph_target) + merger.get_final_weight(merger_target);
+ result.set_final_weight(retval, final_weight);
+ }
+ return retval;
+ }
+
+
+ static bool is_list_symbol(const C & transition_data)
+ {
+ return false;
+ }
+
+ static bool is_list_match(const C & graph_transition_data, const C & merger_transition_data)
+ {
+ return false;
+ }
+
+ // A recursive function used by function intersect.
+ //
+ // @param graph The first transducer argument of intersection.
+ // @param graph_state The current state of \a graph1.
+ // @param merger The second transducer argument of intersection.
+ // @param merger_state The current state of \a graph2.
+ // @param result The intersection of \a graph1 and \a graph2.
+ // @param result_state The current state of \a intersection.
+ // @param state_map State pairs from \a graph and \a merger mapped to states in \a result.
+ // @param agenda States in \a result already handled or scheduled to be handled.
+ //
+ // @pre \a graph and \a merger must be arc-sorted (via sort_arcs()) to make transition matching faster.
+ // @pre \a graph and \a merger must be deterministic. (todo: handle equivalent transitions, maybe even epsilons?)
+ static void find_matches_for_merge
+ (HfstTransitionGraph & graph, HfstState graph_state, HfstTransitionGraph & merger, HfstState merger_state,
+ HfstTransitionGraph & result, HfstState result_state, StateMap & state_map, std::set<HfstState> & agenda)
+ {
+ agenda.insert(result_state); // do not handle \a result_state twice
+ HfstTransitions & graph_transitions = graph.state_vector[graph_state]; // transitions of graph
+ HfstTransitions & merger_transitions = merger.state_vector[merger_state]; // transitions of merger
+
+ if (graph_transitions.size() == 0)
+ {
+ return; // we cannot proceed as no matches are possible
+ }
+
+ // Go through all transitions in state \a graph_state of \a graph.
+ for (unsigned int i=0; i < graph_transitions.size(); i++)
+ {
+ HfstTransition <C> & graph_transition = graph_transitions[i];
+ const C & graph_transition_data = graph_transition.get_transition_data();
+
+ // List symbols must be checked separately
+ if (is_list_symbol(graph_transition_data))
+ {
+ bool list_match_found=false;
+ // Find all matches
+ for(unsigned int j=0; j < merger_transitions.size(); j++)
+ {
+ HfstTransition <C> & merger_transition = merger_transitions[j];
+ const C & merger_transition_data = merger_transition.get_transition_data();
+
+ if (is_list_match(graph_transition_data, merger_transition_data))
+ {
+ list_match_found=true;
+ HfstState target = handle_list_match(graph, graph_transition, merger, merger_transition, result, result_state, state_map);
+ if (agenda.find(target) == agenda.end())
+ {
+ find_matches_for_merge(graph, graph_transition.get_target_state(), merger, merger_transition.get_target_state(), result, target, state_map, agenda);
+ }
+ }
+ }
+ if (list_match_found)
+ {
+ continue;
+ }
+ }
+ // If symbol is not a list symbol or no match was found for it, copy symbol as such
+ // We use merger_state as merger transition target state
+ HfstState target = handle_non_list_match(graph, graph_transition, merger, merger_state, result, result_state, state_map);
+ if (agenda.find(target) == agenda.end())
+ {
+ find_matches_for_merge(graph, graph_transition.get_target_state(), merger, /*merger_transition.get_target_state()*/ merger_state, result, target, state_map, agenda);
+ }
+ // --- A transition in graph compared for all corresponding transitions in merger, compare next transition. ---
+ }
+ // *** All transitions in state gone through. ***
+ return;
+ }
+
+ static HfstTransitionGraph merge
+ (HfstTransitionGraph & graph, HfstTransitionGraph & merger)
+ {
+ HfstTransitionGraph result;
+ StateMap state_map;
+ std::set<HfstState> agenda;
+ graph.sort_arcs();
+ merger.sort_arcs();
+ state_map[StatePair(0, 0)] = 0; // initial states
+
+ if (graph.is_final_state(0) && merger.is_final_state(0))
+ {
+ float final_weight = graph.get_final_weight(0) + merger.get_final_weight(0);
+ result.set_final_weight(0, final_weight);
+ }
+
+ find_matches_for_merge(graph, 0, merger, 0, result, 0, state_map, agenda);
+
+ return result;
+ }
+
+
+
+
+
+
+
+
+
/* /\** @brief Determine whether this graph has input-epsilon cycles. */
/* *\/ */
/* bool has_input_epsilon_cycles(void) */
diff --git a/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h b/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
index 5a08738..2c96f67 100644
--- a/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
+++ b/libhfst/src/implementations/HfstTropicalTransducerTransitionData.h
@@ -19,7 +19,7 @@ namespace hfst {
/** @brief One implementation of template class C in
HfstTransition.
-
+
A HfstTropicalTransducerTransitionData has an input symbol and an
output symbol of type SymbolType (string) and a weight of type
WeightType (float).
@@ -29,7 +29,7 @@ namespace hfst {
implementation is hidden from the user.
The class has two static maps and functions that take care of conversion
between strings and internal numbers.
-
+
@see HfstTransition HfstBasicTransition */
class HfstTropicalTransducerTransitionData {
public:
@@ -47,17 +47,17 @@ namespace hfst {
static SymbolType get_epsilon()
{
- return SymbolType("@_EPSILON_SYMBOL_@");
+ return SymbolType("@_EPSILON_SYMBOL_@");
}
static SymbolType get_unknown()
{
- return SymbolType("@_UNKNOWN_SYMBOL_@");
+ return SymbolType("@_UNKNOWN_SYMBOL_@");
}
static SymbolType get_identity()
{
- return SymbolType("@_IDENTITY_SYMBOL_@");
+ return SymbolType("@_IDENTITY_SYMBOL_@");
}
public: /* FIXME: Should be private. */
@@ -70,63 +70,63 @@ namespace hfst {
/* Get the biggest number used to represent a symbol. */
static unsigned int get_max_number() {
- return max_number;
+ return max_number;
}
/*
- Get a vector that defines how numbers of a transducer must
- be changed, i.e. harmonized, so that it follows the same
- number-to-string encoding as all transducers that use the datatype
- HfstTropicalTransducerTransitionData.
-
- \a symbols defines how numbers are mapped to strings in the
- original transducer so that each index in \a symbols
- is the number that corresponds to the string at that index.
- An empty string at an index means that the index is not
- used in the original transducer.
-
- The result is a vector whose each index is the number that
- must be replaced by the number at that index when a
- transducer is harmonized. If an index is not used in the
- transducer, the result will contain a zero at that index.
+ Get a vector that defines how numbers of a transducer must
+ be changed, i.e. harmonized, so that it follows the same
+ number-to-string encoding as all transducers that use the datatype
+ HfstTropicalTransducerTransitionData.
+
+ \a symbols defines how numbers are mapped to strings in the
+ original transducer so that each index in \a symbols
+ is the number that corresponds to the string at that index.
+ An empty string at an index means that the index is not
+ used in the original transducer.
+
+ The result is a vector whose each index is the number that
+ must be replaced by the number at that index when a
+ transducer is harmonized. If an index is not used in the
+ transducer, the result will contain a zero at that index.
*/
static std::vector<unsigned int> get_harmonization_vector
- (const std::vector<SymbolType> &symbols)
+ (const std::vector<SymbolType> &symbols)
{
- std::vector<unsigned int> harmv;
- harmv.reserve(symbols.size());
- harmv.resize(symbols.size(), 0);
- for (unsigned int i=0; i<symbols.size(); i++)
- {
- if (symbols.at(i) != "")
- harmv.at(i) = get_number(symbols.at(i));
- }
- return harmv;
+ std::vector<unsigned int> harmv;
+ harmv.reserve(symbols.size());
+ harmv.resize(symbols.size(), 0);
+ for (unsigned int i=0; i<symbols.size(); i++)
+ {
+ if (symbols.at(i) != "")
+ harmv.at(i) = get_number(symbols.at(i));
+ }
+ return harmv;
}
static std::vector<unsigned int> get_reverse_harmonization_vector
- (const std::map<SymbolType, unsigned int> &symbols)
+ (const std::map<SymbolType, unsigned int> &symbols)
{
- std::vector<unsigned int> harmv;
- harmv.reserve(max_number+1);
- harmv.resize(max_number+1, 0);
- for (unsigned int i=0; i<harmv.size(); i++)
- {
- std::map<SymbolType, unsigned int>::const_iterator it
- = symbols.find(get_symbol(i));
- if (it != symbols.end())
- harmv.at(i) = it->second;
- }
- return harmv;
+ std::vector<unsigned int> harmv;
+ harmv.reserve(max_number+1);
+ harmv.resize(max_number+1, 0);
+ for (unsigned int i=0; i<harmv.size(); i++)
+ {
+ std::map<SymbolType, unsigned int>::const_iterator it
+ = symbols.find(get_symbol(i));
+ if (it != symbols.end())
+ harmv.at(i) = it->second;
+ }
+ return harmv;
}
protected:
/* Get the symbol that is mapped as \a number */
static const std::string &get_symbol(unsigned int number)
- {
+ {
if (number >= number2symbol_map.size()) {
std::string message("HfstTropicalTransducerTransitionData: "
- "number ");
+ "number ");
std::ostringstream oss;
oss << number;
message.append(oss.str());
@@ -140,27 +140,27 @@ namespace hfst {
/* Get the number that is used to represent \a symbol */
static unsigned int get_number(const std::string &symbol)
{
- if(symbol == "") { // FAIL
- Symbol2NumberMap::iterator it = symbol2number_map.find(symbol);
- if (it == symbol2number_map.end()) {
- std::cerr << "ERROR: No number for the empty symbol\n"
- << std::endl;
- }
- else {
- std::cerr << "ERROR: The empty symbol corresdponds to number "
- << it->second << std::endl;
- }
- assert(false);
- }
-
+ if(symbol == "") { // FAIL
+ Symbol2NumberMap::iterator it = symbol2number_map.find(symbol);
+ if (it == symbol2number_map.end()) {
+ std::cerr << "ERROR: No number for the empty symbol\n"
+ << std::endl;
+ }
+ else {
+ std::cerr << "ERROR: The empty symbol corresdponds to number "
+ << it->second << std::endl;
+ }
+ assert(false);
+ }
+
Symbol2NumberMap::iterator it = symbol2number_map.find(symbol);
if (it == symbol2number_map.end())
- {
- max_number++;
- symbol2number_map[symbol] = max_number;
- number2symbol_map.push_back(symbol);
- return max_number;
- }
+ {
+ max_number++;
+ symbol2number_map[symbol] = max_number;
+ number2symbol_map.push_back(symbol);
+ return max_number;
+ }
return it->second;
}
@@ -212,12 +212,12 @@ namespace hfst {
}
HfstTropicalTransducerTransitionData
- (unsigned int inumber,
- unsigned int onumber,
- WeightType weight) {
- input_number = inumber;
- output_number = onumber;
- this->weight = weight;
+ (unsigned int inumber,
+ unsigned int onumber,
+ WeightType weight) {
+ input_number = inumber;
+ output_number = onumber;
+ this->weight = weight;
}
/** @brief Get the input symbol. */
@@ -231,11 +231,11 @@ namespace hfst {
}
unsigned int get_input_number() const {
- return input_number;
+ return input_number;
}
unsigned int get_output_number() const {
- return output_number;
+ return output_number;
}
/** @brief Get the weight. */
@@ -254,19 +254,19 @@ namespace hfst {
return (symbol.compare("@_IDENTITY_SYMBOL_@") == 0);
}
static bool is_valid_symbol(const SymbolType &symbol) {
- if (symbol == "")
- return false;
- return true;
+ if (symbol == "")
+ return false;
+ return true;
}
static SymbolType get_marker(const SymbolTypeSet &sts) {
- (void)sts;
- return SymbolType("@_MARKER_SYMBOL_@");
+ (void)sts;
+ return SymbolType("@_MARKER_SYMBOL_@");
}
/** @brief Whether this transition is less than transition
\a another.
-
+
/internal is it too slow if string comparison is used instead?
*/
bool operator<(const HfstTropicalTransducerTransitionData &another)
@@ -281,13 +281,27 @@ namespace hfst {
return false;
return (weight < another.weight);
}
+
+ // same as operator< but weight is ignored
+ bool less_than_ignore_weight(const HfstTropicalTransducerTransitionData &another)
+ const {
+ if (input_number < another.input_number )
+ return true;
+ if (input_number > another.input_number)
+ return false;
+ if (output_number < another.output_number)
+ return true;
+ if (output_number > another.output_number)
+ return false;
+ return false;
+ }
void operator=(const HfstTropicalTransducerTransitionData &another)
- {
- input_number = another.input_number;
- output_number = another.output_number;
- weight = another.weight;
- }
+ {
+ input_number = another.input_number;
+ output_number = another.output_number;
+ weight = another.weight;
+ }
friend class Number2SymbolVectorInitializer;
friend class Symbol2NumberMapInitializer;
diff --git a/libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc b/libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc
index 22c99db..ddc95c7 100644
--- a/libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc
+++ b/libhfst/src/implementations/optimized-lookup/find_epsilon_loops.cc
@@ -16,14 +16,11 @@ namespace hfst_ol {
// Define an operation for checking state equivalence for the
// purpose of detecting the same situation happening twice
-bool TraversalState::operator==(TraversalState & rhs)
+bool TraversalState::operator==(const TraversalState & rhs) const
{
if (this->index != rhs.index) {
return false;
}
- if (this->flags.size() != rhs.flags.size()) {
- return false;
- }
for (size_t i = 0; i < this->flags.size(); ++i) {
if (this->flags[i] != rhs.flags[i]) {
return false;
@@ -32,33 +29,62 @@ bool TraversalState::operator==(TraversalState & rhs)
return true;
}
+bool TraversalState::operator<(const TraversalState & rhs) const
+{
+ if (this->index < rhs.index) {
+ return true;
+ }
+ if (this->index > rhs.index) {
+ return false;
+ }
+ for (size_t i = 0; i < this->flags.size(); ++i) {
+ if (this->flags[i] < rhs.flags[i]) {
+ return true;
+ }
+ if (this->flags[i] > rhs.flags[i]) {
+ return false;
+ }
+ }
+ return false;
+}
+
void Transducer::find_loop_epsilon_transitions(
unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states)
+ TransitionTableIndex i)
{
+ FlagDiacriticState flags = flag_state.get_values();
while (true)
{
+ TransitionTableIndex target = tables->get_transition_target(i);
+ TraversalState epsilon_reachable(target, flags);
if (tables->get_transition_input(i) == 0) // epsilon
{
- find_loop(input_pos,
- tables->get_transition_target(i),
- position_states);
+ // We try to trap non-progressing loops
+ if (traversal_states.count(epsilon_reachable) == 1) {
+ // We've been here before
+ throw true;
+ }
+ traversal_states.insert(epsilon_reachable);
+ find_loop(input_pos, target);
+ traversal_states.erase(epsilon_reachable);
found_transition = true;
++i;
} else if (alphabet->is_flag_diacritic(
tables->get_transition_input(i))) {
- std::vector<short> old_values(flag_state.get_values());
+
if (flag_state.apply_operation(
*(alphabet->get_operation(
tables->get_transition_input(i))))) {
// flag diacritic allowed
- find_loop(input_pos,
- tables->get_transition_target(i),
- position_states);
- found_transition = true;
+ if (traversal_states.count(epsilon_reachable) == 1) {
+ // We've been here before
+ throw true;
+ }
+ traversal_states.insert(epsilon_reachable);
+ find_loop(input_pos, target);
+ traversal_states.erase(epsilon_reachable);
}
- flag_state.assign_values(old_values);
+ flag_state.assign_values(flags);
++i;
} else { // it's not epsilon and it's not a flag, so nothing to do
return;
@@ -67,30 +93,27 @@ void Transducer::find_loop_epsilon_transitions(
}
void Transducer::find_loop_epsilon_indices(unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states)
+ TransitionTableIndex i)
{
if (tables->get_index_input(i) == 0)
{
- find_loop_epsilon_transitions(input_pos,
- tables->get_index_target(i) -
- TRANSITION_TARGET_TABLE_START,
- position_states);
+ find_loop_epsilon_transitions(
+ input_pos,
+ tables->get_index_target(i) - TRANSITION_TARGET_TABLE_START);
found_transition = true;
}
}
void Transducer::find_loop_transitions(SymbolNumber input,
unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states)
+ TransitionTableIndex i)
{
while (tables->get_transition_input(i) != NO_SYMBOL_NUMBER) {
if (tables->get_transition_input(i) == input) {
- find_loop(input_pos,
- tables->get_transition_target(i),
- position_states);
+ // We're not going to find an epsilon / flag loop
+ traversal_states.clear();
+ find_loop(input_pos, tables->get_transition_target(i));
found_transition = true;
} else {
return;
@@ -101,16 +124,14 @@ void Transducer::find_loop_transitions(SymbolNumber input,
void Transducer::find_loop_index(SymbolNumber input,
unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states)
+ TransitionTableIndex i)
{
if (tables->get_index_input(i+input) == input)
{
find_loop_transitions(input,
input_pos,
tables->get_index_target(i+input) -
- TRANSITION_TARGET_TABLE_START,
- position_states);
+ TRANSITION_TARGET_TABLE_START);
found_transition = true;
}
}
@@ -119,37 +140,14 @@ void Transducer::find_loop_index(SymbolNumber input,
void Transducer::find_loop(unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states)
+ TransitionTableIndex i)
{
found_transition = false;
- TraversalState this_position(i, flag_state.get_values());
- if (indexes_transition_table(i)) {
- this_position.index -= TRANSITION_TARGET_TABLE_START;
- }
- if (input_pos == position_states.size()) {
- // first time at this input
- std::vector<TraversalState> v;
- position_states.push_back(v);
- } else {
- // see if the same state has been visited already at this input
- for(std::vector<TraversalState>::iterator it = position_states[input_pos].begin();
- it != position_states[input_pos].end(); ++it) {
- if (this_position == *it) {
- throw true;
- }
- }
- }
- // if we didn't throw a loop exception, record this state
- position_states[input_pos].push_back(this_position);
-
if (indexes_transition_table(i))
{
i -= TRANSITION_TARGET_TABLE_START;
- find_loop_epsilon_transitions(input_pos,
- i+1,
- position_states);
+ find_loop_epsilon_transitions(input_pos, i+1);
// input-string ended.
if (input_tape[input_pos] == NO_SYMBOL_NUMBER)
@@ -160,22 +158,16 @@ void Transducer::find_loop(unsigned int input_pos,
SymbolNumber input = input_tape[input_pos];
++input_pos;
- find_loop_transitions(input,
- input_pos,
- i+1,
- position_states);
+ find_loop_transitions(input, input_pos, i+1);
if (alphabet->get_default_symbol() != NO_SYMBOL_NUMBER &&
!found_transition) {
find_loop_transitions(alphabet->get_default_symbol(),
- input_pos, i+1,
- position_states);
+ input_pos, i+1);
}
}
else
{
- find_loop_epsilon_indices(input_pos,
- i+1,
- position_states);
+ find_loop_epsilon_indices(input_pos, i+1);
if (input_tape[input_pos] == NO_SYMBOL_NUMBER)
{ // input-string ended.
@@ -185,16 +177,12 @@ void Transducer::find_loop(unsigned int input_pos,
SymbolNumber input = input_tape[input_pos];
++input_pos;
- find_loop_index(input,
- input_pos,
- i+1,
- position_states);
+ find_loop_index(input, input_pos, i+1);
// If we have a default symbol defined and we didn't find an index,
// check for that
if (alphabet->get_default_symbol() != NO_SYMBOL_NUMBER && !found_transition) {
find_loop_index(alphabet->get_default_symbol(),
- input_pos, i+1,
- position_states);
+ input_pos, i+1);
}
}
}
diff --git a/libhfst/src/implementations/optimized-lookup/pmatch.cc b/libhfst/src/implementations/optimized-lookup/pmatch.cc
index 38a2777..5f7d1e7 100644
--- a/libhfst/src/implementations/optimized-lookup/pmatch.cc
+++ b/libhfst/src/implementations/optimized-lookup/pmatch.cc
@@ -22,7 +22,6 @@ PmatchAlphabet::PmatchAlphabet(std::istream & inputstream,
for (SymbolNumber i = 1; i < symbol_table.size(); ++i) {
add_special_symbol(symbol_table[i], i);
}
-
}
PmatchAlphabet::PmatchAlphabet(void):
@@ -32,7 +31,7 @@ PmatchAlphabet::PmatchAlphabet(void):
bool PmatchAlphabet::is_printable(SymbolNumber symbol)
{
if (symbol == 0 || symbol == NO_SYMBOL_NUMBER ||
- is_flag_diacritic(symbol) || is_end_tag(symbol)) {
+ is_flag_diacritic(symbol) || is_end_tag(symbol) || is_guard(symbol)) {
return false;
}
for (std::map<SpecialSymbol, SymbolNumber>::const_iterator it = special_symbols.begin();
@@ -78,8 +77,9 @@ void PmatchAlphabet::add_special_symbol(const std::string & str,
str.size() - (sizeof("@PMATCH_ENDTAG_@") - 1));
} else if (is_insertion(str)) {
rtn_names[name_from_insertion(str)] = symbol_number;
+ } else if (is_guard(str)) {
+ guards.push_back(symbol_number);
}
-
}
SymbolNumberVector PmatchAlphabet::get_specials(void) const
@@ -222,6 +222,35 @@ bool PmatchAlphabet::is_insertion(const std::string & symbol)
return symbol.find("@I.") == 0 && symbol.rfind("@") == symbol.size() - 1;
}
+bool PmatchAlphabet::is_guard(const std::string & symbol)
+{
+ return symbol.find("@PMATCH_GUARD_") == 0 && symbol.rfind("@") == symbol.size() - 1;
+}
+
+bool PmatchAlphabet::is_special(const std::string & symbol)
+{
+ if (symbol.size() == 0) {
+ return false;
+ }
+ if (is_insertion(symbol) || symbol == "@BOUNDARY@") {
+// || symbol == "@_UNKNOWN_SYMBOL_@" || symbol == "@_IDENTITY_SYMBOL_@"
+ return true;
+ } else {
+ return symbol.find("@PMATCH") == 0 && symbol.at(symbol.size() - 1) == '@';
+ }
+}
+
+bool PmatchAlphabet::is_guard(const SymbolNumber symbol) const
+{
+ for(SymbolNumberVector::const_iterator it = guards.begin();
+ it != guards.end(); ++it) {
+ if(symbol == *it) {
+ return true;
+ }
+ }
+ return false;
+}
+
std::string PmatchAlphabet::name_from_insertion(const std::string & symbol)
{
return symbol.substr(sizeof("@I.") - 1, symbol.size() - (sizeof("@I.@") - 1));
@@ -457,7 +486,8 @@ std::string PmatchAlphabet::stringify(const DoubleTape & str)
}
retval.insert(pos, start_tag(output));
retval.append(end_tag(output));
- } else if (output == special_symbols[boundary]) {
+ } else if (output == special_symbols[boundary]
+ || is_guard(output)) {
continue;
} else {
if (!extract_tags || start_tag_pos.size() != 0) {
diff --git a/libhfst/src/implementations/optimized-lookup/pmatch.h b/libhfst/src/implementations/optimized-lookup/pmatch.h
index f30efce..3bf523d 100644
--- a/libhfst/src/implementations/optimized-lookup/pmatch.h
+++ b/libhfst/src/implementations/optimized-lookup/pmatch.h
@@ -90,7 +90,9 @@ namespace hfst_ol {
std::map<SpecialSymbol, SymbolNumber> special_symbols;
std::map<SymbolNumber, std::string> end_tag_map;
std::map<std::string, SymbolNumber> rtn_names;
+ SymbolNumberVector guards;
bool is_end_tag(const SymbolNumber symbol) const;
+ bool is_guard(const SymbolNumber symbol) const;
std::string end_tag(const SymbolNumber symbol);
std::string start_tag(const SymbolNumber symbol);
bool extract_tags;
@@ -101,6 +103,8 @@ namespace hfst_ol {
~PmatchAlphabet(void);
static bool is_end_tag(const std::string & symbol);
static bool is_insertion(const std::string & symbol);
+ static bool is_guard(const std::string & symbol);
+ static bool is_special(const std::string & symbol);
static std::string name_from_insertion(
const std::string & symbol);
bool is_printable(SymbolNumber symbol);
diff --git a/libhfst/src/implementations/optimized-lookup/transducer.cc b/libhfst/src/implementations/optimized-lookup/transducer.cc
index 4c93341..5ca763e 100644
--- a/libhfst/src/implementations/optimized-lookup/transducer.cc
+++ b/libhfst/src/implementations/optimized-lookup/transducer.cc
@@ -29,6 +29,7 @@ TransducerAlphabet::TransducerAlphabet(std::istream& is,
bool preserve_diacritic_strings)
{
unknown_symbol = NO_SYMBOL_NUMBER;
+ identity_symbol = NO_SYMBOL_NUMBER;
default_symbol = NO_SYMBOL_NUMBER;
for(SymbolNumber i=0; i<symbol_count; i++)
{
@@ -51,6 +52,7 @@ TransducerAlphabet::TransducerAlphabet(std::istream& is,
}
symbol_table.push_back(str.c_str());
}
+ orig_symbol_count = symbol_table.size();
}
void TransducerAlphabet::add_symbol(char * symbol)
@@ -63,6 +65,7 @@ TransducerAlphabet::TransducerAlphabet(const SymbolTable& st):
{
unknown_symbol = NO_SYMBOL_NUMBER;
default_symbol = NO_SYMBOL_NUMBER;
+ identity_symbol = NO_SYMBOL_NUMBER;
for(SymbolNumber i=0; i<symbol_table.size(); i++)
{
if(hfst::FdOperation::is_diacritic(symbol_table[i])) {
@@ -71,8 +74,11 @@ TransducerAlphabet::TransducerAlphabet(const SymbolTable& st):
unknown_symbol = i;
} else if (hfst::is_default(symbol_table[i])) {
default_symbol = i;
+ } else if (hfst::is_identity(symbol_table[i])) {
+ identity_symbol = i;
}
}
+ orig_symbol_count = symbol_table.size();
}
SymbolNumber TransducerAlphabet::symbol_from_string(
@@ -290,9 +296,9 @@ bool Transducer::is_lookup_infinitely_ambiguous(const std::string & s)
if (!initialize_input(s.c_str())) {
return false;
}
- PositionStates position_states;
+ traversal_states.clear();
try {
- find_loop(0, 0, position_states);
+ find_loop(0, 0);
} catch (bool e) {
current_weight = 0.0;
flag_state = alphabet->get_fd_table();
@@ -320,6 +326,7 @@ HfstOneLevelPaths * Transducer::lookup_fd(const char * s, ssize_t limit)
lookup_paths = NULL;
return results;
}
+ traversal_states.clear();
//current_weight += s.second;
get_analyses(0, 0, 0);
//current_weight -= s.second;
@@ -333,32 +340,41 @@ void Transducer::try_epsilon_transitions(unsigned int input_pos,
{
while (true)
{
- if (tables->get_transition_input(i) == 0) // epsilon
+ SymbolNumber input = tables->get_transition_input(i);
+ SymbolNumber output = tables->get_transition_output(i);
+ TransitionTableIndex target = tables->get_transition_target(i);
+ Weight weight = tables->get_weight(i);
+ if (input == 0) // epsilon
{
- output_tape.write(output_pos, tables->get_transition_output(i));
- current_weight += tables->get_weight(i);
- get_analyses(input_pos,
- output_pos + 1,
- tables->get_transition_target(i));
+ output_tape.write(output_pos, output);
+ current_weight += weight;
+ get_analyses(input_pos, output_pos + 1, target);
found_transition = true;
- current_weight -= tables->get_weight(i);
+ current_weight -= weight;
++i;
- } else if (alphabet->is_flag_diacritic(
- tables->get_transition_input(i))) {
- std::vector<short> old_values(flag_state.get_values());
+ } else if (alphabet->is_flag_diacritic(input)) {
+ FlagDiacriticState flags = flag_state.get_values();
if (flag_state.apply_operation(
- *(alphabet->get_operation(
- tables->get_transition_input(i))))) {
+ *(alphabet->get_operation(input)))) {
// flag diacritic allowed
- output_tape.write(output_pos, tables->get_transition_output(i));
- current_weight += tables->get_weight(i);
- get_analyses(input_pos,
- output_pos + 1,
- tables->get_transition_target(i));
+
+ TraversalState flag_reachable(target, flags);
+ if (traversal_states.count(flag_reachable) == 1) {
+ // We've been here before at this input, back out
+ flag_state.assign_values(flags);
+ ++i;
+ continue;
+ }
+
+ traversal_states.insert(flag_reachable);
+ output_tape.write(output_pos, output);
+ current_weight += weight;
+ get_analyses(input_pos, output_pos + 1, target);
found_transition = true;
- current_weight -= tables->get_weight(i);
+ current_weight -= weight;
+ traversal_states.erase(flag_reachable);
}
- flag_state.assign_values(old_values);
+ flag_state.assign_values(flags);
++i;
} else { // it's not epsilon and it's not a flag, so nothing to do
return;
@@ -390,10 +406,14 @@ void Transducer::find_transitions(SymbolNumber input,
{
if (tables->get_transition_input(i) == input)
{
+ // We're not going to find an epsilon / flag loop
+ traversal_states.clear();
SymbolNumber output = tables->get_transition_output(i);
- if (input == alphabet->get_default_symbol()) {
- // we got here via default, so look back in the
- // input tape to find the symbol we want to write
+ if (output == alphabet->get_default_symbol()
+ || output == alphabet->get_identity_symbol()
+ || output == alphabet->get_unknown_symbol()) {
+ // we got here via default, identity or unknown, so look
+ // back in the input tape to find the symbol we want to write
output = input_tape[input_pos - 1];
}
output_tape.write(output_pos, output);
@@ -476,10 +496,22 @@ void Transducer::get_analyses(unsigned int input_pos,
SymbolNumber input = input_tape[input_pos];
++input_pos;
- find_transitions(input,
- input_pos,
- output_pos,
- i+1);
+ if (input < alphabet->get_orig_symbol_count()) {
+ // Input is in the alphabet
+ find_transitions(input,
+ input_pos,
+ output_pos,
+ i+1);
+ } else {
+ if (alphabet->get_identity_symbol() != NO_SYMBOL_NUMBER) {
+ find_transitions(alphabet->get_identity_symbol(),
+ input_pos, output_pos, i+1);
+ }
+ if (alphabet->get_unknown_symbol() != NO_SYMBOL_NUMBER) {
+ find_transitions(alphabet->get_unknown_symbol(),
+ input_pos, output_pos, i+1);
+ }
+ }
if (alphabet->get_default_symbol() != NO_SYMBOL_NUMBER &&
!found_transition) {
find_transitions(alphabet->get_default_symbol(),
@@ -511,10 +543,19 @@ void Transducer::get_analyses(unsigned int input_pos,
SymbolNumber input = input_tape[input_pos];
++input_pos;
- find_index(input,
- input_pos,
- output_pos,
- i+1);
+ if (input < alphabet->get_orig_symbol_count()) {
+ // Input is in the alphabet
+ find_index(input, input_pos, output_pos, i+1);
+ } else {
+ if (alphabet->get_identity_symbol() != NO_SYMBOL_NUMBER) {
+ find_index(alphabet->get_identity_symbol(),
+ input_pos, output_pos, i+1);
+ }
+ if (alphabet->get_unknown_symbol() != NO_SYMBOL_NUMBER) {
+ find_index(alphabet->get_unknown_symbol(),
+ input_pos, output_pos, i+1);
+ }
+ }
// If we have a default symbol defined and we didn't find an index,
// check for that
if (alphabet->get_default_symbol() != NO_SYMBOL_NUMBER && !found_transition) {
diff --git a/libhfst/src/implementations/optimized-lookup/transducer.h b/libhfst/src/implementations/optimized-lookup/transducer.h
index 8c7babc..25dd01b 100644
--- a/libhfst/src/implementations/optimized-lookup/transducer.h
+++ b/libhfst/src/implementations/optimized-lookup/transducer.h
@@ -69,9 +69,11 @@ struct TraversalState
FlagDiacriticState flags;
TraversalState(TransitionTableIndex i, FlagDiacriticState f):
index(i), flags(f) {}
- bool operator==(TraversalState & rhs);
+ bool operator==(const TraversalState & rhs) const;
+ bool operator<(const TraversalState & rhs) const;
+
};
-typedef std::vector<std::vector<TraversalState> > PositionStates;
+typedef std::set<TraversalState> TraversalStates;
const SymbolNumber NO_SYMBOL_NUMBER = std::numeric_limits<SymbolNumber>::max();
const TransitionTableIndex NO_TABLE_INDEX =
@@ -360,7 +362,7 @@ protected:
SymbolNumber unknown_symbol;
SymbolNumber default_symbol;
SymbolNumber identity_symbol;
-
+ SymbolNumber orig_symbol_count;
public:
TransducerAlphabet()
@@ -369,6 +371,7 @@ public:
unknown_symbol = NO_SYMBOL_NUMBER;
default_symbol = NO_SYMBOL_NUMBER;
identity_symbol = NO_SYMBOL_NUMBER;
+ orig_symbol_count = 1;
}
TransducerAlphabet(std::istream& is,
SymbolNumber symbol_count,
@@ -414,6 +417,8 @@ public:
{ return default_symbol; }
SymbolNumber get_identity_symbol(void) const
{ return identity_symbol; }
+ SymbolNumber get_orig_symbol_count(void) const
+ { return orig_symbol_count; }
void add_symbol(char * symbol);
};
@@ -809,10 +814,7 @@ class Transducer
protected:
TransducerHeader* header;
TransducerAlphabet* alphabet;
-
TransducerTablesInterface* tables;
-
-
void load_tables(std::istream& is);
// for lookup
@@ -824,6 +826,9 @@ protected:
hfst::FdState<SymbolNumber> flag_state;
// This is to keep track of whether we're going to take a default transition
bool found_transition;
+ // For keeping a tally of previously epsilon-visited states to control
+ // going into loops
+ TraversalStates traversal_states;
ssize_t max_lookups;
unsigned int recursion_depth_left;
@@ -850,6 +855,19 @@ protected:
unsigned int output_tape_pos,
TransitionTableIndex i);
+ void find_loop_epsilon_transitions(unsigned int input_pos,
+ TransitionTableIndex i);
+ void find_loop_epsilon_indices(unsigned int input_pos,
+ TransitionTableIndex i);
+ void find_loop_transitions(SymbolNumber input,
+ unsigned int input_pos,
+ TransitionTableIndex i);
+ void find_loop_index(SymbolNumber input,
+ unsigned int input_pos,
+ TransitionTableIndex i);
+ void find_loop(unsigned int input_pos,
+ TransitionTableIndex i);
+
public:
Transducer(std::istream& is);
@@ -901,24 +919,7 @@ public:
bool is_lookup_infinitely_ambiguous(const StringVector & s);
bool is_lookup_infinitely_ambiguous(const std::string & input);
- void find_loop_epsilon_transitions(unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states);
- void find_loop_epsilon_indices(unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states);
- void find_loop_transitions(SymbolNumber input,
- unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states);
- void find_loop_index(SymbolNumber input,
- unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states);
- void find_loop(unsigned int input_pos,
- TransitionTableIndex i,
- PositionStates position_states);
-
+
TransducerTable<TransitionWIndex> & copy_windex_table();
TransducerTable<TransitionW> & copy_transitionw_table();
TransducerTable<TransitionIndex> & copy_index_table();
diff --git a/libhfst/src/parsers/XreCompiler.h b/libhfst/src/parsers/XreCompiler.h
index 9a30959..b1b2153 100644
--- a/libhfst/src/parsers/XreCompiler.h
+++ b/libhfst/src/parsers/XreCompiler.h
@@ -89,7 +89,7 @@ class XreCompiler
//! @brief Compile a transducer defined by @a xre and set the value of @a
//! as the number of characters read from @a xre. The characters after
- //! the regular expression that was succesfully parsed are ignored.
+ //! the regular expression that was successfully parsed are ignored.
//! May return a pointer to @e empty transducer on non-fatal error.
//! A null pointer is returned on fatal error, if abort is not called.
HfstTransducer* compile_first(const std::string& xre, unsigned int & chars_read);
diff --git a/libhfst/src/parsers/pmatch_lex.ll b/libhfst/src/parsers/pmatch_lex.ll
index db099d4..ae88028 100644
--- a/libhfst/src/parsers/pmatch_lex.ll
+++ b/libhfst/src/parsers/pmatch_lex.ll
@@ -43,7 +43,7 @@ A7 [\x00-\x7e]
/* special meaning in pmatch */
A7RESTRICTED [- |<>%^:;@0~\\&?$+*/_(){}\]\[-]
/* non-restricted ASCII */
-A7UNRESTRICTED [\x21-\x7e]{-}[- |<>%^:;,@~\\&?$+*/(){}\]\[-]
+A7UNRESTRICTED [\x21-\x7e]{-}[- |<>%^:;,@~\\&?$+*/(){}\]\[]
WEIGHT [0-9]+(\.[0-9]+)?
@@ -59,6 +59,10 @@ LWSP [\t\r\n ]
return DEFINE;
}
+"regex" {
+return REGEX;
+}
+
"DefIns" {
return DEFINS;
}
@@ -293,7 +297,12 @@ LWSP [\t\r\n ]
"[]" { return EPSILON_TOKEN; }
"0" { return EPSILON_TOKEN; }
"?" { return ANY_TOKEN; }
-"#" { return BOUNDARY_MARKER; }
+"#"|".#." { return BOUNDARY_MARKER; }
+
+{EC} {
+ pmatchlval.label = hfst::pmatch::strip_percents(pmatchtext);
+ return QUOTED_LITERAL;
+}
{NAME_CH}+ {
pmatchlval.label = hfst::pmatch::strip_percents(pmatchtext);
@@ -302,7 +311,8 @@ LWSP [\t\r\n ]
{NAME_CH}+"(" {
char * label = (char *) malloc(strlen(pmatchtext));;
- strncpy(label, pmatchtext, strlen(pmatchtext) - 1);
+ strncpy(label, pmatchtext, strlen(pmatchtext));
+ label[strlen(pmatchtext) - 1] = '\0';
pmatchlval.label = hfst::pmatch::strip_percents(label);
free(label);
return SYMBOL_WITH_LEFT_PAREN;
diff --git a/libhfst/src/parsers/pmatch_parse.yy b/libhfst/src/parsers/pmatch_parse.yy
index bde1ab7..e3d5b40 100644
--- a/libhfst/src/parsers/pmatch_parse.yy
+++ b/libhfst/src/parsers/pmatch_parse.yy
@@ -1,11 +1,12 @@
%{
-#define YYDEBUG 1
+#define YYDEBUG 0
#include <stdio.h>
#include <assert.h>
#include <iostream>
-
+#include <sstream>
+
#include "HfstTransducer.h"
#include "HfstInputStream.h"
#include "HfstXeroxRules.h"
@@ -15,20 +16,18 @@
using namespace hfst::xeroxRules;
using namespace hfst::implementations;
-
#include "pmatch_utils.h"
-
+ using hfst::pmatch::PmatchAstNode;
+
extern void pmatcherror(const char * text);
extern int pmatchlex();
-
-
-
+ extern int pmatchlineno;
%}
%name-prefix="pmatch"
%error-verbose
- %debug
+// %debug
@@ -73,10 +72,12 @@
%type <transducerDefinitions> PMATCH
%type <transducerDefinition> DEFINITION BINDING FUNCTION
%type <transducer> REGEXP1 REGEXP2 REGEXP4 REGEXP5 REGEXP6 REGEXP7
-REGEXP8 REGEXP9 REGEXP10 REGEXP11 REGEXP12 LABEL
-REPLACE REGEXP3 FUNCALL MAP
-%type <ast_node> FUNCBODY1 FUNCBODY2 FUNCBODY3 FUNCBODY4 FUNCBODY5 FUNCBODY6
+REGEXP8 REGEXP9 REGEXP10 REGEXP11 REGEXP12 LABEL_PAIR
+REPLACE REGEXP3 FUNCALL MAP FUNCALL_ARG
+%type <label> LABEL
+%type <ast_node> FUNCBODY1 FUNCBODY2 FUNCBODY3 FUNCBODY4 FUNCBODY5 FUNCBODY6 FUNC_LABEL_PAIR
%type <string_vector> ARGLIST
+%type <transducerVector> FUNCALL_ARGLIST
%type <replaceRuleVectorWithArrow> PARALLEL_RULES
%type <replaceRuleWithArrow> RULE
@@ -139,7 +140,7 @@ PAIR_SEPARATOR_WO_RIGHT PAIR_SEPARATOR_WO_LEFT
%token EPSILON_TOKEN ANY_TOKEN BOUNDARY_MARKER
%token LEXER_ERROR
-%nonassoc DEFINE DEFINS DEFFUN ALPHA LOWERALPHA UPPERALPHA NUM PUNCT WHITESPACE
+%nonassoc DEFINE REGEX DEFINS DEFFUN ALPHA LOWERALPHA UPPERALPHA NUM PUNCT WHITESPACE
OPTCAP_LEFT TOLOWER_LEFT TOUPPER_LEFT INS_LEFT DEFINE_LEFT ENDTAG_LEFT LC_LEFT
RC_LEFT NLC_LEFT NRC_LEFT MAP_LEFT SYM_LEFT OR_LEFT AND_LEFT
%%
@@ -149,16 +150,44 @@ RC_LEFT NLC_LEFT NRC_LEFT MAP_LEFT SYM_LEFT OR_LEFT AND_LEFT
PMATCH: DEFINITION {
if ($1->first.compare("@_PMATCH_DUMMY_@")) {
+ if(hfst::pmatch::definitions.count($1->first) != 0) {
+ std::stringstream warning;
+ warning << "definition of " << $1->first << " on line "
+ << pmatchlineno << " shadowed by earlier definition\n";
+ hfst::pmatch::warn(warning.str());
+ }
hfst::pmatch::definitions.insert(*$1);
- }
-// $$ = &hfst::pmatch::definitions;
+ if (hfst::pmatch::verbose) {
+ std::cerr << std::setiosflags(std::ios::fixed) << std::setprecision(2);
+ double duration = (clock() - hfst::pmatch::timer) /
+ (double) CLOCKS_PER_SEC;
+ hfst::pmatch::timer = clock();
+ std::cerr << "compiled " << $1->first << " in " << duration << " seconds\n";
+ hfst::pmatch::print_size_info($1->second);
+ std::cerr << std::endl;
+ }
+ }
delete $1;
} |
PMATCH DEFINITION {
if ($2->first.compare("@_PMATCH_DUMMY_@")) {
+ if(hfst::pmatch::definitions.count($2->first) != 0) {
+ std::stringstream warning;
+ warning << "definition of " << $2->first << " on line "
+ << pmatchlineno << " shadowed by earlier definition\n";
+ hfst::pmatch::warn(warning.str());
+ }
hfst::pmatch::definitions.insert(*$2);
+ if (hfst::pmatch::verbose) {
+ std::cerr << std::setiosflags(std::ios::fixed) << std::setprecision(2);
+ double duration = (clock() - hfst::pmatch::timer) /
+ (double) CLOCKS_PER_SEC;
+ hfst::pmatch::timer = clock();
+ std::cerr << "compiled " << $2->first << " in " << duration << " seconds\n";
+ hfst::pmatch::print_size_info($2->second);
+ std::cerr << std::endl;
+ }
}
-// $$ = &hfst::pmatch::definitions;
delete $2;
}
;
@@ -173,46 +202,77 @@ DEFINITION: DEFINE BINDING { $$ = $2; }
hfst::pmatch::def_insed_transducers[$2->first] = ins_t;
$$ = $2;
}
+| REGEX REGEXP1 {
+ $2->set_name("TOP");
+ if (hfst::pmatch::need_delimiters) {
+ $2 = hfst::pmatch::add_pmatch_delimiters($2);
+ }
+ hfst::pmatch::need_delimiters = false;
+ $2->minimize();
+ $$ = new std::pair<std::string, hfst::HfstTransducer*>("TOP", $2);
+ }
| DEFFUN FUNCTION { $$ = $2; }
;
BINDING: SYMBOL REGEXP1 {
- $2->set_name($1);
- if (hfst::pmatch::verbose) {
- std::cerr << std::setiosflags(std::ios::fixed) << std::setprecision(2);
- double duration = (clock() - hfst::pmatch::timer) /
- (double) CLOCKS_PER_SEC;
- hfst::pmatch::timer = clock();
- std::cerr << "compiled " << $1 << " in " << duration << " seconds\n";
- hfst::pmatch::print_size_info($2);
- std::cerr << std::endl;
+ if (hfst::pmatch::need_delimiters) {
+ $2 = hfst::pmatch::add_pmatch_delimiters($2);
}
- $2 = hfst::pmatch::add_pmatch_delimiters($2);
+ hfst::pmatch::need_delimiters = false;
+ $2->set_name($1);
$2->minimize();
$$ = new std::pair<std::string, hfst::HfstTransducer*>($1, $2);
- };
+ }
+;
FUNCTION: SYMBOL_WITH_LEFT_PAREN ARGLIST RIGHT_PARENTHESIS FUNCBODY1 END_OF_EXPRESSION {
- hfst::pmatch::PmatchFunction fun(* $2, $4);
-// std::pair<std::string, PmatchFunction> name_fun_pair($1, fun);
+ PmatchAstNode * function_body;
+ if (hfst::pmatch::need_delimiters) {
+ function_body = new PmatchAstNode($4, hfst::pmatch::AstAddDelimiters);
+ } else {
+ function_body = $4;
+ }
+ hfst::pmatch::need_delimiters = false;
+ hfst::pmatch::PmatchFunction fun(* $2, function_body);
+ if(hfst::pmatch::functions.count($1) != 0) {
+ std::stringstream warning;
+ warning << "definition of function" << $1 << " on line "
+ << pmatchlineno << " shadowed by earlier definition\n";
+ hfst::pmatch::warn(warning.str());
+ }
hfst::pmatch::functions[$1] = fun;
// Pass a dummy transducer, since function registration is separate
HfstTransducer * dummy = new HfstTransducer(hfst::pmatch::format);
dummy->set_name("@_PMATCH_DUMMY_@");
$$ = new std::pair<std::string, hfst::HfstTransducer*>("@_PMATCH_DUMMY_@", dummy);
- };
+ if (hfst::pmatch::verbose) {
+ std::cerr << std::setiosflags(std::ios::fixed) << std::setprecision(2);
+ double duration = (clock() - hfst::pmatch::timer) /
+ (double) CLOCKS_PER_SEC;
+ hfst::pmatch::timer = clock();
+ std::cerr << "defined function" << $1 << " in " << duration << " seconds\n";
+ std::cerr << std::endl;
+ }
+ }
+;
ARGLIST: ARGS {
$$ = new std::vector<std::string>(hfst::pmatch::tmp_collected_funargs);
hfst::pmatch::tmp_collected_funargs.clear();
- };
+ }
+|
+{
+ $$ = new std::vector<std::string>();
+ hfst::pmatch::tmp_collected_funargs.clear();
+};
ARGS: SYMBOL {
hfst::pmatch::tmp_collected_funargs.push_back($1);
};
| SYMBOL COMMA ARGS {
hfst::pmatch::tmp_collected_funargs.push_back($1);
- };
+ }
+;
REGEXP1: REGEXP2 END_OF_EXPRESSION { }
| REGEXP2 END_OF_WEIGHTED_EXPRESSION {
@@ -228,49 +288,38 @@ REGEXP1: REGEXP2 END_OF_EXPRESSION { }
;
FUNCBODY1: FUNCBODY2 { }
-| FUNCBODY1 ENDTAG_LEFT QUOTED_LITERAL RIGHT_PARENTHESIS {
-
- HfstTransducer * end_tag = hfst::pmatch::make_end_tag($3);
- free($3);
- hfst::pmatch::PmatchAstNode * concatenation =
- new hfst::pmatch::PmatchAstNode($1,
- new hfst::pmatch::PmatchAstNode(end_tag),
- hfst::pmatch::AstConcatenate);
- $$ = new hfst::pmatch::PmatchAstNode(concatenation,
- hfst::pmatch::AstAddDelimiters);
- }
| FUNCBODY1 COMPOSITION FUNCBODY2 {
- $$ = new hfst::pmatch::PmatchAstNode($1, $3, hfst::pmatch::AstCompose);
+ $$ = new PmatchAstNode($1, $3, hfst::pmatch::AstCompose);
}
| FUNCBODY1 CROSS_PRODUCT FUNCBODY2 {
- $$ = new hfst::pmatch::PmatchAstNode($1, $3, hfst::pmatch::AstCrossProduct);
+ $$ = new PmatchAstNode($1, $3, hfst::pmatch::AstCrossProduct);
}
| FUNCBODY1 LENIENT_COMPOSITION FUNCBODY2 {
- $$ = new hfst::pmatch::PmatchAstNode($1, $3, hfst::pmatch::AstLenientCompose);
+ $$ = new PmatchAstNode($1, $3, hfst::pmatch::AstLenientCompose);
}
| FUNCBODY1 FUN_RIGHT_CONTEXT {
- $$ = new hfst::pmatch::PmatchAstNode($1, $2, hfst::pmatch::AstConcatenate);
+ $$ = new PmatchAstNode($1, $2, hfst::pmatch::AstConcatenate);
}
| FUNCBODY1 FUN_LEFT_CONTEXT {
- $$ = new hfst::pmatch::PmatchAstNode($2, $1, hfst::pmatch::AstConcatenate);
+ $$ = new PmatchAstNode($2, $1, hfst::pmatch::AstConcatenate);
}
| FUNCBODY1 FUN_NEGATIVE_RIGHT_CONTEXT {
- $$ = new hfst::pmatch::PmatchAstNode($1, $2, hfst::pmatch::AstConcatenate);
+ $$ = new PmatchAstNode($1, $2, hfst::pmatch::AstConcatenate);
}
| FUNCBODY1 FUN_NEGATIVE_LEFT_CONTEXT {
- $$ = new hfst::pmatch::PmatchAstNode($2, $1, hfst::pmatch::AstConcatenate);
+ $$ = new PmatchAstNode($2, $1, hfst::pmatch::AstConcatenate);
}
;
FUNCBODY2: FUNCBODY3 { }
| FUNCBODY2 UNION FUNCBODY3 {
- $$ = new hfst::pmatch::PmatchAstNode($1, $3, hfst::pmatch::AstDisjunct);
+ $$ = new PmatchAstNode($1, $3, hfst::pmatch::AstDisjunct);
}
| FUNCBODY2 INTERSECTION FUNCBODY3 {
- $$ = new hfst::pmatch::PmatchAstNode($1, $3, hfst::pmatch::AstIntersect);
+ $$ = new PmatchAstNode($1, $3, hfst::pmatch::AstIntersect);
}
| FUNCBODY2 MINUS FUNCBODY3 {
- $$ = new hfst::pmatch::PmatchAstNode($1, $3, hfst::pmatch::AstSubtract);
+ $$ = new PmatchAstNode($1, $3, hfst::pmatch::AstSubtract);
}
// | REGEXP5 UPPER_MINUS REGEXP6 {
// pmatcherror("No upper minus");
@@ -298,7 +347,7 @@ FUNCBODY2: FUNCBODY3 { }
FUNCBODY3: FUNCBODY4 { }
| FUNCBODY3 FUNCBODY4 {
- $$ = new hfst::pmatch::PmatchAstNode($1, $2, hfst::pmatch::AstConcatenate);
+ $$ = new PmatchAstNode($1, $2, hfst::pmatch::AstConcatenate);
};
FUNCBODY4: FUNCBODY5
@@ -306,27 +355,27 @@ FUNCBODY4: FUNCBODY5
$$ = $2;
}
| LEFT_PARENTHESIS FUNCBODY1 RIGHT_PARENTHESIS {
- $$ = new hfst::pmatch::PmatchAstNode($2, hfst::pmatch::AstOptionalize);
+ $$ = new PmatchAstNode($2, hfst::pmatch::AstOptionalize);
}
| ALPHA {
- $$ = new hfst::pmatch::PmatchAstNode(hfst::pmatch::get_utils()->latin1_alpha_acceptor);
+ $$ = new PmatchAstNode(hfst::pmatch::get_utils()->latin1_alpha_acceptor);
}
| LOWERALPHA {
- $$ = new hfst::pmatch::PmatchAstNode(hfst::pmatch::get_utils()->latin1_lowercase_acceptor);
+ $$ = new PmatchAstNode(hfst::pmatch::get_utils()->latin1_lowercase_acceptor);
}
| UPPERALPHA {
- $$ = new hfst::pmatch::PmatchAstNode(hfst::pmatch::get_utils()->latin1_uppercase_acceptor);
+ $$ = new PmatchAstNode(hfst::pmatch::get_utils()->latin1_uppercase_acceptor);
}
| NUM {
- $$ = new hfst::pmatch::PmatchAstNode(hfst::pmatch::get_utils()->latin1_numeral_acceptor);
+ $$ = new PmatchAstNode(hfst::pmatch::get_utils()->latin1_numeral_acceptor);
}
| PUNCT {
- $$ = new hfst::pmatch::PmatchAstNode(hfst::pmatch::get_utils()->latin1_punct_acceptor);
+ $$ = new PmatchAstNode(hfst::pmatch::get_utils()->latin1_punct_acceptor);
}
| WHITESPACE {
- $$ = new hfst::pmatch::PmatchAstNode(hfst::pmatch::get_utils()->latin1_whitespace_acceptor);
+ $$ = new PmatchAstNode(hfst::pmatch::get_utils()->latin1_whitespace_acceptor);
}
-| INSERT { $$ = new hfst::pmatch::PmatchAstNode($1); }
+| INSERT { $$ = new PmatchAstNode($1); }
| FUN_OPTCAP { }
| FUN_TOUPPER { }
| FUN_TOLOWER { }
@@ -335,37 +384,37 @@ FUNCBODY4: FUNCBODY5
FUNCBODY5: FUNCBODY6 { }
| FUNCBODY5 STAR {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstRepeatStar);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstRepeatStar);
}
| FUNCBODY5 PLUS {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstRepeatPlus);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstRepeatPlus);
}
| FUNCBODY5 REVERSE {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstReverse);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstReverse);
}
| FUNCBODY5 INVERT {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstInvert);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstInvert);
}
| FUNCBODY5 UPPER {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstInputProject);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstInputProject);
}
| FUNCBODY5 LOWER {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstOutputProject);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstOutputProject);
}
| FUNCBODY5 CATENATE_N {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstRepeatN);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstRepeatN);
$$->push_numeric_arg($2);
}
| FUNCBODY5 CATENATE_N_PLUS {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstRepeatNPlus);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstRepeatNPlus);
$$->push_numeric_arg($2 + 1);
}
| FUNCBODY5 CATENATE_N_MINUS {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstRepeatNMinus);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstRepeatNMinus);
$$->push_numeric_arg($2 - 1);
}
| FUNCBODY5 CATENATE_N_TO_K {
- $$ = new hfst::pmatch::PmatchAstNode($1, hfst::pmatch::AstRepeatNToK);
+ $$ = new PmatchAstNode($1, hfst::pmatch::AstRepeatNToK);
$$->push_numeric_arg($2[0]);
$$->push_numeric_arg($2[1]);
free($2);
@@ -374,26 +423,68 @@ FUNCBODY5: FUNCBODY6 { }
-FUNCBODY6: QUOTED_LITERAL {
- HfstTokenizer tok;
- HfstTransducer * literal = new HfstTransducer($1, tok, hfst::pmatch::format);
- free($1);
- $$ = new hfst::pmatch::PmatchAstNode(literal);
-}
+FUNCBODY6: FUNC_LABEL_PAIR { }
| SYMBOL {
- $$ = new hfst::pmatch::PmatchAstNode($1);
+ $$ = new PmatchAstNode($1);
free($1);
}
| BOUNDARY_MARKER {
- $$ = new hfst::pmatch::PmatchAstNode(new HfstTransducer("@BOUNDARY@", "@BOUNDARY@", hfst::pmatch::format));
+ $$ = new PmatchAstNode(new HfstTransducer("@BOUNDARY@", "@BOUNDARY@", hfst::pmatch::format));
+ }
+| ENDTAG_LEFT SYMBOL RIGHT_PARENTHESIS {
+ $$ = new PmatchAstNode(hfst::pmatch::make_end_tag($2));
+ hfst::pmatch::need_delimiters = true;
+ free($2);
+ }
+| ENDTAG_LEFT QUOTED_LITERAL RIGHT_PARENTHESIS {
+ $$ = new PmatchAstNode(hfst::pmatch::make_end_tag($2));
+ hfst::pmatch::need_delimiters = true;
+ free($2);
+ }
+| READ_BIN {
+ HfstTransducer * read;
+ try {
+ hfst::HfstInputStream instream($1);
+ read = new HfstTransducer(instream);
+ instream.close();
+ } catch(HfstException) {
+ std::string ermsg =
+ std::string("Couldn't read transducer from ") +
+ std::string($1);
+ free($1);
+ pmatcherror(ermsg.c_str());
+ }
+ if (read->get_type() != hfst::pmatch::format) {
+ read->convert(hfst::pmatch::format);
+ }
+ $$ = new PmatchAstNode(read);
+ free($1);
+ }
+| READ_TEXT {
+ $$ = new PmatchAstNode(hfst::pmatch::read_text($1));
+ free($1);
+ }
+| READ_LEXC {
+ $$ = new PmatchAstNode(hfst::HfstTransducer::read_lexc_ptr($1, hfst::TROPICAL_OPENFST_TYPE, hfst::pmatch::verbose));
+ free($1);
}
;
+ FUNC_LABEL_PAIR:
+LABEL {
+ $$ = new PmatchAstNode(new HfstTransducer($1, hfst::pmatch::format));
+ free($1);
+} |
+CURLY_LITERAL {
+ HfstTokenizer tok;
+ $$ = new PmatchAstNode(new HfstTransducer($1, tok, hfst::pmatch::format));
+ free($1);
+}
+;
+
REGEXP2: REPLACE
-{
-// std::cerr << "regexp2:replace \n"<< std::endl;
-}
+{ }
| REGEXP2 COMPOSITION REPLACE {
$$ = & $1->compose(*$3);
@@ -407,6 +498,7 @@ REGEXP2: REPLACE
$$ = $1;
delete $3;
}
+
;
////////////////////////////
@@ -446,7 +538,6 @@ REPLACE : REGEXP3 {}
}
delete $1;
- $$->insert_to_alphabet(hfst::pmatch::all_pmatch_symbols);
}
;
@@ -748,7 +839,6 @@ REGEXP5: REGEXP6 { }
delete $3;
}
| REGEXP5 MINUS REGEXP6 {
- $1->insert_to_alphabet(hfst::pmatch::all_pmatch_symbols);
$$ = & $1->subtract(*$3);
delete $3;
}
@@ -892,7 +982,6 @@ REGEXP10: REGEXP11 { }
hfst::internal_identity,
hfst::pmatch::format);
$$ = & ( any->subtract(*$2));
- $$->insert_to_alphabet(hfst::pmatch::all_pmatch_symbols);
delete $2;
}
| SUBSTITUTE_LEFT REGEXP10 COMMA REGEXP10 COMMA REGEXP10 RIGHT_BRACKET {
@@ -911,11 +1000,6 @@ REGEXP11: REGEXP12 { }
| LEFT_PARENTHESIS REGEXP2 RIGHT_PARENTHESIS {
$$ = & $2->optionalize();
}
-| CURLY_LITERAL {
- HfstTokenizer tok;
- $$ = new HfstTransducer($1, tok, hfst::pmatch::format);
- free($1);
- }
| ALPHA {
$$ = new HfstTransducer(*hfst::pmatch::get_utils()->latin1_alpha_acceptor);
}
@@ -957,21 +1041,21 @@ TOUPPER: TOUPPER_LEFT REGEXP11 RIGHT_PARENTHESIS {
;
FUN_OPTCAP: OPTCAP_LEFT FUNCBODY4 RIGHT_PARENTHESIS {
- $$ = new hfst::pmatch::PmatchAstNode($2, hfst::pmatch::AstOptCap);
+ $$ = new PmatchAstNode($2, hfst::pmatch::AstOptCap);
}
;
FUN_TOLOWER: TOLOWER_LEFT FUNCBODY4 RIGHT_PARENTHESIS {
- $$ = new hfst::pmatch::PmatchAstNode($2, hfst::pmatch::AstToLower);
+ $$ = new PmatchAstNode($2, hfst::pmatch::AstToLower);
}
;
FUN_TOUPPER: TOUPPER_LEFT FUNCBODY4 RIGHT_PARENTHESIS {
- $$ = new hfst::pmatch::PmatchAstNode($2, hfst::pmatch::AstToUpper);
+ $$ = new PmatchAstNode($2, hfst::pmatch::AstToUpper);
}
;
-REGEXP12: LABEL { }
+REGEXP12: LABEL_PAIR { }
| READ_BIN {
try {
hfst::HfstInputStream instream($1);
@@ -1008,60 +1092,32 @@ REGEXP12: LABEL { }
}
;
-// There follows a cartesian product of {quoted_literal, epsilon_token and any_token}
-LABEL: QUOTED_LITERAL PAIR_SEPARATOR QUOTED_LITERAL {
+LABEL_PAIR: LABEL PAIR_SEPARATOR LABEL {
$$ = new HfstTransducer($1, $3, hfst::pmatch::format);
- free($1);
- free($3);
+ free($1); free($3);
}
-| EPSILON_TOKEN PAIR_SEPARATOR EPSILON_TOKEN {
- $$ = new HfstTransducer(hfst::internal_epsilon,
- hfst::internal_epsilon, hfst::pmatch::format);
- }
+
| ANY_TOKEN PAIR_SEPARATOR ANY_TOKEN {
$$ = new HfstTransducer(hfst::internal_unknown, hfst::internal_unknown,
hfst::pmatch::format);
- // Insert special symbols we don't want to have expanded when this
- // interacts with anything else
- $$->insert_to_alphabet(hfst::pmatch::all_pmatch_symbols);
-
}
-| QUOTED_LITERAL PAIR_SEPARATOR EPSILON_TOKEN {
- $$ = new HfstTransducer($1, hfst::internal_epsilon, hfst::pmatch::format);
- free($1);
-}
-| QUOTED_LITERAL PAIR_SEPARATOR ANY_TOKEN {
+| LABEL PAIR_SEPARATOR ANY_TOKEN {
$$ = new HfstTransducer($1, hfst::internal_unknown, hfst::pmatch::format);
free($1);
}
-| EPSILON_TOKEN PAIR_SEPARATOR QUOTED_LITERAL {
- $$ = new HfstTransducer(hfst::internal_epsilon, $3, hfst::pmatch::format);
- free($3);
-}
-| EPSILON_TOKEN PAIR_SEPARATOR ANY_TOKEN {
- $$ = new HfstTransducer(hfst::internal_epsilon, hfst::internal_unknown,
- hfst::pmatch::format);
- }
-| ANY_TOKEN PAIR_SEPARATOR QUOTED_LITERAL {
+| ANY_TOKEN PAIR_SEPARATOR LABEL {
$$ = new HfstTransducer(hfst::internal_unknown, $3, hfst::pmatch::format);
+ free($3);
}
-| ANY_TOKEN PAIR_SEPARATOR EPSILON_TOKEN {
- $$ = new HfstTransducer(hfst::internal_unknown, hfst::internal_epsilon,
- hfst::pmatch::format);
- }
-| QUOTED_LITERAL PAIR_SEPARATOR_WO_RIGHT {
+| LABEL PAIR_SEPARATOR_WO_RIGHT {
$$ = new HfstTransducer($1, hfst::internal_unknown, hfst::pmatch::format);
free($1);
}
-| EPSILON_TOKEN PAIR_SEPARATOR_WO_RIGHT {
- $$ = new HfstTransducer(hfst::internal_epsilon, hfst::internal_unknown,
- hfst::pmatch::format);
- }
| ANY_TOKEN PAIR_SEPARATOR_WO_RIGHT {
$$ = new HfstTransducer(hfst::internal_unknown, hfst::internal_unknown,
hfst::pmatch::format);
}
-| PAIR_SEPARATOR_WO_LEFT QUOTED_LITERAL {
+| PAIR_SEPARATOR_WO_LEFT LABEL {
$$ = new HfstTransducer(hfst::internal_unknown, $2, hfst::pmatch::format);
free($2);
}
@@ -1069,16 +1125,6 @@ LABEL: QUOTED_LITERAL PAIR_SEPARATOR QUOTED_LITERAL {
$$ = new HfstTransducer(hfst::internal_unknown, hfst::internal_unknown,
hfst::pmatch::format);
}
-| PAIR_SEPARATOR_WO_LEFT EPSILON_TOKEN {
- $$ = new HfstTransducer(hfst::internal_unknown, hfst::internal_epsilon,
- hfst::pmatch::format);
- }
-| EPSILON_TOKEN PAIR_SEPARATOR SYMBOL {
- $$ = new HfstTransducer(hfst::internal_epsilon, $3, hfst::pmatch::format);
-}
-| SYMBOL PAIR_SEPARATOR EPSILON_TOKEN {
- $$ = new HfstTransducer($1, hfst::internal_epsilon, hfst::pmatch::format);
-}
| SYMBOL {
if (hfst::pmatch::definitions.count($1) != 0) {
if (!hfst::pmatch::flatten &&
@@ -1096,7 +1142,6 @@ LABEL: QUOTED_LITERAL PAIR_SEPARATOR QUOTED_LITERAL {
hfst::pmatch::used_definitions.insert($1);
}
$$ = new HfstTransducer(*hfst::pmatch::definitions[$1]);
- $$->insert_to_alphabet(hfst::pmatch::all_pmatch_symbols);
}
} else {
if (strlen($1) == 0) {
@@ -1112,38 +1157,82 @@ LABEL: QUOTED_LITERAL PAIR_SEPARATOR QUOTED_LITERAL {
$$ = new HfstTransducer(hfst::internal_unknown, hfst::internal_unknown,
hfst::pmatch::format);
}
-| EPSILON_TOKEN {
- $$ = new HfstTransducer(hfst::internal_epsilon, hfst::internal_epsilon,
+| LABEL {
+ $$ = new HfstTransducer($1, $1,
hfst::pmatch::format);
+ free($1);
}
| ANY_TOKEN {
$$ = new HfstTransducer(hfst::internal_identity,
hfst::pmatch::format);
- // Insert special symbols we don't want to have expanded when this
- // interacts with anything else
- $$->insert_to_alphabet(hfst::pmatch::all_pmatch_symbols);
}
-| QUOTED_LITERAL {
- $$ = new HfstTransducer($1, hfst::pmatch::format);
+| CURLY_LITERAL {
+ HfstTokenizer tok;
+ $$ = new HfstTransducer($1, tok, hfst::pmatch::format);
free($1);
- }
-| BOUNDARY_MARKER {
- $$ = new HfstTransducer("@BOUNDARY@", "@BOUNDARY@", hfst::pmatch::format);
- }
+ }
+| CURLY_LITERAL PAIR_SEPARATOR CURLY_LITERAL {
+ HfstTokenizer tok;
+ HfstTransducer * left = new HfstTransducer($1, tok, hfst::pmatch::format);
+ HfstTransducer * right = new HfstTransducer($3, tok, hfst::pmatch::format);
+ HfstTransducer * destroy = new HfstTransducer(
+ hfst::internal_unknown, hfst::internal_epsilon, hfst::pmatch::format);
+ HfstTransducer * construct = new HfstTransducer(
+ hfst::internal_epsilon, hfst::internal_unknown, hfst::pmatch::format);
+ left->compose(destroy->repeat_star());
+ left->compose(construct->repeat_star());
+ left->compose(*right);
+ $$ = left;
+ delete destroy; delete construct; delete right;
+ free($1); free($3);
+}
+| LABEL PAIR_SEPARATOR CURLY_LITERAL {
+ HfstTokenizer tok;
+ HfstTransducer * left = new HfstTransducer(
+ $1, hfst::internal_epsilon, hfst::pmatch::format);
+ HfstTransducer * right = new HfstTransducer($3, tok, hfst::pmatch::format);
+ HfstTransducer * construct = new HfstTransducer(
+ hfst::internal_epsilon, hfst::internal_unknown, hfst::pmatch::format);
+ left->compose(construct->repeat_star());
+ left->compose(*right);
+ $$ = left;
+ delete construct; delete right;
+ free($1); free($3);
+}
+| CURLY_LITERAL PAIR_SEPARATOR LABEL {
+ HfstTokenizer tok;
+ HfstTransducer * left = new HfstTransducer($1, tok, hfst::pmatch::format);
+ HfstTransducer * right = new HfstTransducer(
+ hfst::internal_epsilon, $3, hfst::pmatch::format);
+ HfstTransducer * destroy = new HfstTransducer(
+ hfst::internal_unknown, hfst::internal_epsilon, hfst::pmatch::format);
+ left->compose(destroy->repeat_star());
+ left->compose(*right);
+ $$ = left;
+ delete destroy; delete right;
+ free($1); free($3);
+}
| FUNCALL { }
| MAP { }
| CONTEXT_CONDITION { }
| ENDTAG_LEFT SYMBOL RIGHT_PARENTHESIS {
$$ = hfst::pmatch::make_end_tag($2);
+ hfst::pmatch::need_delimiters = true;
}
| ENDTAG_LEFT QUOTED_LITERAL RIGHT_PARENTHESIS {
$$ = hfst::pmatch::make_end_tag($2);
+ hfst::pmatch::need_delimiters = true;
}
;
+LABEL: QUOTED_LITERAL { }
+| EPSILON_TOKEN { $$ = strdup(hfst::internal_epsilon.c_str()); }
+| BOUNDARY_MARKER { $$ = strdup("@BOUNDARY@"); }
+;
+
CONTEXT_CONDITION:
-P_CONTEXT { }
+P_CONTEXT { $$ = $1; hfst::pmatch::need_delimiters = true; }
| OR_CONTEXT { }
| AND_CONTEXT { };
@@ -1161,6 +1250,9 @@ OR_CONTEXT: OR_LEFT CONTEXT_CONDITIONS RIGHT_PARENTHESIS
$$->disjunct(*it);
}
delete $2;
+ // Zero the counter for making minimization
+ // guards for disjuncted negative contexts
+ hfst::pmatch::zero_minimization_guard();
};
AND_CONTEXT: AND_LEFT CONTEXT_CONDITIONS RIGHT_PARENTHESIS
@@ -1184,7 +1276,7 @@ CONTEXT_CONDITION {
$$ = $3;
};
-FUNCALL: SYMBOL_WITH_LEFT_PAREN ARGLIST RIGHT_PARENTHESIS {
+FUNCALL: SYMBOL_WITH_LEFT_PAREN FUNCALL_ARGLIST RIGHT_PARENTHESIS {
if (hfst::pmatch::functions.count($1) == 0) {
std::string errstring = "Function not defined: " + std::string($1);
pmatcherror(errstring.c_str());
@@ -1197,18 +1289,49 @@ FUNCALL: SYMBOL_WITH_LEFT_PAREN ARGLIST RIGHT_PARENTHESIS {
}
std::map<std::string, HfstTransducer*> caller_args;
for (int i = 0; i < $2->size(); ++i) {
- if (hfst::pmatch::definitions.count($2->at(i)) != 0) {
- caller_args[callee_args[i]] = hfst::pmatch::definitions[$2->at(i)];
- } else if (hfst::pmatch::def_insed_transducers.count($1) != 0) {
- caller_args[callee_args[i]] = hfst::pmatch::def_insed_transducers[$2->at(i)];
- } else {
- std::string errstring = "Unknown definition: " + std::string($2->at(i));
- pmatcherror(errstring.c_str());
- }
+ caller_args[callee_args[i]] = new HfstTransducer($2->at(i));
}
$$ = hfst::pmatch::functions[$1].evaluate(caller_args);
+ delete $2;
};
+FUNCALL_ARGLIST:
+FUNCALL_ARG {
+ $$ = new HfstTransducerVector();
+ $$->push_back(HfstTransducer(*$1));
+ delete $1;
+}
+| FUNCALL_ARG COMMA FUNCALL_ARGLIST {
+ $3->push_back(HfstTransducer(*$1));
+ delete $1;
+}
+| { $$ = new HfstTransducerVector(); }
+;
+
+FUNCALL_ARG:
+SYMBOL {
+ if (hfst::pmatch::definitions.count($1) == 1) {
+ $$ = new HfstTransducer(* hfst::pmatch::definitions[$1]);
+ } else if (hfst::pmatch::def_insed_transducers.count($1) == 0) {
+ $$ = new HfstTransducer(* hfst::pmatch::def_insed_transducers[$1]);
+ } else {
+ std::string errstring = "Unknown definition: " + std::string($1);
+ free($1);
+ pmatcherror(errstring.c_str());
+ }
+ free($1);
+}
+| CURLY_LITERAL {
+ HfstTokenizer tok;
+ $$ = new HfstTransducer($1, tok, hfst::pmatch::format);
+ free($1);
+ }
+| QUOTED_LITERAL {
+ $$ = new HfstTransducer($1, hfst::pmatch::format);
+ free($1);
+ }
+;
+
MAP: MAP_LEFT SYMBOL COMMA READ_TEXT RIGHT_PARENTHESIS {
if (hfst::pmatch::functions.count($2) == 0) {
std::string errstring = "Function not defined: " + std::string($2);
@@ -1297,6 +1420,7 @@ RIGHT_CONTEXT: RC_LEFT REPLACE RIGHT_PARENTHESIS {
;
NEGATIVE_RIGHT_CONTEXT: NRC_LEFT REPLACE RIGHT_PARENTHESIS {
+ $$ = hfst::pmatch::get_minimization_guard();
HfstTransducer * nrc_entry = new HfstTransducer(
hfst::internal_epsilon, hfst::pmatch::NRC_ENTRY_SYMBOL, hfst::pmatch::format);
HfstTransducer * nrc_exit = new HfstTransducer(
@@ -1305,9 +1429,8 @@ NEGATIVE_RIGHT_CONTEXT: NRC_LEFT REPLACE RIGHT_PARENTHESIS {
nrc_entry->concatenate(*nrc_exit);
nrc_entry->disjunct(HfstTransducer("@PMATCH_PASSTHROUGH@",
hfst::internal_epsilon, hfst::pmatch::format));
- $$ = nrc_entry;
- delete $2;
- delete nrc_exit;
+ $$->concatenate(*nrc_entry);
+ delete $2; delete nrc_entry; delete nrc_exit;
}
;
@@ -1319,12 +1442,12 @@ LEFT_CONTEXT: LC_LEFT REPLACE RIGHT_PARENTHESIS {
lc_entry->concatenate($2->reverse());
lc_entry->concatenate(*lc_exit);
$$ = lc_entry;
- delete $2;
- delete lc_exit;
+ delete $2; delete lc_exit;
}
;
NEGATIVE_LEFT_CONTEXT: NLC_LEFT REPLACE RIGHT_PARENTHESIS {
+ $$ = hfst::pmatch::get_minimization_guard();
HfstTransducer * nlc_entry = new HfstTransducer(
hfst::internal_epsilon, hfst::pmatch::NLC_ENTRY_SYMBOL, hfst::pmatch::format);
HfstTransducer * nlc_exit = new HfstTransducer(
@@ -1333,20 +1456,19 @@ NEGATIVE_LEFT_CONTEXT: NLC_LEFT REPLACE RIGHT_PARENTHESIS {
nlc_entry->concatenate(*nlc_exit);
nlc_entry->disjunct(HfstTransducer("@PMATCH_PASSTHROUGH@",
hfst::internal_epsilon, hfst::pmatch::format));
- $$ = nlc_entry;
- delete $2;
- delete nlc_exit;
+ $$->concatenate(*nlc_entry);
+ delete $2; delete nlc_entry; delete nlc_exit;
}
;
FUN_RIGHT_CONTEXT: RC_LEFT FUNCBODY2 RIGHT_PARENTHESIS {
- hfst::pmatch::PmatchAstNode * rc_entry =
- new hfst::pmatch::PmatchAstNode(
+ PmatchAstNode * rc_entry =
+ new PmatchAstNode(
new HfstTransducer(hfst::internal_epsilon,
hfst::pmatch::RC_ENTRY_SYMBOL,
hfst::pmatch::format),
$2, hfst::pmatch::AstConcatenate);
- $$ = new hfst::pmatch::PmatchAstNode(rc_entry,
+ $$ = new PmatchAstNode(rc_entry,
new HfstTransducer(
hfst::internal_epsilon,
hfst::pmatch::RC_EXIT_SYMBOL,
@@ -1356,20 +1478,20 @@ FUN_RIGHT_CONTEXT: RC_LEFT FUNCBODY2 RIGHT_PARENTHESIS {
;
FUN_NEGATIVE_RIGHT_CONTEXT: NRC_LEFT FUNCBODY2 RIGHT_PARENTHESIS {
- hfst::pmatch::PmatchAstNode * nrc_entry =
- new hfst::pmatch::PmatchAstNode(
+ PmatchAstNode * nrc_entry =
+ new PmatchAstNode(
new HfstTransducer(hfst::internal_epsilon,
hfst::pmatch::NRC_ENTRY_SYMBOL,
hfst::pmatch::format),
$2, hfst::pmatch::AstConcatenate);
- hfst::pmatch::PmatchAstNode * nrc_main_branch =
- new hfst::pmatch::PmatchAstNode(nrc_entry,
+ PmatchAstNode * nrc_main_branch =
+ new PmatchAstNode(nrc_entry,
new HfstTransducer(
hfst::internal_epsilon,
hfst::pmatch::NRC_EXIT_SYMBOL,
hfst::pmatch::format),
hfst::pmatch::AstConcatenate);
- $$ = new hfst::pmatch::PmatchAstNode(
+ $$ = new PmatchAstNode(
nrc_main_branch,
new HfstTransducer("@PMATCH_PASSTHROUGH@",
hfst::internal_epsilon, hfst::pmatch::format),
@@ -1378,7 +1500,7 @@ FUN_NEGATIVE_RIGHT_CONTEXT: NRC_LEFT FUNCBODY2 RIGHT_PARENTHESIS {
;
FUN_LEFT_CONTEXT: LC_LEFT FUNCBODY2 RIGHT_PARENTHESIS {
- hfst::pmatch::PmatchAstNode * reverse = new hfst::pmatch::PmatchAstNode(
+ PmatchAstNode * reverse = new PmatchAstNode(
$2, hfst::pmatch::AstReverse);
HfstTransducer * lc_entry = new HfstTransducer(
@@ -1386,31 +1508,31 @@ FUN_LEFT_CONTEXT: LC_LEFT FUNCBODY2 RIGHT_PARENTHESIS {
HfstTransducer * lc_exit = new HfstTransducer(
hfst::internal_epsilon, hfst::pmatch::LC_EXIT_SYMBOL, hfst::pmatch::format);
- hfst::pmatch::PmatchAstNode * entry = new hfst::pmatch::PmatchAstNode(
+ PmatchAstNode * entry = new PmatchAstNode(
lc_entry, reverse, hfst::pmatch::AstConcatenate);
- $$ = new hfst::pmatch::PmatchAstNode(
+ $$ = new PmatchAstNode(
entry, lc_exit, hfst::pmatch::AstConcatenate);
}
;
FUN_NEGATIVE_LEFT_CONTEXT: NLC_LEFT FUNCBODY2 RIGHT_PARENTHESIS {
- hfst::pmatch::PmatchAstNode * reverse = new hfst::pmatch::PmatchAstNode(
+ PmatchAstNode * reverse = new PmatchAstNode(
$2, hfst::pmatch::AstReverse);
-
+
HfstTransducer * nlc_entry = new HfstTransducer(
hfst::internal_epsilon, hfst::pmatch::NLC_ENTRY_SYMBOL, hfst::pmatch::format);
HfstTransducer * nlc_exit = new HfstTransducer(
hfst::internal_epsilon, hfst::pmatch::NLC_EXIT_SYMBOL, hfst::pmatch::format);
-
- hfst::pmatch::PmatchAstNode * entry = new hfst::pmatch::PmatchAstNode(
+
+ PmatchAstNode * entry = new PmatchAstNode(
nlc_entry, reverse, hfst::pmatch::AstConcatenate);
- hfst::pmatch::PmatchAstNode * main_branch = new hfst::pmatch::PmatchAstNode(
+ PmatchAstNode * main_branch = new PmatchAstNode(
entry, nlc_exit, hfst::pmatch::AstConcatenate);
-
- $$ = new hfst::pmatch::PmatchAstNode(main_branch,
- new HfstTransducer("@PMATCH_PASSTHROUGH@",
- hfst::internal_epsilon, hfst::pmatch::format),
- hfst::pmatch::AstDisjunct);
+
+ $$ = new PmatchAstNode(main_branch,
+ new HfstTransducer("@PMATCH_PASSTHROUGH@",
+ hfst::internal_epsilon, hfst::pmatch::format),
+ hfst::pmatch::AstDisjunct);
}
;
diff --git a/libhfst/src/parsers/pmatch_utils.cc b/libhfst/src/parsers/pmatch_utils.cc
index 72eeda4..95304fe 100644
--- a/libhfst/src/parsers/pmatch_utils.cc
+++ b/libhfst/src/parsers/pmatch_utils.cc
@@ -7,6 +7,7 @@
#include <cassert>
#include <cstdlib>
#include <cstring>
+#include <sstream>
#include "pmatch_utils.h"
#include "HfstTransducer.h"
@@ -62,17 +63,15 @@ size_t len;
bool verbose;
bool flatten;
clock_t timer;
+int minimization_guard_count;
+bool need_delimiters;
std::map<std::string, hfst::HfstTransducer> named_transducers;
PmatchUtilityTransducers* utils=NULL;
-std::set<std::string> all_pmatch_symbols;
-void add_to_pmatch_symbols(StringSet symbols)
+void warn(std::string warning)
{
- for(StringSet::const_iterator it = symbols.begin();
- it != symbols.end(); ++it) {
- all_pmatch_symbols.insert(*it);
- }
+ std::cerr << "pmatch: warning: " << warning;
}
PmatchUtilityTransducers*
@@ -85,6 +84,23 @@ get_utils()
return utils;
}
+void zero_minimization_guard(void)
+{
+ minimization_guard_count = 0;
+}
+
+HfstTransducer * get_minimization_guard(void)
+{
+ std::stringstream guard;
+ if(minimization_guard_count == 0) {
+ guard << hfst::internal_epsilon;
+ } else {
+ guard << "@PMATCH_GUARD_" << minimization_guard_count << "@";
+ }
+ ++minimization_guard_count;
+ return new HfstTransducer(hfst::internal_epsilon, guard.str(), format);
+}
+
int*
get_n_to_k(const char* s)
{
@@ -186,7 +202,6 @@ get_Ins_transition(const char *s)
rv = strcpy(rv, "@I.");
rv = strcat(rv, s);
rv = strcat(rv, "@");
- all_pmatch_symbols.insert(rv);
return rv;
}
@@ -197,7 +212,6 @@ get_RC_transition(const char *s)
rv = strcpy(rv, "@RC.");
rv = strcat(rv, s);
rv = strcat(rv, "@");
- all_pmatch_symbols.insert(rv);
return rv;
}
@@ -208,7 +222,6 @@ get_LC_transition(const char *s)
rv = strcpy(rv, "@LC.");
rv = strcat(rv, s);
rv = strcat(rv, "@");
- all_pmatch_symbols.insert(rv);
return rv;
}
@@ -227,19 +240,17 @@ HfstTransducer * add_pmatch_delimiters(HfstTransducer * regex)
void add_end_tag(HfstTransducer * regex, std::string tag)
{
- HfstTransducer end_tag(hfst::internal_epsilon,
- "@PMATCH_ENDTAG_" + tag + "@",
- regex->get_type());
- all_pmatch_symbols.insert("@PMATCH_ENDTAG_" + tag + "@");
- regex->concatenate(end_tag);
+ HfstTransducer * end_tag = make_end_tag(tag);
+ regex->concatenate(*end_tag);
+ delete end_tag;
}
HfstTransducer * make_end_tag(std::string tag)
{
- HfstTransducer * end_tag = new HfstTransducer(hfst::internal_epsilon,
- "@PMATCH_ENDTAG_" + tag + "@",
- format);
- all_pmatch_symbols.insert("@PMATCH_ENDTAG_" + tag + "@");
+ HfstTransducer * end_tag = new HfstTransducer(
+ hfst::internal_epsilon,
+ "@PMATCH_ENDTAG_" + tag + "@",
+ format);
return end_tag;
}
@@ -272,8 +283,8 @@ char * unescape_delimited(char *s, char delim)
char * read = s;
char * write = s;
while (*read != '\0') {
- if (*read == '\\' && *(read + 1) == delim) {
- *write = delim;
+ if (*read == '\\' && (*(read + 1) == delim || *(read + 1) == '\\')) {
+ *write = *(read + 1);
read += 2;
write += 1;
} else {
@@ -282,7 +293,7 @@ char * unescape_delimited(char *s, char delim)
++write;
}
}
- *write == '\0';
+ *write = '\0';
return s;
}
@@ -471,21 +482,8 @@ void init_globals(void)
used_definitions.clear();
functions.clear();
tmp_collected_funargs.clear();
-
- all_pmatch_symbols.clear();
- all_pmatch_symbols.insert(RC_ENTRY_SYMBOL);
- all_pmatch_symbols.insert(RC_EXIT_SYMBOL);
- all_pmatch_symbols.insert(LC_ENTRY_SYMBOL);
- all_pmatch_symbols.insert(LC_EXIT_SYMBOL);
- all_pmatch_symbols.insert(NRC_ENTRY_SYMBOL);
- all_pmatch_symbols.insert(NRC_EXIT_SYMBOL);
- all_pmatch_symbols.insert(NLC_ENTRY_SYMBOL);
- all_pmatch_symbols.insert(NLC_EXIT_SYMBOL);
- all_pmatch_symbols.insert(PASSTHROUGH_SYMBOL);
- all_pmatch_symbols.insert(BOUNDARY_SYMBOL);
- all_pmatch_symbols.insert(ENTRY_SYMBOL);
- all_pmatch_symbols.insert(EXIT_SYMBOL);
-
+ zero_minimization_guard();
+ need_delimiters = false;
}
std::map<std::string, HfstTransducer*>
@@ -540,17 +538,12 @@ compile(const string& pmatch, map<string,HfstTransducer*>& defs,
}
HfstTransducer dummy(format);
- dummy.insert_to_alphabet(all_pmatch_symbols);
// We keep TOP and any inserted transducers
std::map<std::string, hfst::HfstTransducer *>::iterator defs_itr;
for (defs_itr = definitions.begin(); defs_itr != definitions.end();
++defs_itr) {
if (defs_itr->first.compare("TOP") == 0 ||
inserted_transducers.count(defs_itr->first) != 0) {
- // In order to avoid expanding special pmatch markers, insert
- // them first
- hfst::StringSet symbols_so_far = dummy.get_alphabet();
- defs_itr->second->insert_to_alphabet(symbols_so_far);
dummy.harmonize(*defs_itr->second);
}
}
@@ -561,8 +554,6 @@ compile(const string& pmatch, map<string,HfstTransducer*>& defs,
++defs_itr) {
if (defs_itr->first.compare("TOP") == 0 ||
inserted_transducers.count(defs_itr->first) != 0) {
- hfst::StringSet symbols_so_far = dummy.get_alphabet();
- defs_itr->second->insert_to_alphabet(symbols_so_far);
dummy.harmonize(*defs_itr->second);
retval.insert(std::pair<std::string, hfst::HfstTransducer*>(
defs_itr->first,
diff --git a/libhfst/src/parsers/pmatch_utils.h b/libhfst/src/parsers/pmatch_utils.h
index b3d3260..18f1258 100644
--- a/libhfst/src/parsers/pmatch_utils.h
+++ b/libhfst/src/parsers/pmatch_utils.h
@@ -32,6 +32,8 @@ extern ImplementationType format;
extern bool verbose;
extern bool flatten;
extern clock_t timer;
+extern int minimization_guard_count;
+extern bool need_delimiters;
struct PmatchUtilityTransducers;
//extern PmatchUtilityTransducers* utils;
@@ -48,11 +50,11 @@ const std::string BOUNDARY_SYMBOL = "@BOUNDARY@";
const std::string ENTRY_SYMBOL = "@PMATCH_ENTRY@";
const std::string EXIT_SYMBOL = "@PMATCH_EXIT@";
-extern std::set<std::string> all_pmatch_symbols;
-
void add_to_pmatch_symbols(StringSet symbols);
-
+void warn(std::string warning);
PmatchUtilityTransducers* get_utils();
+void zero_minimization_guard(void);
+HfstTransducer * get_minimization_guard(void);
/**
* @brief input handling function for flex that parses strings.
diff --git a/man/Makefile.am b/man/Makefile.am
index 2f0c9da..767441d 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -30,15 +30,31 @@ hfst_tools=hfst-compare.1 hfst-compose.1 \
hfst-calculate.1 hfst-compose-intersect.1 \
hfst-lookup.1 \
hfst-substitute.1 hfst-format.1 hfst-name.1 hfst-lexc.1 \
- hfst-lexc-wrapper.1
+ hfst-lexc-wrapper.1 \
+ hfst-affix-guessify.1 hfst-edit-metadata.1 hfst-expand-equivalences.1 \
+ hfst-grep.1 hfst-guess.1 hfst-guessify.1 hfst-info.1 hfst-multiply.1 \
+ hfst-pair-test.1 hfst-prune-alphabet.1 hfst-reweight.1 hfst-shuffle.1 \
+ hfst-traverse.1 hfst-proc2.1 hfst-pmatch.1 hfst-pmatch2fst.1 \
+ hfst-optimized-lookup.1
hfst_xfst=hfst-xfst.1
hfst_apertium_proc=hfst-apertium-proc.1
hfst_foma=hfst-foma.1 # hand-written
+hfst_foma_wrapper=hfst-foma-wrapper.1 # hand-written
+hfst_tag_and_reweight=hfst-tag.1 hfst-reweight-tagger.1
+hfst_train_tagger=hfst-train-tagger.1
+hfst_train_tagger_subprograms=hfst-open-input-file-for-tagger.1 \
+ hfst_tagger_compute_data_statistics.py.1 hfst-build-tagger.1 # hand-written
+hfst_twolc=hfst-twolc.1
+hfst_twolc_subprograms= htwolcpre1.1 htwolcpre2.1 htwolcpre3.1 # hand-written
aliases= hfst-determinise.1 hfst-expand.1 hfst-intersect.1 hfst-minimise.1 \
- hfst-minus.1 hfst-sfstpl2fst.1 hfst-summarise.1 hfst-union.1 hfst-proc.1
+ hfst-minus.1 hfst-sfstpl2fst.1 hfst-summarise.1 hfst-union.1 hfst-proc.1 \
+ hfst-optimised-lookup.1 hfst-train-tagger-loc.1 hfst-train-tagger-system.1 \
+ hfst-twolc-loc.1 hfst-twolc-system.1
-dist_man1_MANS=$(hfst_tools) $(hfst_xfst) $(hfst_apertium_proc) $(hfst_foma) $(aliases)
+dist_man1_MANS=$(hfst_tools) $(hfst_xfst) $(hfst_apertium_proc) $(hfst_foma) $(hfst_foma_wrapper) \
+ $(hfst_tag_and_reweight) $(hfst_train_tagger) $(hfst_train_tagger_subprograms) $(hfst_twolc) \
+ $(hfst_twolc_subprograms) $(aliases)
LN_S=ln -s -f
@@ -57,6 +73,9 @@ hfst-minimise.1:
hfst-minus.1:
$(LN_S) hfst-subtract.1 hfst-minus.1
+hfst-optimised-lookup.1:
+ $(LN_S) hfst-optimized-lookup.1 hfst-optimised-lookup.1
+
hfst-proc.1:
$(LN_S) hfst-apertium-proc.1 hfst-proc.1
@@ -66,6 +85,18 @@ hfst-sfstpl2fst.1:
hfst-summarise.1:
$(LN_S) hfst-summarize.1 hfst-summarise.1
+hfst-train-tagger-loc.1:
+ $(LN_S) hfst-train-tagger.1 hfst-train-tagger-loc.1
+
+hfst-train-tagger-system.1:
+ $(LN_S) hfst-train-tagger.1 hfst-train-tagger-system.1
+
+hfst-twolc-loc.1:
+ $(LN_S) hfst-twolc.1 hfst-twolc-loc.1
+
+hfst-twolc-system.1:
+ $(LN_S) hfst-twolc.1 hfst-twolc-system.1
+
hfst-union.1:
$(LN_S) hfst-disjunct.1 hfst-union.1
@@ -94,6 +125,24 @@ bootstrap:
-n="$$N" \
-s 1 -o $$f -N -S HFST `echo $$f | sed 's/.1//' | sed 's/hfst/..\/tools\/src\/hfst-proc\/hfst/'`; \
done;
+ for f in $(hfst_tag_and_reweight); do \
+ N=`echo $$f | sed 's/.1/ \-\-help/' | sed 's/hfst/..\/tools\/src\/hfst-tagger\/src\/hfst/' | sh | sed -n 2p`; \
+ help2man --no-discard-stderr \
+ -n="$$N" \
+ -s 1 -o $$f -N -S HFST `echo $$f | sed 's/.1//' | sed 's/hfst/..\/tools\/src\/hfst-tagger\/src\/hfst/'`; \
+ done;
+ for f in $(hfst_train_tagger); do \
+ N=`echo "../tools/src/hfst-tagger/src/hfst-open-input-file-for-tagger --help" | sh | sed -n 2p`; \
+ help2man --no-discard-stderr \
+ -n="$$N" \
+ -s 1 -o $$f -N -S HFST `echo "../tools/src/hfst-tagger/src/hfst-train-tagger"`; \
+ done;
+ for f in $(hfst_twolc); do \
+ N=`echo "../tools/src/hfst-twolc/src/hfst-twolc --help" | sh | sed -n 2p`; \
+ help2man --no-discard-stderr \
+ -n="$$N" \
+ -s 1 -o $$f -N -S HFST `echo "../tools/src/hfst-twolc/src/hfst-twolc"`; \
+ done;
# hfst-foma man page is hand-written
# Create symbolic links to aliases
diff --git a/man/hfst-format.1 b/man/hfst-affix-guessify.1
similarity index 66%
copy from man/hfst-format.1
copy to man/hfst-affix-guessify.1
index c590c9f..a1fec79 100644
--- a/man/hfst-format.1
+++ b/man/hfst-affix-guessify.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FORMAT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-AFFIX-GUESSIFY "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-format \- =determine HFST transducer format
+hfst-affix-guessify \- =Create weighted affix guesser from automaton
.SH SYNOPSIS
-.B hfst-format
+.B hfst-affix-guessify
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-determine HFST transducer format
+Create weighted affix guesser from automaton
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -30,33 +30,30 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
-.SS "Tool-specific options:"
+.SS "Guesser parameters:"
.TP
-\fB\-l\fR, \fB\-\-list\-formats\fR
-List available transducer formats
-and print them to standard output
+\fB\-D\fR, \fB\-\-direction\fR=\fIDIR\fR
+set direction of guessing
.TP
-\fB\-t\fR, \fB\-\-test\-format\fR FMT
-Whether the format FMT is available,
-exits with 0 if it is, else with 1
+\fB\-w\fR, \fB\-\-weight\fR=\fIWEIGHT\fR
+set weight difference of affix lengths
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
+DIR is either suffix or prefix, or suffix if omitted.
+WEIGHT is a weight of each arc not in the known suffix or prefix being guessed, as parsed with strtod(3), or 1.0 if omitted.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-format home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstFormat>
+hfst\-affix\-guessify home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstAffixGuessify>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
-.br
-Transducers in (null) are of type SFST (1.4 compatible)
.SH COPYRIGHT
Copyright \(co 2010 University of Helsinki,
License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
-Transducers in (null) are of type SFST (1.4 compatible)
diff --git a/man/hfst-apertium-proc.1 b/man/hfst-apertium-proc.1
index 3819733..0600a96 100644
--- a/man/hfst-apertium-proc.1
+++ b/man/hfst-apertium-proc.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-APERTIUM-PROC "1" "February 2014" "HFST" "User Commands"
+.TH HFST-APERTIUM-PROC "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-apertium-proc \- =Usage: hfst-proc [-a [-p|-C|-x] [-k]|-g|-n|-d|-t] [-W] [-n N] [-c|-w] [-z] [-v|-q|]
.SH SYNOPSIS
@@ -10,6 +10,7 @@ hfst-apertium-proc \- =Usage: hfst-proc [-a [-p|-C|-x] [-k]|-g|-n|-d|-t] [-W] [-
transducer_file [input_file [output_file]]
.PP
Perform a transducer lookup on a text stream, tokenizing on the fly
+Transducer must be in HFST optimized lookup format
.TP
\fB\-a\fR, \fB\-\-analysis\fR
Morphological analysis (default)
@@ -85,7 +86,7 @@ case, ``superblanks'' or anything else!!!
.SH "REPORTING BUGS"
Report bugs to hfst\-bugs at helsinki.fi
.PP
-hfst\-proc 0.0 (hfst 3.6.1)
+hfst\-proc 0.0 (hfst 3.8.0)
.br
-Feb 13 2014 16:05:42
+Oct 3 2014 13:50:55
copyright (C) 2009\-2011 University of Helsinki
diff --git a/man/hfst-build-tagger.1 b/man/hfst-build-tagger.1
new file mode 100644
index 0000000..ba6e98e
--- /dev/null
+++ b/man/hfst-build-tagger.1
@@ -0,0 +1,15 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST-BUILD-TAGGER "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst-build-tagger \- =
+.SH SYNOPSIS
+.B hfst-build-tagger
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+A program used by hfst-train-tagger, not intended to be used directly.
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/hfst-calculate.1 b/man/hfst-calculate.1
index 1e30f46..73a99bf 100644
--- a/man/hfst-calculate.1
+++ b/man/hfst-calculate.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SFSTPL2FST "1" "February 2014" "HFST" "User Commands"
+.TH HFST-SFSTPL2FST "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-sfstpl2fst \- =Compile a file written with SFST programming language into a transducer.
.SH SYNOPSIS
diff --git a/man/hfst-compare.1 b/man/hfst-compare.1
index fabb22e..ea6f339 100644
--- a/man/hfst-compare.1
+++ b/man/hfst-compare.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-COMPARE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-COMPARE "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-compare \- =Compare two transducers
.SH SYNOPSIS
diff --git a/man/hfst-compose-intersect.1 b/man/hfst-compose-intersect.1
index 5f19229..54cf6e4 100644
--- a/man/hfst-compose-intersect.1
+++ b/man/hfst-compose-intersect.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-COMPOSE-INTERSECT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-COMPOSE-INTERSECT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-compose-intersect \- =Compose a lexicon with one or more rule transducers.
.SH SYNOPSIS
@@ -40,6 +40,10 @@ Compose the intersection of the
rules with the lexicon instead
of composing the lexicon with
the intersection of the rules.
+.TP
+\fB\-e\fR, \fB\-\-encode\-weights\fR
+Encode weights when minimizing
+(default is false).
.PP
If OUTFILE, or either INFILE1 or INFILE2 is missing or \-, standard
streams will be used. INFILE1, INFILE2, or both, must be specified
diff --git a/man/hfst-compose.1 b/man/hfst-compose.1
index 7f57f3b..a006915 100644
--- a/man/hfst-compose.1
+++ b/man/hfst-compose.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-COMPOSE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-COMPOSE "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-compose \- =Compose two transducers
.SH SYNOPSIS
diff --git a/man/hfst-concatenate.1 b/man/hfst-concatenate.1
index fb10d8e..be9cc4a 100644
--- a/man/hfst-concatenate.1
+++ b/man/hfst-concatenate.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-CONCATENATE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-CONCATENATE "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-concatenate \- =Concatenate two transducers
.SH SYNOPSIS
diff --git a/man/hfst-conjunct.1 b/man/hfst-conjunct.1
index f25bde4..76c505f 100644
--- a/man/hfst-conjunct.1
+++ b/man/hfst-conjunct.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-CONJUNCT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-CONJUNCT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-conjunct \- =Conjunct (intersect, AND) two transducers
.SH SYNOPSIS
diff --git a/man/hfst-determinise.1 b/man/hfst-determinise.1
new file mode 120000
index 0000000..31d9799
--- /dev/null
+++ b/man/hfst-determinise.1
@@ -0,0 +1 @@
+hfst-determinize.1
\ No newline at end of file
diff --git a/man/hfst-determinize.1 b/man/hfst-determinize.1
index 087dd0c..ec27723 100644
--- a/man/hfst-determinize.1
+++ b/man/hfst-determinize.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-DETERMINIZE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-DETERMINIZE "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-determinize \- =Determinize a transducer
.SH SYNOPSIS
@@ -30,6 +30,11 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
+.SS "Command-specific options:"
+.TP
+\fB\-E\fR, \fB\-\-encode\-weights\fR
+Encode weights when determinizing
+(default is false).
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
diff --git a/man/hfst-disjunct.1 b/man/hfst-disjunct.1
index f85c45c..1824cde 100644
--- a/man/hfst-disjunct.1
+++ b/man/hfst-disjunct.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-DISJUNCT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-DISJUNCT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-disjunct \- =Disjunct (union, OR) two transducers
.SH SYNOPSIS
diff --git a/man/hfst-name.1 b/man/hfst-edit-metadata.1
similarity index 75%
copy from man/hfst-name.1
copy to man/hfst-edit-metadata.1
index 30461e4..1f32f3c 100644
--- a/man/hfst-name.1
+++ b/man/hfst-edit-metadata.1
@@ -1,22 +1,22 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-NAME "1" "February 2014" "HFST" "User Commands"
+.TH HFST-EDIT-METADATA "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-name \- =Name a transducer
+hfst-edit-metadata \- =Name a transducer
.SH SYNOPSIS
-.B hfst-name
+.B hfst-edit-metadata
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
Name a transducer
.SS "Name options:"
.TP
-\fB\-n\fR, \fB\-\-name\fR=\fINAME\fR
-Name the transducer NAME
+\fB\-a\fR, \fB\-\-add\fR=\fIANAME=VALUE\fR
+add or replace property ANAMEwith VALUE
.TP
-\fB\-p\fR, \fB\-\-print\-name\fR
-Only print the current name
+\fB\-p\fR, \fB\-\-print\fR[=\fINAME\fR]
+print the current PNAME
.TP
\fB\-t\fR, \fB\-\-truncate_length\fR=\fILEN\fR
-Truncate name length to LEN
+truncate added properties' lengths to LEN
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -43,12 +43,13 @@ Write output transducer to OUTFILE
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
+If PNAME is omitted, all values are printed
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-name home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstName>
+hfst\-edit\-metadata home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstEditMetadata>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-expand-equivalences.1 b/man/hfst-expand-equivalences.1
new file mode 100644
index 0000000..30decd0
--- /dev/null
+++ b/man/hfst-expand-equivalences.1
@@ -0,0 +1,63 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST-EXPAND-EQUIVALENCES "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst-expand-equivalences \- =Extend transducer arcs for equivalence classes
+.SH SYNOPSIS
+.B hfst-expand-equivalences
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+Extend transducer arcs for equivalence classes
+.SS "Common options:"
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Print help message
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Print version info
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Print verbosely while processing
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Only print fatal erros and requested output
+.TP
+\fB\-s\fR, \fB\-\-silent\fR
+Alias of \fB\-\-quiet\fR
+.SS "Eqv. class extension options:"
+.TP
+\fB\-f\fR, \fB\-\-from\fR=\fIISYM\fR
+convert single symbol ISYM to allow OSYM
+.TP
+\fB\-t\fR, \fB\-\-to\fR=\fIOSYM\fR
+convert to OSYM
+.TP
+\fB\-a\fR, \fB\-\-acx\fR=\fIACXFILE\fR
+read extensions in acx format from ACXFILE
+.TP
+\fB\-T\fR, \fB\-\-tsv\fR=\fITSVFILE\fR
+read extensions in tsv format from TSVFILE
+.TP
+\fB\-l\fR, \fB\-\-level\fR=\fILEVEL\fR
+perform extensions on LEVEL of fsa
+.PP
+Either ACXFILE, TSVFILE or both ISYM and OSYM must be specified.
+LEVEL should be either {upper, first, 1, input, surface}, {lower, second, 2, output, analysis} or both.
+If LEVEL is omitted, default is first.
+Examples:
+.TP
+hfst\-expand\-equivalences \fB\-o\fR rox.hfst \fB\-a\fR romanian.acx ro.hfst
+extend romanian charequivalences
+.SH "REPORTING BUGS"
+Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
+<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+hfst\-expand\-equivalences home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstExpandEquivalences>
+.br
+General help using HFST software:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/hfst-expand.1 b/man/hfst-expand.1
new file mode 120000
index 0000000..53f5e0f
--- /dev/null
+++ b/man/hfst-expand.1
@@ -0,0 +1 @@
+hfst-fst2strings.1
\ No newline at end of file
diff --git a/man/hfst-foma-wrapper.1 b/man/hfst-foma-wrapper.1
new file mode 100644
index 0000000..2c3112f
--- /dev/null
+++ b/man/hfst-foma-wrapper.1
@@ -0,0 +1,11 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH UNKNOWN "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+Unknown \- =
+.SH SYNOPSIS
+.B hfst-foma-wrapper.sh
+[\fI-h|-V\fR] [\fI-v|-s|-q\fR] [\fI-f FORMAT\fR] [\fI-F STACK_FILES\fR] [\fISCRIPTS\fR]
+.SH DESCRIPTION
+Unknown command line switch , try \fB\-h\fR for help
+.PP
+Usage: ../scripts/hfst\-foma\-wrapper.sh [\-h|\-V] [\-v|\-s|\-q] [\-f FORMAT] [\-F STACK_FILES] [SCRIPTS]
diff --git a/man/hfst-format.1 b/man/hfst-format.1
index c590c9f..72f061b 100644
--- a/man/hfst-format.1
+++ b/man/hfst-format.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FORMAT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-FORMAT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-format \- =determine HFST transducer format
.SH SYNOPSIS
diff --git a/man/hfst-fst2fst.1 b/man/hfst-fst2fst.1
index 442e2a2..f950c5e 100644
--- a/man/hfst-fst2fst.1
+++ b/man/hfst-fst2fst.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FST2FST "1" "February 2014" "HFST" "User Commands"
+.TH HFST-FST2FST "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-fst2fst \- =Convert transducers between binary formats
.SH SYNOPSIS
diff --git a/man/hfst-fst2strings.1 b/man/hfst-fst2strings.1
index 87ac61e..ffc50d5 100644
--- a/man/hfst-fst2strings.1
+++ b/man/hfst-fst2strings.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FST2STRINGS "1" "February 2014" "HFST" "User Commands"
+.TH HFST-FST2STRINGS "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-fst2strings \- =Display the strings recognized by a transducer
.SH SYNOPSIS
diff --git a/man/hfst-fst2txt.1 b/man/hfst-fst2txt.1
index 9c1d630..142c6c6 100644
--- a/man/hfst-fst2txt.1
+++ b/man/hfst-fst2txt.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-FST2TXT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-FST2TXT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-fst2txt \- =Print transducer in AT&T, dot, prolog or pckimmo format
.SH SYNOPSIS
diff --git a/man/hfst-grep.1 b/man/hfst-grep.1
new file mode 100644
index 0000000..8ca2087
--- /dev/null
+++ b/man/hfst-grep.1
@@ -0,0 +1,174 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST-GREP "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst-grep \- =Search for PATTERN in each FILE or standard input.
+.SH SYNOPSIS
+.B hfst-grep
+[\fIOPTIONS\fR...] \fIPATTERN \fR[\fIFILE\fR...]
+.SH DESCRIPTION
+Search for PATTERN in each FILE or standard input.
+Pattern is, by default, a Xerox regular expression (XRE).
+Example: hfst\-grep 'h e l l o % w o r l d' menu.h menu.c
+.SS "Common options:"
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Print help message
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Print version info
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Print verbosely while processing
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Only print fatal erros and requested output
+.TP
+\fB\-s\fR, \fB\-\-silent\fR
+Alias of \fB\-\-quiet\fR
+.TP
+\fB\-9\fR, \fB\-\-format\fR=\fITYPE\fR
+compile expressions to TYPE automata
+.SS "Regexp selection and interpretation:"
+.TP
+\fB\-E\fR, \fB\-\-extended\-regexp\fR
+PATTERN is an extended regular expression (ERE)
+.TP
+\fB\-F\fR, \fB\-\-fixed\-strings\fR
+PATTERN is a set of newline\-separated fixed strings
+.TP
+\fB\-G\fR, \fB\-\-basic\-regexp\fR
+PATTERN is a basic regular expression (BRE)
+.TP
+\fB\-P\fR, \fB\-\-perl\-regexp\fR
+PATTERN is a Perl regular expression
+.TP
+\fB\-X\fR, \fB\-\-xerox\-regexp\fR
+PATTERN is a Xerox regulare expression
+.TP
+\fB\-e\fR, \fB\-\-regexp\fR=\fIPATTERN\fR
+use PATTERN for matching
+.TP
+\fB\-f\fR, \fB\-\-file\fR=\fIFILE\fR
+obtain PATTERN from FILE
+.TP
+\fB\-I\fR, \fB\-\-ignore\-case\fR
+ignore case distinctions
+.TP
+\fB\-w\fR, \fB\-\-word\-regexp\fR
+force PATTERN to match only whole words
+.TP
+\fB\-x\fR, \fB\-\-line\-regexp\fR
+force PATTERN to match only whole lines
+.TP
+\fB\-z\fR, \fB\-\-null\-data\fR
+a data line ends in 0 byte, not newline
+.SS "Miscellaneous options:"
+.TP
+\fB\-\-no\-messages\fR
+suppress error messages
+.TP
+\fB\-\-invert\-match\fR
+select non\-matching lines
+.SS "Output control:"
+.TP
+\fB\-m\fR, \fB\-\-max\-count\fR=\fINUM\fR
+stop after NUM matches\en \fB\-b\fR, \fB\-\-byte\-offset\fR print the byte offset with output lines
+.TP
+\fB\-n\fR, \fB\-\-line\-number\fR
+print line number with output lines
+.TP
+\fB\-\-line\-buffered\fR
+flush output on every line
+.TP
+\fB\-H\fR, \fB\-\-with\-filename\fR
+print the filename for each match
+.TP
+\fB\-h\fR, \fB\-\-no\-filename\fR
+suppress the prefixing filename on output
+.TP
+\fB\-\-label\fR=\fILABEL\fR
+print LABEL as filename for standard input
+.TP
+\fB\-o\fR, \fB\-\-only\-matching\fR
+show only the part of a line matching PATTERN
+.TP
+\fB\-\-binary\-files\fR=\fITYPE\fR
+assume that binary files are TYPE;
+TYPE is `binary', `text', or `without\-match'
+.TP
+\fB\-a\fR, \fB\-\-text\fR
+equivalent to \fB\-\-binary\-files\fR=\fItext\fR
+.TP
+\fB\-d\fR, \fB\-\-directories\fR=\fIACTION\fR
+how to handle directories;
+ACTION is `read', `recurse', or `skip'
+.TP
+\fB\-D\fR, \fB\-\-devices\fR=\fIACTION\fR
+how to handle devices, FIFOs and sockets;
+ACTION is `read' or `skip'
+.TP
+\fB\-R\fR, \fB\-r\fR, \fB\-\-recursive\fR
+equivalent to \fB\-\-directories\fR=\fIrecurse\fR
+.TP
+\fB\-\-include\fR=\fIFILE_PATTERN\fR
+search only files that match FILE_PATTERN
+.TP
+\fB\-\-exclude\fR=\fIFILE_PATTERN\fR
+skip files and directories matching FILE_PATTERN
+.TP
+\fB\-\-exclude\-from\fR=\fIFILE\fR
+skip files matching any file pattern from FILE
+.TP
+\fB\-\-exclude\-dir\fR=\fIPATTERN\fR
+directories that match PATTERN will be skipped
+.TP
+\fB\-L\fR, \fB\-\-files\-without\-match\fR
+print only names of FILEs containing no match
+.TP
+\fB\-l\fR, \fB\-\-files\-with\-matches\fR
+print only names of FILEs containing matches
+.TP
+\fB\-c\fR, \fB\-\-count\fR
+print only a count of matching lines per FILE
+.TP
+\fB\-T\fR, \fB\-\-initial\-tab\fR
+make tabs line up (if needed)
+.TP
+\fB\-Z\fR, \fB\-\-null\fR
+print 0 byte after FILE name
+.SS "Context control:"
+.TP
+\fB\-B\fR, \fB\-\-before\-context\fR=\fINUM\fR
+print NUM lines of leading context
+.TP
+\fB\-A\fR, \fB\-\-after\-context\fR=\fINUM\fR
+print NUM lines of trailing context
+.TP
+\fB\-C\fR, \fB\-\-context\fR=\fINUM\fR
+print NUM lines of output context
+.HP
+\fB\-\-color\fR[=\fIWHEN\fR],
+.TP
+\fB\-\-colour\fR[=\fIWHEN\fR]
+use markers to highlight the matching strings;
+WHEN is `always', `never', or `auto'
+.TP
+\fB\-U\fR, \fB\-\-binary\fR
+do not strip CR characters at EOL (MSDOS)
+.TP
+\fB\-u\fR, \fB\-\-unix\-byte\-offsets\fR
+report offsets as if CRs were not there (MSDOS)
+.SH "REPORTING BUGS"
+Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
+<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+hfst\-grep home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstGrep>
+.br
+General help using HFST software:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/hfst-txt2fst.1 b/man/hfst-guess.1
similarity index 50%
copy from man/hfst-txt2fst.1
copy to man/hfst-guess.1
index 8a71ef0..9d17464 100644
--- a/man/hfst-txt2fst.1
+++ b/man/hfst-guess.1
@@ -1,12 +1,13 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TXT2FST "1" "February 2014" "HFST" "User Commands"
+.TH HFST-GUESS "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-txt2fst \- =Convert AT&T or prolog format into a binary transducer
+hfst-guess \- =Use a guesser (and generator) to guess analyses or inflectional
.SH SYNOPSIS
-.B hfst-txt2fst
+.B hfst-guess
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Convert AT&T or prolog format into a binary transducer
+Use a guesser (and generator) to guess analyses or inflectional
+paradigms of unknown words.
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -30,31 +31,43 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
-.SS "Text and format options:"
+.SS "Guesser options:"
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFMT\fR
-Write result using FMT as backend format
+\fB\-f\fR, \fB\-\-model\-form\-filename\fR
+Inflectional information for
+generated model forms is read
+from this file.
.TP
-\fB\-e\fR, \fB\-\-epsilon\fR=\fIEPS\fR
-Interpret string EPS as epsilon in att format
+\fB\-n\fR, \fB\-\-max\-number\-of\-guesses\fR
+Maximal number of analysis
+per word form (5 by default).
.TP
-\fB\-p\fR, \fB\-\-prolog\fR
-Read prolog format instead of att
+\fB\-m\fR \fB\-\-max\-number\-of\-forms\fR
+Maximal number of generated model
+forms per guess (2 by default).
+.TP
+\fB\-g\fR \fB\-\-generate\-threshold\fR
+Generate only forms whose weight
+is better than the weight of the
+of the best form plus this threshold.
+(50 by default).
.PP
-If OUTFILE or INFILE is missing or \-, standard streams will be used.
-If FMT is not given, OpenFst's tropical format will be used.
-The possible values for FMT are { foma, openfst\-tropical, openfst\-log,
-sfst, optimized\-lookup\-weighted, optimized\-lookup\-unweighted }.
-If EPS is not given, @0@ will be used.
+The guesser and generator should be constructed using the tool
+hfst\-guessify, which can compile a guesser and generator from a
+morphological analyzer. hfst\-guessify packages the guesser and
+generator in the same fst\-file.
.PP
-Space in transition symbols must be escaped as '@_SPACE_@' when using
-att format.
+If option \fB\-f\fR is used, but a generator has not been compiled
+with the guesser, a generator will be compiled, which will
+increase load time.
+.PP
+If OUTFILE or INFILE is missing or \-, standard streams will be used.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-txt2fst home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstTxt2Fst>
+hfst\-guess home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstGuess>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-txt2fst.1 b/man/hfst-guessify.1
similarity index 51%
copy from man/hfst-txt2fst.1
copy to man/hfst-guessify.1
index 8a71ef0..9c1e3d7 100644
--- a/man/hfst-txt2fst.1
+++ b/man/hfst-guessify.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TXT2FST "1" "February 2014" "HFST" "User Commands"
+.TH HFST-GUESSIFY "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-txt2fst \- =Convert AT&T or prolog format into a binary transducer
+hfst-guessify \- =Compile a morphological analyzer into a guesser and generator.
.SH SYNOPSIS
-.B hfst-txt2fst
+.B hfst-guessify
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Convert AT&T or prolog format into a binary transducer
+Compile a morphological analyzer into a guesser and generator.
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -30,31 +30,36 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
-.SS "Text and format options:"
+.SS "Guesser options:"
.TP
-\fB\-f\fR, \fB\-\-format\fR=\fIFMT\fR
-Write result using FMT as backend format
+\fB\-p\fR, \fB\-\-default\-penalty\fR
+Give penalty for skipping one
+symbol of input (1.0 by default).
.TP
-\fB\-e\fR, \fB\-\-epsilon\fR=\fIEPS\fR
-Interpret string EPS as epsilon in att format
-.TP
-\fB\-p\fR, \fB\-\-prolog\fR
-Read prolog format instead of att
+\fB\-G\fR, \fB\-\-do\-not\-compile\-generator\fR
+When compiling the guesser, do
+not compile a model form
+generator.
.PP
-If OUTFILE or INFILE is missing or \-, standard streams will be used.
-If FMT is not given, OpenFst's tropical format will be used.
-The possible values for FMT are { foma, openfst\-tropical, openfst\-log,
-sfst, optimized\-lookup\-weighted, optimized\-lookup\-unweighted }.
-If EPS is not given, @0@ will be used.
+All analyses in the morphological analyzer should have the form:
+w o r d f o r m POS [GUESS_CATEGORY=CLASS] X Y Z ...
+where POS is the part\-of\-speech tag, [GUESS_CATEGORY=CLASS]
+is an inflectional category marker and X, Y and Z are inflectional
+markers. The form of the inflectional category marker is fixed.
+CLASS can be any string, which doesn't contain "]".
.PP
-Space in transition symbols must be escaped as '@_SPACE_@' when using
-att format.
+Using the option \fB\-d\fR will reduce the size of the guesser file by
+approximately half, but may substantially increase the load time of
+the guesser when generating model forms. If you only need to guess
+analyses of unknown word forms, \fB\-d\fR has no effect on load time.
+.PP
+If OUTFILE or INFILE is missing or \-, standard streams will be used.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-txt2fst home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstTxt2Fst>
+hfst\-guessify home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstGuessify>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-head.1 b/man/hfst-head.1
index 6611d3c..90a7770 100644
--- a/man/hfst-head.1
+++ b/man/hfst-head.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-HEAD "1" "February 2014" "HFST" "User Commands"
+.TH HFST-HEAD "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-head \- =Get first transducers from an archive
.SH SYNOPSIS
diff --git a/man/hfst-name.1 b/man/hfst-info.1
similarity index 56%
copy from man/hfst-name.1
copy to man/hfst-info.1
index 30461e4..5816b91 100644
--- a/man/hfst-name.1
+++ b/man/hfst-info.1
@@ -1,22 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-NAME "1" "February 2014" "HFST" "User Commands"
+.TH HFST-INFO "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-name \- =Name a transducer
+hfst-info \- =show or test HFST versions and features
.SH SYNOPSIS
-.B hfst-name
+.B hfst-info
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Name a transducer
-.SS "Name options:"
-.TP
-\fB\-n\fR, \fB\-\-name\fR=\fINAME\fR
-Name the transducer NAME
-.TP
-\fB\-p\fR, \fB\-\-print\-name\fR
-Only print the current name
-.TP
-\fB\-t\fR, \fB\-\-truncate_length\fR=\fILEN\fR
-Truncate name length to LEN
+show or test HFST versions and features
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -33,22 +23,28 @@ Only print fatal erros and requested output
.TP
\fB\-s\fR, \fB\-\-silent\fR
Alias of \fB\-\-quiet\fR
-.SS "Input/Output options:"
+.SS "Test features:"
+.TP
+\fB\-a\fR, \fB\-\-atleast\-version\fR=\fIMVER\fR
+require at least MVER version of HFST
+.TP
+\fB\-e\fR, \fB\-\-exact\-version\fR=\fIEVER\fR
+require exactly EVER version of HFST
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
-Read input transducer from INFILE
+\fB\-m\fR, \fB\-\-max\-version\fR=\fIUVER\fR
+require at most UVER version of HFST
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
-Write output transducer to OUTFILE
+\fB\-f\fR, \fB\-\-requirefeature\fR=\fIFEAT\fR
+require named FEAT support from HFST
.PP
-If OUTFILE or INFILE is missing or \-, standard streams will be used.
-Format of result depends on format of INFILE
+MVER, EVER or UVER version vectors must be composed of one to three full stop separated runs of digits.
+FEAT should be name of feature supported by HFST, such as SFST, foma or openfst
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-name home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstName>
+hfst\-info home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstInfo>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-intersect.1 b/man/hfst-intersect.1
new file mode 120000
index 0000000..4f9c28b
--- /dev/null
+++ b/man/hfst-intersect.1
@@ -0,0 +1 @@
+hfst-conjunct.1
\ No newline at end of file
diff --git a/man/hfst-invert.1 b/man/hfst-invert.1
index 8cb2a4f..d8ef2d2 100644
--- a/man/hfst-invert.1
+++ b/man/hfst-invert.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-INVERT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-INVERT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-invert \- =Invert a transducer
.SH SYNOPSIS
diff --git a/man/hfst-lexc-wrapper.1 b/man/hfst-lexc-wrapper.1
index 91cea3a..0156695 100644
--- a/man/hfst-lexc-wrapper.1
+++ b/man/hfst-lexc-wrapper.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-LEXC-WRAPPER "1" "February 2014" "HFST" "User Commands"
+.TH HFST-LEXC-WRAPPER "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-lexc-wrapper \- =Compile lexc files into transducer or imitate Xerox lexc
.SH SYNOPSIS
diff --git a/man/hfst-lexc.1 b/man/hfst-lexc.1
index 0cf10c4..685591f 100644
--- a/man/hfst-lexc.1
+++ b/man/hfst-lexc.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-LEXC "1" "February 2014" "HFST" "User Commands"
+.TH HFST-LEXC "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-lexc \- =Compile lexc files into transducer
.SH SYNOPSIS
@@ -34,10 +34,28 @@ write result into OUTFILE
.TP
\fB\-F\fR, \fB\-\-withFlags\fR
use flags to hyperminimize result
+.TP
+\fB\-M\fR, \fB\-\-minimizeFlags\fR
+if \fB\-\-withFlags\fR is used, minimize the number of flags
+.TP
+\fB\-R\fR, \fB\-\-renameFlags\fR
+if \fB\-\-withFlags\fR and \fB\-\-minimizeFlags\fR are used, rename
+flags (for testing)
+.TP
+\fB\-x\fR, \fB\-\-xerox\-composition\fR=\fIVALUE\fR Whether flag diacritics are treated as ordinary
+symbols in composition (default is true).
+.TP
+\fB\-X\fR, \fB\-\-xfst\fR=\fIVARIABLE\fR
+toggle xfst compatibility option VARIABLE.
+.TP
+\fB\-W\fR, \fB\-\-Werror\fR
+treat warnings as errors
.PP
If INFILE or OUTFILE are omitted or \-, standard streams will be used
The possible values for FORMAT are { sfst, openfst\-tropical, openfst\-log,
foma, optimized\-lookup\-unweighted, optimized\-lookup\-weighted }.
+VALUEs recognized are {true,ON,yes} and {false,OFF,no}.
+Xfst variables are {flag\-is\-epsilon (default OFF)}.
.SH EXAMPLES
.TP
hfst\-lexc \fB\-o\fR cat.hfst cat.lexc
diff --git a/man/hfst-lookup.1 b/man/hfst-lookup.1
index 608bb38..b669f88 100644
--- a/man/hfst-lookup.1
+++ b/man/hfst-lookup.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-LOOKUP "1" "February 2014" "HFST" "User Commands"
+.TH HFST-LOOKUP "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-lookup \- =perform transducer lookup (apply)
.SH SYNOPSIS
@@ -55,6 +55,9 @@ Toggle xfst VARIABLE
.TP
\fB\-c\fR, \fB\-\-cycles\fR=\fIINT\fR
How many times to follow input epsilon cycles
+.TP
+\fB\-P\fR, \fB\-\-progress\fR
+Show neat progress bar if possible
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
diff --git a/man/hfst-minimise.1 b/man/hfst-minimise.1
new file mode 120000
index 0000000..5b8343d
--- /dev/null
+++ b/man/hfst-minimise.1
@@ -0,0 +1 @@
+hfst-minimize.1
\ No newline at end of file
diff --git a/man/hfst-minimize.1 b/man/hfst-minimize.1
index a8bf190..36d26b8 100644
--- a/man/hfst-minimize.1
+++ b/man/hfst-minimize.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-MINIMIZE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-MINIMIZE "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-minimize \- =Minimize a transducer
.SH SYNOPSIS
@@ -30,6 +30,11 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
+.SS "Command-specific options:"
+.TP
+\fB\-E\fR, \fB\-\-encode\-weights\fR
+Encode weights when minimizing
+(default is false).
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
diff --git a/man/hfst-minus.1 b/man/hfst-minus.1
new file mode 120000
index 0000000..8ec56ec
--- /dev/null
+++ b/man/hfst-minus.1
@@ -0,0 +1 @@
+hfst-subtract.1
\ No newline at end of file
diff --git a/man/hfst-project.1 b/man/hfst-multiply.1
similarity index 74%
copy from man/hfst-project.1
copy to man/hfst-multiply.1
index b0ff57a..c7939cf 100644
--- a/man/hfst-project.1
+++ b/man/hfst-multiply.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PROJECT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-MULTIPLY "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-project \- =Project (extract a level) transducer
+hfst-multiply \- =Use first transducer of an archive repeatedly
.SH SYNOPSIS
-.B hfst-project
+.B hfst-multiply
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Project (extract a level) transducer
+Use first transducer of an archive repeatedly
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -30,20 +30,20 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
-.SS "Projection options:"
+.SS "Archive options:"
.TP
-\fB\-p\fR, \fB\-\-project\fR=\fILEVEL\fR
-project extracting tape LEVEL
+\fB\-n\fR, \fB\-\-n\-last\fR=\fINUMBER\fR
+Duplicate each transducer NUMBER times
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
-LEVEL must be one of upper, input, first, analysis or lower, output, second, generation
+NUMBER must be a positive integer as parsed by strtoul base 10
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-project home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstProject>
+hfst\-multiply home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstDuplicate>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-name.1 b/man/hfst-name.1
index 30461e4..d735f76 100644
--- a/man/hfst-name.1
+++ b/man/hfst-name.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-NAME "1" "February 2014" "HFST" "User Commands"
+.TH HFST-NAME "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-name \- =Name a transducer
.SH SYNOPSIS
diff --git a/man/hfst-open-input-file-for-tagger.1 b/man/hfst-open-input-file-for-tagger.1
new file mode 100644
index 0000000..a5f5292
--- /dev/null
+++ b/man/hfst-open-input-file-for-tagger.1
@@ -0,0 +1,15 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST-OPEN-INPUT-FILE-FOR-TAGGER "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst-open-input-file-for-tagger \- =
+.SH SYNOPSIS
+.B hfst-open-input-file-for-tagger
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+A program used by hfst-train-tagger, not intended to be used directly.
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/hfst-optimised-lookup.1 b/man/hfst-optimised-lookup.1
new file mode 120000
index 0000000..931382b
--- /dev/null
+++ b/man/hfst-optimised-lookup.1
@@ -0,0 +1 @@
+hfst-optimized-lookup.1
\ No newline at end of file
diff --git a/man/hfst-optimized-lookup.1 b/man/hfst-optimized-lookup.1
new file mode 100644
index 0000000..45b8dee
--- /dev/null
+++ b/man/hfst-optimized-lookup.1
@@ -0,0 +1,58 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST-OPTIMIZED-LOOKUP "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst-optimized-lookup \- =Usage: hfst-optimized-lookup [OPTIONS] TRANSDUCER
+.SH SYNOPSIS
+.B hfst-optimized-lookup
+[\fIOPTIONS\fR] \fITRANSDUCER\fR
+.SH DESCRIPTION
+Run a transducer on standard input (one word per line) and print analyses
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Print this help message
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Print version information
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Be verbose
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Don't be verbose (default)
+.TP
+\fB\-s\fR, \fB\-\-silent\fR
+Same as quiet
+.TP
+\fB\-e\fR, \fB\-\-echo\fR
+Echo inputs
+(useful if redirecting lots of output to a file)
+.TP
+\fB\-w\fR, \fB\-\-show\-weights\fR
+Print final analysis weights (if any)
+.TP
+\fB\-u\fR, \fB\-\-unique\fR
+Suppress duplicate analyses
+.TP
+\fB\-n\fR N, \fB\-\-analyses\fR=\fIN\fR
+Output no more than N analyses
+(if the transducer is weighted, the N best analyses)
+.TP
+\fB\-x\fR, \fB\-\-xerox\fR
+Xerox output format (default)
+.TP
+\fB\-f\fR, \fB\-\-fast\fR
+Be as fast as possible.
+(with this option enabled \fB\-u\fR and \fB\-n\fR don't work and
+output won't be ordered by weight).
+.PP
+Note that hfst\-optimized\-lookup is *not* guaranteed to behave identically to
+hfst\-lookup (although it almost always does): input\-side multicharacter symbols
+are not fully supported. If the first character of such a symbol is an ASCII
+symbol also matching a single\-character symbol, it will be tokenized as such.
+.SH "REPORTING BUGS"
+Report bugs to hfst\-bugs at helsinki.fi
+.PP
+hfst\-optimized\-lookup 1.2
+.br
+Oct 3 2014 13:53:38
+copyright (C) 2009 University of Helsinki
diff --git a/man/hfst-pair-test.1 b/man/hfst-pair-test.1
new file mode 100644
index 0000000..1927624
--- /dev/null
+++ b/man/hfst-pair-test.1
@@ -0,0 +1,88 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST-PAIR-TEST "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst-pair-test \- =pair test for a twolc rule file.
+.SH SYNOPSIS
+.B hfst-pair-test
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+pair test for a twolc rule file.
+.SS "Common options:"
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Print help message
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Print version info
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Print verbosely while processing
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Only print fatal erros and requested output
+.TP
+\fB\-s\fR, \fB\-\-silent\fR
+Alias of \fB\-\-quiet\fR
+.SS "Input/Output options:"
+.TP
+\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+Read input rule file from INFILE
+.TP
+\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+Write test output to OUTFILE
+.TP
+\fB\-N\fR \fB\-\-negative\-test\fR
+Test fails if any of the pair strings is
+accepted.
+.SS "Pair test options:"
+.TP
+\fB\-I\fR, \fB\-\-input\-strings\fR=\fISFILE\fR
+Read pair test strings from
+SFILE
+.PP
+If SFILE is missing, the test pair strings are read from STDIN.
+If OUTFILE is missing, test output is written to STDOUT.
+.PP
+The rule file is tested using correspondences given as
+pair strings, e.g. "e a r l y:i e r". Every pair string is
+tested using every rule and the program prints information
+about correspondences that are incorrectly allowed or
+disallowed.
+.PP
+The test pair string files contain one pair string/line. Lines
+where the first non\-white\-space character is "!" are
+considered comment lines and skipped.
+.PP
+There are two test modes positive and negative. In positive
+mode, all of the pair strings should be allowed and in negative
+mode they should be disallowed.
+.PP
+Ordinarily, positive test mode is in use. Option \fB\-N\fR switches to
+negative test mode. The exit code for a successful test is 0.
+The exit code is 1 otherwise. A successful test will print
+"Test passed". A failing test prints "Test failed" and
+information about pair strings that are handled incorrectly.
+.PP
+In positive test mode (i.e. without option \fB\-N\fR), if a pair
+string is not accepted, the names of the rules that reject
+it are printed as well as the positions in the string where the
+rules run out of possible transitions. In negative mode, only
+the strings that are allowed are printed.
+.PP
+In silent mode (\fB\-s\fR), the program won't print anything. Only the
+exit code tells whether the test was successful or not.
+.SH "REPORTING BUGS"
+Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
+<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+.PP
+hfst\-pair\-test home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstPairTest>
+.br
+General help using HFST software:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/hfst-invert.1 b/man/hfst-pmatch.1
similarity index 63%
copy from man/hfst-invert.1
copy to man/hfst-pmatch.1
index 8cb2a4f..9042e50 100644
--- a/man/hfst-invert.1
+++ b/man/hfst-pmatch.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-INVERT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-PMATCH "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-invert \- =Invert a transducer
+hfst-pmatch \- =perform matching/lookup on text streams
.SH SYNOPSIS
-.B hfst-invert
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.B hfst-pmatch
+[\fIOPTIONS\fR...] \fITRANSDUCER\fR
.SH DESCRIPTION
-Invert a transducer
+perform matching/lookup on text streams
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -23,22 +23,23 @@ Only print fatal erros and requested output
.TP
\fB\-s\fR, \fB\-\-silent\fR
Alias of \fB\-\-quiet\fR
-.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
-Read input transducer from INFILE
+\fB\-n\fR \fB\-\-newline\fR
+Newline as input separator (default is blank line)
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
-Write output transducer to OUTFILE
+\fB\-x\fR \fB\-\-extract\-tags\fR
+Only print tagged parts in output
+.TP
+\fB\-l\fR \fB\-\-locate\fR
+Only print locations of matches
.PP
-If OUTFILE or INFILE is missing or \-, standard streams will be used.
-Format of result depends on format of INFILE
+Use standard streams for input and output.
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-invert home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstInvert>
+hfst\-pmatch home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstPmatch>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-name.1 b/man/hfst-pmatch2fst.1
similarity index 62%
copy from man/hfst-name.1
copy to man/hfst-pmatch2fst.1
index 30461e4..226aea3 100644
--- a/man/hfst-name.1
+++ b/man/hfst-pmatch2fst.1
@@ -1,22 +1,14 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-NAME "1" "February 2014" "HFST" "User Commands"
+.TH HFST-PMATCH2FST "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-name \- =Name a transducer
+hfst-pmatch2fst \- =Compile regular expressions into transducer(s)
.SH SYNOPSIS
-.B hfst-name
+.B hfst-pmatch2fst
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Name a transducer
-.SS "Name options:"
-.TP
-\fB\-n\fR, \fB\-\-name\fR=\fINAME\fR
-Name the transducer NAME
-.TP
-\fB\-p\fR, \fB\-\-print\-name\fR
-Only print the current name
-.TP
-\fB\-t\fR, \fB\-\-truncate_length\fR=\fILEN\fR
-Truncate name length to LEN
+Compile regular expressions into transducer(s)
+.IP
+(Experimental version)
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -40,15 +32,29 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
+.SS "String and format options:"
+.TP
+\fB\-e\fR, \fB\-\-epsilon\fR=\fIEPS\fR
+Map EPS as zero
+.TP
+\fB\-\-flatten\fR
+Compile in all RTNs
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
-Format of result depends on format of INFILE
+If EPS is not defined, the default representation of 0 is used
+Weights are currently not implemented.
+.SH EXAMPLES
+.TP
+echo "Define TOP
+UppercaseAlpha Alpha* LC(\e"professor\e") EndTag(ProfName);" | hfst\-pmatch2fst
+.IP
+create matcher that tags "professor Chomsky" as "professor <ProfName>Chomsky</ProfName>"
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-name home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstName>
+hfst\-pmatch2fst home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//Pmatch2Fst>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-invert.1 b/man/hfst-proc2.1
similarity index 63%
copy from man/hfst-invert.1
copy to man/hfst-proc2.1
index 8cb2a4f..07e408d 100644
--- a/man/hfst-invert.1
+++ b/man/hfst-proc2.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-INVERT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-PROC2 "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-invert \- =Invert a transducer
+hfst-proc2 \- =perform matching/lookup on text streams
.SH SYNOPSIS
-.B hfst-invert
-[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.B hfst-proc2
+[\fIOPTIONS\fR...] \fITOKENIZER\fR
.SH DESCRIPTION
-Invert a transducer
+perform matching/lookup on text streams
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -23,22 +23,20 @@ Only print fatal erros and requested output
.TP
\fB\-s\fR, \fB\-\-silent\fR
Alias of \fB\-\-quiet\fR
-.SS "Input/Output options:"
.TP
-\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
-Read input transducer from INFILE
+\fB\-n\fR \fB\-\-newline\fR
+Newline as input separator (default is blank line)
.TP
-\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
-Write output transducer to OUTFILE
+\fB\-x\fR \fB\-\-xerox\fR
+Xerox output (default)
.PP
-If OUTFILE or INFILE is missing or \-, standard streams will be used.
-Format of result depends on format of INFILE
+Use standard streams for input and output (for now).
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-invert home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstInvert>
+hfst\-proc2 home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstProc2>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-project.1 b/man/hfst-project.1
index b0ff57a..4ca628e 100644
--- a/man/hfst-project.1
+++ b/man/hfst-project.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PROJECT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-PROJECT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-project \- =Project (extract a level) transducer
.SH SYNOPSIS
diff --git a/man/hfst-name.1 b/man/hfst-prune-alphabet.1
similarity index 73%
copy from man/hfst-name.1
copy to man/hfst-prune-alphabet.1
index 30461e4..350472f 100644
--- a/man/hfst-name.1
+++ b/man/hfst-prune-alphabet.1
@@ -1,22 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-NAME "1" "February 2014" "HFST" "User Commands"
+.TH HFST-PRUNE-ALPHABET "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-name \- =Name a transducer
+hfst-prune-alphabet \- =Prune the alphabet of a transducer
.SH SYNOPSIS
-.B hfst-name
+.B hfst-prune-alphabet
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Name a transducer
-.SS "Name options:"
-.TP
-\fB\-n\fR, \fB\-\-name\fR=\fINAME\fR
-Name the transducer NAME
-.TP
-\fB\-p\fR, \fB\-\-print\-name\fR
-Only print the current name
-.TP
-\fB\-t\fR, \fB\-\-truncate_length\fR=\fILEN\fR
-Truncate name length to LEN
+Prune the alphabet of a transducer
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -40,6 +30,14 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
+.SS "Alphabet pruning options:"
+.TP
+\fB\-f\fR, \fB\-\-force\fR
+force pruning
+.TP
+\fB\-S\fR, \fB\-\-safe\fR
+prune only if no unknown or identity symbols
+are used in the transducer (default)
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
Format of result depends on format of INFILE
@@ -47,8 +45,8 @@ Format of result depends on format of INFILE
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-name home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstName>
+hfst\-prune\-alphabet home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstPruneAlphabet>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-push-weights.1 b/man/hfst-push-weights.1
index 65e1b3b..3e4f227 100644
--- a/man/hfst-push-weights.1
+++ b/man/hfst-push-weights.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-PUSH-WEIGHTS "1" "February 2014" "HFST" "User Commands"
+.TH HFST-PUSH-WEIGHTS "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-push-weights \- =Push weights of transducer
.SH SYNOPSIS
diff --git a/man/hfst-regexp2fst.1 b/man/hfst-regexp2fst.1
index b6079d9..0304a5d 100644
--- a/man/hfst-regexp2fst.1
+++ b/man/hfst-regexp2fst.1
@@ -1,15 +1,13 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REGEXP2FST "1" "February 2014" "HFST" "User Commands"
+.TH HFST-REGEXP2FST "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-regexp2fst \- =Compile regular expressions into transducer(s)
+hfst-regexp2fst \- =Compile (weighted) regular expressions into transducer(s)
.SH SYNOPSIS
.B hfst-regexp2fst
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Compile regular expressions into transducer(s)
-.IP
-(Experimental version)
-.SS "Common options:"
+Compile (weighted) regular expressions into transducer(s)
+Common options:
.TP
\fB\-h\fR, \fB\-\-help\fR
Print help message
@@ -41,24 +39,20 @@ Write result in FMT format
Disjunct all regexps instead of transforming
each regexp into a separate transducer
.TP
-\fB\-\-sum\fR (todo)
-Sum weights of duplicate strings instead of
-taking minimum
-.TP
-\fB\-\-norm\fR (todo)
-Divide each weight by sum of all weights
-.TP
-\fB\-\-log\fR (todo)
-Take negative logarithm of each weight
-.TP
\fB\-l\fR, \fB\-\-line\fR
Input is line separated (default)
.TP
\fB\-S\fR, \fB\-\-semicolon\fR
-Input is semicolon separated (weights not supported)
+Input is semicolon separated
.TP
\fB\-e\fR, \fB\-\-epsilon\fR=\fIEPS\fR
-Map EPS as zero.
+Map EPS as zero, i.e. epsilon.
+.TP
+\fB\-x\fR, \fB\-\-xerox\-composition\fR=\fIVALUE\fR Whether flag diacritics are treated as ordinary
+symbols in composition (default is false).
+.TP
+\fB\-X\fR, \fB\-\-xfst\fR=\fIVARIABLE\fR
+Toggle xfst compatibility option VARIABLE.
.SS "Harmonization:"
.TP
\fB\-H\fR, \fB\-\-do\-not\-harmonize\fR
@@ -66,18 +60,29 @@ Do not expand '?' symbols.
.TP
\fB\-F\fR, \fB\-\-harmonize\-flags\fR
Harmonize flag diacritics.
+.TP
+\fB\-E\fR, \fB\-\-encode\-weights\fR
+Encode weights when minimizing (default is false).
.PP
If OUTFILE or INFILE is missing or \-, standard streams will be used.
FMT must be one of the following: {foma, sfst, openfst\-tropical, openfst\-log}.
If EPS is not defined, the default representation of 0 is used
-Weights are currently not implemented.
+VALUEs recognized are {true,ON,yes} and {false,OFF,no}.
+Xfst variables are {flag\-is\-epsilon (default OFF)}.
.SH EXAMPLES
.TP
-echo " c:d a:o t:g " | hfst\-regexp2fst
+echo " {cat}:{dog} " | hfst\-regexp2fst
create transducer {cat}:{dog}
.TP
-echo " c:d a:o t:g ; 3" | hfst\-regexp2fst
+echo " {cat}:{dog}::3 " | hfst\-regexp2fst
+same but with weight 3
+.TP
+echo " c:d a:o::3 t:g " | hfst\-regexp2fst
same but with weight 3
+in the middle
+.TP
+echo " {cat}:{dog} ; 3 " | hfst\-regexp2fst
+legacy way of defining weights
.TP
echo " cat ; dog ; 3 " | hfst\-regexp2fst \fB\-S\fR
create transducers
diff --git a/man/hfst-remove-epsilons.1 b/man/hfst-remove-epsilons.1
index a8fe781..bc028fd 100644
--- a/man/hfst-remove-epsilons.1
+++ b/man/hfst-remove-epsilons.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REMOVE-EPSILONS "1" "February 2014" "HFST" "User Commands"
+.TH HFST-REMOVE-EPSILONS "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-remove-epsilons \- =Remove epsilons from a transducer
.SH SYNOPSIS
diff --git a/man/hfst-repeat.1 b/man/hfst-repeat.1
index 789086f..c92f779 100644
--- a/man/hfst-repeat.1
+++ b/man/hfst-repeat.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REPEAT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-REPEAT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-repeat \- =Repeat transducer
.SH SYNOPSIS
diff --git a/man/hfst-reverse.1 b/man/hfst-reverse.1
index 160d1ab..29f632a 100644
--- a/man/hfst-reverse.1
+++ b/man/hfst-reverse.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REVERSE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-REVERSE "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-reverse \- =Reverse a transducer
.SH SYNOPSIS
diff --git a/man/hfst-reverse.1 b/man/hfst-reweight-tagger.1
similarity index 77%
copy from man/hfst-reverse.1
copy to man/hfst-reweight-tagger.1
index 160d1ab..58b931b 100644
--- a/man/hfst-reverse.1
+++ b/man/hfst-reweight-tagger.1
@@ -1,12 +1,13 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-REVERSE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-REWEIGHT-TAGGER "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-reverse \- =Reverse a transducer
+hfst-reweight-tagger \- =Reweight a tagger accoring to a configuration file
.SH SYNOPSIS
-.B hfst-reverse
+.B hfst-reweight-tagger
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Reverse a transducer
+Reweight a tagger accoring to a configuration file
+hfst_tagger_config.
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -30,15 +31,12 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
-.PP
-If OUTFILE or INFILE is missing or \-, standard streams will be used.
-Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-reverse home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstReverse>
+hfst\-reweight\-tagger home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstReweightTagger>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-reweight.1 b/man/hfst-reweight.1
new file mode 100644
index 0000000..3b778a1
--- /dev/null
+++ b/man/hfst-reweight.1
@@ -0,0 +1,93 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST-REWEIGHT "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst-reweight \- =Reweight transducer weights simply
+.SH SYNOPSIS
+.B hfst-reweight
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+Reweight transducer weights simply
+.SS "Common options:"
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Print help message
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Print version info
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Print verbosely while processing
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Only print fatal erros and requested output
+.TP
+\fB\-s\fR, \fB\-\-silent\fR
+Alias of \fB\-\-quiet\fR
+.SS "Input/Output options:"
+.TP
+\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+Read input transducer from INFILE
+.TP
+\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+Write output transducer to OUTFILE
+.SS "Reweighting options:"
+.TP
+\fB\-a\fR, \fB\-\-addition\fR=\fIAVAL\fR
+add AVAL to matching weights
+.TP
+\fB\-b\fR, \fB\-\-multiplier\fR=\fIBVAL\fR
+multiply matching weights by BVAL
+.TP
+\fB\-F\fR, \fB\-\-function\fR=\fIFNAME\fR
+operate matching weights by FNAME
+.TP
+\fB\-l\fR, \fB\-\-lower\-bound\fR=\fILVAL\fR
+match weights greater than LVAL
+.TP
+\fB\-u\fR, \fB\-\-upper\-bound\fR=\fIUVAL\fR
+match weights less than UVAL
+.TP
+\fB\-I\fR, \fB\-\-input\-symbol\fR=\fIISYM\fR
+match arcs with input symbol ISYM
+.TP
+\fB\-O\fR, \fB\-\-output\-symbol\fR=\fIOSYM\fR
+match arcs with output symbol OSYM
+.TP
+\fB\-S\fR, \fB\-\-symbol\fR=\fISYM\fR
+match arcs havins symbol SYM
+.TP
+\fB\-e\fR, \fB\-\-end\-states\-only\fR
+match end states only, no arcs
+.TP
+\fB\-T\fR, \fB\-\-tsv\-file\fR=\fITFILE\fR
+read reweighting rules from TFILE
+.PP
+If OUTFILE or INFILE is missing or \-, standard streams will be used.
+Format of result depends on format of INFILE
+If AVAL, BVAL or FNAME are omitted, they default to neutral elements of addition, multiplication or identity function.
+If LVAL or UVAL are omitted, they default to minimum and maximum values of the weight structure.
+If ISYM, OSYM or SYM are omitted, they default to value that matches all arcs.
+Float values are parsed with strtod(3) and integers strtoul(3)
+The functions allowed for FNAME are <cmath> float functions with parameter count of 1 and a matching return value:
+abs, acos, asin, ... sqrt, tan, tanh
+The precedence of operands follows the formula BVAL * FNAME(w) + AVAL
+The formula is applied iff
+((LVAL <= w) && (w <= UVAL)),
+where w is weight of arc, and
+(ISYM == i) && (OSYM == o) && ((SYM == i) || (SYM == o)) ^^
+(end state && \fB\-e\fR).
+.SH "REPORTING BUGS"
+Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
+<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
+.PP
+hfst\-reweight home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstReweight>
+.br
+General help using HFST software:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/hfst-sfstpl2fst.1 b/man/hfst-sfstpl2fst.1
new file mode 120000
index 0000000..48cc25f
--- /dev/null
+++ b/man/hfst-sfstpl2fst.1
@@ -0,0 +1 @@
+hfst-calculate.1
\ No newline at end of file
diff --git a/man/hfst-disjunct.1 b/man/hfst-shuffle.1
similarity index 78%
copy from man/hfst-disjunct.1
copy to man/hfst-shuffle.1
index f85c45c..6e9029b 100644
--- a/man/hfst-disjunct.1
+++ b/man/hfst-shuffle.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-DISJUNCT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-SHUFFLE "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-disjunct \- =Disjunct (union, OR) two transducers
+hfst-shuffle \- =Shuffle two transducers
.SH SYNOPSIS
-.B hfst-disjunct
+.B hfst-shuffle
[\fIOPTIONS\fR...] [\fIINFILE1 \fR[\fIINFILE2\fR]]
.SH DESCRIPTION
-Disjunct (union, OR) two transducers
+Shuffle two transducers
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -44,21 +44,15 @@ The operation is applied pairwise for INFILE1 and INFILE2
that must have the same number of transducers.
If INFILE2 has only one transducer, the operation is applied for
each transducer in INFILE1 keeping the second transducer constant.
-Harmonization:
-.HP
-\fB\-H\fR, \fB\-\-do\-not\-harmonize\fR Do not harmonize symbols.
-.TP
-\fB\-F\fR, \fB\-\-harmonize\-flags\fR
-Harmonize flag diacritics.
.SH EXAMPLES
.IP
-hfst\-disjunct \fB\-o\fR cat_or_dog.hfst cat.hfst dog.hfst
+hfst\-shuffle \fB\-o\fR shuffled.hfst cat.hfst dog.hfst
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-disjunct home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstDisjunct>
+hfst\-shuffle home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstShuffle>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-split.1 b/man/hfst-split.1
index 896e6d1..10db676 100644
--- a/man/hfst-split.1
+++ b/man/hfst-split.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SPLIT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-SPLIT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-split \- =Extract transducers from archive with systematic file names
.SH SYNOPSIS
diff --git a/man/hfst-strings2fst.1 b/man/hfst-strings2fst.1
index f84a3b5..47ab607 100644
--- a/man/hfst-strings2fst.1
+++ b/man/hfst-strings2fst.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-STRINGS2FST "1" "February 2014" "HFST" "User Commands"
+.TH HFST-STRINGS2FST "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-strings2fst \- =Compile string pairs and pair-strings into transducer(s)
.SH SYNOPSIS
@@ -44,7 +44,10 @@ Divide each weight by sum of all weights
(with option \fB\-j\fR)
.TP
\fB\-\-log\fR
-Take negative logarithm of each weight
+Take negative natural logarithm of each weight
+.TP
+\fB\-\-log10\fR
+Take negative 10\-based logarithm of each weight
.TP
\fB\-p\fR, \fB\-\-pairstrings\fR
Input is in pairstring format
diff --git a/man/hfst-substitute.1 b/man/hfst-substitute.1
index 1f978c2..df319b0 100644
--- a/man/hfst-substitute.1
+++ b/man/hfst-substitute.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SUBSTITUTE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-SUBSTITUTE "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-substitute \- =Relabel transducer arcs
.SH SYNOPSIS
diff --git a/man/hfst-subtract.1 b/man/hfst-subtract.1
index 971a686..35d5fa3 100644
--- a/man/hfst-subtract.1
+++ b/man/hfst-subtract.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SUBTRACT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-SUBTRACT "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-subtract \- =Subtract (minus) two transducers
.SH SYNOPSIS
diff --git a/man/hfst-summarise.1 b/man/hfst-summarise.1
new file mode 120000
index 0000000..3317fde
--- /dev/null
+++ b/man/hfst-summarise.1
@@ -0,0 +1 @@
+hfst-summarize.1
\ No newline at end of file
diff --git a/man/hfst-summarize.1 b/man/hfst-summarize.1
index 36088ff..fb42c33 100644
--- a/man/hfst-summarize.1
+++ b/man/hfst-summarize.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-SUMMARIZE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-SUMMARIZE "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-summarize \- =Calculate the properties of a transducer
.SH SYNOPSIS
diff --git a/man/hfst-invert.1 b/man/hfst-tag.1
similarity index 77%
copy from man/hfst-invert.1
copy to man/hfst-tag.1
index 8cb2a4f..bb57167 100644
--- a/man/hfst-invert.1
+++ b/man/hfst-tag.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-INVERT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-TAG "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-invert \- =Invert a transducer
+hfst-tag \- =Tag a text file given from stdin using an hfst tagger.
.SH SYNOPSIS
-.B hfst-invert
+.B hfst-tag
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Invert a transducer
+Tag a text file given from stdin using an hfst tagger.
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -30,15 +30,12 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
-.PP
-If OUTFILE or INFILE is missing or \-, standard streams will be used.
-Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-invert home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstInvert>
+hfst\-tag home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstTag>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-tail.1 b/man/hfst-tail.1
index 993892f..76f094c 100644
--- a/man/hfst-tail.1
+++ b/man/hfst-tail.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TAIL "1" "February 2014" "HFST" "User Commands"
+.TH HFST-TAIL "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-tail \- =Get last transducers from an archive
.SH SYNOPSIS
diff --git a/man/hfst-train-tagger-loc.1 b/man/hfst-train-tagger-loc.1
new file mode 120000
index 0000000..d893136
--- /dev/null
+++ b/man/hfst-train-tagger-loc.1
@@ -0,0 +1 @@
+hfst-open-input-file-for-tagger.1
\ No newline at end of file
diff --git a/man/hfst-train-tagger-system.1 b/man/hfst-train-tagger-system.1
new file mode 120000
index 0000000..d893136
--- /dev/null
+++ b/man/hfst-train-tagger-system.1
@@ -0,0 +1 @@
+hfst-open-input-file-for-tagger.1
\ No newline at end of file
diff --git a/man/hfst-invert.1 b/man/hfst-train-tagger.1
similarity index 77%
copy from man/hfst-invert.1
copy to man/hfst-train-tagger.1
index 8cb2a4f..9720164 100644
--- a/man/hfst-invert.1
+++ b/man/hfst-train-tagger.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-INVERT "1" "February 2014" "HFST" "User Commands"
+.TH HFST-TRAIN-TAGGER "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-invert \- =Invert a transducer
+hfst-train-tagger \- =
.SH SYNOPSIS
-.B hfst-invert
+.B hfst-train-tagger
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Invert a transducer
+Compile training data file into an hfst part\-of\-speech tagger.
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -30,15 +30,12 @@ Read input transducer from INFILE
.TP
\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
Write output transducer to OUTFILE
-.PP
-If OUTFILE or INFILE is missing or \-, standard streams will be used.
-Format of result depends on format of INFILE
.SH "REPORTING BUGS"
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-invert home page:
-<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstInvert>
+hfst\-train\-tagger home page:
+<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstTrainTagger>
.br
General help using HFST software:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstHome>
diff --git a/man/hfst-determinize.1 b/man/hfst-traverse.1
similarity index 87%
copy from man/hfst-determinize.1
copy to man/hfst-traverse.1
index 087dd0c..f958459 100644
--- a/man/hfst-determinize.1
+++ b/man/hfst-traverse.1
@@ -1,12 +1,12 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-DETERMINIZE "1" "February 2014" "HFST" "User Commands"
+.TH HFST-TRAVERSE "1" "October 2014" "HFST" "User Commands"
.SH NAME
-hfst-determinize \- =Determinize a transducer
+hfst-traverse \- =Walk through the transducer arc by arc
.SH SYNOPSIS
-.B hfst-determinize
+.B hfst-traverse
[\fIOPTIONS\fR...] [\fIINFILE\fR]
.SH DESCRIPTION
-Determinize a transducer
+Walk through the transducer arc by arc
.SS "Common options:"
.TP
\fB\-h\fR, \fB\-\-help\fR
@@ -37,7 +37,7 @@ Format of result depends on format of INFILE
Report bugs to <hfst\-bugs at helsinki.fi> or directly to our bug tracker at:
<https://sourceforge.net/tracker/?atid=1061990&group_id=224521&func=browse>
.PP
-hfst\-determinize home page:
+hfst\-traverse home page:
<https://kitwiki.csc.fi/twiki/bin/view/KitWiki//HfstDeterminize>
.br
General help using HFST software:
diff --git a/man/hfst-twolc-loc.1 b/man/hfst-twolc-loc.1
new file mode 120000
index 0000000..f1c9142
--- /dev/null
+++ b/man/hfst-twolc-loc.1
@@ -0,0 +1 @@
+hfst-twolc.1
\ No newline at end of file
diff --git a/man/hfst-twolc-system.1 b/man/hfst-twolc-system.1
new file mode 120000
index 0000000..f1c9142
--- /dev/null
+++ b/man/hfst-twolc-system.1
@@ -0,0 +1 @@
+hfst-twolc.1
\ No newline at end of file
diff --git a/man/hfst-twolc.1 b/man/hfst-twolc.1
new file mode 100644
index 0000000..3ff2620
--- /dev/null
+++ b/man/hfst-twolc.1
@@ -0,0 +1,66 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST-TWOLC "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst-twolc \- =
+.SH SYNOPSIS
+.B hfst-twolc
+[\fIOPTIONS\fR...] \fIINFILE\fR
+.SH DESCRIPTION
+Usage: hfst\-twolc [OPTIONS...] \fB\-i\fR INFILE
+Usage: hfst\-twolc [OPTIONS...] \fB\-\-input\fR=\fIINFILE\fR
+Usage: cat INFILE | hfst\-twolc [OPTIONS...]
+An input file has to be given either using the option \fB\-i\fR or
+\fB\-\-input\fR, as the last commandline argument or from STDIN.
+.PP
+Read a twolc grammar, compile it and store it. If INFILE is
+missing, the grammar is read from STDIN. If there is no output
+file given using \fB\-o\fR or \fB\-\-output\fR, the compiled grammar is
+written to STDOUT.
+.SS "Common options:"
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Print help message
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Print version info
+.TP
+\fB\-u\fR, \fB\-\-usage\fR
+Print usage
+.TP
+\fB\-v\fR, \fB\-\-verbose\fR
+Print verbosely while processing
+.TP
+\fB\-q\fR, \fB\-\-quiet\fR
+Do not print output
+.TP
+\fB\-s\fR, \fB\-\-silent\fR
+Alias of \fB\-\-quiet\fR
+.SS "Input/Output options:"
+.TP
+\fB\-i\fR, \fB\-\-input\fR=\fIINFILE\fR
+Read input transducer from INFILE
+.TP
+\fB\-o\fR, \fB\-\-output\fR=\fIOUTFILE\fR
+Write output transducer to OUTFILE
+.SS "TwolC grammar options:"
+.TP
+\fB\-R\fR, \fB\-\-resolve\fR
+Resolve left\-arrow conflicts.
+.HP
+\fB\-D\fR, \fB\-\-dont\-resolve\-right\fR Don't resolve right\-arrow conflicts.
+.TP
+\fB\-f\fR, \fB\-\-format\fR=\fIFORMAT\fR
+Store result in format FORMAT.
+.PP
+Format may be one of openfst\-log, openfst\-tropical, foma or sfst.
+.PP
+By default format is openfst\-tropical. By default right arrow
+conflicts are resolved and left arrow conflicts are not resolved.
+.PP
+hfst\-twolc 0 (hfst 3.7.1)
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3
+<http://gnu.org/licenses/gpl.html>
+This is free software: you are free to change and
+redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/hfst-txt2fst.1 b/man/hfst-txt2fst.1
index 8a71ef0..0ee69ad 100644
--- a/man/hfst-txt2fst.1
+++ b/man/hfst-txt2fst.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-TXT2FST "1" "February 2014" "HFST" "User Commands"
+.TH HFST-TXT2FST "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-txt2fst \- =Convert AT&T or prolog format into a binary transducer
.SH SYNOPSIS
diff --git a/man/hfst-union.1 b/man/hfst-union.1
new file mode 120000
index 0000000..085b7e4
--- /dev/null
+++ b/man/hfst-union.1
@@ -0,0 +1 @@
+hfst-disjunct.1
\ No newline at end of file
diff --git a/man/hfst-xfst.1 b/man/hfst-xfst.1
index a8c2ffe..5699e99 100644
--- a/man/hfst-xfst.1
+++ b/man/hfst-xfst.1
@@ -1,5 +1,5 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
-.TH HFST-XFST "1" "February 2014" "HFST" "User Commands"
+.TH HFST-XFST "1" "October 2014" "HFST" "User Commands"
.SH NAME
hfst-xfst \- =Compile XFST scripts or execute XFST commands interactively
.SH SYNOPSIS
@@ -43,6 +43,9 @@ Read commands from standard input (non\-interactive)
\fB\-r\fR, \fB\-\-no\-readline\fR
Do not use readline library for input
.TP
+\fB\-w\fR, \fB\-\-print\-weight\fR
+Print weights for each operation
+.TP
\fB\-k\fR, \fB\-\-output\-to\-console\fR
Output directly to console (Windows\-specific)
.PP
diff --git a/man/hfst_tagger_compute_data_statistics.py.1 b/man/hfst_tagger_compute_data_statistics.py.1
new file mode 100644
index 0000000..29bc8cb
--- /dev/null
+++ b/man/hfst_tagger_compute_data_statistics.py.1
@@ -0,0 +1,15 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HFST_TAGGER_COMPUTE_DATA_STATISTICS.PY "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+hfst_tagger_compute_data_statistics.py \- =
+.SH SYNOPSIS
+.B hfst_tagger_compute_data_statistics.py
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+A script used by hfst-train-tagger, not intended to be used directly.
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/htwolcpre1.1 b/man/htwolcpre1.1
new file mode 100644
index 0000000..cab35e1
--- /dev/null
+++ b/man/htwolcpre1.1
@@ -0,0 +1,15 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HTWOLCPRE1 "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+htwolcpre1 \- =
+.SH SYNOPSIS
+.B htwolcpre1
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+A program used by hfst-twolc, not intended to be used directly.
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/htwolcpre2.1 b/man/htwolcpre2.1
new file mode 100644
index 0000000..1809d19
--- /dev/null
+++ b/man/htwolcpre2.1
@@ -0,0 +1,15 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HTWOLCPRE2 "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+htwolcpre2 \- =
+.SH SYNOPSIS
+.B htwolcpre2
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+A program used by hfst-twolc, not intended to be used directly.
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/man/htwolcpre3.1 b/man/htwolcpre3.1
new file mode 100644
index 0000000..0381d54
--- /dev/null
+++ b/man/htwolcpre3.1
@@ -0,0 +1,15 @@
+.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.40.4.
+.TH HTWOLCPRE3 "1" "October 2014" "HFST" "User Commands"
+.SH NAME
+htwolcpre3 \- =
+.SH SYNOPSIS
+.B htwolcpre3
+[\fIOPTIONS\fR...] [\fIINFILE\fR]
+.SH DESCRIPTION
+A program used by hfst-twolc, not intended to be used directly.
+.SH COPYRIGHT
+Copyright \(co 2010 University of Helsinki,
+License GPLv3: GNU GPL version 3 <http://gnu.org/licenses/gpl.html>
+.br
+This is free software: you are free to change and redistribute it.
+There is NO WARRANTY, to the extent permitted by law.
diff --git a/scripts/hfst-fst2tesseract.xfst b/scripts/hfst-fst2tesseract.xfst
new file mode 100644
index 0000000..7d9bab9
--- /dev/null
+++ b/scripts/hfst-fst2tesseract.xfst
@@ -0,0 +1,10 @@
+!!! hfst-xfst script for converting a morphological analyzer to a Tesseract !!!
+!!! OCR word model !!!
+
+!!! USAGE: hfst-xfst -e "load <analyzer name>" -F hfst-fst2tesseract.xfst !!!
+
+set quit-on-fail ON
+upper-side
+eliminate flags
+lookup-optimize
+save LAN.word-hfst-fsm
\ No newline at end of file
diff --git a/swig/doc/libhfst.py b/swig/doc/libhfst.py
index 18908ff..aae8860 100644
--- a/swig/doc/libhfst.py
+++ b/swig/doc/libhfst.py
@@ -1630,7 +1630,7 @@ def detokenize_and_purge_paths(tokenized_paths):
# print "ERROR: Stream cannot be read."
# exit(1)
# t = libhfst.HfstTransducer(instr)
-# print "One transducer succesfully read."
+# print "One transducer successfully read."
# transducers_read++
#
# print "Read %i transducers in total" % transducers_read
diff --git a/swig/hfstBot.py b/swig/hfstBot.py
index 2a16aab..69b5dbb 100644
--- a/swig/hfstBot.py
+++ b/swig/hfstBot.py
@@ -82,7 +82,7 @@ class HfstBot(irc.IRCClient):
# callbacks for events
def signedOn(self):
- """Called when bot has succesfully signed on to server."""
+ """Called when bot has successfully signed on to server."""
self.join(self.factory.channel)
def joined(self, channel):
diff --git a/swig/setup.py b/swig/setup.py
index 2efd084..6ebcb85 100644
--- a/swig/setup.py
+++ b/swig/setup.py
@@ -26,7 +26,7 @@ libhfst_module = Extension('_libhfst',
# ["libhfst-NN.dll", "libgcc_s_seh-1.dll"] or
# ["libhfst-NN.dll", "libgcc_s_dw2-1.dll"] or
setup(name = 'libhfst_swig',
- version = '3.7.1_beta',
+ version = '3.8.1_beta',
author = 'HFST team',
author_email = 'hfst-bugs at helsinki.fi',
url = 'http://hfst.sourceforge.net',
diff --git a/swig/test/test_examples.py b/swig/test/test_examples.py
index 1649734..7258a66 100644
--- a/swig/test/test_examples.py
+++ b/swig/test/test_examples.py
@@ -376,7 +376,7 @@ while not instr.is_eof():
print("ERROR: Stream cannot be read.")
exit(1)
t = libhfst.HfstTransducer(instr)
- print("One transducer succesfully read.")
+ print("One transducer successfully read.")
transducers_read += 1
print("Read {0} transducers in total".format(transducers_read))
diff --git a/test/libhfst/test_transducer_functions.cc b/test/libhfst/test_transducer_functions.cc
index 534ce18..0330bfd 100644
--- a/test/libhfst/test_transducer_functions.cc
+++ b/test/libhfst/test_transducer_functions.cc
@@ -84,6 +84,7 @@ bool compare_string_vectors(const StringVector &v1, const StringVector &v2,
for (unsigned int i=0; i<v2.size(); i++) {
v2_string.append(v2[i]);
}
+ // fprintf(stderr, "####### comparing strings '%s' and '%s'...\n", v1_string.c_str(), v2_string.c_str() ); // debug
return (v1_string.compare(v2_string) == 0);
}
@@ -117,8 +118,10 @@ bool do_hfst_lookup_paths_contain(const HfstOneLevelPaths &results,
weight = it->first;
}
}
- if (found == false)
+ if (found == false) {
+ // fprintf(stderr, "######## returning false...\n"); // debug
return false;
+ }
if (not test_path_weight)
return true;
@@ -422,6 +425,10 @@ int main(int argc, char **argv)
/* convert to optimized lookup format */
HfstTransducer animals_ol(animals);
+ // debug
+ // std::cerr << "####### animals_ol:" << std::endl;
+ // std::cerr << animals_ol << std::endl;
+
if (types[i] == TROPICAL_OPENFST_TYPE ||
types[i] == LOG_OPENFST_TYPE) {
animals_ol.convert(HFST_OLW_TYPE); }
diff --git a/test/tools/Makefile.am b/test/tools/Makefile.am
index a9dccd9..794ade4 100644
--- a/test/tools/Makefile.am
+++ b/test/tools/Makefile.am
@@ -68,6 +68,9 @@ endif
if WANT_LOOKUP
TESTS += lookup-functionality.sh
endif
+if WANT_OPTIMIZED_LOOKUP
+TESTS += optimized-lookup-functionality.sh
+endif
if WANT_MINIMIZE
TESTS += minimize-functionality.sh
endif
diff --git a/test/tools/optimized-lookup-functionality.sh b/test/tools/optimized-lookup-functionality.sh
new file mode 100755
index 0000000..0e1d174
--- /dev/null
+++ b/test/tools/optimized-lookup-functionality.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+TOOLDIR=../../tools/src
+if ! $TOOLDIR/hfst-optimized-lookup cat2dog.hfstol < $srcdir/cat.strings > test.lookups ; then
+ exit 1
+fi
+
+# Bug 123
+echo > empty
+if ! $TOOLDIR/hfst-optimized-lookup cat2dog.hfstol < empty > test.lookups ; then
+ exit 1
+fi
+
+rm test.lookups empty
diff --git a/test/tools/pmatch-tester.sh b/test/tools/pmatch-tester.sh
index 1dd086f..99c531c 100755
--- a/test/tools/pmatch-tester.sh
+++ b/test/tools/pmatch-tester.sh
@@ -14,22 +14,29 @@ logpass=
logfail=
logverbose=
logfile=$srcdir/$progname.log
-number_tests=
+number_tests=1
+include_tests=
+exclude_tests=
+truncate_lines=800
+truncate_log_lines=1600
-compiler=hfst-pmatch2fst
+compiler_name=hfst-pmatch2fst
compiler_opts=
-runner=hfst-pmatch
+runner_name=hfst-pmatch
runner_opts=
preamble=
testset_descr=
test_descr=
+skip_test=
input=
output=
template=
+print_cmd=echo
testcount=0
failcount=0
+skipcount=0
subtestcount=0
subfailcount=0
@@ -67,22 +74,36 @@ getopts_func () {
usage () {
cat <<EOF
- cat <<EOF
Usage: $progname [options]
Test the HFST Pmatch tool
Options:
--help show this help
+ --include-tests INCLUDE_REGEX
+ include only tests with name matching the regular expression
+ INCLUDE_REGEX
+ --exclude-tests EXCLUDE_REGEX
+ exclude tests with name matching the regular expression
+ EXCLUDE_REGEX; if --include-tests is also specified, run only
+ tests matching INCLUDE_REGEX and not matching EXCLUDE_REGEX
--tmpdir DIR use DIR for temporary files (default: .)
- --tooldir DIR use hfst-pmatch2fst and hfst-pmatch found in DIR (default: .)
+ --tooldir DIR use hfst-pmatch2fst and hfst-pmatch found in DIR (default:
+ $tooldir)
--log none|failed|all|verbose
log the results of no tests, failed tests, all tests or all
tests verbosely (default: none)
- --logfile FILE write the log output to FILE (default:
+ --logfile FILE write the log output to FILE (default: $logfile)
--verbose print verbose output of the tests to stdout
- --number-tests show test and subtest numbers
+ --no-number-tests
+ do not show test and subtest numbers
--quiet do not print any output
--debug do not remove temporary files
+ --truncate-lines MAXLEN
+ truncate (verbose) output lines to at most MAXLEN characters;
+ 0 for no truncation (default: 800)
+ --truncate-log-lines MAXLEN
+ truncate log file output lines to at most MAXLEN characters;
+ 0 for no truncation (default: 1600)
EOF
exit 0
}
@@ -93,7 +114,7 @@ cmdline="$0 ""$@"
getopt -T > /dev/null
if [ $? -eq 4 ]; then
# This requires GNU getopt
- args=`getopt -o "h" -l "help,debug,log:,logfile:,number-tests,quiet,tmpdir:,tooldir:,verbose" -n "$progname" -- "$@"`
+ args=`getopt -o "h" -l "help,debug,exclude-tests:,include-tests:,log:,logfile:,no-number-tests,quiet,tmpdir:,tooldir:,truncate-lines:,truncate-log-lines:,verbose" -n "$progname" -- "$@"`
if [ $? -ne 0 ]; then
exit 1
fi
@@ -125,7 +146,6 @@ while [ "x$1" != "x" ] ; do
--tooldir )
shift
tooldir="$1"
- tooldir_set=1
;;
--log )
shift
@@ -156,8 +176,24 @@ while [ "x$1" != "x" ] ; do
shift
logfile="$1"
;;
- --number-tests )
- number_tests=1
+ --no-number-tests )
+ number_tests=
+ ;;
+ --include-tests )
+ shift
+ include_tests="$1"
+ ;;
+ --exclude-tests )
+ shift
+ exclude_tests="$1"
+ ;;
+ --truncate-lines )
+ shift
+ truncate_lines="$1"
+ ;;
+ --truncate-log-lines )
+ shift
+ truncate_log_lines="$1"
;;
-- )
shift
@@ -173,13 +209,27 @@ while [ "x$1" != "x" ] ; do
shift
done
-for prog in $compiler $runner; do
- if test ! -x $tooldir/$prog; then
- msg="Executable $prog not found in $tooldir"
- if [ "x$tooldir_set" = "x" ]; then
- msg="$msg; please specify directory with --tooldir"
+if [ "x$tooldir" != "x" ]; then
+ compiler="$tooldir/$compiler_name"
+ runner="$tooldir/$runner_name"
+else
+ compiler=`which $compiler_name`
+ runner=`which $runner_name`
+fi
+
+for progtype in compiler runner; do
+ eval "prog_name=\$${progtype}_name"
+ eval "prog=\$$progtype"
+ if [ "x$prog" = "x" ] || [ ! -e "$prog" ]; then
+ msg="Executable $prog_name not found"
+ if [ "x$tooldir" = "x" ]; then
+ msg="$msg on PATH; please specify directory with --tooldir"
+ else
+ msg="$msg in $tooldir"
fi
error "$msg"
+ elif [ ! -x "$prog" ]; then
+ error "Cannot run $prog"
fi
done
@@ -199,6 +249,47 @@ else
error "$msg"
fi
+# Find a printf supporting \xHH escape sequences (the built-in printf
+# in Dash does not support them)
+for printf in printf `which printf`; do
+ if [ `$printf "\x41"` = "A" ]; then
+ break
+ fi
+done
+
+
+# Cat a file; if print_cmd is not echo (i.e., printf), double
+# backslashes to escape them from expanding in further processing.
+catf () {
+ if [ "x$print_cmd" = "xecho" ]; then
+ cat "$@"
+ else
+ cat "$@" | sed -e 's/\\/\\\\/'
+ fi
+}
+
+echo_truncate () {
+ maxlen=$1
+ shift
+ if [ "$maxlen" = "0" ]; then
+ $print_cmd "$@"
+ else
+ $print_cmd "$@" |
+ awk 'BEGIN {
+ maxlen = '$maxlen';
+ taillen = int (maxlen / 5);
+ extraskip = 35;
+ }
+ {
+ len = length ($0);
+ if (len > maxlen) {
+ print substr ($0, i, maxlen - taillen - extraskip) "[... " len - maxlen + extraskip " characters skipped ...]" substr ($0, len - taillen)
+ } else {
+ print
+ }
+ }'
+ fi
+}
log () {
logtype=$1
@@ -221,7 +312,7 @@ log () {
;;
esac
if [ "x$outputlog" != "x" ]; then
- echo "$@" >> $logfile
+ echo_truncate $truncate_log_lines "$@" >> $logfile
fi
}
@@ -231,7 +322,7 @@ echo_log () {
log "$@"
shift
if [ "$verbosity" -ge "$verbosity_level" ]; then
- echo "$@"
+ echo_truncate $truncate_lines "$@"
fi
}
@@ -250,7 +341,11 @@ subtest_number () {
report_results () {
verbosity_level=$1
shift
- result="`expr $2 - $3` / $2 $1 passed"
+ skipped=$4
+ if [ "x$skipped" = "x" ]; then
+ skipped=0
+ fi
+ result="`expr $2 - $3 - $skipped` / `expr $2 - $skipped` $1 passed"
if test "x$3" != "x0"; then
result_type=fail
result="$result, $3 FAILED"
@@ -261,8 +356,11 @@ report_results () {
result_type=pass
result="PASS: All $result"
fi
+ if [ "x$4" != "x" ]; then
+ result="$result, $4 $1 skipped"
+ fi
if [ "x$1" = "xsubtests" ]; then
- result="*`test_number` $result"
+ result="* Test`test_number` $result"
fi
echo_log $verbosity_level $result_type "$result
"
@@ -282,12 +380,12 @@ testset_end () {
if [ "x$subtestcount" != "x0" ]; then
test_end
fi
- report_results 1 "tests" $testcount $failcount
+ report_results 1 "tests" $testcount $failcount $skipcount
report_results 2 "subtests in total" $subtestcount_total $subfailcount_total
if test "x$failcount" != "x0"; then
- exit 1
+ exitval=1
else
- exit 0
+ exitval=0
fi
}
@@ -309,34 +407,62 @@ testset_begin () {
subfailcount_total=0
}
+check_skip_test () {
+ skip_test=
+ if [ "x$include_tests" != "x" ]; then
+ if echo "$test_descr" | egrep -qe "$include_tests"; then
+ :
+ else
+ skip_test=1
+ fi
+ fi
+ if [ "x$exclude_tests" != "x" ] \
+ && echo "$test_descr" | egrep -qe "$exclude_tests"; then
+ skip_test=1
+ fi
+ if [ "x$skip_test" != "x" ]; then
+ skipcount=`expr $skipcount + 1`
+ fi
+}
+
test_begin () {
if [ "x$subtestcount" != "x0" ]; then
test_end
fi
test_descr="$1"
testcount=`expr $testcount + 1`
- echo_log 1 meta "Test`test_number`: $test_descr"
+ check_skip_test
+ if [ "x$skip_test" != "x" ]; then
+ msg="Skipping test`test_number`: $test_descr
+"
+ else
+ msg="Test`test_number`: $test_descr"
+ fi
+ echo_log 1 meta "$msg"
subtestcount=0
subfailcount=0
}
cleanup () {
- if [ "x$testcount" != "x0" ]; then
- testset_end
- testcount=0
- fi
if [ "x$debug" = "x" ]; then
rm -f $pmatch_file $err_file $out_file $expect_out_file $file_base.src
rm -f $file_base-*.pmatch $file_base-*.src $file_base-*.descr
fi
- echo_log 3 meta "
-Test completed at "`date +"%Y-%m-%d %H:%M:%S"`"
+ if [ "x$exitval" != "x77" ] && [ "x$testcount" != "x0" ]; then
+ testset_end
+ testcount=0
+ fi
+ end_time=`date +"%s"`
+ duration=`expr $end_time - $start_time`
+ echo_log 3 meta "Test completed at "`date +"%Y-%m-%d %H:%M:%S"`" in $duration seconds
"
+ exit $exitval
}
cleanup_abort () {
echo_log 1 meta "
Caught a signal; aborting tests."
+ exitval=77
exit 77
}
@@ -344,6 +470,7 @@ Caught a signal; aborting tests."
trap cleanup 0
trap cleanup_abort 1 2 13 15
+start_time=`date +"%s"`
echo_log 3 meta "Test run on "`uname -n`" at "`date +"%Y-%m-%d %H:%M:%S"`"
"
echo_log 3 meta "Command line: $cmdline
@@ -363,6 +490,9 @@ common_preamble () {
}
compile_to () {
+ if [ "x$skip_test" != "x" ]; then
+ return 0
+ fi
compile_only=
if [ "x$1" = "x--compile-only" ]; then
compile_only=1
@@ -375,19 +505,20 @@ compile_to () {
$code"
fi
srcfile=`basename $codefile .pmatch`.src
- echo "$code" > $srcfile
- $tooldir/$compiler $compiler_opts $srcfile > $codefile 2> $err_file
+ $print_cmd "$code" > $srcfile
+ $compiler $compiler_opts $srcfile > $codefile 2> $err_file
if test $? != 0 || test -s $err_file; then
if [ "x$compile_only" != "x" ]; then
msg="* Subtest`subtest_number` compilation FAILED: $descr"
+ subfailcount=`expr $subfailcount + 1`
else
msg="* Compilation FAILED"
fi
echo_log 1 fail "$msg"
echo_log 2 fail "--- Source:
$code
---- Errors in running $tooldir/$compiler:
-`cat $err_file`"
+--- Errors in running $compiler:
+`catf $err_file`"
rm $err_file
return 1
else
@@ -408,22 +539,27 @@ compile_template () {
opts=
# FIXME: "shift n" is not portable
getopts_func opts "$@" || shift $?
- compile $opts "`echo "$template" | sed -e "s/@1/$1/g" -e "s/@2/$2/g" -e "s/@3/$3/g"`"
+ compile $opts "`$print_cmd "$template" | sed -e "s/@1/$1/g" -e "s/@2/$2/g" -e "s/@3/$3/g"`"
return $?
}
check_with_file () {
subtestcount=`expr $subtestcount + 1`
+ _prev_print_cmd_check_with_file=$print_cmd
if [ "x$1" = "x--printf" ]; then
- print_fn=printf
+ print_cmd=$printf
shift
- else
- print_fn=echo
fi
_code_file="$1"
_descr="$2"
shift
shift
+ # Treat an empty file as a sign of failed compilation
+ if [ ! -s $_code_file ]; then
+ echo_log 1 fail "* Subtest`subtest_number` FAILED compilation: $_descr"
+ subfailcount=`expr $subfailcount + 1`
+ return 1
+ fi
srcfile=`basename $codefile .pmatch`.src
if test "$#" -lt 2; then
_use_input="$input"
@@ -432,35 +568,36 @@ check_with_file () {
_use_input="$1"
_use_expect_output="$2"
fi
- $print_fn "$_use_expect_output" > $expect_out_file
- $print_fn "$_use_input" |
- $tooldir/$runner $runner_opts $_code_file > $out_file 2> $err_file
+ $print_cmd "$_use_expect_output" > $expect_out_file
+ $print_cmd "$_use_input" |
+ $runner $runner_opts $_code_file > $out_file 2> $err_file
diff -q $expect_out_file $out_file > /dev/null
if test $? != 0 || test -s $err_file; then
echo_log 1 fail "* Subtest`subtest_number` FAILED: $_descr"
echo_log 3 fail "--- Source:
-`cat $srcfile`"
+`catf $srcfile`"
subfailcount=`expr $subfailcount + 1`
if test -s $err_file; then
echo_log 2 fail "--- Errors in running $runner $runner_opts:
-`cat $err_file`"
+`catf $err_file`"
else
echo_log 2 fail "--- Input:
$_use_input"
echo_log 2 fail "--- Expected output:
-`cat $expect_out_file`"
+`catf $expect_out_file`"
echo_log 2 fail "--- Actual output:
-`cat $out_file`"
+`catf $out_file`"
fi
else
echo_log 2 pass "* Subtest`subtest_number` passed: $_descr"
echo_log 3 verbose "--- Source:
-`cat $srcfile`"
+`catf $srcfile`"
echo_log 3 verbose "--- Input:
$_use_input"
echo_log 3 verbose "--- Output:
-`cat $expect_out_file`"
+`catf $expect_out_file`"
fi
+ print_cmd=$_prev_print_cmd_check_with_file
}
check () {
@@ -487,6 +624,9 @@ check_compile_run_single () {
}
check_compile_aux () {
+ if [ "x$skip_test" != "x" ]; then
+ return 0
+ fi
func=$1
shift
opts=
@@ -495,23 +635,27 @@ check_compile_aux () {
descr=$1
shift
subtestcount=`expr $subtestcount + 1`
- $func $opts "$@"
+ $func --compile-only $opts "$@"
}
check_compile () {
- check_compile_aux compile --compile-only "$@"
+ check_compile_aux compile "$@"
}
check_compile_template () {
- check_compile_aux compile_template --compile-only "$@"
+ check_compile_aux compile_template "$@"
}
subst_templ_arg () {
+ subst_bs='\\\\\\\\'
+ # if [ "x$print_cmd" = "x$printf" ]; then
+ # subst_bs="$subst_bs$subst_bs"
+ # fi
templ="$1"
argnum=$2
arg=$3
- arg="`echo "$arg" | sed -e 's/\\\\/\\\\\\\\/g'`"
- echo "$templ" | sed -e "s/@$argnum@/$arg/g"
+ arg="`$print_cmd "$arg" | sed -e 's/\\\\/'$subst_bs'/g'`"
+ $print_cmd "$templ" | sed -e "s/@$argnum@/$arg/g"
}
check_compile_run_multi () {
@@ -522,7 +666,6 @@ check_compile_run_multi () {
descr=
fname=
common_input=
- compile_failed_count=0
while [ "x$1" != "x" ]; do
optname=$1
case "$1" in
@@ -532,13 +675,10 @@ check_compile_run_multi () {
error "Expected a description and code after $optname"
fi
fname=$file_base-$codenum.pmatch
- if compile_to "$2" "$fname"; then
- echo "$1" > $file_base-$codenum.descr
- codefiles="$codefiles $fname"
- codenum=$(($codenum + 1))
- else
- compile_failed_count=$(($compile_failed_count + 1))
- fi
+ $print_cmd "$1" > $file_base-$codenum.descr
+ compile_to "$2" "$fname" || cat /dev/null > "$fname"
+ codefiles="$codefiles $fname"
+ codenum=$(($codenum + 1))
shift
shift
;;
@@ -584,6 +724,8 @@ check_compile_run_multi () {
;;
* )
codetempl_filled="`subst_templ_arg "$codetempl_filled" $argnum "$1"`"
+ # Should we only expand descr if it is
+ # descrtempl or only if it contans @N@?
descr="`subst_templ_arg "$descr" $argnum "$1"`"
argnum=$(($argnum + 1))
;;
@@ -591,13 +733,11 @@ check_compile_run_multi () {
shift
done
fname=$file_base-$codenum.pmatch
- if compile_to "$codetempl_filled" "$fname"; then
- echo "$descr" > $file_base-$codenum.descr
- codefiles="$codefiles $fname"
- codenum=$(($codenum + 1))
- else
- compile_failed_count=$(($compile_failed_count + 1))
- fi
+ $print_cmd "$descr" > $file_base-$codenum.descr
+ compile_to "$codetempl_filled" "$fname" \
+ || cat /dev/null > "$fname"
+ codefiles="$codefiles $fname"
+ codenum=$(($codenum + 1))
;;
--outtemplargs | --output-template-arguments )
shift
@@ -614,9 +754,9 @@ check_compile_run_multi () {
descr="$1"
shift
fi
- expected_output=$1
+ expected_output="$1"
shift
- codetempl_filled=$codetempl
+ codetempl_filled="$codetempl"
argnum=1
while [ "$#" -gt 0 ]; do
case "$1" in
@@ -655,13 +795,11 @@ check_compile_run_multi () {
codetempl_filled="`subst_templ_arg "$codetempl" 1 "$1"`"
descr="`subst_templ_arg "$descr" 1 "$1"`"
fname=$file_base-$codenum.pmatch
- if compile_to "$codetempl_filled" "$fname"; then
- echo "$descr" > $file_base-$codenum.descr
- codefiles="$codefiles $fname"
- codenum=$(($codenum + 1))
- else
- compile_failed_count=$(($compile_failed_count + 1))
- fi
+ $print_cmd "$descr" > $file_base-$codenum.descr
+ compile_to "$codetempl_filled" "$fname" \
+ || cat /dev/null > "$fname"
+ codefiles="$codefiles $fname"
+ codenum=$(($codenum + 1))
;;
esac
shift
@@ -673,13 +811,13 @@ check_compile_run_multi () {
error "Expected a description, input and expected output after $optname"
fi
opts=
- if [ "x$1" = "x--printf" ]; then
- opts=$1
+ if [ "x$1" = "x--printf" ] || [ "$print_cmd" = "printf" ]; then
+ opts=--printf
shift
fi
for codefile in $codefiles; do
descrfile=`basename $codefile .pmatch`.descr
- descr="`cat $descrfile`"
+ descr="`catf $descrfile`"
if [ "x$descr" = "x" ]; then
descr=$1
elif [ "x$1" != "x" ]; then
@@ -687,8 +825,6 @@ check_compile_run_multi () {
fi
check_with_file $opts $codefile "$descr" "$2" "$3"
done
- subtestcount=$(($subtestcount + $compile_failed_count))
- subfailcount=$(($subfailcount + $compile_failed_count))
shift
shift
shift
@@ -698,7 +834,7 @@ check_compile_run_multi () {
if [ "$#" -lt 1 ]; then
error "Expected an input string after --input"
fi
- common_input=$1
+ common_input="$1"
shift
;;
--* )
@@ -718,6 +854,14 @@ check_compile_run_multi () {
}
check_compile_run () {
+ if [ "x$skip_test" != "x" ]; then
+ return 0
+ fi
+ _prev_print_cmd_check_compile_run=$print_cmd
+ if [ "x$1" = "x--printf" ]; then
+ print_cmd=$printf
+ shift
+ fi
case "$1" in
--* )
check_compile_run_multi "$@"
@@ -726,4 +870,5 @@ check_compile_run () {
check_compile_run_single "$@"
;;
esac
+ print_cmd=$_prev_print_cmd_check_compile_run
}
diff --git a/test/tools/pmatch-tests.sh b/test/tools/pmatch-tests.sh
index b04da4c..b4ef178 100755
--- a/test/tools/pmatch-tests.sh
+++ b/test/tools/pmatch-tests.sh
@@ -23,17 +23,17 @@ test_begin "Compilation"
check_compile "Commented-out definition" \
'! Define foo bar;
-Define TOP foo;'
+Define TOP "foo";'
# Bug (reported 2013-01-22)
check_compile 'Comment at the end of file' \
- 'Define TOP foo;
+ 'Define TOP "foo";
! comment'
check_compile 'Semicolon in the middle of a comment' \
'! foo; bar
-Define TOP foo;'
+Define TOP "foo";'
# Bug/feature (reported 2013-01-22)
@@ -47,7 +47,7 @@ check_compile_run 'Define name containing a 0' \
test_begin "String literal memory allocation"
for s in "xxxxxxxxxxxxxxxxxxxxxxxx" "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; do
- check_compile "String literal $s" 'Define TOP "'$s'";'
+ check_compile "String literal $s" 'Define TOP {'$s'};'
done
@@ -58,7 +58,7 @@ test_begin 'Semicolons'
check_compile_run \
--codetempl 'Define TOP @1@ EndTag(X);' \
--descrtempl 'Semicolon to be marked: @1@' \
- --templarg-single '[";"]' '(";")' '";"' '"\;"' '%;'\
+ --templarg-single '[{;}]' '({;})' '{;}' '";"' '%;'\
--inout '' 'a;b' 'a<X>;</X>b'
@@ -80,7 +80,7 @@ test_begin "Referring to a double quote in a regexp"
check_compile_run \
--codetempl 'Define TOP @1@ EndTag(Q);' \
--descrtempl 'Double quote as @1@' \
- --templarg-single '"\x22"' '"\""' '%"' \
+ --templarg-single '"\x22"' '"\""' '{"}' '%"' \
--inout '' 'a "b" c' 'a <Q>"</Q>b<Q>"</Q> c'
@@ -90,7 +90,7 @@ check_compile_run \
test_begin "RC() with no character between the recognized string and context"
check_compile_run 'RC() separating space' \
- 'Define TOP "a" RC("b") EndTag(AB);' 'aba b' '<AB>a</AB>ba b'
+ 'Define TOP {a} RC({b}) EndTag(AB);' 'aba b' '<AB>a</AB>ba b'
# Bug (reported 2013-05-30): Ins() with a name containing "name" does
@@ -111,33 +111,36 @@ Define TOP [ Ins(@1@) ] ;' \
test_begin "Different tag in different contexts"
check_compile_run '' \
- 'Define TOP "a" [[RC("b") EndTag(AB)] | [RC("c") EndTag(AC)]];' \
+ 'Define TOP {a} [[RC({b}) EndTag(AB)] | [RC({c}) EndTag(AC)]];' \
'ab ac' '<AB>a</AB>b <AC>a</AC>c'
-# Bug? (reported 2013-01-15)
+# Not a bug (reported 2013-01-15; Sam clarified 2013-08-16)
test_begin "Multiple EndTags in a single define"
check_compile_run \
--code 'EndTag, string, EndTag' \
- 'Define TOP ["a" EndTag(A)] "bcd" ["e" EndTag(B)];' \
+ 'Define TOP Define( [{a} EndTag(A)] ) {bcd} Define( [{e} EndTag(B)] );' \
+ --code 'EndTag+RC, any, EndTag' \
+ 'Define TOP Define( [{a} EndTag(A) RC({bcd})] ) ?* Define( [{e} EndTag(B)] );' \
+ --code 'EndTag, any, LC+EndTag' \
+ 'Define TOP Define( [{a} EndTag(A)] ) ?* Define( [{e} EndTag(B) LC({bcd})] );' \
+ --code 'EndTag+RC, any, LC+EndTag' \
+ 'Define TOP Define( [{a} EndTag(A) RC({bc})] ) ?* Define( [{e} EndTag(B) LC({d})] );' \
+ --code 'EndTag+RC, any, string, any, LC+EndTag' \
+ 'Define TOP Define( [{a} EndTag(A) RC({b})] ) ?* {c} ?* Define( [{e} EndTag(B) LC({d})] );' \
+ --code 'EndTag+RC, any, string, EndTag' \
+ 'Define TOP Define( [{a} EndTag(A) RC({b})] ) ?* {cd} Define( [{e} EndTag(B)] );' \
+ --code 'EndTag, string, any, EndTag+LC' \
+ 'Define TOP Define( [{a} EndTag(A)] ) {bc} ?* Define( [{e} EndTag(B) LC({d})] );' \
+ --code 'EndTag+RC, any, EndTag+LC, RC+LC overlapping' \
+ 'Define TOP Define( [{a} EndTag(A) RC({bc})] ) ?* Define( [{e} EndTag(B) LC({cd})] );' \
+ --code 'EndTag+RC, any, EndTag+LC, RC+LC equal coverage' \
+ 'Define TOP Define( [{a} EndTag(A) RC({bcd})] ) ?* Define( [{e} EndTag(B) LC({bcd})] );' \
--inout 'No matches' 'ab' 'ab' \
--inout 'Two matches' 'abcde' '<A>a</A>bcd<B>e</B>'
-# --code 'EndTag+RC, EndTag' \
-# 'Define TOP ["a" EndTag(A) RC("bcd")] ["e" EndTag(B)];' \
-# --code 'EndTag, LC+EndTag' \
-# 'Define TOP ["a" EndTag(A)] ["e" EndTag(B) LC("bcd")];' \
-# --code 'EndTag+RC, LC+EndTag' \
-# 'Define TOP ["a" EndTag(A) RC("bc")] ["e" EndTag(B) LC("d")];' \
-# --code 'EndTag+RC, string, LC+EndTag' \
-# 'Define TOP ["a" EndTag(A) RC("b")] "c" ["e" EndTag(B) LC("d")];' \
-# --code 'EndTag+RC, string, EndTag' \
-# 'Define TOP ["a" EndTag(A) RC("b")] "cd" ["e" EndTag(B)];' \
-# --code 'EndTag, string, EndTag+LC' \
-# 'Define TOP ["a" EndTag(A)] "bc" ["e" EndTag(B) LC("d")];' \
-
# Bug? (reported 2013-01-15)
test_begin "Removing tags by using transductions"
@@ -146,13 +149,13 @@ in='X <A>y</A>'
out='<X>X y</X>'
check_compile_run \
- --codetempl 'Define TOP "X " ["<A>" -> 0] @1 at + ["</A>" -> 0] EndTag(X);' \
+ --codetempl 'Define TOP {X } [{<A>} -> 0] @1 at + [{</A>} -> 0] EndTag(X);' \
--descrtempl 'Remove tags (@1@)' \
--templarg-single Alpha LowercaseAlpha \
--inout '' "$in" "$out"
check_compile_run 'Remove tags only if they exist (Alpha)' \
- 'Define TOP "X " ["<A>" .o. ["<A>" -> 0]] Alpha+ ["</A>" .o. ["</A>" -> 0]] EndTag(X);' \
+ 'Define TOP {X } [{<A>} .o. [{<A>} -> 0]] Alpha+ [{</A>} .o. [{</A>} -> 0]] EndTag(X);' \
"$in" "$out"
@@ -163,18 +166,18 @@ test_begin "Order of contexts and EndTag"
check_compile_run \
--codetempl 'Define TOP [ Alpha+ @1@ ] ;' \
--templarg-single \
- 'Simple, LC before EndTag' 'LC(" ") EndTag(A)' \
- 'Simple, LC after EndTag' 'EndTag(A) LC(" ")' \
+ 'Simple, LC before EndTag' 'LC({ }) EndTag(A)' \
+ 'Simple, LC after EndTag' 'EndTag(A) LC({ })' \
--inout 'No matches' 'abc' 'abc' \
--inout 'Single match' 'a abc' 'a <A>abc</A>'
check_compile_run \
- --codetempl 'Define T1 [ Num [" " Num]* @1@ EndTag(A) @2@ ] ;
-Define T2 [ "xx" [" " Alpha+]+ EndTag(A) @3@ ] ;
-Define TOP [ [T1 | T2] LC(" ") ] ;' \
- --templargs 'Two-level, LC before EndTag' 'LC(" ")' '' '' \
- --templargs 'Two-level, LC after EndTag' '' 'LC(" ")' '' \
- --templargs 'Two-level, same context in TOP' '' 'LC(" ")' 'LC(" ")' \
+ --codetempl 'Define T1 [ Num [{ } Num]* @1@ EndTag(A) @2@ ] ;
+Define T2 [ {xx} [{ } Alpha+]+ EndTag(A) @3@ ] ;
+Define TOP [ [T1 | T2] LC({ }) ] ;' \
+ --templargs 'Two-level, LC before EndTag' 'LC({ })' '' '' \
+ --templargs 'Two-level, LC after EndTag' '' 'LC({ })' '' \
+ --templargs 'Two-level, same context in TOP' '' 'LC({ })' 'LC({ })' \
--templargs 'Context only in TOP' '' '' '' \
--inout 'No matches' 'a abc def' 'a abc def' \
--inout 'First rule matches' '1 2 3' '1 <A>2 3</A>' \
@@ -183,8 +186,8 @@ Define TOP [ [T1 | T2] LC(" ") ] ;' \
check_compile_run \
--codetempl 'Define T [ Alpha+ EndTag(A) ] ;
Define TOP [ T | [T @1@] ] ;' \
- --templargs 'LC before EndTag' 'LC(" ") EndTag(B)' \
- --templargs 'LC after EndTag' 'EndTag(B) LC(" ")' \
+ --templargs 'LC before EndTag' 'LC({ }) EndTag(B)' \
+ --templargs 'LC after EndTag' 'EndTag(B) LC({ })' \
--inout 'Three-character match' 'abc' '<A>abc</A>' \
--inout 'Single-character match' 'a' '<A>a</A>' \
--inout 'Space-separated matches' 'ab abc' '<A>ab</A> <B><A>abc</A></B>' \
@@ -198,10 +201,10 @@ test_begin "Variable-length right context"
check_compile_run \
--codetempl 'Define TOP [ Alpha+ @1@ ] ;' \
- --templargs 'RC before EndTag, star' 'RC(" " Alpha*) EndTag(A)' \
- --templargs 'RC before EndTag, plus' 'RC(" " Alpha+) EndTag(A)' \
- --templargs 'RC after EndTag, star' 'EndTag(A) RC(" " Alpha*)' \
- --templargs 'RC after EndTag, plus' 'EndTag(A) RC(" " Alpha+)' \
+ --templargs 'RC before EndTag, star' 'RC({ } Alpha*) EndTag(A)' \
+ --templargs 'RC before EndTag, plus' 'RC({ } Alpha+) EndTag(A)' \
+ --templargs 'RC after EndTag, star' 'EndTag(A) RC({ } Alpha*)' \
+ --templargs 'RC after EndTag, plus' 'EndTag(A) RC({ } Alpha+)' \
--inout 'Two matching words' 'abc def ghi' '<A>abc</A> <A>def</A> ghi' \
--inout 'One matching word' 'abc defghi' '<A>abc</A> defghi' \
@@ -211,26 +214,26 @@ check_compile_run \
test_begin "A context as an affix of another"
check_compile_run \
- --codetempl 'Define T1 [ "c" @1@ EndTag(A) ];
-Define T2 [ "d" LC("b") EndTag(A) ];
+ --codetempl 'Define T1 [ {c} @1@ EndTag(A) ];
+Define T2 [ {d} LC({b}) EndTag(A) ];
Define TOP [ T1 | T2 ];' \
- --templargs 'Same context (LC)' 'LC("b")' \
- --templargs 'Context as a suffix of another (LC)' 'LC("ab")' \
+ --templargs 'Same context (LC)' 'LC({b})' \
+ --templargs 'Context as a suffix of another (LC)' 'LC({ab})' \
--inout 'Longer context matches' 'abc' 'ab<A>c</A>' \
--inout 'Only short context matches' 'abd' 'ab<A>d</A>'
check_compile_run \
--code 'Non-affix contexts' \
-'Define T1 [ "c" LC("a") EndTag(A) ];
-Define T2 [ "d" LC("b") EndTag(A) ];
+'Define T1 [ {c} LC({a}) EndTag(A) ];
+Define T2 [ {d} LC({b}) EndTag(A) ];
Define TOP [ T1 | T2 ];' \
--inout 'First matches' 'bac' 'ba<A>c</A>' \
--inout 'Second matches' 'abd' 'ab<A>d</A>'
check_compile_run \
--code 'Context as a prefix of another (RC)' \
-'Define T1 [ "c" EndTag(A) RC("ba") ];
-Define T2 [ "d" EndTag(A) RC("b") ];
+'Define T1 [ {c} EndTag(A) RC({ba}) ];
+Define T2 [ {d} EndTag(A) RC({b}) ];
Define TOP [ T1 | T2 ];' \
--inout 'Longer context matches' 'cba' '<A>c</A>ba' \
--inout 'Only short context matches' 'dba' '<A>d</A>ba'
@@ -241,17 +244,17 @@ Define TOP [ T1 | T2 ];' \
test_begin "Fix of a bug causing segmentation faults"
check_compile_run \
- --code 'No errors' 'Define TOP "a" EndTag(A);' \
+ --code 'No errors' 'Define TOP {a} EndTag(A);' \
--inout 'Single character (ok)' 'a' '<A>a</A>' \
--inout 'Two characters, single match (ok)' 'ab' '<A>a</A>b'
check_compile_run \
- --code 'Errors with EndTag' 'Define TOP "a"+ EndTag(A);' \
+ --code 'Errors with EndTag' 'Define TOP {a}+ EndTag(A);' \
--inout 'Single non-match character (ok)' 'b' 'b' \
--inout 'Single character match (segfault)' 'a' '<A>a</A>'
check_compile_run \
- --code 'Errors without EndTag' 'Define TOP "a"+;' \
+ --code 'Errors without EndTag' 'Define TOP {a}+;' \
--inout 'Single character' 'b' 'b' \
--inout 'Single character (segfault)' 'a' 'a'
@@ -262,17 +265,18 @@ check_compile_run \
test_begin "Converting tags with replacement"
-code_preamble='Define AlphaToUpper [a:A|b:B|c:C|d:D|e:E|f:F|g:G|h:H|i:I|j:J|k:K|l:L|m:M|
- n:N|o:O|p:P|q:Q|r:R|s:S|t:T|u:U|v:V|w:W|x:X|y:Y|z:Z|
- UppercaseAlpha] ;
+code_preamble='Define AlphaToUpper
+ ["a":"A" | "b":"B" | "c":"C" | "d":"D" | "e":"E" | "f":"F" | "g":"G" |
+ "h":"H" | "i":"I" | "j":"J" | "k":"K" | "l":"L" | "m":"M" | "n":"N" |
+ "o":"O" | "p":"P" | "q":"Q" | "r":"R" | "s":"S" | "t":"T" | "u":"U" |
+ "v":"V" | "w":"W" | "x":"X" | "y":"Y" | "z":"Z" | UppercaseAlpha] ;
Define AlphaToUpper3 [AlphaToUpper AlphaToUpper AlphaToUpper] ;
-Define MainTagName ["ENAMEX" | "NUMEX" | "TIMEX"] ;
-Define ConvertStartTag ["<" MainTagName
- ["" -> " TYPE=" %"] AlphaToUpper3
- ["" -> %" " SBT=" %"]
-AlphaToUpper3
- ["" -> %"] ">"] ;
-Define ConvertEndTag ["</" MainTagName [Alpha+ .o. [Alpha+ -> ""]] ">"] ;'
+Define MainTagName [{ENAMEX} | {NUMEX} | {TIMEX}] ;
+Define ConvertStartTag [{<} MainTagName
+ [[] -> { TYPE=" }] AlphaToUpper3
+ [[] -> {" SBT="}] AlphaToUpper3
+ [[] -> {"}] {>}] ;
+Define ConvertEndTag [{</} MainTagName [Alpha+ .o. [Alpha+ -> []]] {>}] ;'
in='<ENAMEXPrsHum>Johan</ENAMEXPrsHum>'
@@ -300,13 +304,13 @@ Define TOP [ConvertStartTag | ConvertEndTag] ;" \
test_begin "Regex^n"
check_compile_run \
- --code 'Regex^n' 'Define TOP "a"^3 EndTag(A);' \
+ --code 'Regex^n' 'Define TOP {a}^3 EndTag(A);' \
--inout 'No match' 'a' 'a' \
--inout 'One match' 'aaa' '<A>aaa</A>' \
--inout 'Two matches' 'aaaaaa' '<A>aaa</A><A>aaa</A>'
check_compile_run \
- --codetempl 'Define TOP "a"@1@ EndTag(A);' \
+ --codetempl 'Define TOP {a}@1@ EndTag(A);' \
--descrtempl '@1@' \
--templarg-single '^2,4' '^{2,4}' \
--inout 'No match' 'a' 'a' \
@@ -317,13 +321,13 @@ check_compile_run \
--inout 'Two matches' 'aaaaaa' '<A>aaaa</A><A>aa</A>'
check_compile_run \
- --code 'Regex^<n' 'Define TOP "a"^<3 EndTag(A);' \
+ --code 'Regex^<n' 'Define TOP {a}^<3 EndTag(A);' \
--inout 'One match 1' 'a' '<A>a</A>' \
--inout 'One match 2' 'aa' '<A>aa</A>' \
--inout 'Two matches' 'aaa' '<A>aa</A><A>a</A>' \
check_compile_run \
- --code 'Regex^>n' 'Define TOP "a"^>3 EndTag(A);' \
+ --code 'Regex^>n' 'Define TOP {a}^>3 EndTag(A);' \
--inout 'No match 3' 'aaa' 'aaa' \
--inout 'Match 4' 'aaaa' '<A>aaaa</A>' \
--inout 'Match 5' 'aaaaa' '<A>aaaaa</A>'
@@ -337,23 +341,46 @@ test_begin "Character literal escapes"
# Pmatch
check_compile_run \
- --codetempl 'Define TOP ["@1@" EndTag(A)];' \
- --templarg-single '\xHH' '\x65' \
+ --codetempl 'Define TOP [@1@ EndTag(A)];' \
+ --descrtempl 'Matching @1@' \
+ --templarg-single '"\x65"' \
--inout '' 'ABCDEabcde' 'ABCDEabcd<A>e</A>'
check_compile_run \
- --codetempl 'Define TOP ["a" -> "@1@"];' \
- --templarg-single '\xHH' '\x65' \
+ --codetempl 'Define TOP [{a} -> @1@];' \
+ --descrtempl 'Replacement @1@' \
+ --templarg-single '"\x65"' \
--inout '' 'a' 'e'
+# Bug (reported 2014-10-03) and features
+
+test_begin "Backslash in string literals"
+
+# In string literals {...}, backslash escapes } and itself, otherwise
+# it is literal.
+
+# Use printf here, since the processing of \ varies between shells;
+# thus, double all backslashes to that should be literal for printf,
+# and add an explicit trailing newline (\n) to the expected output.
+check_compile_run \
+ --printf \
+ --codetempl 'Define TOP [{@1@} EndTag(A)];' \
+ --input '\t\\t\\x65\\}' \
+ --outtemplargs 'Backslash escapes }' '\t\\t\\x65\\<A>}</A>\n' '\\}' \
+ --outtemplargs 'Literal backslash: @1@' '\t<A>\\t</A>\\x65\\}\n' '\\t' \
+ --outtemplargs 'Literal backslash: @1@' '\t\\t<A>\\x65</A>\\}\n' '\\x65' \
+ --outtemplargs 'Literal backslash: @1@' '\t<A>\\</A>t<A>\\</A>x65<A>\\</A>}\n' '\\\\' \
+ --outtemplargs 'Literal backslash followed by }' '\t\\t\\x65<A>\\}</A>\n' '\\\\\\}'
+
+
# Bug (reported 2013-02-22)
test_begin "Input boundary"
check_compile_run \
--code 'Input boundary' \
- 'Define TOP [ # "a" | "b" # | ["c" LC(#)] | ["d" RC(#)] EndTag(A)];' \
+ 'Define TOP [ [# {a} | {b} # | [{c} LC(#)] | [{d} RC(#)]] EndTag(A) ];' \
--inout 'Beginning' 'a x' '<A>a</A> x' \
--inout 'End' 'x b' 'x <A>b</A>' \
--inout 'LC' 'c x' '<A>c</A> x' \
@@ -365,7 +392,7 @@ check_compile_run \
test_begin "Replace semantics"
check_compile_run \
- --codetempl 'Define TOP [ ["a"+ "b"+] @1@ "X" ];' \
+ --codetempl 'Define TOP [ [{a}+ {b}+] @1@ {X} ];' \
--input 'aabb' \
--outtemplargs 'Bare replacement ->' 'Xb' '->' \
--outtemplargs 'Leftmost longest replacement @->' 'X' '@->' \
@@ -379,7 +406,7 @@ check_compile_run \
test_begin "Add characters with replacement or transduction"
check_compile_run \
- --codetempl 'Define TOP [ "<" Alpha+ [@1@ -> ">"] ] ;' \
+ --codetempl 'Define TOP [ {<} Alpha+ [@1@ -> {>}] ] ;' \
--descrtempl 'Replacing @1@' \
--templarg-single '""' '[..]' \
--inout 'Non-match' '<1' '<1' \
@@ -391,7 +418,7 @@ check_compile_run \
--inout 'Consecutive occurrences' '<aaa<aaa<aaa' '<aaa><aaa><aaa>'
check_compile_run \
- --codetempl 'Define TOP [ "<" Alpha+ [@1@ -> ">"] "::" ] ;' \
+ --codetempl 'Define TOP [ {<} Alpha+ [@1@ -> {>}] {::} ] ;' \
--descrtempl 'Replacing @1@, characters after replacement' \
--templarg-single '""' '[..]' \
--inout 'Non-match prefix' '<1::' '<1::' \
@@ -402,30 +429,30 @@ check_compile_run \
--inout 'Three replacements, consecutive' '<aaa::<aaa::<aaa::' '<aaa>::<aaa>::<aaa>::'
check_compile_run \
- --codetempl 'Define TOP ["<" Alpha+ @2@ ">"] ;' \
+ --codetempl 'Define TOP [{<} Alpha+ @2@ {>}] ;' \
--descrtempl 'Single character after replacement, @1@' \
- --templargs 'replacing ""' '["" -> "x"]' \
- --templargs 'replacing [..]' '[[..] -> "x"]' \
- --templargs 'transduction from 0' '0:x' \
+ --templargs 'replacing ""' '["" -> {x}]' \
+ --templargs 'replacing [..]' '[[..] -> {x}]' \
+ --templargs 'transduction from 0' '0:"x"' \
--inout '' '<AB>B</AB>' '<ABx>B</AB>'
check_compile_run \
- --codetempl 'Define TOP [["<" Alpha+ @2@ ">"] | ["</" Alpha+ ">"]] ;' \
+ --codetempl 'Define TOP [[{<} Alpha+ @2@ {>}] | [{</} Alpha+ {>}]] ;' \
--descrtempl 'Single character after replacement, disjunction, @1@' \
- --templargs 'replacing ""' '["" -> "x"]' \
- --templargs 'replacing [..]' '[[..] -> "x"]' \
- --templargs 'transduction from 0' '0:x' \
+ --templargs 'replacing ""' '["" -> {x}]' \
+ --templargs 'replacing [..]' '[[..] -> {x}]' \
+ --templargs 'transduction from 0' '0:"x"' \
--inout '' '<AB>B</AB>' '<ABx>B</AB>'
check_compile_run \
- --code 'Transduction' 'Define TOP [ "<" Alpha+ 0:%> ] ;' \
+ --code 'Transduction' 'Define TOP [ {<} Alpha+ 0:">" ] ;' \
--inout 'Non-match' '<1' '<1' \
--inout 'Single 1-char occurrence' '<a' '<a>' \
--inout 'Single 3-char occurrence' '<aaa' '<aaa>' \
--inout 'Two 2-char occurrences' '<aa<aa' '<aa><aa>'
check_compile_run \
- --code 'Transduction, suffix required' 'Define TOP [ "<" Alpha+ 0:%> "::" ] ;' \
+ --code 'Transduction, suffix required' 'Define TOP [ {<} Alpha+ 0:">" {::} ] ;' \
--inout 'Non-match prefix' '<1' '<1' \
--inout 'No matching suffix' '<a' '<a' \
--inout 'Single 1-char occurrence' '<a::' '<a>::' \
@@ -440,8 +467,8 @@ test_begin "Transductions and replacements with EndTag"
check_compile_run \
--codetempl 'Define TOP [ Alpha+ @1@ EndTag(A) ] ;' \
- --templarg-single 'Transductions' '[1:X | 2:X | 3:X]+' \
- 'Composition and replacement' '[Num+ .o. [Num -> "X"]]' \
+ --templarg-single 'Transductions' '["1":"X" | "2":"X" | "3":"X"]+' \
+ 'Composition and replacement' '[Num+ .o. [Num -> {X}]]' \
--inout 'No match (Alpha)' 'abc' 'abc' \
--inout 'No match (Num)' '123' '123' \
--inout 'Single-char transduction' 'abc1' '<A>abcX</A>' \
@@ -451,7 +478,7 @@ check_compile_run \
check_compile_run \
--code 'Replacement without composition' \
- 'Define TOP [ Alpha+ [Num -> "X"]+ EndTag(A) ] ;' \
+ 'Define TOP [ Alpha+ [Num -> {X}]+ EndTag(A) ] ;' \
--inout 'Non-replacement only' 'abc' '<A>abc</A>' \
--inout 'Replacement only' '123' '123' \
--inout 'Single-char replacement' 'abc1' '<A>abcX</A>' \
@@ -461,7 +488,7 @@ check_compile_run \
# CHECK: The following outputs are as Pmatch outputs but are they
# correct?
check_compile_run \
- --codetempl 'Define TOP [ [Num -> "X"]@1@ EndTag(A) ] ;' \
+ --codetempl 'Define TOP [ [Num -> {X}]@1@ EndTag(A) ] ;' \
--templargs 'Only replacement in EndTag' '' \
--templargs 'Only replacement in EndTag, with +' '+' \
--inout 'Non-match' 'abc' 'abc' \
@@ -478,17 +505,17 @@ test_begin "Transductions and replacements in EndTag context conditions (unsure
check_compile_run \
--codetempl 'Define TOP [ Alpha+ LC(@1@) EndTag(A) ] ;' \
--templargs 'Composition and replacement in LC, plus outside' \
- '[Num .o. [Num -> "X"]]+' \
+ '[Num .o. [Num -> {X}]]+' \
--templargs 'Composition and replacement in LC, plus inside' \
- '[Num+ .o. [Num -> "X"]]' \
+ '[Num+ .o. [Num -> {X}]]' \
--templargs 'Composition and transductions in LC, plus outside' \
- '[Num .o. [1:X | 2:X | 3:X | 4:X]]+' \
+ '[Num .o. ["1":"X" | "2":"X" | "3":"X" | "4":"X"]]+' \
--templargs 'Composition and transductions in LC, plus inside' \
- '[Num+ .o. [1:X | 2:X | 3:X | 4:X]]' \
+ '[Num+ .o. ["1":"X" | "2":"X" | "3":"X" | "4":"X"]]' \
--templargs 'Transductions without composition in LC, plus outside' \
- '[1:X | 2:X | 3:X | 4:X]+' \
+ '["1":"X" | "2":"X" | "3":"X" | "4":"X"]+' \
--templargs 'Transductions without composition in LC, no plus' \
- '[1:X | 2:X | 3:X | 4:X]' \
+ '["1":"X" | "2":"X" | "3":"X" | "4":"X"]' \
--inout 'No context match' 'abc' 'abc' \
--inout '1-char context' '1abc' '1<A>abc</A>' \
--inout '2-char context' '12abc' '12<A>abc</A>' \
@@ -498,9 +525,9 @@ check_compile_run \
check_compile_run \
--codetempl 'Define TOP [ Alpha+ LC(@1@) EndTag(A) ] ;' \
--templargs 'Replacement without composition in LC, plus outside' \
- '[Num -> "X"]+' \
+ '[Num -> {X}]+' \
--templargs 'Replacement without composition in LC, no plus' \
- '[Num -> "X"]' \
+ '[Num -> {X}]' \
--inout 'No context match' 'abc' '<A>abc</A>' \
--inout '1-char context' '1abc' '1<A>abc</A>' \
--inout '2-char context' '12abc' '12<A>abc</A>' \
@@ -510,17 +537,17 @@ check_compile_run \
check_compile_run \
--codetempl 'Define TOP [ Alpha+ RC(@1@) EndTag(A) ] ;' \
--templargs 'Composition and replacement in RC, plus outside' \
- '[Num .o. [Num -> "X"]]+' \
+ '[Num .o. [Num -> {X}]]+' \
--templargs 'Composition and replacement in RC, plus inside' \
- '[Num+ .o. [Num -> "X"]]' \
+ '[Num+ .o. [Num -> {X}]]' \
--templargs 'Composition and transductions in RC, plus outside' \
- '[Num .o. [1:X | 2:X | 3:X | 4:X]]+' \
+ '[Num .o. ["1":"X" | "2":"X" | "3":"X" | "4":"X"]]+' \
--templargs 'Composition and transductions in RC, plus inside' \
- '[Num+ .o. [1:X | 2:X | 3:X | 4:X]]' \
+ '[Num+ .o. ["1":"X" | "2":"X" | "3":"X" | "4":"X"]]' \
--templargs 'Transductions without composition in RC, plus outside' \
- '[1:X | 2:X | 3:X | 4:X]+' \
+ '["1":"X" | "2":"X" | "3":"X" | "4":"X"]+' \
--templargs 'Transductions without composition in RC, no plus' \
- '[1:X | 2:X | 3:X | 4:X]' \
+ '["1":"X" | "2":"X" | "3":"X" | "4":"X"]' \
--inout 'No context match' 'abc' 'abc' \
--inout '1-char context' 'abc1' '<A>abc</A>1' \
--inout '2-char context' 'abc12' '<A>abc</A>12' \
@@ -530,9 +557,9 @@ check_compile_run \
check_compile_run \
--codetempl 'Define TOP [ Alpha+ RC(@1@) EndTag(A) ] ;' \
--templargs 'Replacement without composition in RC, plus outside' \
- '[Num -> "X"]+' \
+ '[Num -> {X}]+' \
--templargs 'Replacement without composition in RC, no plus' \
- '[Num -> "X"]' \
+ '[Num -> {X}]' \
--inout 'No context match' 'abc' '<A>abc</A>' \
--inout '1-char context' 'abc1' '<A>abc</A>1' \
--inout '2-char context' 'abc12' '<A>abc</A>12' \
@@ -597,8 +624,8 @@ LC(Whitespace);' \
check_compile_run \
--codetempl 'Define TOP [ [UppercaseAlpha+ @1@] | [LowercaseAlpha+ @2@] ]
LC(Whitespace);' \
- --templargs 'Different LC before second EndTag' "$eta" "LC(\".\") $etb" \
- --templargs 'Different LC after second EndTag' "$eta" "$etb LC(\".\")" \
+ --templargs 'Different LC before second EndTag' "$eta" "LC({.}) $etb" \
+ --templargs 'Different LC after second EndTag' "$eta" "$etb LC({.})" \
--inout '' 'A BC de .' 'A <A>BC</A> de .'
check_compile_run \
@@ -616,8 +643,8 @@ RC(Whitespace);' \
check_compile_run \
--codetempl 'Define TOP [ [UppercaseAlpha+ @1@] | [LowercaseAlpha+ @2@] ]
RC(Whitespace);' \
- --templargs 'Different RC before second EndTag' "$eta" "RC(\".\") $etb" \
- --templargs 'Different RC after second EndTag' "$eta" "$etb RC(\".\")" \
+ --templargs 'Different RC before second EndTag' "$eta" "RC({.}) $etb" \
+ --templargs 'Different RC after second EndTag' "$eta" "$etb RC({.})" \
--inout '' 'A BC de .' '<A>A</A> <A>BC</A> de .'
check_compile_run \
@@ -635,15 +662,15 @@ RC(Whitespace);' \
# Bug (reported 2013-04-15)
-test_begin "Handling trailing newline on the last line of input"
+test_begin "The last line of input without a trailing newline"
check_compile_run \
--code '' 'Define TOP [ Alpha+ EndTag(A) ];' \
--inout 'Trailing newline in input' '1abc2' '1<A>abc</A>2' \
--inout --printf 'No trailing newline in input, match ends before end of line' \
- '1abc2' '1<A>abc</A>2' \
+ '1abc2' '1<A>abc</A>2\n' \
--inout --printf 'No trailing newline in input, match ends at end of line' \
- '1abc' '1<A>abc</A>'
+ '1abc' '1<A>abc</A>\n'
# Bug/feature (reported 2013-05-30)
@@ -651,7 +678,7 @@ check_compile_run \
test_begin "OptCap, ToUpper, ToLower"
check_compile_run \
- --code 'OptCap' 'Define TOP OptCap(["a"|"ä"|"š"| LowercaseAlpha LowercaseAlpha+]) EndTag(A) ;' \
+ --code 'OptCap' 'Define TOP OptCap([{a}|{ä}|{š}| LowercaseAlpha LowercaseAlpha+]) EndTag(A) ;' \
--inout 'ASCII' 'aA' '<A>a</A><A>A</A>' \
--inout 'Multi-letter ASCII word' \
'.abc.Abc.ABC.' '.<A>abc</A>.<A>Abc</A>.<A>A</A>BC.' \
@@ -659,7 +686,7 @@ check_compile_run \
--inout 'Non-Latin 1' 'šŠ' '<A>š</A><A>Š</A>' \
check_compile_run \
- --code 'ToUpper' 'Define TOP ToUpper(["a"|"ä"|"š"| LowercaseAlpha LowercaseAlpha+]) EndTag(A) ;' \
+ --code 'ToUpper' 'Define TOP ToUpper([{a}|{ä}|{š}| LowercaseAlpha LowercaseAlpha+]) EndTag(A) ;' \
--inout 'ASCII' 'aA' 'a<A>A</A>' \
--inout 'Multi-letter ASCII word' \
'.abc.Abc.ABC.' '.abc.<A>A</A>bc.<A>ABC</A>.' \
@@ -667,7 +694,7 @@ check_compile_run \
--inout 'Non-Latin 1' 'šŠ' 'š<A>Š</A>'
check_compile_run \
- --code 'ToLower' 'Define TOP ToLower(["A"|"Ä"|"Š"| UppercaseAlpha UppercaseAlpha+]) EndTag(A) ;' \
+ --code 'ToLower' 'Define TOP ToLower([{A}|{Ä}|{Š}| UppercaseAlpha UppercaseAlpha+]) EndTag(A) ;' \
--inout 'ASCII' 'aA' '<A>a</A>A' \
--inout 'Multi-letter ASCII word' \
'.abc.Abc.ABC.' '.<A>abc</A>.A<A>bc</A>.ABC.' \
@@ -679,8 +706,8 @@ check_fn_singlechar () {
shift
check_compile_run \
--codetempl "Define TOP $fn([@1@]) EndTag(A);" \
- --templargs "$fn uppercase single-char pattern" '"B"' \
- --templargs "$fn lowercase single-char pattern" '"b"' \
+ --templargs "$fn uppercase single-char pattern" '{B}' \
+ --templargs "$fn lowercase single-char pattern" '{b}' \
--templargs "$fn single LowercaseAlpha" 'LowercaseAlpha' \
--templargs "$fn single UppercaseAlpha" 'UppercaseAlpha' \
--templargs "$fn single Alpha" 'Alpha' \
@@ -692,14 +719,14 @@ check_fn_multichar () {
shift
check_compile_run \
--codetempl "Define TOP $fn([@1@]) EndTag(A);" \
- --templargs "$fn uppercase multi-char pattern" '"BCDE"' \
- --templargs "$fn lowercase multi-char pattern" '"bcde"' \
+ --templargs "$fn uppercase multi-char pattern" '{BCDE}' \
+ --templargs "$fn lowercase multi-char pattern" '{bcde}' \
--templargs "$fn multiple LowercaseAlpha" \
'LowercaseAlpha LowercaseAlpha+' \
--templargs "$fn multiple UppercaseAlpha" \
'UppercaseAlpha UppercaseAlpha+' \
--templargs "$fn multiple Alpha" 'Alpha Alpha+' \
- --templargs "$fn mixed-case pattern" '"bCdE"' \
+ --templargs "$fn mixed-case pattern" '{bCdE}' \
"$@"
}
@@ -708,9 +735,9 @@ check_fn_multiword () {
shift
check_compile_run \
--codetempl "Define TOP $fn([@1@]) EndTag(A);" \
- --templargs "$fn uppercase multi-word pattern" '"BCDE FGHI"' \
- --templargs "$fn lowercase multi-word pattern" '"bcde fghi"' \
- --templargs "$fn mixed-case pattern" '"bCdE fGhI"' \
+ --templargs "$fn uppercase multi-word pattern" '{BCDE FGHI}' \
+ --templargs "$fn lowercase multi-word pattern" '{bcde fghi}' \
+ --templargs "$fn mixed-case pattern" '{bCdE fGhI}' \
"$@"
}
@@ -719,7 +746,7 @@ check_fn_other () {
shift
check_compile_run \
--code "$fn other single-char pattern" \
- "Define TOP $fn(".") EndTag(A);" \
+ "Define TOP $fn({.}) EndTag(A);" \
--inout 'Uppercase input' 'B' 'B' \
--inout 'Lowercase input' 'b' 'b' \
--inout 'Other input' '.' '<A>.</A>' \
@@ -765,18 +792,18 @@ check_fn_other ToLower
check_compile_run \
--codetempl "Define TOP ToLower([@1@]) EndTag(A);" \
--input 'Bcde' \
- --outtemplargs "ToLower uppercase multi-char pattern, Capitalized input" 'Bcde' '"BCDE"' \
- --outtemplargs "ToLower lowercase multi-char pattern, Capitalized input" 'Bcde' '"bcde"' \
+ --outtemplargs "ToLower uppercase multi-char pattern, Capitalized input" 'Bcde' '{BCDE}' \
+ --outtemplargs "ToLower lowercase multi-char pattern, Capitalized input" 'Bcde' '{bcde}' \
--outtemplargs "ToLower multiple LowercaseAlpha, Capitalized input" 'B<A>cde</A>' \
'LowercaseAlpha LowercaseAlpha+' \
--outtemplargs "ToLower multiple UppercaseAlpha, Capitalized input" 'B<A>cde</A>' \
'UppercaseAlpha UppercaseAlpha+' \
--outtemplargs "ToLower multiple Alpha, Capitalized input" 'B<A>cde</A>' 'Alpha Alpha+' \
- --outtemplargs "ToLower capitalized multi-char pattern, Capitalized input" 'Bcde' '"Bcde"' \
- --outtemplargs "ToLower mixed-case pattern, Capitalized input" 'Bcde' '"bCdE"'
+ --outtemplargs "ToLower capitalized multi-char pattern, Capitalized input" 'Bcde' '{Bcde}' \
+ --outtemplargs "ToLower mixed-case pattern, Capitalized input" 'Bcde' '{bCdE}'
check_compile_run \
- --codetempl 'Define TOP @1@(["+"|"+ +"]) EndTag(X);' \
+ --codetempl 'Define TOP @1@([{+}|{+ +}]) EndTag(X);' \
--descrtempl '@1@ punct pattern' \
--templarg-single 'OptCap' 'ToUpper' 'ToLower' \
--inout 'Single char' '+' '<X>+</X>' \
@@ -784,7 +811,7 @@ check_compile_run \
check_compile_run \
--codetempl "Define TOP OptCap([@1@]) EndTag(A);" \
- --templargs "OptCap lowercase single-char pattern" '"b"' \
+ --templargs "OptCap lowercase single-char pattern" '{b}' \
--templargs "OptCap single LowercaseAlpha" 'LowercaseAlpha' \
--templargs "OptCap single Alpha" 'Alpha' \
--inout 'Uppercase input' 'B' '<A>B</A>' \
@@ -793,7 +820,7 @@ check_compile_run \
check_compile_run \
--codetempl "Define TOP OptCap([@1@]) EndTag(A);" \
- --templargs "OptCap uppercase single-char pattern" '"B"' \
+ --templargs "OptCap uppercase single-char pattern" '{B}' \
--templargs "OptCap single UppercaseAlpha" 'UppercaseAlpha' \
--inout 'Uppercase input' 'B' '<A>B</A>' \
--inout 'Lowercase input' 'b' '<A>b</A>' \
@@ -801,8 +828,8 @@ check_compile_run \
check_compile_run \
--codetempl "Define TOP OptCap([@1@]) EndTag(A);" \
- --templargs "OptCap lowercase multi-char pattern" '"bcde"' \
- --templargs "OptCap capitalized multi-char pattern" '"Bcde"' \
+ --templargs "OptCap lowercase multi-char pattern" '{bcde}' \
+ --templargs "OptCap capitalized multi-char pattern" '{Bcde}' \
--inout 'Lowercase input' 'bcde' '<A>bcde</A>' \
--inout 'Capitalized input' 'Bcde' '<A>Bcde</A>' \
--inout 'Uppercase input' 'BCDE' 'BCDE' \
@@ -820,7 +847,7 @@ check_compile_run \
check_compile_run \
--codetempl 'Define TOP OptCap([@1@]) EndTag(A);' \
- --templargs "OptCap uppercase multi-char pattern" '"BCDE"' \
+ --templargs "OptCap uppercase multi-char pattern" '{BCDE}' \
--templargs "OptCap multiple UppercaseAlpha" \
'UppercaseAlpha UppercaseAlpha+' \
--inout 'Uppercase input' 'BCDE' '<A>BCDE</A>' \
@@ -832,7 +859,7 @@ check_compile_run \
--codetempl 'Define TOP OptCap([@1@]) EndTag(A);' \
--input 'bCdE' \
--outtemplargs "OptCap uppercase multi-char pattern, Mixed-case input" \
- 'bCdE' '"BCDE"' \
+ 'bCdE' '{BCDE}' \
--outtemplargs "OptCap multiple UppercaseAlpha, Mixed-case input" \
'<A>bC</A><A>dE</A>' 'UppercaseAlpha UppercaseAlpha+'
@@ -845,7 +872,7 @@ check_compile_run \
--inout 'Other input' '++' '++'
check_compile_run \
- --code 'OptCap mixed case pattern' 'Define TOP OptCap(["bCdE"]) EndTag(A);' \
+ --code 'OptCap mixed case pattern' 'Define TOP OptCap([{bCdE}]) EndTag(A);' \
--inout 'Uppercase input' 'BCDE' 'BCDE' \
--inout 'Lowercase input' 'bcde' 'bcde' \
--inout 'Capitalized input' 'Bcde' 'Bcde' \
@@ -854,7 +881,7 @@ check_compile_run \
check_compile_run \
--code 'OptCap uppercase multi-word pattern' \
- 'Define TOP OptCap(["BCDE FGHI"]) EndTag(A);' \
+ 'Define TOP OptCap([{BCDE FGHI}]) EndTag(A);' \
--inout 'Uppercase input' 'BCDE FGHI' '<A>BCDE FGHI</A>' \
--inout 'Lowercase input' 'bcde fghi' 'bcde fghi' \
--inout 'Capitalized input' 'Bcde Fghi' 'Bcde Fghi' \
@@ -862,7 +889,7 @@ check_compile_run \
check_compile_run \
--code 'OptCap lowercase multi-word pattern' \
- 'Define TOP OptCap(["bcde fghi"]) EndTag(A);' \
+ 'Define TOP OptCap([{bcde fghi}]) EndTag(A);' \
--inout 'Uppercase input' 'BCDE FGHI' 'BCDE FGHI' \
--inout 'Lowercase input' 'bcde fghi' '<A>bcde fghi</A>' \
--inout 'Capitalized input' 'Bcde Fghi' '<A>Bcde Fghi</A>' \
@@ -870,7 +897,7 @@ check_compile_run \
check_compile_run \
--code 'OptCap capitalized multi-word pattern' \
- 'Define TOP OptCap(["Bcde Fghi"]) EndTag(A);' \
+ 'Define TOP OptCap([{Bcde Fghi}]) EndTag(A);' \
--inout 'Uppercase input' 'BCDE FGHI' 'BCDE FGHI' \
--inout 'Lowercase input' 'bcde fghi' 'bcde fghi' \
--inout 'Capitalized input' 'Bcde Fghi' '<A>Bcde Fghi</A>' \
@@ -878,8 +905,138 @@ check_compile_run \
check_compile_run \
--code 'OptCap mixed-case multi-word pattern' \
- 'Define TOP OptCap(["bCdE fGhI"]) EndTag(A);' \
+ 'Define TOP OptCap([{bCdE fGhI}]) EndTag(A);' \
--inout 'Uppercase input' 'BCDE FGHI' 'BCDE FGHI' \
--inout 'Lowercase input' 'bcde fghi' 'bcde fghi' \
--inout 'Capitalized input' 'Bcde Fghi' 'Bcde Fghi' \
--inout 'Mixed-case input' 'bCdE fGhI' '<A>bCdE fGhI</A>'
+
+
+# Feature (reported 2013-08-24)
+
+test_begin "Ins maximizing globally"
+
+check_compile_run \
+ --code 'Ins followed by a character contained in Ins expression' \
+ 'Define A [Alpha+]; Define TOP [Ins(A) {a} EndTag(A)];' \
+ --inout '' 'aa' 'aa'
+
+check_compile_run \
+ --code 'Ins followed by a character not contained in Ins expression' \
+ 'Define A [Alpha+]; Define TOP [Ins(A) {1} EndTag(A)];' \
+ --inout '' 'aa1' '<A>aa1</A>' \
+
+
+# Bug (reported 2013-08-24)
+
+test_begin "Difference and character sets in named expressions"
+
+check_compile_run \
+ --codetempl '@1@ Define TOP [[@2@ {a}] - {Ca}] EndTag(A);' \
+ --templargs 'Alpha as named expression' 'Define Any Alpha;' 'Any' \
+ --templargs 'Directly Alpha' '' 'Alpha' \
+ --templargs '[Alpha|Num] as named expression' \
+ 'Define Any [Alpha|Num];' 'Any' \
+ --templargs 'Directly [Alpha|Num]' '' '[Alpha|Num]' \
+ --inout '' 'Aa Ba Ca' '<A>Aa</A> <A>Ba</A> Ca'
+
+
+# Bug (reported 2013-09-03)
+
+test_begin "Named expressions in OptCap, ToUpper"
+
+check_compile_run \
+ --code 'Literals in OptCap' \
+ 'Define TOP [ OptCap([{aa} | {bb}]) ] EndTag(A);' \
+ --code 'Named expression in OptCap' \
+ 'Define AB [{aa} | {bb}]; Define TOP [ OptCap(AB) ] EndTag(A);' \
+ --code 'Named expression and literal in OptCap' \
+ 'Define A [{aa}]; Define TOP [ OptCap([A | {bb}]) ] EndTag(A);' \
+ --inout '' 'aa Aa AA bb Bb BB' \
+ '<A>aa</A> <A>Aa</A> AA <A>bb</A> <A>Bb</A> BB'
+
+check_compile_run \
+ --code 'Literals in ToUpper' \
+ 'Define TOP [ ToUpper([{aa} | {bb}]) ] EndTag(A);' \
+ --code 'Named expression in ToUpper' \
+ 'Define AB [{aa} | {bb}]; Define TOP [ ToUpper(AB) ] EndTag(A);' \
+ --code 'Named expression and literal in ToUpper' \
+ 'Define A [{aa}]; Define TOP [ ToUpper([A | {bb}]) ] EndTag(A);' \
+ --inout '' 'aa Aa AA bb Bb BB' 'aa Aa <A>AA</A> bb Bb <A>BB</A>'
+
+
+# Bug (reported 2013-09-03)
+
+test_begin "Named expressions in replace"
+
+check_compile_run \
+ --code 'Named expression in replace source' \
+ 'Define A {a}; Define TOP [A -> {b}];' \
+ --code 'Named expression in replace target' \
+ 'Define B {b}; Define TOP [{a} -> B];' \
+ --code 'Unnamed expressions in replace' 'Define TOP [{a} -> {b}];' \
+ --inout '' 'ab' 'bb'
+
+
+# Bug (reported 2013-09-10)
+
+test_begin "Long input lines"
+
+gen_input () {
+ perl -e 'print "'$1'" x '$2' . "\n"'
+}
+
+a9999=`gen_input a 9999`
+a10000=`gen_input a 10000`
+a10001=`gen_input a 10001`
+# The exact threshold for causing segfaults seemed to vary
+a58178=`gen_input a 58178`
+a58200=`gen_input a 58200`
+a100001=`gen_input a 100001`
+
+check_compile_run \
+ --code '' 'Define TOP [Alpha+ EndTag(A)];' \
+ --inout '9999 characters' "$a9999" "<A>$a9999</A>" \
+ --inout '10000 characters' "$a10000" "<A>$a10000</A>" \
+ --inout '10001 characters' "$a10001" "<A>$a10001</A>" \
+ --inout '58178 characters' "$a58178" "<A>$a58178</A>" \
+ --inout '58200 characters' "$a58200" "<A>$a58200</A>" \
+ --inout '100001 characters' "$a100001" "<A>$a100001</A>"
+
+
+# Bug (reported 2013-10-21)
+
+test_begin "Ins should not throw std::out_of_range"
+
+check_compile_run \
+ --code 'Four empty defines, Ins in TOP' \
+ 'Define A ""; Define B ""; Define C ""; Define D ""; Define TOP [ Ins(A) ];' \
+ --code 'Four defines; two Inses in TOP' \
+ 'Define C ""; Define CA ""; Define B {b}; Define D [ {d} EndTag(A) ]; Define TOP [ Ins(B) | Ins(D) ];' \
+ --inout '' 'aa' 'aa'
+
+
+# Bug(?) (reported 2013-09-22 by Juha)
+
+test_begin "Disjunction of two Ins expressions"
+
+check_compile_run \
+ --codetempl 'Define S { }; Define NS [? - S]; Define A [NS* {a} NS*]; Define B [NS* {b} NS*]; Define C {c}; Define TOP [[[@1@] S C]+ EndTag(AB)];' \
+ --templargs 'Without Ins' 'A | B' \
+ --templargs 'With Ins' 'Ins(A) | Ins(B)' \
+ --inout 'No match' 'k c' 'k c' \
+ --inout 'Match first disjunct' 'a c' '<AB>a c</AB>' \
+ --inout 'Match second disjunct' 'b c' '<AB>b c</AB>' \
+ --inout 'Match X + first disjunct' 'ka c' '<AB>ka c</AB>' \
+ --inout 'Match first disjunct + X' 'ak c' '<AB>ak c</AB>' \
+ --inout 'Match X + first disjunct + X' 'kak c' '<AB>kak c</AB>' \
+ --inout 'Match first disjunct * 2' 'aa c' '<AB>aa c</AB>' \
+ --inout 'Match second disjunct * 2' 'bb c' '<AB>bb c</AB>' \
+ --inout 'Match first + second disjunct' 'ab c' '<AB>ab c</AB>' \
+ --inout 'Match second + first disjunct' 'ba c' '<AB>ba c</AB>' \
+ --inout 'Match first + second + first disjucnt' 'aba c' '<AB>aba c</AB>' \
+ --inout 'Match second + first + second disjucnt' 'bab c' '<AB>bab c</AB>' \
+ --inout 'Match X + first + X + second disjunct' 'kakb c' '<AB>kakb c</AB>' \
+ --inout 'Multiple matches' \
+ 'k c a c b c ka c ak c kak c aa c bb c ab c ba c aba c bab c kakb c' \
+ 'k c <AB>a c</AB> <AB>b c</AB> <AB>ka c</AB> <AB>ak c</AB> <AB>kak c</AB> <AB>aa c</AB> <AB>bb c</AB> <AB>ab c</AB> <AB>ba c</AB> <AB>aba c</AB> <AB>bab c</AB> <AB>kakb c</AB>'
diff --git a/tools/src/HfstAlphabet.h b/tools/src/HfstAlphabet.h
index a1c3eb2..1187234 100644
--- a/tools/src/HfstAlphabet.h
+++ b/tools/src/HfstAlphabet.h
@@ -1,7 +1,21 @@
#ifndef _HFST_ALPHABET_H_
#define _HFST_ALPHABET_H_
-#include <ext/hash_map>
+#if HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#if HAVE_BACKWARD_HASH_MAP
+# include <backward/hash_map>
+#elif HAVE_EXT_HASH_MAP
+# include <ext/hash_map>
+#elif HAVE_HASH_MAP
+# include <hash_map>
+#else
+# warning "unknown hash_map"
+# include <hash_map>
+#endif
+
#include <set>
#include <vector>
#include <string.h>
@@ -19,7 +33,11 @@ namespace hfst {
public:
typedef std::pair<unsigned int,unsigned int> NumberPair;
// hash table used to map the codes back to the symbols
- typedef __gnu_cxx::hash_map<unsigned int, char*> CharMap;
+#ifdef __GNUC__
+ typedef __gnu_cxx::hash_map<unsigned int, char*> CharMap;
+#else
+ typedef std::hash_map<unsigned int, char*> CharMap;
+#endif
private:
// string comparison operators needed to stored strings in a hash table
@@ -30,7 +48,11 @@ namespace hfst {
};
// hash table used to map the symbols to their codes
+#ifdef __GNUC__
typedef __gnu_cxx::hash_map<const char*, unsigned int, __gnu_cxx::hash<const char*>,eqstr> SymbolMap;
+#else
+ typedef std::hash_map<const char*, unsigned int, std::hash<const char*>,eqstr> SymbolMap;
+#endif
// set of symbol pairs
typedef std::set<NumberPair> NumberPairSet;
diff --git a/tools/src/hfst-expand-equivalences.cc b/tools/src/hfst-expand-equivalences.cc
index 0023757..809d076 100644
--- a/tools/src/hfst-expand-equivalences.cc
+++ b/tools/src/hfst-expand-equivalences.cc
@@ -423,11 +423,12 @@ int main( int argc, char **argv )
#endif
hfst_set_program_name(argv[0], "0.1", "HfstExpandEquivalences");
int retval = parse_options(argc, argv);
- check_options(argc, argv);
if (retval != EXIT_CONTINUE)
{
return retval;
}
+ check_options(argc, argv);
+
// close buffers, we use streams
if (inputfile != stdin)
{
diff --git a/tools/src/hfst-guess.cc b/tools/src/hfst-guess.cc
index f9b05d6..c3fb85c 100644
--- a/tools/src/hfst-guess.cc
+++ b/tools/src/hfst-guess.cc
@@ -373,7 +373,7 @@ int main( int argc, char **argv )
delete instream;
if (outfile != stdout)
- { assert(out != std::cout); delete out; }
+ { assert(out != &std::cout); delete out; }
free(inputfilename);
free(outfilename);
diff --git a/tools/src/hfst-lexc-wrapper.cc b/tools/src/hfst-lexc-wrapper.cc
index fdf6e5f..0c61902 100644
--- a/tools/src/hfst-lexc-wrapper.cc
+++ b/tools/src/hfst-lexc-wrapper.cc
@@ -359,7 +359,7 @@ int main( int argc, char **argv ) {
free(outfilename);
if (tempfilename != 0)
{
- verbose_printf("Deleting temporary files on succesful exit\n");
+ verbose_printf("Deleting temporary files on successful exit\n");
hfst_remove(tempfilename);
}
return retval;
diff --git a/tools/src/hfst-optimized-lookup.cc b/tools/src/hfst-optimized-lookup.cc
index 6df07bf..73cc34d 100644
--- a/tools/src/hfst-optimized-lookup.cc
+++ b/tools/src/hfst-optimized-lookup.cc
@@ -385,7 +385,7 @@ void runTransducer (genericTransducer T)
while(std::cin.getline(str,MAX_IO_STRING))
{
// Carriage returns in Windows..
- unsigned int last_char_index = (unsigned int)(std::cin.gcount()) - 2;
+ unsigned int last_char_index = (unsigned int) (std::cin.gcount() > 2)? (std::cin.gcount()) - 2: 0;
if (str[last_char_index] == '\r')
str[last_char_index] = '\0';
diff --git a/tools/src/hfst-tagger/src/hfst-reweight-tagger.cc b/tools/src/hfst-tagger/src/hfst-reweight-tagger.cc
index 7d4e2ec..150005c 100644
--- a/tools/src/hfst-tagger/src/hfst-reweight-tagger.cc
+++ b/tools/src/hfst-tagger/src/hfst-reweight-tagger.cc
@@ -52,8 +52,8 @@ print_usage()
{
// c.f. http://www.gnu.org/prep/standards/standards.html#g_t_002d_002dhelp
fprintf(message_out, "Usage: %s [OPTIONS...] [INFILE]\n"
- "Reweight a tagger accoring to a configuration file\n"
- "hfst_tagger_config.\n"
+ "Reweight a tagger accoring to a configuration file\n"
+ "hfst_tagger_config.\n"
"\n", program_name);
print_common_program_options(message_out);
@@ -110,8 +110,8 @@ float reweight(float w)
{ return coeff*w; }
void reweight_fst(HfstInputStream &in,
- HfstOutputStream &out,
- float coefficient)
+ HfstOutputStream &out,
+ float coefficient)
{
HfstTransducer fst(in);
coeff = coefficient;
@@ -120,8 +120,8 @@ void reweight_fst(HfstInputStream &in,
}
int process_input_data(const std::string &tagger_file_name,
- const std::string &output_file_name,
- const FloatVector &coefficients)
+ const std::string &output_file_name,
+ const FloatVector &coefficients)
{
// Read sequence model fsts and reweight.
verbose_printf("Reading models and rewighting.");
@@ -138,7 +138,7 @@ int process_input_data(const std::string &tagger_file_name,
++it)
{
if (not seq_in.is_good())
- { error(EXIT_FAILURE, 0, "Config file has too many patterns."); }
+ { error(EXIT_FAILURE, 0, "Config file has too many patterns."); }
reweight_fst(seq_in, seq_out, *it);
}
@@ -155,7 +155,7 @@ int process_input_data(const std::string &tagger_file_name,
HfstTransducer lex(lex_in);
HfstOutputStream lex_out(output_file_name + ".lex",
- HFST_OLW_TYPE);
+ HFST_OLW_TYPE);
lex_out << lex;
return EXIT_SUCCESS;
@@ -196,12 +196,12 @@ FloatVector parse_coefficients(void)
std::getline(coeff_in,line);
if (line == "")
- { continue; }
+ { continue; }
float coefficient = get_coefficient(line);
if (coefficient == -1)
- { error(EXIT_FAILURE, 0, "Invalid configuration file."); }
+ { error(EXIT_FAILURE, 0, "Invalid configuration file."); }
coefficients.push_back(coefficient);
}
@@ -211,9 +211,14 @@ FloatVector parse_coefficients(void)
int main(int argc, char * argv[])
{
-
- parse_options(argc, argv);
+ hfst_set_program_name(argv[0], "0.1", "HfstReweightTagger");
+ int retval = parse_options(argc, argv);
+ if (retval != EXIT_CONTINUE)
+ {
+ return retval;
+ }
+
std::string tagger_file_name = inputfilename;
std::string output_file_name = outfilename;
diff --git a/tools/src/hfst-tagger/src/use_model_src/DataTypes.h b/tools/src/hfst-tagger/src/use_model_src/DataTypes.h
index 5735db3..57e3a61 100644
--- a/tools/src/hfst-tagger/src/use_model_src/DataTypes.h
+++ b/tools/src/hfst-tagger/src/use_model_src/DataTypes.h
@@ -5,7 +5,12 @@
# include <config.h>
#endif
-#include <tr1/unordered_map>
+#ifdef USE_TR1_UNORDERED_MAP
+ #include <tr1/unordered_map>
+#else
+ #include <unordered_map>
+#endif
+
#include <string>
#include <vector>
@@ -19,18 +24,33 @@ struct TransitionData
State target;
};
-typedef std::tr1::unordered_map<std::string,Symbol> Symbol2NumberMap;
+#ifdef USE_TR1_UNORDERED_MAP
+ typedef std::tr1::unordered_map<std::string,Symbol> Symbol2NumberMap;
+#else
+ typedef std::unordered_map<std::string,Symbol> Symbol2NumberMap;
+#endif
+
typedef std::vector<std::string> Number2SymbolMap;
typedef std::vector<Weight> StateFinalWeightMap;
-typedef std::tr1::unordered_map<Symbol,TransitionData> Symbol2TransitionDataMap;
+#ifdef USE_TR1_UNORDERED_MAP
+ typedef std::tr1::unordered_map<Symbol,TransitionData> Symbol2TransitionDataMap;
+#else
+ typedef std::unordered_map<Symbol,TransitionData> Symbol2TransitionDataMap;
+#endif
+
typedef std::vector<Symbol2TransitionDataMap> TransitionMap;
typedef std::pair<size_t,Weight> IdWeightPair;
-typedef std::tr1::unordered_map<std::string,IdWeightPair> StringWeightMap;
-typedef std::tr1::unordered_map<std::string,float> ProbabilityMap;
+#ifdef USE_TR1_UNORDERED_MAP
+ typedef std::tr1::unordered_map<std::string,IdWeightPair> StringWeightMap;
+ typedef std::tr1::unordered_map<std::string,float> ProbabilityMap;
+#else
+ typedef std::unordered_map<std::string,IdWeightPair> StringWeightMap;
+ typedef std::unordered_map<std::string,float> ProbabilityMap;
+#endif
#define DEFAULT_SYMBOL "<NONE>"
#define DEFAULT 1
diff --git a/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h b/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h
index fef5e56..397e589 100644
--- a/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h
+++ b/tools/src/hfst-tagger/src/use_model_src/NewLexicalModel.h
@@ -6,8 +6,19 @@
#endif
#include "HfstTransducer.h"
-#include <tr1/unordered_map>
-#include <tr1/unordered_set>
+
+#ifdef USE_TR1_UNORDERED_MAP
+ #include <tr1/unordered_map>
+#else
+ #include <unordered_map>
+#endif
+
+#ifdef USE_TR1_UNORDERED_SET
+ #include <tr1/unordered_set>
+#else
+ #include <unordered_set>
+#endif
+
#include <iostream>
#include "DataTypes.h"
@@ -24,14 +35,19 @@ class NewLexicalModel
{
public:
NewLexicalModel(const std::string &filename,
- std::istream * paradigm_guess_stream = 0);
+ std::istream * paradigm_guess_stream = 0);
const WeightedStringVector &operator[] (const std::string &word);
const WeightedStringVector &get_first_word_analysis(const std::string &word);
bool is_oov(const std::string &word);
bool is_lexicon_oov(const std::string &word);
private:
+#ifdef USE_TR1_UNORDERED_MAP
typedef std::tr1::unordered_map<std::string,WeightedStringVector>
AnalysisCache;
+#else
+ typedef std::unordered_map<std::string,WeightedStringVector>
+ AnalysisCache;
+#endif
AnalysisCache analysis_cache;
AnalysisCache upper_case_suffix_cache;
@@ -55,15 +71,20 @@ class NewLexicalModel
size_t id;
bool lexical_model_is_broken;
std::istream * paradigm_guess_stream;
+#ifdef USE_TR1_UNORDERED_SET
std::tr1::unordered_set<std::string> o_o_v_words;
std::tr1::unordered_set<std::string> lexicon_o_o_v_words;
+#else
+ std::unordered_set<std::string> o_o_v_words;
+ std::unordered_set<std::string> lexicon_o_o_v_words;
+#endif
void initialize_tag_probabilities(void);
const WeightedStringVector &cache_word_analyses(const std::string &word);
const WeightedStringVector &guess(const std::string &word,
- bool upper_case,
- const StringVector &rev_tokenized_word);
+ bool upper_case,
+ const StringVector &rev_tokenized_word);
const WeightedStringVector &cache_analyses
(const std::string &word,
HfstOneLevelPaths * paths,
@@ -77,14 +98,14 @@ class NewLexicalModel
bool upper_case);
void merge_analyses(const WeightedStringVector &suffix_analyses,
- WeightedStringVector &word_analyses,
- bool upper_case);
+ WeightedStringVector &word_analyses,
+ bool upper_case);
void bayesian_invert(WeightedStringVector &word_analyses,
- const std::string &suffix,
- bool upper_case);
+ const std::string &suffix,
+ bool upper_case);
size_t get_suffix_length(StringVector::const_iterator begin,
- StringVector::const_iterator end);
+ StringVector::const_iterator end);
const WeightedStringVector &cache_suffix_analyses
(const std::string &word,
const std::string &suffix,
@@ -95,13 +116,13 @@ class NewLexicalModel
float get_tag_penalty(const std::string &tag, bool upper_case);
std::string to_string(StringVector::const_iterator begin,
- StringVector::const_iterator end);
+ StringVector::const_iterator end);
float get_prob(float weight);
float get_penalty(float prob);
std::string to_lower_case(const std::string &word) const;
const WeightedStringVector ¶digm_guess(const std::string &word,
- const StringVector &guesses);
+ const StringVector &guesses);
StringVector split(const std::string &guesses);
float get_weight(const std::string &weight_string);
};
diff --git a/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc b/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc
index af917e7..cbcfccc 100644
--- a/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc
+++ b/tools/src/hfst-twolc/src/commandline_src/CommandLine.cc
@@ -191,7 +191,7 @@ int CommandLine::parse_options(int argc, char** argv)
else
{
std::cerr << "Unknown format \"" << optarg << "\"."
- << "Try runnning with option -h or --help."
+ << "Try running with option -h or --help."
<< std::endl;
exit(1);
}
diff --git a/tools/src/parsers/XfstCompiler.cc b/tools/src/parsers/XfstCompiler.cc
index 8d632bc..dae3c9e 100644
--- a/tools/src/parsers/XfstCompiler.cc
+++ b/tools/src/parsers/XfstCompiler.cc
@@ -68,10 +68,10 @@ extern void hxfst_delete_buffer(YY_BUFFER_STATE);
using hfst::implementations::HfstBasicTransducer;
using hfst::implementations::HfstBasicTransition;
-#define GET_TOP(x) HfstTransducer * x = this->top(); if ((x) == NULL) { return *this; }
+#define GET_TOP(x) HfstTransducer * x = this->top(); if ((x) == NULL) { xfst_lesser_fail(); return *this; }
#define PROMPT_AND_RETURN_THIS prompt(); return *this;
#define PRINT_INFO_PROMPT_AND_RETURN_THIS print_transducer_info(); prompt(); return *this;
-#define IF_NULL_PROMPT_AND_RETURN_THIS(x) if (x == NULL) { prompt(); return *this; }
+#define IF_NULL_PROMPT_AND_RETURN_THIS(x) if (x == NULL) { if(variables_["quit-on-fail"] == "ON") { exit(EXIT_FAILURE); } prompt(); return *this; }
#define MAYBE_MINIMIZE(x) if (variables_["minimal"] == "ON") { x->minimize(); }
#define MAYBE_ASSERT(assertion, value) if (!value && ((variables_["assert"] == "ON" || assertion) && (variables_["quit-on-fail"] == "ON"))) { exit(EXIT_FAILURE); }
#define MAYBE_QUIT if(variables_["quit-on-fail"] == "ON") { exit(EXIT_FAILURE); }
@@ -291,6 +291,15 @@ namespace xfst {
}
}
+ void XfstCompiler::xfst_lesser_fail()
+ {
+ if (variables_["quit-on-fail"] == "ON" && !read_interactive_text_from_stdin_)
+ {
+ exit(EXIT_FAILURE);
+ }
+ }
+
+
XfstCompiler&
XfstCompiler::add_prop_line(char* line)
{
@@ -565,6 +574,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
prompt();
return *this;
}
@@ -619,6 +629,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
prompt();
return *this;
}
@@ -667,6 +678,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
prompt();
return *this;
}
@@ -809,6 +821,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
prompt();
return *this;
}
@@ -1001,6 +1014,14 @@ namespace xfst {
XfstCompiler&
XfstCompiler::define_list(const char* name, const char* values)
{
+ if (definitions_.find(name) != definitions_.end())
+ {
+ fprintf(warnstream_, "Error: '%s' has already been defined as a transducer variable.\n"
+ "It cannot have an incompatible definition as a list.\n"
+ "Please undefine the definition first.\n", name);
+ MAYBE_QUIT;
+ PROMPT_AND_RETURN_THIS;
+ }
list<string> l;
char* p = strdup(values);
char* token = strtok(p, " ");
@@ -1053,6 +1074,16 @@ namespace xfst {
// When calling this function, the regex \a indata should already have
// been compiled into a transducer which should have been stored to
// the variable latest_regex_compiled.
+
+ if (lists_.find(name) != lists_.end())
+ {
+ fprintf(warnstream_, "Error: '%s' has already been defined as a list variable.\n"
+ "It cannot have an incompatible definition as a transducer.\n"
+ "Please undefine the variable first.\n", name);
+ MAYBE_QUIT;
+ PROMPT_AND_RETURN_THIS;
+ }
+
/*else*/ if (latest_regex_compiled != NULL)
{
bool was_defined = xre_.is_definition(name);
@@ -1504,6 +1535,7 @@ namespace xfst {
{
hfst_fprintf(warnstream_, "warning: loaded transducer definition "
"has no name, skipping it\n");
+ // xfst_lesser_fail(); ???
return *this;
}
std::map<std::string, HfstTransducer*>::const_iterator it
@@ -1512,6 +1544,7 @@ namespace xfst {
{
hfst_fprintf(warnstream_, "warning: a definition named '%s' "
"already exists, overwriting it\n", def_name.c_str());
+ // xfst_lesser_fail(); ???
definitions_.erase(def_name);
}
definitions_[def_name] = t;
@@ -1723,6 +1756,7 @@ namespace xfst {
else
{
hfst_fprintf(warnstream_, "no such variable: '%s'\n", name);
+ // xfst_lesser_fail(); ???
PROMPT_AND_RETURN_THIS;
}
}
@@ -1777,6 +1811,7 @@ namespace xfst {
if (variables_.find(name) == variables_.end())
{
hfst_fprintf(warnstream_, "no such variable: '%s'\n", name);
+ // xfst_lesser_fail(); ???
PROMPT_AND_RETURN_THIS;
}
char* num = static_cast<char*>(malloc(sizeof(char)*31));
@@ -1791,6 +1826,7 @@ namespace xfst {
if (variables_.find(name) == variables_.end())
{
hfst_fprintf(warnstream_, "no such variable: '%s'\n", name);
+ // xfst_lesser_fail(); ???
PROMPT_AND_RETURN_THIS;
}
hfst_fprintf(outstream_, "variable %s = %s\n", name, variables_[name].c_str());
@@ -1822,6 +1858,7 @@ namespace xfst {
{
hfst_fprintf(stderr, "Not enough networks on stack. "
"Operation requires at least 2.\n");
+ xfst_lesser_fail();
return *this;
}
HfstTransducer* first = stack_.top();
@@ -1849,6 +1886,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
prompt();
return NULL;
}
@@ -1988,6 +2026,7 @@ namespace xfst {
{
hfst_fprintf(stderr, "Not enough networks on stack. "
"Operation requires at least 2.\n");
+ xfst_lesser_fail();
PROMPT_AND_RETURN_THIS;
}
std::stack<HfstTransducer*> copied_stack(stack_);
@@ -3081,6 +3120,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
PROMPT_AND_RETURN_THIS;
}
std::stack<hfst::HfstTransducer*> reverse_stack;
@@ -3177,6 +3217,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
return *this;
}
@@ -3374,6 +3415,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
return *this;
}
PRINT_INFO_PROMPT_AND_RETURN_THIS;
@@ -3479,6 +3521,7 @@ namespace xfst {
{
hfst_fprintf(stderr, "Not enough networks on stack. "
"Operation requires at least 2.\n");
+ xfst_lesser_fail();
return *this;
}
HfstTransducer * result = stack_.top();
@@ -3526,12 +3569,15 @@ namespace xfst {
XfstCompiler&
XfstCompiler::apply_binary_operation_iteratively(BinaryOperation operation)
{
- if (stack_.size() < 1)
+ if (stack_.size() < 2)
{
- hfst_fprintf(stderr, "Empty stack.\n");
+ hfst_fprintf(stderr, "Not enough transducers on stack, operation requires at least 2.\n");
+ xfst_lesser_fail();
return *this;
}
HfstTransducer* result = stack_.top();
+ //HfstBasicTransducer fsm(*result);
+
stack_.pop();
while (!stack_.empty())
{
@@ -3539,6 +3585,12 @@ namespace xfst {
switch (operation)
{
case INTERSECT_NET:
+ /*{
+ HfstBasicTransducer basic(*t);
+ HfstBasicTransducer merge_tr = HfstBasicTransducer::merge(fsm, basic);
+ fprintf(stderr, "result of merge is:\n");
+ merge_tr.write_in_att_format(stderr);
+ }*/
result->intersect(*t);
break;
case IGNORE_NET:
@@ -3554,6 +3606,7 @@ namespace xfst {
{
hfst_fprintf(warnstream_, "Both composition arguments contain flag diacritics. "
"Set harmonize-flags ON to harmonize them.\n");
+ // xfst_lesser_fail(); ???
}
}
else
@@ -3571,6 +3624,7 @@ namespace xfst {
hfst_fprintf(outstream_, "Error: flag diacritics must be identities in composition if flag-is-epsilon is ON.\n"
"I.e. only FLAG:FLAG is allowed, not FLAG1:FLAG2, FLAG:bar or foo:FLAG\n"
"Apply twosided flag-diacritics (tfd) before composition.\n");
+ xfst_lesser_fail();
prompt();
return *this;
}
@@ -3686,6 +3740,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
return *this;
}
HfstTransducer* t = stack_.top();
@@ -3699,6 +3754,7 @@ namespace xfst {
if (stack_.size() < 1)
{
hfst_fprintf(stderr, "Empty stack.\n");
+ xfst_lesser_fail();
return *this;
}
@@ -4126,19 +4182,144 @@ namespace xfst {
ignore_history_after_index(ind);
PROMPT_AND_RETURN_THIS;
}
+
+ static HfstTransducer * contains_regexps(hfst::xre::XreCompiler & xre_)
+ {
+ HfstTransducer * not_bracket_star = xre_.compile("[? - \"^[\" - \"^]\"]* ;");
+ xre_.define("TempNotBracketStar", *not_bracket_star);
+ // all paths that contain one or more well-formed ^[ ^] expressions
+ HfstTransducer * well_formed = xre_.compile("TempNotBracketStar \"^[\" TempNotBracketStar [ \"^]\" TempNotBracketStar \"^[\" TempNotBracketStar ]* \"^]\" TempNotBracketStar ;");
+ xre_.undefine("TempNotBracketStar");
+ delete not_bracket_star;
+ return well_formed;
+ }
+
+ static bool is_well_formed_for_compile_replace(const HfstTransducer * t, hfst::xre::XreCompiler & xre_)
+ {
+ HfstTransducer * well_formed = contains_regexps(xre_);
+ // subtract those paths from copy of t
+ HfstTransducer tc(*t);
+ tc.subtract(*well_formed).minimize();
+ delete well_formed;
+ // all paths that contain one or more ^[ or ^]
+ HfstTransducer * brackets = xre_.compile("$[ \"^[\" | \"^]\" ] ;");
+
+ // test if the result is empty
+ tc.intersect(*brackets).minimize();
+ delete(brackets);
+ HfstTransducer empty(tc.get_type());
+ bool value = empty.compare(tc, false);
+ return value;
+ }
+
+ static std::string to_literal_regexp(const hfst::StringPairVector & path)
+ {
+ std::string pathstr("[ ");
+ for (hfst::StringPairVector::const_iterator it = path.begin(); it != path.end(); it++)
+ {
+ pathstr.append("\"").append(it->first).append("\" ");
+ }
+ pathstr.append("]");
+ return pathstr;
+ }
+
+ static std::string to_regexp(const hfst::StringPairVector & path)
+ {
+ std::string pathstr("[ ");
+ for (hfst::StringPairVector::const_iterator it = path.begin(); it != path.end(); it++)
+ {
+ pathstr.append(it->first).append(" ");
+ }
+ pathstr.append("]");
+ return pathstr;
+ }
+
XfstCompiler&
- XfstCompiler::compile_replace_upper_net()
+ XfstCompiler::compile_replace_net(Level level)
{
- hfst_fprintf(stderr, "missing compile_replace_upper net %s:%d\n", __FILE__, __LINE__);
+ assert(level != BOTH_LEVELS);
+ using hfst::implementations::HfstState;
+ using hfst::implementations::HfstReplacements;
+ using hfst::implementations::HfstReplacementsMap;
+
+ GET_TOP(tmp);
+ if (is_well_formed_for_compile_replace(tmp, xre_))
+ {
+ fprintf(stderr, "Network is well-formed.\n");
+ }
+ else
+ {
+ fprintf(stderr, "Network is not well-formed.\n");
+ }
+ HfstBasicTransducer fsm(*tmp);
+ try
+ {
+ HfstReplacementsMap replacement_map = fsm.find_replacements();
+
+ for (HfstReplacementsMap::const_iterator it = replacement_map.begin();
+ it != replacement_map.end(); it++)
+ {
+ HfstState start_state = it->first;
+ HfstReplacements replacements = it->second;
+ for (HfstReplacements::const_iterator rit = replacements.begin();
+ rit != replacements.end(); rit++)
+ {
+ HfstState end_state = rit->first;
+ std::string CPR(""); // Cross-Product Regexp
+ if (level == LOWER_LEVEL)
+ {
+ CPR = to_literal_regexp(rit->second) + std::string(" .x. ") + to_regexp(rit->second);
+ CPR = std::string("\"^[\":0") + std::string(" [") + CPR + std::string("] ") + std::string("\"^]\":0 ;");
+ }
+ else
+ {
+ CPR = to_regexp(rit->second) + std::string(" .x. ") + to_literal_regexp(rit->second);
+ CPR = std::string("0:\"^[\"") + std::string(" [") + CPR + std::string("] ") + std::string("0:\"^]\" ;");
+ }
+ char * cpr = strdup(CPR.c_str());
+ fprintf(stderr, "compiling replacement '%s'...\n", cpr);
+ HfstTransducer * replacement = xre_.compile(cpr);
+ assert(replacement != NULL); // todo
+ replacement->minimize();
+ HfstBasicTransducer repl(*replacement);
+ // DEBUG
+ std::cerr << "inserting transducer:" << std::endl << *replacement << std::endl << "between states " << start_state << " and " << end_state << "." << std::endl ;
+ delete replacement;
+ fsm.insert_transducer(start_state, end_state, repl);
+ }
+ }
+ }
+ catch(const char * msg)
+ {
+ fprintf(stderr, "compile_replace threw an error: '%s'\n", msg);
+ }
+ HfstTransducer * result = new HfstTransducer(fsm, format_);
+
+ std::cerr << "result from compile-replace is:" << std::endl << *result << std::endl;
+
+ HfstTransducer * cr = contains_regexps(xre_);
+ result->subtract(*cr).minimize();
+ delete cr;
+ stack_.pop();
+ delete tmp;
+ stack_.push(result);
+
PROMPT_AND_RETURN_THIS;
}
+
XfstCompiler&
XfstCompiler::compile_replace_lower_net()
{
- hfst_fprintf(stderr, "missing compile_replace_lower net %s:%d\n", __FILE__, __LINE__);
- PROMPT_AND_RETURN_THIS;
+ return compile_replace_net(LOWER_LEVEL);
+ }
+
+ XfstCompiler&
+ XfstCompiler::compile_replace_upper_net()
+ {
+ return compile_replace_net(UPPER_LEVEL);
}
+
XfstCompiler&
XfstCompiler::hfst(const char* text)
{
diff --git a/tools/src/parsers/XfstCompiler.h b/tools/src/parsers/XfstCompiler.h
index 5350b77..7495231 100644
--- a/tools/src/parsers/XfstCompiler.h
+++ b/tools/src/parsers/XfstCompiler.h
@@ -438,11 +438,12 @@ class XfstCompiler
XfstCompiler& inspect_net();
//! @brief Repeat 0..1 times
XfstCompiler& optional_net();
+
+ // internal function
+ XfstCompiler& compile_replace_net(Level level);
//! @brief Compile-replace lower
- //! @todo missing from HFST
XfstCompiler& compile_replace_lower_net();
//! @brief Compile-replace upper
- //! @todo missing from HFST
XfstCompiler& compile_replace_upper_net();
@@ -566,6 +567,10 @@ class XfstCompiler
//! else do nothing.
void xfst_fail();
+ //! @brief Exit with failure status if quit-on-fail is ON and hfst-xfst
+ //! is not used in interactive mode, else do nothing.
+ void xfst_lesser_fail();
+
//! @brief Print alphabet \a alpha to \a outfile. \a unknown and \a identity
//! define whether these symbols occur in the transitions of the transducer
//! whose alphabet we are printing.
diff --git a/tools/src/parsers/xfst-parser.yy b/tools/src/parsers/xfst-parser.yy
index 57dfcc3..619f4c4 100644
--- a/tools/src/parsers/xfst-parser.yy
+++ b/tools/src/parsers/xfst-parser.yy
@@ -199,12 +199,12 @@ COMMAND: ADD_PROPS REDIRECT_IN END_COMMAND {
free($2);
free($4);
}
- | DEFINE_LIST NAMETOKEN NAMETOKEN_LIST END_COMMAND {
+ | LIST NAMETOKEN NAMETOKEN_LIST SEMICOLON END_COMMAND {
hfst::xfst::xfst_->define_list($2, $3);
free($2);
free($3);
}
- | DEFINE_LIST NAMETOKEN RANGE END_COMMAND {
+ | LIST NAMETOKEN RANGE END_COMMAND {
hfst::xfst::xfst_->define_list($2, $3[0], $3[1]);
free($2);
free($3[0]);
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/hfst.git
More information about the debian-science-commits
mailing list